@aexol/spectral 0.9.182 → 0.9.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/memory/hooks/observer-trigger.d.ts.map +1 -1
- package/dist/memory/hooks/observer-trigger.js +11 -6
- package/dist/relay/client.d.ts +40 -4
- package/dist/relay/client.d.ts.map +1 -1
- package/dist/relay/client.js +351 -180
- package/dist/sdk/ai/cache-benchmark/benchmark.d.ts.map +1 -1
- package/dist/sdk/ai/cache-benchmark/benchmark.js +31 -7
- package/dist/sdk/ai/cache-benchmark/strategies.d.ts.map +1 -1
- package/dist/sdk/ai/cache-benchmark/strategies.js +29 -4
- package/dist/sdk/ai/cache-benchmark/types.d.ts +31 -0
- package/dist/sdk/ai/cache-benchmark/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"observer-trigger.d.ts","sourceRoot":"","sources":["../../../src/memory/hooks/observer-trigger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAWpE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"observer-trigger.d.ts","sourceRoot":"","sources":["../../../src/memory/hooks/observer-trigger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAWpE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAY7C,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CA8KjF"}
|
|
@@ -7,23 +7,28 @@ import { estimateEntryTokens, estimateStringTokens } from "../tokens.js";
|
|
|
7
7
|
import { OBSERVATION_CUSTOM_TYPE, reflectionToPromptLine } from "../types.js";
|
|
8
8
|
import { getProjectObsStore } from "../project-observations-store.js";
|
|
9
9
|
import { relayProjectObservations } from "../inter-agent-relay.js";
|
|
10
|
+
/** Write diagnostic lines only when structured debug logging is enabled. */
|
|
11
|
+
function debugStderr(runtime, message) {
|
|
12
|
+
if (runtime.config.debugLog === true)
|
|
13
|
+
process.stderr.write(message);
|
|
14
|
+
}
|
|
10
15
|
export function registerObserverTrigger(ext, runtime) {
|
|
11
16
|
ext.on("turn_end", (_event, ctx) => {
|
|
12
|
-
|
|
17
|
+
debugStderr(runtime, "[obs-mem] turn_end fired\n");
|
|
13
18
|
runtime.ensureConfig(ctx.cwd);
|
|
14
19
|
if (runtime.config.passive === true) {
|
|
15
|
-
|
|
20
|
+
debugStderr(runtime, "[obs-mem] passive=true → skipping\n");
|
|
16
21
|
return;
|
|
17
22
|
}
|
|
18
23
|
if (runtime.observerInFlight) {
|
|
19
|
-
|
|
24
|
+
debugStderr(runtime, "[obs-mem] observer already in flight → skipping\n");
|
|
20
25
|
return;
|
|
21
26
|
}
|
|
22
27
|
const entries = ctx.sessionManager.getBranch();
|
|
23
28
|
const tokens = rawTokensSinceLastBound(entries);
|
|
24
|
-
|
|
29
|
+
debugStderr(runtime, `[obs-mem] tokens since last bound: ${tokens} (threshold: ${runtime.config.observationThresholdTokens})\n`);
|
|
25
30
|
if (tokens < runtime.config.observationThresholdTokens) {
|
|
26
|
-
|
|
31
|
+
debugStderr(runtime, "[obs-mem] below threshold → skipping observer\n");
|
|
27
32
|
return;
|
|
28
33
|
}
|
|
29
34
|
const lastBoundIdx = lastObservationCoverEndIdx(entries);
|
|
@@ -60,7 +65,7 @@ export function registerObserverTrigger(ext, runtime) {
|
|
|
60
65
|
if (capIdx < chunkEntries.length) {
|
|
61
66
|
chunkEntries = chunkEntries.slice(0, capIdx);
|
|
62
67
|
effectiveCoversUpToId = chunkEntries[capIdx - 1]?.id ?? coversUpToId;
|
|
63
|
-
|
|
68
|
+
debugStderr(runtime, `[obs-mem] chunk capped at ~${accumulated.toLocaleString()} tokens ` +
|
|
64
69
|
`(${chunkEntries.length} source entries); remaining unobserved entries deferred\n`);
|
|
65
70
|
debugLog("observer.chunk_capped", {
|
|
66
71
|
originalCount: rawTailEntriesBetween(entries, coversFromId, coversUpToId).length,
|
package/dist/relay/client.d.ts
CHANGED
|
@@ -7,12 +7,16 @@
|
|
|
7
7
|
* machine JWT via `Authorization: Bearer <jwt>`.
|
|
8
8
|
* - Reply `{kind:"pong"}` to backend `{kind:"ping"}`. Backend closes
|
|
9
9
|
* `4408 heartbeat-timeout` if it doesn't hear from us within 90s; we
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* keep this app-level reply in addition to the protocol-level heartbeat
|
|
11
|
+
* below.
|
|
12
|
+
* - **Client heartbeat:** additionally emits a protocol-level `ws` ping
|
|
13
|
+
* every `HEARTBEAT_INTERVAL_MS` and requires a `pong` within
|
|
14
|
+
* `HEARTBEAT_TIMEOUT_MS`. This detects a half-open send path (e.g. after
|
|
15
|
+
* laptop sleep/wake) where `readyState` still reports `OPEN`.
|
|
12
16
|
* - **Watchdog timer:** tracks `lastActivityMs` on every received frame /
|
|
13
17
|
* ping. Every 15 s, if no activity within `WATCHDOG_MS` (120 s = 2×
|
|
14
|
-
* backend timeout),
|
|
15
|
-
*
|
|
18
|
+
* backend timeout), terminates the socket and triggers a reconnect.
|
|
19
|
+
* Detects silent backend death (Docker OOM
|
|
16
20
|
* kill, network partition without TCP RST) where the socket stays open
|
|
17
21
|
* but no data flows.
|
|
18
22
|
* - **Pre-reconnect health check:** before opening a new WS, does a quick
|
|
@@ -104,11 +108,19 @@ export declare class RelayClient extends EventEmitter {
|
|
|
104
108
|
private tightLoopWindowStart;
|
|
105
109
|
private reconnectTimer;
|
|
106
110
|
private sendQueue;
|
|
111
|
+
/** Wire currently waiting on a `ws.send` callback. Only one in flight to preserve FIFO. */
|
|
112
|
+
private sendInFlight;
|
|
113
|
+
private sendTimer;
|
|
107
114
|
/** Set true when the WS error handler sees an HTTP 401/403 upgrade rejection. */
|
|
108
115
|
private authFailed;
|
|
109
116
|
/** Timestamp of last received frame / ping — drives the watchdog. */
|
|
110
117
|
private lastActivityMs;
|
|
111
118
|
private watchdogTimer;
|
|
119
|
+
private heartbeatTimer;
|
|
120
|
+
private heartbeatPongTimer;
|
|
121
|
+
private heartbeatPending;
|
|
122
|
+
/** Guards the async window of `scheduleReconnect` (health check). */
|
|
123
|
+
private reconnectInFlight;
|
|
112
124
|
constructor(opts: RelayClientOptions);
|
|
113
125
|
/** Open the connection. Idempotent — calling twice is a no-op. */
|
|
114
126
|
connect(): void;
|
|
@@ -129,6 +141,30 @@ export declare class RelayClient extends EventEmitter {
|
|
|
129
141
|
private openSocket;
|
|
130
142
|
private startWatchdog;
|
|
131
143
|
private stopWatchdog;
|
|
144
|
+
private startHeartbeat;
|
|
145
|
+
private stopHeartbeat;
|
|
146
|
+
private enqueueWire;
|
|
147
|
+
private clearSendInFlight;
|
|
148
|
+
/**
|
|
149
|
+
* Drain the FIFO send queue over the current socket. Exactly one frame is
|
|
150
|
+
* in flight at a time so ordering is preserved. If `ws.send` never calls
|
|
151
|
+
* its callback (half-open socket), the frame stays at the front of the
|
|
152
|
+
* queue and the connection is force-reconnected; the next `open` flushes
|
|
153
|
+
* it again.
|
|
154
|
+
*/
|
|
155
|
+
private flushSendQueue;
|
|
156
|
+
private setTcpKeepAlive;
|
|
157
|
+
/**
|
|
158
|
+
* Tear down a suspected-dead socket and schedule a reconnect immediately.
|
|
159
|
+
* Unlike the `close` path, this does not wait for a (possibly never
|
|
160
|
+
* arriving) `close` event — `terminate()` destroys the socket directly.
|
|
161
|
+
*/
|
|
162
|
+
private forceReconnect;
|
|
163
|
+
/**
|
|
164
|
+
* Single disconnect entry point for both real `close` events and forced
|
|
165
|
+
* reconnects. Ignored for sockets that have already been replaced.
|
|
166
|
+
*/
|
|
167
|
+
private handleDisconnect;
|
|
132
168
|
/**
|
|
133
169
|
* Pre-reconnect health check: quick HTTP GET to `<backendUrl>/health`.
|
|
134
170
|
* Returns true when the backend is reachable (or when no backendUrl is
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/relay/client.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/relay/client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,SAAS,MAAM,IAAI,CAAC;AAoC3B,MAAM,WAAW,kBAAkB;IACjC,4DAA4D;IAC5D,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,SAAS,CAAC;IACjC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB;;OAEG;IACH,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;IACjD,2DAA2D;IAC3D,WAAW,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C;;;;OAIG;IACH,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC;CAChC;AAED,6EAA6E;AAC7E,MAAM,WAAW,UAAU;IACzB,uDAAuD;IACvD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AASD,qBAAa,WAAY,SAAQ,YAAY;IAC3C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAmB;IACtC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0C;IACjE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA0B;IAC/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA+C;IAE3E,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,gBAAgB,CAAK;IAC7B,mEAAmE;IACnE,OAAO,CAAC,UAAU,CAAK;IACvB,qEAAqE;IACrE,OAAO,CAAC,sBAAsB,CAAK;IACnC,mEAAmE;IACnE,OAAO,CAAC,oBAAoB,CAAK;IACjC,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,SAAS,CAAgB;IACjC,2FAA2F;IAC3F,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,SAAS,CAA+B;IAChD,iFAAiF;IACjF,OAAO,CAAC,UAAU,CAAS;IAE3B,qEAAqE;IACrE,OAAO,CAAC,cAAc,CAAa;IACnC,OAAO,CAAC,aAAa,CAA+B;IACpD,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,kBAAkB,CAA+B;IACzD,OAAO,CAAC,gBAAgB,CAAS;IACjC,qEAAqE;IACrE,OAAO,CAAC,iBAAiB,CAAS;gBAEtB,IAAI,EAAE,kBAAkB;IAYpC,kEAAkE;IAClE,OAAO,IAAI,IAAI;IAMf;;;;;;;OAOG;IACH,IAAI,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,GAAG,OAAO;IAQzC;;;OAGG;IACH,OAAO,IAAI,IAAI;IA6Bf,OAAO,CAAC,UAAU;IAwFlB,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,cAAc;IAgCtB,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,WAAW;IAYnB,OAAO,CAAC,iBAAiB;IAQzB;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAqDtB,OAAO,CAAC,eAAe;IAOvB;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAgBtB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IA+GxB;;;;;OAKG;YACW,WAAW;YA8BX,iBAAiB;CAyDhC"}
|
package/dist/relay/client.js
CHANGED
|
@@ -7,12 +7,16 @@
|
|
|
7
7
|
* machine JWT via `Authorization: Bearer <jwt>`.
|
|
8
8
|
* - Reply `{kind:"pong"}` to backend `{kind:"ping"}`. Backend closes
|
|
9
9
|
* `4408 heartbeat-timeout` if it doesn't hear from us within 90s; we
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* keep this app-level reply in addition to the protocol-level heartbeat
|
|
11
|
+
* below.
|
|
12
|
+
* - **Client heartbeat:** additionally emits a protocol-level `ws` ping
|
|
13
|
+
* every `HEARTBEAT_INTERVAL_MS` and requires a `pong` within
|
|
14
|
+
* `HEARTBEAT_TIMEOUT_MS`. This detects a half-open send path (e.g. after
|
|
15
|
+
* laptop sleep/wake) where `readyState` still reports `OPEN`.
|
|
12
16
|
* - **Watchdog timer:** tracks `lastActivityMs` on every received frame /
|
|
13
17
|
* ping. Every 15 s, if no activity within `WATCHDOG_MS` (120 s = 2×
|
|
14
|
-
* backend timeout),
|
|
15
|
-
*
|
|
18
|
+
* backend timeout), terminates the socket and triggers a reconnect.
|
|
19
|
+
* Detects silent backend death (Docker OOM
|
|
16
20
|
* kill, network partition without TCP RST) where the socket stays open
|
|
17
21
|
* but no data flows.
|
|
18
22
|
* - **Pre-reconnect health check:** before opening a new WS, does a quick
|
|
@@ -66,6 +70,14 @@ const WATCHDOG_INTERVAL_MS = 15_000;
|
|
|
66
70
|
const WATCHDOG_SILENCE_MS = 120_000;
|
|
67
71
|
/** HTTP timeout for the pre-reconnect health check. */
|
|
68
72
|
const HEALTH_CHECK_TIMEOUT_MS = 5_000;
|
|
73
|
+
/** TCP keepalive initial delay (ms). Lets the OS detect a dead path after sleep. */
|
|
74
|
+
const TCP_KEEPALIVE_MS = 30_000;
|
|
75
|
+
/** Client-originated protocol-level ping interval (ms). */
|
|
76
|
+
const HEARTBEAT_INTERVAL_MS = 30_000;
|
|
77
|
+
/** Client pong timeout before forcing a reconnect (ms). */
|
|
78
|
+
const HEARTBEAT_TIMEOUT_MS = 10_000;
|
|
79
|
+
/** Outbound write acknowledgement timeout (ms). */
|
|
80
|
+
const SEND_TIMEOUT_MS = 10_000;
|
|
69
81
|
export class RelayClient extends EventEmitter {
|
|
70
82
|
relayUrl;
|
|
71
83
|
machineJwt;
|
|
@@ -86,11 +98,19 @@ export class RelayClient extends EventEmitter {
|
|
|
86
98
|
tightLoopWindowStart = 0;
|
|
87
99
|
reconnectTimer = null;
|
|
88
100
|
sendQueue = [];
|
|
101
|
+
/** Wire currently waiting on a `ws.send` callback. Only one in flight to preserve FIFO. */
|
|
102
|
+
sendInFlight = null;
|
|
103
|
+
sendTimer = null;
|
|
89
104
|
/** Set true when the WS error handler sees an HTTP 401/403 upgrade rejection. */
|
|
90
105
|
authFailed = false;
|
|
91
106
|
/** Timestamp of last received frame / ping — drives the watchdog. */
|
|
92
107
|
lastActivityMs = 0;
|
|
93
108
|
watchdogTimer = null;
|
|
109
|
+
heartbeatTimer = null;
|
|
110
|
+
heartbeatPongTimer = null;
|
|
111
|
+
heartbeatPending = false;
|
|
112
|
+
/** Guards the async window of `scheduleReconnect` (health check). */
|
|
113
|
+
reconnectInFlight = false;
|
|
94
114
|
constructor(opts) {
|
|
95
115
|
super();
|
|
96
116
|
this.relayUrl = opts.relayUrl;
|
|
@@ -106,7 +126,7 @@ export class RelayClient extends EventEmitter {
|
|
|
106
126
|
connect() {
|
|
107
127
|
if (this.disposed)
|
|
108
128
|
return;
|
|
109
|
-
if (this.ws)
|
|
129
|
+
if (this.ws || this.reconnectTimer || this.reconnectInFlight)
|
|
110
130
|
return;
|
|
111
131
|
this.openSocket();
|
|
112
132
|
}
|
|
@@ -122,22 +142,8 @@ export class RelayClient extends EventEmitter {
|
|
|
122
142
|
if (this.disposed)
|
|
123
143
|
return false;
|
|
124
144
|
const wire = typeof frame === "string" ? frame : JSON.stringify(frame);
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
try {
|
|
128
|
-
ws.send(wire);
|
|
129
|
-
return true;
|
|
130
|
-
}
|
|
131
|
-
catch (err) {
|
|
132
|
-
this.emit("error", err);
|
|
133
|
-
return false;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
// Queue it.
|
|
137
|
-
if (this.sendQueue.length >= SEND_QUEUE_CAP) {
|
|
138
|
-
this.sendQueue.shift();
|
|
139
|
-
}
|
|
140
|
-
this.sendQueue.push(wire);
|
|
145
|
+
this.enqueueWire(wire);
|
|
146
|
+
this.flushSendQueue();
|
|
141
147
|
return true;
|
|
142
148
|
}
|
|
143
149
|
/**
|
|
@@ -156,6 +162,8 @@ export class RelayClient extends EventEmitter {
|
|
|
156
162
|
clearInterval(this.watchdogTimer);
|
|
157
163
|
this.watchdogTimer = null;
|
|
158
164
|
}
|
|
165
|
+
this.stopHeartbeat();
|
|
166
|
+
this.clearSendInFlight();
|
|
159
167
|
const ws = this.ws;
|
|
160
168
|
this.ws = null;
|
|
161
169
|
if (ws) {
|
|
@@ -183,25 +191,20 @@ export class RelayClient extends EventEmitter {
|
|
|
183
191
|
this.lastActivityMs = Date.now();
|
|
184
192
|
this.startWatchdog();
|
|
185
193
|
ws.on("open", () => {
|
|
194
|
+
if (this.ws !== ws)
|
|
195
|
+
return;
|
|
186
196
|
this.lastActivityMs = Date.now();
|
|
187
197
|
this.openedAtMs = Date.now();
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
ws.send(wire);
|
|
194
|
-
}
|
|
195
|
-
catch (err) {
|
|
196
|
-
// Re-queue on failure and stop draining; the next open will retry.
|
|
197
|
-
this.sendQueue = queued.slice(queued.indexOf(wire));
|
|
198
|
-
this.emit("error", err);
|
|
199
|
-
return;
|
|
200
|
-
}
|
|
201
|
-
}
|
|
198
|
+
this.setTcpKeepAlive(ws);
|
|
199
|
+
this.startHeartbeat();
|
|
200
|
+
// Flush any queued frames with write acknowledgements and timeouts.
|
|
201
|
+
this.clearSendInFlight();
|
|
202
|
+
this.flushSendQueue();
|
|
202
203
|
this.emit("open");
|
|
203
204
|
});
|
|
204
205
|
ws.on("message", (data) => {
|
|
206
|
+
if (this.ws !== ws)
|
|
207
|
+
return;
|
|
205
208
|
this.lastActivityMs = Date.now();
|
|
206
209
|
let parsed;
|
|
207
210
|
try {
|
|
@@ -213,12 +216,7 @@ export class RelayClient extends EventEmitter {
|
|
|
213
216
|
}
|
|
214
217
|
// Heartbeat: reply pong inline, do NOT propagate.
|
|
215
218
|
if (parsed && parsed.kind === "ping") {
|
|
216
|
-
|
|
217
|
-
ws.send(JSON.stringify({ kind: "pong" }));
|
|
218
|
-
}
|
|
219
|
-
catch (err) {
|
|
220
|
-
this.emit("error", err);
|
|
221
|
-
}
|
|
219
|
+
this.send({ kind: "pong" });
|
|
222
220
|
return;
|
|
223
221
|
}
|
|
224
222
|
if (parsed && parsed.kind === "welcome") {
|
|
@@ -230,7 +228,21 @@ export class RelayClient extends EventEmitter {
|
|
|
230
228
|
}
|
|
231
229
|
this.emit("frame", parsed);
|
|
232
230
|
});
|
|
231
|
+
ws.on("pong", () => {
|
|
232
|
+
if (this.ws !== ws)
|
|
233
|
+
return;
|
|
234
|
+
this.lastActivityMs = Date.now();
|
|
235
|
+
if (this.heartbeatPending) {
|
|
236
|
+
this.heartbeatPending = false;
|
|
237
|
+
if (this.heartbeatPongTimer) {
|
|
238
|
+
clearTimeout(this.heartbeatPongTimer);
|
|
239
|
+
this.heartbeatPongTimer = null;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
});
|
|
233
243
|
ws.on("error", (err) => {
|
|
244
|
+
if (this.ws !== ws)
|
|
245
|
+
return;
|
|
234
246
|
// Detect HTTP auth failures during the WS upgrade handshake.
|
|
235
247
|
// The `ws` library surfaces these as `Error: Unexpected server
|
|
236
248
|
// response: 401` (or 403). Mark `authFailed` so the close handler
|
|
@@ -240,97 +252,13 @@ export class RelayClient extends EventEmitter {
|
|
|
240
252
|
msg.includes("Unexpected server response: 403")) {
|
|
241
253
|
this.authFailed = true;
|
|
242
254
|
}
|
|
243
|
-
// `ws` emits 'error' before 'close'
|
|
244
|
-
//
|
|
245
|
-
// double-schedule.
|
|
255
|
+
// `ws` emits 'error' before 'close'. Reconnect is driven from the
|
|
256
|
+
// `close` handler (or `forceReconnect` for suspected-dead sockets);
|
|
257
|
+
// scheduling it here would double-schedule.
|
|
246
258
|
this.emit("error", err);
|
|
247
259
|
});
|
|
248
260
|
ws.on("close", (code, reason) => {
|
|
249
|
-
this.ws
|
|
250
|
-
this.stopWatchdog();
|
|
251
|
-
// Stable-open gate: only reset backoff if the socket was open long
|
|
252
|
-
// enough to be considered healthy. A connect→immediate-die loop (1006,
|
|
253
|
-
// proxy kill, duplicate instance) must escalate through the full
|
|
254
|
-
// schedule instead of pinning at RECONNECT_SCHEDULE[0] = 1s forever.
|
|
255
|
-
const openDuration = this.openedAtMs > 0 ? Date.now() - this.openedAtMs : 0;
|
|
256
|
-
this.openedAtMs = 0;
|
|
257
|
-
if (openDuration >= STABLE_OPEN_MS) {
|
|
258
|
-
this.reconnectAttempt = 0;
|
|
259
|
-
this.consecutiveRapidCloses = 0;
|
|
260
|
-
}
|
|
261
|
-
else {
|
|
262
|
-
// Tight-loop detection: if the socket dies repeatedly right after
|
|
263
|
-
// open, bail out with an actionable message instead of reconnecting
|
|
264
|
-
// forever.
|
|
265
|
-
const now = Date.now();
|
|
266
|
-
if (now - this.tightLoopWindowStart > TIGHT_LOOP_WINDOW_MS) {
|
|
267
|
-
this.tightLoopWindowStart = now;
|
|
268
|
-
this.consecutiveRapidCloses = 1;
|
|
269
|
-
}
|
|
270
|
-
else {
|
|
271
|
-
this.consecutiveRapidCloses++;
|
|
272
|
-
}
|
|
273
|
-
if (this.consecutiveRapidCloses >= TIGHT_LOOP_MAX_CLOSES) {
|
|
274
|
-
this.logger.error("\n✗ Relay connection dropped repeatedly right after opening.");
|
|
275
|
-
this.logger.error(" This usually means another `spectral serve` process is running for the same machine,");
|
|
276
|
-
this.logger.error(" or a reverse proxy / firewall is closing the WebSocket immediately.");
|
|
277
|
-
this.logger.error(' Check: `ps aux | grep "spectral serve"` and your proxy WS timeout settings.\n');
|
|
278
|
-
this.dispose();
|
|
279
|
-
this.exit(1);
|
|
280
|
-
return;
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
const reasonStr = reason?.toString() ?? "";
|
|
284
|
-
this.emit("close", { code, reason: reasonStr });
|
|
285
|
-
if (this.disposed)
|
|
286
|
-
return;
|
|
287
|
-
// Backend evicts older WS sessions when a newer registration arrives
|
|
288
|
-
// for the same machineId (see backend `registry.ts: register()`).
|
|
289
|
-
// If we reconnect here we'll just kick out the new instance, which
|
|
290
|
-
// will then reconnect and kick us out — an infinite ping-pong.
|
|
291
|
-
// Exit instead so the operator (or process supervisor) notices.
|
|
292
|
-
if (code === 1000 && reasonStr === "replaced-by-newer-registration") {
|
|
293
|
-
this.logger.error("\n✗ Another `spectral serve` instance has registered for this machine.");
|
|
294
|
-
this.logger.error(" This instance is exiting to avoid a reconnect loop.");
|
|
295
|
-
this.logger.error(" If you want to run multiple instances, use distinct `--machine-name` values.\n");
|
|
296
|
-
this.dispose();
|
|
297
|
-
this.exit(1);
|
|
298
|
-
return;
|
|
299
|
-
}
|
|
300
|
-
// Machine was revoked by the team admin from the Aexol Studio panel
|
|
301
|
-
// (backend sets `KnownMachine.revokedAt` and closes the socket with
|
|
302
|
-
// 4001 "machine-revoked"). Exit immediately — the machine JWT is now
|
|
303
|
-
// invalid and re-registration requires a fresh `spectral serve`.
|
|
304
|
-
if (code === 4001 && reasonStr === "machine-revoked") {
|
|
305
|
-
this.logger.error("\n✗ This machine has been disconnected from the Aexol Studio panel.");
|
|
306
|
-
// Delete the machine identity file so the next `spectral serve`
|
|
307
|
-
// forces a fresh registration rather than reusing the revoked JWT.
|
|
308
|
-
unlink(getMachineFile()).catch(() => { });
|
|
309
|
-
this.logger.error(" Run `spectral login` then `spectral serve` to re-register.\n");
|
|
310
|
-
this.dispose();
|
|
311
|
-
this.exit(1);
|
|
312
|
-
return;
|
|
313
|
-
}
|
|
314
|
-
if (reasonStr === "team-changed") {
|
|
315
|
-
// Team was changed via Studio — the machine_team_changed frame handler
|
|
316
|
-
// in serve.ts already saved the new JWT. Just reconnect normally.
|
|
317
|
-
if (this.authFailed) {
|
|
318
|
-
this.emit("auth-failed");
|
|
319
|
-
return;
|
|
320
|
-
}
|
|
321
|
-
this.scheduleReconnect();
|
|
322
|
-
return;
|
|
323
|
-
}
|
|
324
|
-
// Auth failure at the transport layer (HTTP 401/403 on WS upgrade).
|
|
325
|
-
// The machine JWT is invalid (expired, malformed, or revoked by key
|
|
326
|
-
// rotation) but the backend didn't send a structured close reason.
|
|
327
|
-
// Signal the caller so it can attempt re-registration instead of
|
|
328
|
-
// entering an infinite reconnect loop with a dead token.
|
|
329
|
-
if (this.authFailed) {
|
|
330
|
-
this.emit("auth-failed");
|
|
331
|
-
return;
|
|
332
|
-
}
|
|
333
|
-
this.scheduleReconnect();
|
|
261
|
+
this.handleDisconnect(ws, code, reason?.toString() ?? "");
|
|
334
262
|
});
|
|
335
263
|
}
|
|
336
264
|
// --- watchdog -------------------------------------------------------------
|
|
@@ -343,16 +271,7 @@ export class RelayClient extends EventEmitter {
|
|
|
343
271
|
const elapsed = Date.now() - this.lastActivityMs;
|
|
344
272
|
if (elapsed > WATCHDOG_SILENCE_MS) {
|
|
345
273
|
this.logger.warn(`Watchdog: no relay activity for ${Math.round(elapsed / 1000)}s, forcing reconnect`);
|
|
346
|
-
|
|
347
|
-
this.ws = null;
|
|
348
|
-
if (ws) {
|
|
349
|
-
try {
|
|
350
|
-
ws.close(4001, "watchdog-timeout");
|
|
351
|
-
}
|
|
352
|
-
catch {
|
|
353
|
-
// ignore
|
|
354
|
-
}
|
|
355
|
-
}
|
|
274
|
+
this.forceReconnect("watchdog-timeout");
|
|
356
275
|
}
|
|
357
276
|
}, WATCHDOG_INTERVAL_MS);
|
|
358
277
|
}
|
|
@@ -362,6 +281,250 @@ export class RelayClient extends EventEmitter {
|
|
|
362
281
|
this.watchdogTimer = null;
|
|
363
282
|
}
|
|
364
283
|
}
|
|
284
|
+
// --- heartbeat -------------------------------------------------------------
|
|
285
|
+
startHeartbeat() {
|
|
286
|
+
if (this.heartbeatTimer)
|
|
287
|
+
return;
|
|
288
|
+
this.heartbeatTimer = setInterval(() => {
|
|
289
|
+
if (this.disposed)
|
|
290
|
+
return;
|
|
291
|
+
const ws = this.ws;
|
|
292
|
+
if (!ws || ws.readyState !== WebSocket.OPEN || this.heartbeatPending) {
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
this.heartbeatPending = true;
|
|
296
|
+
this.heartbeatPongTimer = setTimeout(() => {
|
|
297
|
+
this.heartbeatPongTimer = null;
|
|
298
|
+
if (!this.heartbeatPending)
|
|
299
|
+
return;
|
|
300
|
+
this.heartbeatPending = false;
|
|
301
|
+
this.logger.warn("Heartbeat: no pong from relay, forcing reconnect");
|
|
302
|
+
this.forceReconnect("heartbeat-timeout");
|
|
303
|
+
}, HEARTBEAT_TIMEOUT_MS);
|
|
304
|
+
try {
|
|
305
|
+
ws.ping();
|
|
306
|
+
}
|
|
307
|
+
catch (err) {
|
|
308
|
+
this.heartbeatPending = false;
|
|
309
|
+
if (this.heartbeatPongTimer) {
|
|
310
|
+
clearTimeout(this.heartbeatPongTimer);
|
|
311
|
+
this.heartbeatPongTimer = null;
|
|
312
|
+
}
|
|
313
|
+
this.emit("error", err);
|
|
314
|
+
this.forceReconnect("heartbeat-error");
|
|
315
|
+
}
|
|
316
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
317
|
+
}
|
|
318
|
+
stopHeartbeat() {
|
|
319
|
+
if (this.heartbeatTimer) {
|
|
320
|
+
clearInterval(this.heartbeatTimer);
|
|
321
|
+
this.heartbeatTimer = null;
|
|
322
|
+
}
|
|
323
|
+
if (this.heartbeatPongTimer) {
|
|
324
|
+
clearTimeout(this.heartbeatPongTimer);
|
|
325
|
+
this.heartbeatPongTimer = null;
|
|
326
|
+
}
|
|
327
|
+
this.heartbeatPending = false;
|
|
328
|
+
}
|
|
329
|
+
// --- outbound send safety ---------------------------------------------------
|
|
330
|
+
enqueueWire(wire) {
|
|
331
|
+
if (this.sendQueue.length >= SEND_QUEUE_CAP) {
|
|
332
|
+
// Never evict the frame currently awaiting a write callback — it is
|
|
333
|
+
// still at the front of the queue and must survive a reconnect.
|
|
334
|
+
const dropIndex = this.sendInFlight !== null ? 1 : 0;
|
|
335
|
+
if (this.sendQueue.length > dropIndex) {
|
|
336
|
+
this.sendQueue.splice(dropIndex, 1);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
this.sendQueue.push(wire);
|
|
340
|
+
}
|
|
341
|
+
clearSendInFlight() {
|
|
342
|
+
this.sendInFlight = null;
|
|
343
|
+
if (this.sendTimer) {
|
|
344
|
+
clearTimeout(this.sendTimer);
|
|
345
|
+
this.sendTimer = null;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
/**
|
|
349
|
+
* Drain the FIFO send queue over the current socket. Exactly one frame is
|
|
350
|
+
* in flight at a time so ordering is preserved. If `ws.send` never calls
|
|
351
|
+
* its callback (half-open socket), the frame stays at the front of the
|
|
352
|
+
* queue and the connection is force-reconnected; the next `open` flushes
|
|
353
|
+
* it again.
|
|
354
|
+
*/
|
|
355
|
+
flushSendQueue() {
|
|
356
|
+
if (this.disposed)
|
|
357
|
+
return;
|
|
358
|
+
const ws = this.ws;
|
|
359
|
+
if (!ws ||
|
|
360
|
+
ws.readyState !== WebSocket.OPEN ||
|
|
361
|
+
this.sendInFlight !== null ||
|
|
362
|
+
this.sendQueue.length === 0) {
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
const wire = this.sendQueue[0];
|
|
366
|
+
this.sendInFlight = wire;
|
|
367
|
+
this.sendTimer = setTimeout(() => {
|
|
368
|
+
this.sendTimer = null;
|
|
369
|
+
if (this.sendInFlight !== wire || this.ws !== ws)
|
|
370
|
+
return;
|
|
371
|
+
this.sendInFlight = null;
|
|
372
|
+
this.logger.warn("Relay send timed out; forcing reconnect");
|
|
373
|
+
this.forceReconnect("send-timeout");
|
|
374
|
+
}, SEND_TIMEOUT_MS);
|
|
375
|
+
try {
|
|
376
|
+
ws.send(wire, (err) => {
|
|
377
|
+
if (this.sendInFlight !== wire)
|
|
378
|
+
return;
|
|
379
|
+
this.sendInFlight = null;
|
|
380
|
+
if (this.sendTimer) {
|
|
381
|
+
clearTimeout(this.sendTimer);
|
|
382
|
+
this.sendTimer = null;
|
|
383
|
+
}
|
|
384
|
+
if (err) {
|
|
385
|
+
this.logger.warn(`Relay send failed: ${err.message}`);
|
|
386
|
+
this.forceReconnect("send-error");
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
if (this.sendQueue[0] === wire) {
|
|
390
|
+
this.sendQueue.shift();
|
|
391
|
+
}
|
|
392
|
+
this.flushSendQueue();
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
catch (err) {
|
|
396
|
+
if (this.sendInFlight === wire) {
|
|
397
|
+
this.sendInFlight = null;
|
|
398
|
+
if (this.sendTimer) {
|
|
399
|
+
clearTimeout(this.sendTimer);
|
|
400
|
+
this.sendTimer = null;
|
|
401
|
+
}
|
|
402
|
+
this.emit("error", err);
|
|
403
|
+
this.forceReconnect("send-error");
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
setTcpKeepAlive(ws) {
|
|
408
|
+
const socket = ws._socket;
|
|
409
|
+
if (socket && typeof socket.setKeepAlive === "function") {
|
|
410
|
+
socket.setKeepAlive(true, TCP_KEEPALIVE_MS);
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Tear down a suspected-dead socket and schedule a reconnect immediately.
|
|
415
|
+
* Unlike the `close` path, this does not wait for a (possibly never
|
|
416
|
+
* arriving) `close` event — `terminate()` destroys the socket directly.
|
|
417
|
+
*/
|
|
418
|
+
forceReconnect(reason) {
|
|
419
|
+
if (this.disposed)
|
|
420
|
+
return;
|
|
421
|
+
const ws = this.ws;
|
|
422
|
+
if (!ws) {
|
|
423
|
+
void this.scheduleReconnect();
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
try {
|
|
427
|
+
ws.terminate();
|
|
428
|
+
}
|
|
429
|
+
catch (err) {
|
|
430
|
+
this.emit("error", err);
|
|
431
|
+
}
|
|
432
|
+
this.handleDisconnect(ws, 1006, reason);
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Single disconnect entry point for both real `close` events and forced
|
|
436
|
+
* reconnects. Ignored for sockets that have already been replaced.
|
|
437
|
+
*/
|
|
438
|
+
handleDisconnect(ws, code, reason) {
|
|
439
|
+
if (this.ws !== ws)
|
|
440
|
+
return;
|
|
441
|
+
this.ws = null;
|
|
442
|
+
this.stopWatchdog();
|
|
443
|
+
this.stopHeartbeat();
|
|
444
|
+
this.clearSendInFlight();
|
|
445
|
+
// Stable-open gate: only reset backoff if the socket was open long
|
|
446
|
+
// enough to be considered healthy. A connect→immediate-die loop (1006,
|
|
447
|
+
// proxy kill, duplicate instance) must escalate through the full
|
|
448
|
+
// schedule instead of pinning at RECONNECT_SCHEDULE[0] = 1s forever.
|
|
449
|
+
const openDuration = this.openedAtMs > 0 ? Date.now() - this.openedAtMs : 0;
|
|
450
|
+
this.openedAtMs = 0;
|
|
451
|
+
if (openDuration >= STABLE_OPEN_MS) {
|
|
452
|
+
this.reconnectAttempt = 0;
|
|
453
|
+
this.consecutiveRapidCloses = 0;
|
|
454
|
+
}
|
|
455
|
+
else {
|
|
456
|
+
// Tight-loop detection: if the socket dies repeatedly right after
|
|
457
|
+
// open, bail out with an actionable message instead of reconnecting
|
|
458
|
+
// forever.
|
|
459
|
+
const now = Date.now();
|
|
460
|
+
if (now - this.tightLoopWindowStart > TIGHT_LOOP_WINDOW_MS) {
|
|
461
|
+
this.tightLoopWindowStart = now;
|
|
462
|
+
this.consecutiveRapidCloses = 1;
|
|
463
|
+
}
|
|
464
|
+
else {
|
|
465
|
+
this.consecutiveRapidCloses++;
|
|
466
|
+
}
|
|
467
|
+
if (this.consecutiveRapidCloses >= TIGHT_LOOP_MAX_CLOSES) {
|
|
468
|
+
this.logger.error("\n✗ Relay connection dropped repeatedly right after opening.");
|
|
469
|
+
this.logger.error(" This usually means another `spectral serve` process is running for the same machine,");
|
|
470
|
+
this.logger.error(" or a reverse proxy / firewall is closing the WebSocket immediately.");
|
|
471
|
+
this.logger.error(' Check: `ps aux | grep "spectral serve"` and your proxy WS timeout settings.\n');
|
|
472
|
+
this.dispose();
|
|
473
|
+
this.exit(1);
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
this.emit("close", { code, reason });
|
|
478
|
+
if (this.disposed)
|
|
479
|
+
return;
|
|
480
|
+
// Backend evicts older WS sessions when a newer registration arrives
|
|
481
|
+
// for the same machineId (see backend `registry.ts: register()`).
|
|
482
|
+
// If we reconnect here we'll just kick out the new instance, which
|
|
483
|
+
// will then reconnect and kick us out — an infinite ping-pong.
|
|
484
|
+
// Exit instead so the operator (or process supervisor) notices.
|
|
485
|
+
if (code === 1000 && reason === "replaced-by-newer-registration") {
|
|
486
|
+
this.logger.error("\n✗ Another `spectral serve` instance has registered for this machine.");
|
|
487
|
+
this.logger.error(" This instance is exiting to avoid a reconnect loop.");
|
|
488
|
+
this.logger.error(" If you want to run multiple instances, use distinct `--machine-name` values.\n");
|
|
489
|
+
this.dispose();
|
|
490
|
+
this.exit(1);
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
// Machine was revoked by the team admin from the Aexol Studio panel
|
|
494
|
+
// (backend sets `KnownMachine.revokedAt` and closes the socket with
|
|
495
|
+
// 4001 "machine-revoked"). Exit immediately — the machine JWT is now
|
|
496
|
+
// invalid and re-registration requires a fresh `spectral serve`.
|
|
497
|
+
if (code === 4001 && reason === "machine-revoked") {
|
|
498
|
+
this.logger.error("\n✗ This machine has been disconnected from the Aexol Studio panel.");
|
|
499
|
+
// Delete the machine identity file so the next `spectral serve`
|
|
500
|
+
// forces a fresh registration rather than reusing the revoked JWT.
|
|
501
|
+
unlink(getMachineFile()).catch(() => { });
|
|
502
|
+
this.logger.error(" Run `spectral login` then `spectral serve` to re-register.\n");
|
|
503
|
+
this.dispose();
|
|
504
|
+
this.exit(1);
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
if (reason === "team-changed") {
|
|
508
|
+
// Team was changed via Studio — the machine_team_changed frame handler
|
|
509
|
+
// in serve.ts already saved the new JWT. Just reconnect normally.
|
|
510
|
+
if (this.authFailed) {
|
|
511
|
+
this.emit("auth-failed");
|
|
512
|
+
return;
|
|
513
|
+
}
|
|
514
|
+
void this.scheduleReconnect();
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
// Auth failure at the transport layer (HTTP 401/403 on WS upgrade).
|
|
518
|
+
// The machine JWT is invalid (expired, malformed, or revoked by key
|
|
519
|
+
// rotation) but the backend didn't send a structured close reason.
|
|
520
|
+
// Signal the caller so it can attempt re-registration instead of
|
|
521
|
+
// entering an infinite reconnect loop with a dead token.
|
|
522
|
+
if (this.authFailed) {
|
|
523
|
+
this.emit("auth-failed");
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
void this.scheduleReconnect();
|
|
527
|
+
}
|
|
365
528
|
/**
|
|
366
529
|
* Pre-reconnect health check: quick HTTP GET to `<backendUrl>/health`.
|
|
367
530
|
* Returns true when the backend is reachable (or when no backendUrl is
|
|
@@ -400,54 +563,62 @@ export class RelayClient extends EventEmitter {
|
|
|
400
563
|
async scheduleReconnect() {
|
|
401
564
|
if (this.disposed)
|
|
402
565
|
return;
|
|
403
|
-
if (this.reconnectTimer)
|
|
566
|
+
if (this.reconnectTimer || this.reconnectInFlight)
|
|
404
567
|
return;
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
568
|
+
this.reconnectInFlight = true;
|
|
569
|
+
try {
|
|
570
|
+
// Pre-reconnect health check — avoids wasting time on a slow TCP
|
|
571
|
+
// connect when the backend is down. The health endpoint is a cheap
|
|
572
|
+
// HTTP GET; if even that fails, skip the WS attempt entirely.
|
|
573
|
+
const healthy = await this.healthCheck();
|
|
574
|
+
if (this.disposed)
|
|
575
|
+
return;
|
|
576
|
+
if (!healthy) {
|
|
577
|
+
// Backend unreachable — use a fixed 5 s retry instead of advancing
|
|
578
|
+
// the backoff schedule (this is an environment problem, not a WS
|
|
579
|
+
// handshake problem).
|
|
580
|
+
const idx = Math.min(this.reconnectAttempt, RECONNECT_SCHEDULE.length - 1);
|
|
581
|
+
const delay = RECONNECT_SCHEDULE[idx];
|
|
582
|
+
this.reconnectAttempt++;
|
|
583
|
+
this.emit("reconnect-scheduled", { delayMs: delay, attempt: this.reconnectAttempt });
|
|
584
|
+
this.reconnectTimer = setTimeout(() => {
|
|
585
|
+
this.reconnectTimer = null;
|
|
586
|
+
void this.scheduleReconnect().catch(() => { });
|
|
587
|
+
}, delay);
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
413
590
|
const idx = Math.min(this.reconnectAttempt, RECONNECT_SCHEDULE.length - 1);
|
|
414
|
-
const
|
|
591
|
+
const base = RECONNECT_SCHEDULE[idx];
|
|
592
|
+
const jitter = base * JITTER_RATIO * (Math.random() * 2 - 1);
|
|
593
|
+
const delay = Math.max(0, Math.round(base + jitter));
|
|
415
594
|
this.reconnectAttempt++;
|
|
416
595
|
this.emit("reconnect-scheduled", { delayMs: delay, attempt: this.reconnectAttempt });
|
|
596
|
+
// IMPORTANT: do NOT `unref()` this timer. `spectral serve` is a long-
|
|
597
|
+
// lived daemon and the reconnect timer is frequently the ONLY thing
|
|
598
|
+
// keeping the event loop alive between a WS close and the next open
|
|
599
|
+
// (the SessionStreamManager holds nothing while idle). An unref'd
|
|
600
|
+
// timer lets Node decide the loop is empty and exit the process,
|
|
601
|
+
// which manifests as the daemon silently dying after a network blip.
|
|
602
|
+
// Tests dispose the client explicitly via `dispose()`, which clears
|
|
603
|
+
// this timer, so they still terminate cleanly.
|
|
417
604
|
this.reconnectTimer = setTimeout(() => {
|
|
418
605
|
this.reconnectTimer = null;
|
|
419
|
-
|
|
606
|
+
if (this.disposed)
|
|
607
|
+
return;
|
|
608
|
+
try {
|
|
609
|
+
this.openSocket();
|
|
610
|
+
}
|
|
611
|
+
catch (err) {
|
|
612
|
+
// A synchronous throw from the WebSocket constructor (DNS, bad
|
|
613
|
+
// URL, etc.) must not kill the daemon — log it and re-schedule
|
|
614
|
+
// so the backoff continues forever.
|
|
615
|
+
this.emit("error", err);
|
|
616
|
+
void this.scheduleReconnect().catch(() => { });
|
|
617
|
+
}
|
|
420
618
|
}, delay);
|
|
421
|
-
return;
|
|
422
619
|
}
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
const delay = Math.max(0, Math.round(base + jitter));
|
|
427
|
-
this.reconnectAttempt++;
|
|
428
|
-
this.emit("reconnect-scheduled", { delayMs: delay, attempt: this.reconnectAttempt });
|
|
429
|
-
// IMPORTANT: do NOT `unref()` this timer. `spectral serve` is a long-
|
|
430
|
-
// lived daemon and the reconnect timer is frequently the ONLY thing
|
|
431
|
-
// keeping the event loop alive between a WS close and the next open
|
|
432
|
-
// (the SessionStreamManager holds nothing while idle). An unref'd
|
|
433
|
-
// timer lets Node decide the loop is empty and exit the process,
|
|
434
|
-
// which manifests as the daemon silently dying after a network blip.
|
|
435
|
-
// Tests dispose the client explicitly via `dispose()`, which clears
|
|
436
|
-
// this timer, so they still terminate cleanly.
|
|
437
|
-
this.reconnectTimer = setTimeout(() => {
|
|
438
|
-
this.reconnectTimer = null;
|
|
439
|
-
if (this.disposed)
|
|
440
|
-
return;
|
|
441
|
-
try {
|
|
442
|
-
this.openSocket();
|
|
443
|
-
}
|
|
444
|
-
catch (err) {
|
|
445
|
-
// A synchronous throw from the WebSocket constructor (DNS, bad
|
|
446
|
-
// URL, etc.) must not kill the daemon — log it and re-schedule
|
|
447
|
-
// so the backoff continues forever.
|
|
448
|
-
this.emit("error", err);
|
|
449
|
-
void this.scheduleReconnect().catch(() => { });
|
|
450
|
-
}
|
|
451
|
-
}, delay);
|
|
620
|
+
finally {
|
|
621
|
+
this.reconnectInFlight = false;
|
|
622
|
+
}
|
|
452
623
|
}
|
|
453
624
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../../../src/sdk/ai/cache-benchmark/benchmark.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAgB,aAAa,EAAE,MAAM,YAAY,CAAC;AAMhG,wBAAgB,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,eAAe,GAAG,eAAe,
|
|
1
|
+
{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../../../src/sdk/ai/cache-benchmark/benchmark.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAgB,aAAa,EAAE,MAAM,YAAY,CAAC;AAMhG,wBAAgB,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,eAAe,GAAG,eAAe,CA0F9F;AAED,oFAAoF;AACpF,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,aAAa,EAAE,EAAE,MAAM,EAAE,eAAe,GAAG,eAAe,EAAE,CASzG;AASD,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM,CAuBxE"}
|
|
@@ -8,6 +8,7 @@ function resolvePricing(config) {
|
|
|
8
8
|
export function runBenchmark(strategy, config) {
|
|
9
9
|
const pricing = resolvePricing(config);
|
|
10
10
|
const minCacheTokens = config.minCacheTokens ?? 1024;
|
|
11
|
+
const reReadFraction = config.reReadFraction ?? 0;
|
|
11
12
|
const systemText = fillerText(Math.max(0, config.systemPromptTokens * 4), 0);
|
|
12
13
|
const toolsText = fillerText(Math.max(0, config.toolsTokens * 4), 1);
|
|
13
14
|
const simulator = new PrefixCacheSimulator({
|
|
@@ -23,6 +24,10 @@ export function runBenchmark(strategy, config) {
|
|
|
23
24
|
let totalCacheWriteTokens = 0;
|
|
24
25
|
let maxPromptTokens = 0;
|
|
25
26
|
let finalPromptTokens = 0;
|
|
27
|
+
let totalTurnPromptTokens = 0;
|
|
28
|
+
let totalDroppedTokens = 0;
|
|
29
|
+
let totalSummaryTokens = 0;
|
|
30
|
+
let totalReReadTokens = 0;
|
|
26
31
|
for (let turn = 0; turn < config.turns; turn++) {
|
|
27
32
|
const promptTokens = resolvePerTurn(config.turnPromptTokens, turn);
|
|
28
33
|
const outputTokens = resolvePerTurn(config.turnOutputTokens, turn);
|
|
@@ -31,6 +36,13 @@ export function runBenchmark(strategy, config) {
|
|
|
31
36
|
state = { entries: step.entries };
|
|
32
37
|
if (step.cut)
|
|
33
38
|
cutCount++;
|
|
39
|
+
if (step.lost) {
|
|
40
|
+
totalDroppedTokens += step.lost.droppedTokens;
|
|
41
|
+
totalSummaryTokens += step.lost.summaryTokens;
|
|
42
|
+
const hardLost = Math.max(0, step.lost.droppedTokens - step.lost.summaryTokens);
|
|
43
|
+
totalReReadTokens += hardLost * reReadFraction;
|
|
44
|
+
}
|
|
45
|
+
totalTurnPromptTokens += promptTokens;
|
|
34
46
|
const promptText = serializePrompt(systemText, toolsText, step.entries);
|
|
35
47
|
const result = simulator.account(promptText, outputTokens);
|
|
36
48
|
totalPromptTokens += result.promptTokens;
|
|
@@ -41,7 +53,13 @@ export function runBenchmark(strategy, config) {
|
|
|
41
53
|
maxPromptTokens = Math.max(maxPromptTokens, result.promptTokens);
|
|
42
54
|
finalPromptTokens = result.promptTokens;
|
|
43
55
|
}
|
|
44
|
-
const
|
|
56
|
+
const contextLossRate = totalDroppedTokens > 0
|
|
57
|
+
? Math.min(1, Math.max(0, 1 - totalSummaryTokens / totalDroppedTokens))
|
|
58
|
+
: 0;
|
|
59
|
+
const avgTurnPromptTokens = totalTurnPromptTokens / config.turns;
|
|
60
|
+
const reReadTurns = avgTurnPromptTokens > 0 ? totalReReadTokens / avgTurnPromptTokens : 0;
|
|
61
|
+
const reReadCost = (totalReReadTokens * pricing.input) / 1_000_000;
|
|
62
|
+
const totalCost = ((totalInputTokens + totalReReadTokens) * pricing.input +
|
|
45
63
|
totalOutputTokens * pricing.output +
|
|
46
64
|
totalCacheReadTokens * pricing.cacheRead) /
|
|
47
65
|
1_000_000;
|
|
@@ -59,6 +77,12 @@ export function runBenchmark(strategy, config) {
|
|
|
59
77
|
avgPromptTokens: Math.round(totalPromptTokens / config.turns),
|
|
60
78
|
maxPromptTokens,
|
|
61
79
|
finalPromptTokens,
|
|
80
|
+
totalDroppedTokens,
|
|
81
|
+
totalSummaryTokens,
|
|
82
|
+
contextLossRate,
|
|
83
|
+
totalReReadTokens,
|
|
84
|
+
reReadTurns,
|
|
85
|
+
reReadCost,
|
|
62
86
|
totalCost,
|
|
63
87
|
costPerTurn: totalCost / config.turns,
|
|
64
88
|
costVsKeepAllPct: 0,
|
|
@@ -85,14 +109,14 @@ export function formatBenchmarkReport(results) {
|
|
|
85
109
|
const lines = [];
|
|
86
110
|
lines.push("## Cache benchmark (OpenAI-style prefix cache)");
|
|
87
111
|
lines.push("");
|
|
88
|
-
lines.push("cacheHitRate = cacheRead / prompt tokens
|
|
112
|
+
lines.push("cacheHitRate = cacheRead / prompt tokens.");
|
|
113
|
+
lines.push("ctx loss % = 1 - summaryTokens/droppedTokens (100% == hard truncation destroyed everything).");
|
|
114
|
+
lines.push("re-read tokens = hard-lost context re-injected as new uncached input; re-read turns = re-read tokens / avg turn prompt. cost $ includes the re-read penalty.");
|
|
89
115
|
lines.push("");
|
|
90
|
-
lines.push("| strategy | cuts | avg prompt |
|
|
91
|
-
lines.push("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |");
|
|
116
|
+
lines.push("| strategy | cuts | avg prompt | cache hit % | ctx loss % | dropped ctx | re-read tok | re-read turns | cost $ | vs keep-all % |");
|
|
117
|
+
lines.push("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |");
|
|
92
118
|
for (const result of results) {
|
|
93
|
-
|
|
94
|
-
const avgCacheWrite = result.totalCacheWriteTokens / result.turns;
|
|
95
|
-
lines.push(`| ${result.name} | ${result.cutCount} | ${fmt(result.avgPromptTokens, 0)} | ${fmt(avgCacheRead, 0)} | ${fmt(avgCacheWrite, 0)} | ${fmt(result.cacheHitRate * 100, 1)} | ${fmt(result.totalCost, 4)} | ${fmt(result.costPerTurn, 5)} | ${fmt(result.costVsKeepAllPct, 1)} |`);
|
|
119
|
+
lines.push(`| ${result.name} | ${result.cutCount} | ${fmt(result.avgPromptTokens, 0)} | ${fmt(result.cacheHitRate * 100, 1)} | ${fmt(result.contextLossRate * 100, 1)} | ${fmt(result.totalDroppedTokens, 0)} | ${fmt(result.totalReReadTokens, 0)} | ${fmt(result.reReadTurns, 1)} | ${fmt(result.totalCost, 4)} | ${fmt(result.costVsKeepAllPct, 1)} |`);
|
|
96
120
|
}
|
|
97
121
|
lines.push("");
|
|
98
122
|
return lines.join("\n");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../../../src/sdk/ai/cache-benchmark/strategies.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAiB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../../../src/sdk/ai/cache-benchmark/strategies.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAiB,MAAM,YAAY,CAAC;AAU5E,oFAAoF;AACpF,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,WAAW,CAUtF;AAED,wFAAwF;AACxF,wBAAgB,OAAO,IAAI,aAAa,CAMvC;AAED,kFAAkF;AAClF,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,aAAa,CAgBlE;AAED,4FAA4F;AAC5F,wBAAgB,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,aAAa,CAoB3F;AAED,iFAAiF;AACjF,wBAAgB,gBAAgB,CAC/B,aAAa,EAAE,MAAM,EACrB,eAAe,EAAE,MAAM,EACvB,aAAa,EAAE,MAAM,GACnB,aAAa,CAsBf;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACpC,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,MAAM,EACb,gBAAgB,EAAE,MAAM,EACxB,aAAa,EAAE,MAAM,GACnB,aAAa,CAmCf"}
|
|
@@ -2,6 +2,9 @@ import { fillerText } from "./tokens.js";
|
|
|
2
2
|
function append(state, entry) {
|
|
3
3
|
return [...state.entries, entry];
|
|
4
4
|
}
|
|
5
|
+
function droppedTokens(entries) {
|
|
6
|
+
return entries.reduce((sum, entry) => sum + entry.tokens, 0);
|
|
7
|
+
}
|
|
5
8
|
/** Create a deterministic compaction summary that is byte-stable after creation. */
|
|
6
9
|
export function createSummaryEntry(cutTurn, summaryTokens) {
|
|
7
10
|
const header = `summary(cut=${cutTurn}):`;
|
|
@@ -30,7 +33,13 @@ export function truncateSliding(windowTurns) {
|
|
|
30
33
|
step: (state, _turn, entry) => {
|
|
31
34
|
const appended = append(state, entry);
|
|
32
35
|
const entries = appended.slice(-windowTurns);
|
|
33
|
-
|
|
36
|
+
const dropped = appended.slice(0, Math.max(0, appended.length - entries.length));
|
|
37
|
+
const cut = dropped.length > 0;
|
|
38
|
+
return {
|
|
39
|
+
entries,
|
|
40
|
+
cut,
|
|
41
|
+
lost: cut ? { droppedTokens: droppedTokens(dropped), summaryTokens: 0 } : undefined,
|
|
42
|
+
};
|
|
34
43
|
},
|
|
35
44
|
};
|
|
36
45
|
}
|
|
@@ -44,7 +53,13 @@ export function truncateBatched(cutEveryTurns, keepLastTurns) {
|
|
|
44
53
|
// turn is 0-based; cut once a full batch has accumulated.
|
|
45
54
|
if ((turn + 1) % cutEveryTurns === 0) {
|
|
46
55
|
const entries = appended.slice(-keepLastTurns);
|
|
47
|
-
|
|
56
|
+
const dropped = appended.slice(0, Math.max(0, appended.length - entries.length));
|
|
57
|
+
const cut = dropped.length > 0;
|
|
58
|
+
return {
|
|
59
|
+
entries,
|
|
60
|
+
cut,
|
|
61
|
+
lost: cut ? { droppedTokens: droppedTokens(dropped), summaryTokens: 0 } : undefined,
|
|
62
|
+
};
|
|
48
63
|
}
|
|
49
64
|
return { entries: appended, cut: false };
|
|
50
65
|
},
|
|
@@ -65,7 +80,12 @@ export function summarizeBatched(cutEveryTurns, keepRecentTurns, summaryTokens)
|
|
|
65
80
|
return { entries: appended, cut: false };
|
|
66
81
|
}
|
|
67
82
|
const summary = createSummaryEntry(turn + 1, summaryTokens);
|
|
68
|
-
|
|
83
|
+
const dropped = appended.slice(0, appended.length - recent.length);
|
|
84
|
+
return {
|
|
85
|
+
entries: [summary, ...recent],
|
|
86
|
+
cut: true,
|
|
87
|
+
lost: { droppedTokens: droppedTokens(dropped), summaryTokens: summary.tokens },
|
|
88
|
+
};
|
|
69
89
|
},
|
|
70
90
|
};
|
|
71
91
|
}
|
|
@@ -100,7 +120,12 @@ export function summarizeOnTokenRatio(contextWindow, ratio, keepRecentTokens, su
|
|
|
100
120
|
return { entries: appended, cut: false };
|
|
101
121
|
}
|
|
102
122
|
const summary = createSummaryEntry(turn + 1, summaryTokens);
|
|
103
|
-
|
|
123
|
+
const dropped = appended.slice(0, keepFrom);
|
|
124
|
+
return {
|
|
125
|
+
entries: [summary, ...recent],
|
|
126
|
+
cut: true,
|
|
127
|
+
lost: { droppedTokens: droppedTokens(dropped), summaryTokens: summary.tokens },
|
|
128
|
+
};
|
|
104
129
|
},
|
|
105
130
|
};
|
|
106
131
|
}
|
|
@@ -32,12 +32,21 @@ export interface StrategyState {
|
|
|
32
32
|
/** Ordered prompt entries currently in the active prefix (oldest first). */
|
|
33
33
|
entries: PromptEntry[];
|
|
34
34
|
}
|
|
35
|
+
/** Accounting for the context a cut destroyed (feeds quality/duration metrics). */
|
|
36
|
+
export interface LostContext {
|
|
37
|
+
/** Raw prompt tokens removed from the live prefix by this cut. */
|
|
38
|
+
droppedTokens: number;
|
|
39
|
+
/** Tokens of the replacement summary (0 for hard truncation). */
|
|
40
|
+
summaryTokens: number;
|
|
41
|
+
}
|
|
35
42
|
/** Result of one strategy step. */
|
|
36
43
|
export interface StrategyStep {
|
|
37
44
|
/** Ordered entries to send this turn (oldest first). */
|
|
38
45
|
entries: PromptEntry[];
|
|
39
46
|
/** True when this turn dropped or summarized part of the history (a cache breakpoint). */
|
|
40
47
|
cut: boolean;
|
|
48
|
+
/** Present when this turn dropped part of the history. */
|
|
49
|
+
lost?: LostContext;
|
|
41
50
|
}
|
|
42
51
|
export interface CacheStrategy {
|
|
43
52
|
name: string;
|
|
@@ -84,6 +93,21 @@ export interface BenchmarkResult {
|
|
|
84
93
|
avgPromptTokens: number;
|
|
85
94
|
maxPromptTokens: number;
|
|
86
95
|
finalPromptTokens: number;
|
|
96
|
+
/** Raw context tokens dropped by cuts (oldest-first history removed). */
|
|
97
|
+
totalDroppedTokens: number;
|
|
98
|
+
/** Replacement summary tokens injected by cuts. */
|
|
99
|
+
totalSummaryTokens: number;
|
|
100
|
+
/**
|
|
101
|
+
* Fraction of dropped context NOT carried forward by a summary, in [0, 1].
|
|
102
|
+
* 0 == nothing lost, 1 == hard truncation destroyed everything.
|
|
103
|
+
*/
|
|
104
|
+
contextLossRate: number;
|
|
105
|
+
/** Extra uncached input tokens from forced re-reads of hard-lost context. */
|
|
106
|
+
totalReReadTokens: number;
|
|
107
|
+
/** First-order duration proxy: how many extra turns those re-reads add. */
|
|
108
|
+
reReadTurns: number;
|
|
109
|
+
/** Dollar cost of the re-read recovery tokens. */
|
|
110
|
+
reReadCost: number;
|
|
87
111
|
totalCost: number;
|
|
88
112
|
costPerTurn: number;
|
|
89
113
|
/** Percent cost delta vs the keep-all baseline (0 == same cost). */
|
|
@@ -102,5 +126,12 @@ export interface BenchmarkConfig {
|
|
|
102
126
|
/** OpenAI only caches prefixes of at least this many tokens. */
|
|
103
127
|
minCacheTokens?: number;
|
|
104
128
|
sessionId?: string;
|
|
129
|
+
/**
|
|
130
|
+
* When a cut hard-loses context (dropped without a summary), the agent must
|
|
131
|
+
* re-read it. This is the fraction of hard-lost tokens re-injected as new
|
|
132
|
+
* uncached input. 0 disables recovery modeling (default); 1 means the agent
|
|
133
|
+
* fully re-reads everything it destroyed.
|
|
134
|
+
*/
|
|
135
|
+
reReadFraction?: number;
|
|
105
136
|
}
|
|
106
137
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/sdk/ai/cache-benchmark/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC5B,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,qBAAqB,EAAE,YAKnC,CAAC;AAEF,0EAA0E;AAC1E,MAAM,WAAW,WAAW;IAC3B,4DAA4D;IAC5D,EAAE,EAAE,MAAM,CAAC;IACX,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC7B,4EAA4E;IAC5E,OAAO,EAAE,WAAW,EAAE,CAAC;CACvB;AAED,mCAAmC;AACnC,MAAM,WAAW,YAAY;IAC5B,wDAAwD;IACxD,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,0FAA0F;IAC1F,GAAG,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/sdk/ai/cache-benchmark/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC5B,2CAA2C;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,eAAO,MAAM,qBAAqB,EAAE,YAKnC,CAAC;AAEF,0EAA0E;AAC1E,MAAM,WAAW,WAAW;IAC3B,4DAA4D;IAC5D,EAAE,EAAE,MAAM,CAAC;IACX,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CACzB;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC7B,4EAA4E;IAC5E,OAAO,EAAE,WAAW,EAAE,CAAC;CACvB;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IAC3B,kEAAkE;IAClE,aAAa,EAAE,MAAM,CAAC;IACtB,iEAAiE;IACjE,aAAa,EAAE,MAAM,CAAC;CACtB;AAED,mCAAmC;AACnC,MAAM,WAAW,YAAY;IAC5B,wDAAwD;IACxD,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,0FAA0F;IAC1F,GAAG,EAAE,OAAO,CAAC;IACb,0DAA0D;IAC1D,IAAI,CAAC,EAAE,WAAW,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,IAAI,aAAa,CAAC;IACtB,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,GAAG,YAAY,CAAC;CAC3E;AAED,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gEAAgE;IAChE,cAAc,EAAE,MAAM,CAAC;IACvB,mEAAmE;IACnE,cAAc,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;CAC3C;AAED,2CAA2C;AAC3C,MAAM,WAAW,qBAAqB;IACrC,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,eAAe,EAAE,MAAM,CAAC;IACxB,6FAA6F;IAC7F,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,gBAAgB,EAAE,MAAM,CAAC;IACzB,0CAA0C;IAC1C,YAAY,EAAE,MAAM,CAAC;IACrB,gFAAgF;IAChF,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kEAAkE;IAClE,QAAQ,EAAE,OAAO,CAAC;CAClB;AAED,8DAA8D;AAC9D,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,4CAA4C;IAC5C,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,yEAAyE;IACzE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mDAAmD;IACnD,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,iBAAiB,EAAE,MAAM,CAAC;IAC1B,2EAA2E;IAC3E,WAAW,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,oEAAoE;IACpE,gBAAgB,EAAE,MAAM,CAAC;CACzB;AAED,kDAAkD;AAClD,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,gBAAgB,EAAE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC;IACtD,qEAAqE;IACrE,gBAAgB,EAAE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC;IACtD,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAChC,gEAAgE;IAChE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB"}
|