machine-bridge-mcp 1.2.2 → 1.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -1
- package/README.md +2 -0
- package/SECURITY.md +1 -1
- package/browser-extension/manifest.json +2 -2
- package/docs/AGENT_CONTEXT.md +1 -1
- package/docs/ARCHITECTURE.md +13 -12
- package/docs/AUDIT.md +12 -0
- package/docs/ENGINEERING.md +1 -1
- package/docs/LOGGING.md +6 -6
- package/docs/OPERATIONS.md +6 -3
- package/docs/TESTING.md +3 -3
- package/docs/UPGRADING.md +3 -3
- package/package.json +1 -1
- package/src/local/capability-ranking.mjs +3 -2
- package/src/local/relay-connection.mjs +83 -17
- package/src/local/runtime.mjs +31 -11
- package/src/worker/daemon-liveness.ts +63 -0
- package/src/worker/daemon-sockets.ts +107 -0
- package/src/worker/index.ts +153 -97
- package/src/worker/observability.ts +2 -1
|
@@ -6,6 +6,7 @@ const DEFAULT_HEARTBEAT_INTERVAL_MS = 25_000;
|
|
|
6
6
|
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75_000;
|
|
7
7
|
const DEFAULT_CONNECT_TIMEOUT_MS = 15_000;
|
|
8
8
|
const DEFAULT_HANDSHAKE_TIMEOUT_MS = 10_000;
|
|
9
|
+
const DEFAULT_READINESS_TIMEOUT_MS = 15_000;
|
|
9
10
|
const DEFAULT_OUTAGE_WARN_AFTER_MS = 10_000;
|
|
10
11
|
const DEFAULT_OUTAGE_WARN_REPEAT_MS = 60_000;
|
|
11
12
|
const DEFAULT_OUTAGE_WARN_MAX_REPEAT_MS = 15 * 60_000;
|
|
@@ -43,6 +44,7 @@ export class RelayConnection {
|
|
|
43
44
|
this.heartbeatTimeoutMs = boundedPositiveInteger(options.heartbeatTimeoutMs, DEFAULT_HEARTBEAT_TIMEOUT_MS);
|
|
44
45
|
this.connectTimeoutMs = boundedPositiveInteger(options.connectTimeoutMs, DEFAULT_CONNECT_TIMEOUT_MS);
|
|
45
46
|
this.handshakeTimeoutMs = boundedPositiveInteger(options.handshakeTimeoutMs, DEFAULT_HANDSHAKE_TIMEOUT_MS);
|
|
47
|
+
this.readinessTimeoutMs = boundedPositiveInteger(options.readinessTimeoutMs, DEFAULT_READINESS_TIMEOUT_MS);
|
|
46
48
|
this.outageWarnAfterMs = boundedPositiveInteger(options.outageWarnAfterMs, DEFAULT_OUTAGE_WARN_AFTER_MS);
|
|
47
49
|
this.outageWarnRepeatMs = boundedPositiveInteger(options.outageWarnRepeatMs, DEFAULT_OUTAGE_WARN_REPEAT_MS);
|
|
48
50
|
this.outageWarnMaxRepeatMs = Math.max(
|
|
@@ -52,7 +54,9 @@ export class RelayConnection {
|
|
|
52
54
|
|
|
53
55
|
this.closed = true;
|
|
54
56
|
this.socket = null;
|
|
57
|
+
this.authenticated = false;
|
|
55
58
|
this.ready = false;
|
|
59
|
+
this.readinessProbeDelivered = false;
|
|
56
60
|
this.hasConnected = false;
|
|
57
61
|
this.connectedAt = 0;
|
|
58
62
|
this.lastInboundAt = 0;
|
|
@@ -61,6 +65,7 @@ export class RelayConnection {
|
|
|
61
65
|
this.connectTimer = null;
|
|
62
66
|
this.heartbeatTimer = null;
|
|
63
67
|
this.handshakeTimer = null;
|
|
68
|
+
this.readinessTimer = null;
|
|
64
69
|
this.outageWarnTimer = null;
|
|
65
70
|
this.outageStartedAt = 0;
|
|
66
71
|
this.outageAttempts = 0;
|
|
@@ -79,7 +84,9 @@ export class RelayConnection {
|
|
|
79
84
|
|
|
80
85
|
status() {
|
|
81
86
|
return {
|
|
87
|
+
authenticated: this.authenticated,
|
|
82
88
|
ready: this.ready,
|
|
89
|
+
readiness_probe_delivered: this.readinessProbeDelivered,
|
|
83
90
|
closed: this.closed,
|
|
84
91
|
network_route: this.networkRoute,
|
|
85
92
|
reconnect_attempt: this.reconnectAttempt,
|
|
@@ -89,7 +96,7 @@ export class RelayConnection {
|
|
|
89
96
|
}
|
|
90
97
|
|
|
91
98
|
currentSessionId() {
|
|
92
|
-
return this.
|
|
99
|
+
return this.authenticated ? this.activeSessionId : 0;
|
|
93
100
|
}
|
|
94
101
|
|
|
95
102
|
start() {
|
|
@@ -105,11 +112,14 @@ export class RelayConnection {
|
|
|
105
112
|
|
|
106
113
|
stop() {
|
|
107
114
|
this.closed = true;
|
|
115
|
+
this.authenticated = false;
|
|
108
116
|
this.ready = false;
|
|
117
|
+
this.readinessProbeDelivered = false;
|
|
109
118
|
this.activeSessionId = 0;
|
|
110
119
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
111
120
|
this.clearTimer("connectTimer", "clearTimeout");
|
|
112
121
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
122
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
113
123
|
this.clearTimer("reconnectTimer", "clearTimeout");
|
|
114
124
|
this.clearTimer("outageWarnTimer", "clearTimeout");
|
|
115
125
|
const socket = this.socket;
|
|
@@ -129,10 +139,11 @@ export class RelayConnection {
|
|
|
129
139
|
sendForSession(value, expectedSessionId) {
|
|
130
140
|
const sessionId = Number(expectedSessionId) || 0;
|
|
131
141
|
if (!sessionId || sessionId !== this.activeSessionId) return { ok: false, reason: "session_ended" };
|
|
132
|
-
if (!this.
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
142
|
+
if (!this.authenticated || !this.isSocketOpen(this.socket)) return { ok: false, reason: "transport_unavailable" };
|
|
143
|
+
if (!this.ready && value?.type !== "relay_probe_result") return { ok: false, reason: "transport_unavailable" };
|
|
144
|
+
if (!this.sendOnSocket(this.socket, value)) return { ok: false, reason: "send_failed" };
|
|
145
|
+
if (value?.type === "relay_probe_result") this.readinessProbeDelivered = true;
|
|
146
|
+
return { ok: true, reason: "sent" };
|
|
136
147
|
}
|
|
137
148
|
|
|
138
149
|
interrupt(category = "relay_transport_error") {
|
|
@@ -144,7 +155,7 @@ export class RelayConnection {
|
|
|
144
155
|
|
|
145
156
|
observeWelcome(message = {}) {
|
|
146
157
|
const socket = this.socket;
|
|
147
|
-
if (this.closed || this.
|
|
158
|
+
if (this.closed || this.authenticated || !this.isSocketOpen(socket)) return false;
|
|
148
159
|
const mismatch = welcomeMismatch(message, this.expectedServer, this.expectedVersion);
|
|
149
160
|
if (mismatch) {
|
|
150
161
|
this.logger.debug?.("remote relay welcome rejected", { reason: mismatch });
|
|
@@ -157,26 +168,54 @@ export class RelayConnection {
|
|
|
157
168
|
|
|
158
169
|
acknowledge(message = {}) {
|
|
159
170
|
const socket = this.socket;
|
|
160
|
-
if (this.closed || this.
|
|
171
|
+
if (this.closed || this.authenticated || !this.isSocketOpen(socket)) return false;
|
|
161
172
|
const mismatch = acknowledgementMismatch(message, this.expectedServer, this.expectedVersion);
|
|
162
173
|
if (mismatch) {
|
|
163
174
|
this.logger.debug?.("remote relay acknowledgement rejected", { reason: mismatch });
|
|
164
175
|
this.failPermanently("relay_protocol_mismatch");
|
|
165
176
|
return false;
|
|
166
177
|
}
|
|
167
|
-
this.
|
|
178
|
+
this.authenticated = true;
|
|
179
|
+
this.readinessProbeDelivered = false;
|
|
168
180
|
this.sessionGeneration += 1;
|
|
169
181
|
this.activeSessionId = this.sessionGeneration;
|
|
170
182
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
171
183
|
this.connectedAt = this.now();
|
|
172
184
|
this.lastInboundAt = this.connectedAt;
|
|
173
|
-
this.reconnectAttempt = 0;
|
|
174
185
|
this.startHeartbeat();
|
|
186
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
187
|
+
this.readinessTimer = this.scheduler.setTimeout(() => {
|
|
188
|
+
if (this.socket !== socket || this.closed || this.ready) return;
|
|
189
|
+
this.logger.debug?.("remote relay end-to-end readiness probe timed out", { timeout_ms: this.readinessTimeoutMs });
|
|
190
|
+
this.pendingCloseCategory = "relay_readiness_timeout";
|
|
191
|
+
terminateSocket(socket);
|
|
192
|
+
}, this.readinessTimeoutMs);
|
|
193
|
+
this.readinessTimer?.unref?.();
|
|
194
|
+
return true;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
confirmReady(message = {}) {
|
|
198
|
+
const socket = this.socket;
|
|
199
|
+
if (this.closed || !this.authenticated || this.ready || !this.isSocketOpen(socket)) return false;
|
|
200
|
+
const mismatch = readinessMismatch(message, this.expectedServer, this.expectedVersion);
|
|
201
|
+
if (!mismatch && !this.readinessProbeDelivered) {
|
|
202
|
+
this.logger.debug?.("remote relay declared readiness before the end-to-end probe result was delivered");
|
|
203
|
+
this.failPermanently("relay_protocol_mismatch");
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
if (mismatch) {
|
|
207
|
+
this.logger.debug?.("remote relay readiness acknowledgement rejected", { reason: mismatch });
|
|
208
|
+
this.failPermanently("relay_protocol_mismatch");
|
|
209
|
+
return false;
|
|
210
|
+
}
|
|
211
|
+
this.ready = true;
|
|
212
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
213
|
+
this.reconnectAttempt = 0;
|
|
175
214
|
|
|
176
215
|
if (!this.hasConnected) {
|
|
177
|
-
this.logger.info?.("remote relay connected");
|
|
216
|
+
this.logger.info?.("remote relay connected and end-to-end result delivery verified");
|
|
178
217
|
} else if (this.outageStartedAt > 0) {
|
|
179
|
-
const outageMs = Math.max(0, this.
|
|
218
|
+
const outageMs = Math.max(0, this.now() - this.outageStartedAt);
|
|
180
219
|
if (this.outageNoticeEmitted) {
|
|
181
220
|
this.logger.info?.(`remote relay connection restored after ${formatDuration(outageMs)} (${formatAttempts(this.outageAttempts)})`);
|
|
182
221
|
this.logger.debug?.("remote relay outage recovery details", {
|
|
@@ -204,13 +243,20 @@ export class RelayConnection {
|
|
|
204
243
|
handleServerError(message = {}) {
|
|
205
244
|
const errorCode = sanitizeProtocolErrorCode(message?.error);
|
|
206
245
|
this.logger.debug?.("remote relay reported a protocol error", { error_code: errorCode });
|
|
207
|
-
if (errorCode === "daemon_hello_timeout" && !this.
|
|
246
|
+
if (errorCode === "daemon_hello_timeout" && !this.authenticated) {
|
|
208
247
|
const socket = this.socket;
|
|
209
248
|
if (this.closed || !socket) return true;
|
|
210
249
|
this.pendingCloseCategory = "relay_handshake_timeout";
|
|
211
250
|
terminateSocket(socket);
|
|
212
251
|
return true;
|
|
213
252
|
}
|
|
253
|
+
if (errorCode === "daemon_ready_timeout" && this.authenticated && !this.ready) {
|
|
254
|
+
const socket = this.socket;
|
|
255
|
+
if (this.closed || !socket) return true;
|
|
256
|
+
this.pendingCloseCategory = "relay_readiness_timeout";
|
|
257
|
+
terminateSocket(socket);
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
214
260
|
this.failPermanently("relay_protocol_error");
|
|
215
261
|
return true;
|
|
216
262
|
}
|
|
@@ -272,8 +318,10 @@ export class RelayConnection {
|
|
|
272
318
|
socket.on("message", (data) => {
|
|
273
319
|
if (this.socket !== socket || this.closed) return;
|
|
274
320
|
this.lastInboundAt = this.now();
|
|
321
|
+
// Bind results to the generation that received this message.
|
|
322
|
+
const relayContext = { sessionId: this.activeSessionId, authenticated: this.authenticated, ready: this.ready };
|
|
275
323
|
try {
|
|
276
|
-
const outcome = this.onMessage(data);
|
|
324
|
+
const outcome = this.onMessage(data, relayContext);
|
|
277
325
|
if (outcome && typeof outcome.catch === "function") {
|
|
278
326
|
outcome.catch((error) => this.logger.error?.("daemon message handler failed", { error_class: classifyOperationalError(error) }));
|
|
279
327
|
}
|
|
@@ -285,16 +333,20 @@ export class RelayConnection {
|
|
|
285
333
|
socket.on("close", (code, reason) => {
|
|
286
334
|
if (this.socket !== socket) return;
|
|
287
335
|
const wasReady = this.ready;
|
|
336
|
+
const wasAuthenticated = this.authenticated;
|
|
288
337
|
this.socket = null;
|
|
338
|
+
this.authenticated = false;
|
|
289
339
|
this.ready = false;
|
|
340
|
+
this.readinessProbeDelivered = false;
|
|
290
341
|
this.activeSessionId = 0;
|
|
291
342
|
this.clearTimer("connectTimer", "clearTimeout");
|
|
292
343
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
293
344
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
345
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
294
346
|
const reasonText = sanitizeCloseReason(reason);
|
|
295
347
|
const category = this.pendingCloseCategory || relayCloseCategory(code, reasonText);
|
|
296
348
|
this.pendingCloseCategory = "";
|
|
297
|
-
const connectedForMs =
|
|
349
|
+
const connectedForMs = wasAuthenticated && this.connectedAt > 0 ? Math.max(0, this.now() - this.connectedAt) : 0;
|
|
298
350
|
this.logger.debug?.("remote relay transport closed", {
|
|
299
351
|
close_code: Number(code) || 0,
|
|
300
352
|
close_reason: reasonText || "<none>",
|
|
@@ -318,7 +370,7 @@ export class RelayConnection {
|
|
|
318
370
|
this.closed = true;
|
|
319
371
|
this.clearTimer("reconnectTimer", "clearTimeout");
|
|
320
372
|
this.clearTimer("outageWarnTimer", "clearTimeout");
|
|
321
|
-
this.logger.warn?.("daemon connection was replaced by a newer
|
|
373
|
+
this.logger.warn?.("daemon connection was replaced by a newer verified instance");
|
|
322
374
|
queueMicrotask(() => {
|
|
323
375
|
try { this.onSuperseded(); } catch (error) {
|
|
324
376
|
this.logger.error?.("daemon superseded callback failed", { error_class: classifyOperationalError(error) });
|
|
@@ -348,12 +400,15 @@ export class RelayConnection {
|
|
|
348
400
|
if (this.closed) return;
|
|
349
401
|
const socket = this.socket;
|
|
350
402
|
this.closed = true;
|
|
403
|
+
this.authenticated = false;
|
|
351
404
|
this.ready = false;
|
|
405
|
+
this.readinessProbeDelivered = false;
|
|
352
406
|
this.activeSessionId = 0;
|
|
353
407
|
this.socket = null;
|
|
354
408
|
this.clearTimer("connectTimer", "clearTimeout");
|
|
355
409
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
356
410
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
411
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
357
412
|
this.clearTimer("reconnectTimer", "clearTimeout");
|
|
358
413
|
this.clearTimer("outageWarnTimer", "clearTimeout");
|
|
359
414
|
if (wasReady) {
|
|
@@ -413,7 +468,7 @@ export class RelayConnection {
|
|
|
413
468
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
414
469
|
this.heartbeatTimer = this.scheduler.setInterval(() => {
|
|
415
470
|
const socket = this.socket;
|
|
416
|
-
if (this.closed || !this.
|
|
471
|
+
if (this.closed || !this.authenticated || !this.isSocketOpen(socket)) return;
|
|
417
472
|
const silentForMs = Math.max(0, this.now() - this.lastInboundAt);
|
|
418
473
|
if (silentForMs >= this.heartbeatTimeoutMs) {
|
|
419
474
|
this.logger.debug?.("remote relay heartbeat timed out", { silent_for_ms: silentForMs });
|
|
@@ -505,6 +560,7 @@ export function relayCloseCategory(code, reason = "") {
|
|
|
505
560
|
const reasonText = String(reason || "");
|
|
506
561
|
if (isSupersededClose(numeric, reasonText)) return "superseded";
|
|
507
562
|
if (numeric === 1008 && reasonText === "daemon hello timeout") return "relay_handshake_timeout";
|
|
563
|
+
if (numeric === 1008 && reasonText === "daemon ready timeout") return "relay_readiness_timeout";
|
|
508
564
|
if (numeric === 1008 && ["stale daemon candidate", "expired daemon candidate"].includes(reasonText)) return "relay_restarting_or_unavailable";
|
|
509
565
|
if (numeric === 1008 && ["daemon hello required", "missing daemon attachment", "invalid daemon candidate timestamp"].includes(reasonText)) return "relay_protocol_error";
|
|
510
566
|
if (numeric === 1000) return "normal_close";
|
|
@@ -541,6 +597,7 @@ function relayCloseUserCause(category) {
|
|
|
541
597
|
relay_authentication_failed: "relay authentication failed",
|
|
542
598
|
relay_connect_timeout: "relay connection attempt timed out",
|
|
543
599
|
relay_handshake_timeout: "relay authentication acknowledgement timed out",
|
|
600
|
+
relay_readiness_timeout: "end-to-end relay readiness verification timed out",
|
|
544
601
|
relay_heartbeat_timeout: "relay stopped responding",
|
|
545
602
|
relay_transport_error: "relay transport error",
|
|
546
603
|
relay_protocol_error: "relay protocol error",
|
|
@@ -572,8 +629,17 @@ export function acknowledgementMismatch(message, expectedServer = "", expectedVe
|
|
|
572
629
|
return "";
|
|
573
630
|
}
|
|
574
631
|
|
|
632
|
+
export function readinessMismatch(message, expectedServer = "", expectedVersion = "") {
|
|
633
|
+
if (!message || typeof message !== "object" || Array.isArray(message)) return "invalid_readiness_acknowledgement";
|
|
634
|
+
if (message.type !== "ready_ack") return "unexpected_readiness_acknowledgement_type";
|
|
635
|
+
if (expectedServer && message.server !== expectedServer) return "server_identity_mismatch";
|
|
636
|
+
if (expectedVersion && message.version !== expectedVersion) return "server_version_mismatch";
|
|
637
|
+
if (typeof message.server !== "string" || !message.server || typeof message.version !== "string" || !message.version) return "incomplete_readiness_acknowledgement";
|
|
638
|
+
return "";
|
|
639
|
+
}
|
|
640
|
+
|
|
575
641
|
export function isSupersededClose(code, reason) {
|
|
576
|
-
return Number(code) === 1012 && String(reason || "") === "replaced by
|
|
642
|
+
return Number(code) === 1012 && String(reason || "") === "replaced by verified daemon";
|
|
577
643
|
}
|
|
578
644
|
|
|
579
645
|
export function reconnectDelay(attempt, random = Math.random) {
|
package/src/local/runtime.mjs
CHANGED
|
@@ -302,10 +302,13 @@ export class LocalRuntime {
|
|
|
302
302
|
return;
|
|
303
303
|
}
|
|
304
304
|
if (this.handleRelayControlMessage(message)) return;
|
|
305
|
-
if (message.type
|
|
306
|
-
this.handleRelayProtocolViolation("
|
|
305
|
+
if (message.type === "relay_probe") {
|
|
306
|
+
if (relayContext.ready === true) return this.handleRelayProtocolViolation("unexpected_relay_probe");
|
|
307
|
+
this.handleRelayProbe(message, relayContext);
|
|
307
308
|
return;
|
|
308
309
|
}
|
|
310
|
+
if (message.type !== "tool_call") return this.handleRelayProtocolViolation("unexpected_server_message_type");
|
|
311
|
+
if (relayContext.ready !== true) return this.handleRelayProtocolViolation("tool_call_before_ready");
|
|
309
312
|
await this.handleRelayToolCall(message, relayContext);
|
|
310
313
|
}
|
|
311
314
|
|
|
@@ -318,6 +321,10 @@ export class LocalRuntime {
|
|
|
318
321
|
this.relay?.acknowledge(message);
|
|
319
322
|
return true;
|
|
320
323
|
}
|
|
324
|
+
if (message.type === "ready_ack") {
|
|
325
|
+
this.relay?.confirmReady(message);
|
|
326
|
+
return true;
|
|
327
|
+
}
|
|
321
328
|
if (message.type === "pong") return true;
|
|
322
329
|
if (message.type === "error") {
|
|
323
330
|
this.relay?.handleServerError(message);
|
|
@@ -338,6 +345,13 @@ export class LocalRuntime {
|
|
|
338
345
|
this.logger.error?.("remote relay protocol error; upgrade and redeploy both components, then restart the daemon");
|
|
339
346
|
}
|
|
340
347
|
|
|
348
|
+
handleRelayProbe(message, relayContext = {}) {
|
|
349
|
+
const id = typeof message?.id === "string" && /^probe_[A-Za-z0-9_-]{8,240}$/.test(message.id) ? message.id : "";
|
|
350
|
+
const relaySessionId = Number(relayContext.sessionId) || 0;
|
|
351
|
+
if (!id || !relaySessionId) return this.handleRelayProtocolViolation("invalid_relay_probe");
|
|
352
|
+
this.deliverRelayToolResult({ type: "relay_probe_result", id }, relaySessionId);
|
|
353
|
+
}
|
|
354
|
+
|
|
341
355
|
async handleRelayToolCall(message, relayContext = {}) {
|
|
342
356
|
const envelope = normalizeRelayToolCall(message);
|
|
343
357
|
const relaySessionId = Number(relayContext.sessionId) || 0;
|
|
@@ -387,15 +401,21 @@ export class LocalRuntime {
|
|
|
387
401
|
}
|
|
388
402
|
|
|
389
403
|
deliverRelayToolResult(response, relaySessionId = 0) {
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
:
|
|
404
|
+
const sessionId = Number(relaySessionId) || 0;
|
|
405
|
+
const outcome = this.relay?.sendForSession
|
|
406
|
+
? this.relay.sendForSession(response, sessionId)
|
|
407
|
+
: (this.send(response) ? { ok: true, reason: "sent" } : { ok: false, reason: "transport_unavailable" });
|
|
408
|
+
if (outcome?.ok) return true;
|
|
409
|
+
const reason = String(outcome?.reason || "transport_unavailable");
|
|
410
|
+
const missingSession = reason === "session_ended" && sessionId <= 0;
|
|
411
|
+
this.logger.event?.(missingSession ? "error" : "debug", "relay.tool_result.discarded", {
|
|
412
|
+
call_id: shortCallId(response?.id), reason, relay_session_id: sessionId,
|
|
413
|
+
active_session_id: Number(this.relay?.currentSessionId?.() || 0),
|
|
414
|
+
}, missingSession
|
|
415
|
+
? "Discarded a tool result because the relay session id was missing from the inbound tool_call context"
|
|
416
|
+
: reason === "send_failed"
|
|
417
|
+
? "Could not send a tool result because the relay transport failed"
|
|
418
|
+
: "Discarded a tool result because its relay session had ended");
|
|
399
419
|
if (reason === "send_failed") this.relay?.interrupt?.("relay_transport_error");
|
|
400
420
|
return false;
|
|
401
421
|
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker-side authenticated daemon liveness.
|
|
3
|
+
*
|
|
4
|
+
* Local daemons heartbeat about every 25s and treat 75s of silence as half-open.
|
|
5
|
+
* The Worker must apply a symmetric bound: role=daemon + readyState=OPEN is not
|
|
6
|
+
* enough after Durable Object hibernation or a half-closed transport, because
|
|
7
|
+
* those sockets still look connected while tool_call never returns.
|
|
8
|
+
*/
|
|
9
|
+
export const DAEMON_LIVENESS_TIMEOUT_MS = 90_000;
|
|
10
|
+
export const DAEMON_READY_TIMEOUT_MS = 15_000;
|
|
11
|
+
|
|
12
|
+
export type DaemonRole = "candidate" | "probing" | "expired" | "daemon";
|
|
13
|
+
|
|
14
|
+
export interface DaemonLivenessFields {
|
|
15
|
+
role?: DaemonRole | string;
|
|
16
|
+
connectedAt?: string;
|
|
17
|
+
lastSeenAt?: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function daemonLastSeenMs(attachment: DaemonLivenessFields | undefined | null): number {
|
|
21
|
+
const raw = attachment?.lastSeenAt || attachment?.connectedAt || "";
|
|
22
|
+
return Date.parse(raw);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function isLiveDaemonAttachment(
|
|
26
|
+
attachment: DaemonLivenessFields | undefined | null,
|
|
27
|
+
now = Date.now(),
|
|
28
|
+
): boolean {
|
|
29
|
+
if (!attachment || attachment.role !== "daemon") return false;
|
|
30
|
+
const lastSeen = daemonLastSeenMs(attachment);
|
|
31
|
+
if (!Number.isFinite(lastSeen)) return false;
|
|
32
|
+
const age = now - lastSeen;
|
|
33
|
+
return age >= 0 && age <= DAEMON_LIVENESS_TIMEOUT_MS;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function daemonLivenessDeadlineMs(attachment: DaemonLivenessFields | undefined | null): number {
|
|
37
|
+
const lastSeen = daemonLastSeenMs(attachment);
|
|
38
|
+
if (!Number.isFinite(lastSeen)) return Number.NaN;
|
|
39
|
+
return lastSeen + DAEMON_LIVENESS_TIMEOUT_MS;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function daemonReadyDeadlineMs(attachment: DaemonLivenessFields | undefined | null): number {
|
|
43
|
+
if (!attachment || attachment.role !== "probing") return Number.NaN;
|
|
44
|
+
const connectedAt = Date.parse(attachment.connectedAt || "");
|
|
45
|
+
if (!Number.isFinite(connectedAt)) return Number.NaN;
|
|
46
|
+
return connectedAt + DAEMON_READY_TIMEOUT_MS;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function withDaemonLastSeenAt<T extends DaemonLivenessFields>(
|
|
50
|
+
attachment: T,
|
|
51
|
+
lastSeenAt = new Date().toISOString(),
|
|
52
|
+
): T & { lastSeenAt: string } {
|
|
53
|
+
return { ...attachment, lastSeenAt };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export const DAEMON_HELLO_TIMEOUT_MS = 10_000;
|
|
57
|
+
|
|
58
|
+
export function isFreshDaemonCandidate(connectedAt: string, now = Date.now()): boolean {
|
|
59
|
+
const timestamp = Date.parse(connectedAt);
|
|
60
|
+
if (!Number.isFinite(timestamp)) return false;
|
|
61
|
+
const age = now - timestamp;
|
|
62
|
+
return age >= 0 && age <= DAEMON_HELLO_TIMEOUT_MS;
|
|
63
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { isLiveDaemonAttachment, withDaemonLastSeenAt, type DaemonRole } from "./daemon-liveness.ts";
|
|
2
|
+
import { sanitizeMetadataText } from "./http.ts";
|
|
3
|
+
import { sanitizeDaemonPolicy, sanitizeDaemonTools, type DaemonPolicy } from "./policy.ts";
|
|
4
|
+
|
|
5
|
+
export interface DaemonAttachment {
|
|
6
|
+
role: DaemonRole;
|
|
7
|
+
connectedAt: string;
|
|
8
|
+
lastSeenAt?: string;
|
|
9
|
+
probeId?: string;
|
|
10
|
+
policy?: DaemonPolicy;
|
|
11
|
+
tools?: string[];
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
interface WebSocketContext {
|
|
15
|
+
getWebSockets(): WebSocket[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export class DaemonSocketRegistry {
|
|
19
|
+
constructor(private readonly context: WebSocketContext) {}
|
|
20
|
+
|
|
21
|
+
attachment(socket: WebSocket): DaemonAttachment | undefined {
|
|
22
|
+
const raw = socket.deserializeAttachment();
|
|
23
|
+
if (!raw || typeof raw !== "object") return undefined;
|
|
24
|
+
const candidate = raw as Partial<DaemonAttachment>;
|
|
25
|
+
if (!["candidate", "probing", "expired", "daemon"].includes(String(candidate.role))) return undefined;
|
|
26
|
+
const policy = sanitizeDaemonPolicy(candidate.policy);
|
|
27
|
+
return {
|
|
28
|
+
role: candidate.role as DaemonRole,
|
|
29
|
+
connectedAt: sanitizeMetadataText(candidate.connectedAt, 64) ?? "",
|
|
30
|
+
lastSeenAt: sanitizeMetadataText(candidate.lastSeenAt, 64),
|
|
31
|
+
probeId: sanitizeProbeId(candidate.probeId),
|
|
32
|
+
policy,
|
|
33
|
+
tools: sanitizeDaemonTools(candidate.tools, policy),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
readyAttachment(socket: WebSocket): DaemonAttachment | undefined {
|
|
38
|
+
const attachment = this.attachment(socket);
|
|
39
|
+
return attachment?.role === "daemon" ? attachment : undefined;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
candidateSockets(): WebSocket[] { return this.openSockets("candidate"); }
|
|
43
|
+
probingSockets(): WebSocket[] { return this.openSockets("probing"); }
|
|
44
|
+
readyRoleSockets(): WebSocket[] { return this.openSockets("daemon").sort((left, right) => this.connectedAt(right) - this.connectedAt(left)); }
|
|
45
|
+
readySockets(now = Date.now()): WebSocket[] {
|
|
46
|
+
return this.readyRoleSockets().filter((socket) => isLiveDaemonAttachment(this.readyAttachment(socket), now));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
nonReadySockets(): WebSocket[] {
|
|
50
|
+
return this.context.getWebSockets().filter((socket) => this.attachment(socket)?.role !== "daemon" && socket.readyState === WebSocket.OPEN);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
beginCandidate(socket: WebSocket, connectedAt = new Date().toISOString()): void {
|
|
54
|
+
socket.serializeAttachment({ role: "candidate", connectedAt } satisfies DaemonAttachment);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
beginProbe(socket: WebSocket, values: { connectedAt: string; probeId: string; policy: DaemonPolicy; tools: string[] }): void {
|
|
58
|
+
socket.serializeAttachment({
|
|
59
|
+
role: "probing",
|
|
60
|
+
connectedAt: values.connectedAt,
|
|
61
|
+
lastSeenAt: values.connectedAt,
|
|
62
|
+
probeId: values.probeId,
|
|
63
|
+
policy: values.policy,
|
|
64
|
+
tools: values.tools,
|
|
65
|
+
} satisfies DaemonAttachment);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
promote(socket: WebSocket, lastSeenAt = new Date().toISOString()): DaemonAttachment | undefined {
|
|
69
|
+
const attachment = this.attachment(socket);
|
|
70
|
+
if (attachment?.role !== "probing") return undefined;
|
|
71
|
+
const ready = { ...attachment, role: "daemon" as const, lastSeenAt };
|
|
72
|
+
delete ready.probeId;
|
|
73
|
+
socket.serializeAttachment(ready satisfies DaemonAttachment);
|
|
74
|
+
return ready;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
touch(socket: WebSocket, lastSeenAt = new Date().toISOString()): DaemonAttachment | undefined {
|
|
78
|
+
const attachment = this.attachment(socket);
|
|
79
|
+
if (!attachment || (attachment.role !== "probing" && attachment.role !== "daemon")) return undefined;
|
|
80
|
+
const touched = withDaemonLastSeenAt(attachment, lastSeenAt);
|
|
81
|
+
socket.serializeAttachment(touched satisfies DaemonAttachment);
|
|
82
|
+
return touched;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
expire(socket: WebSocket): void {
|
|
86
|
+
const attachment = this.attachment(socket);
|
|
87
|
+
if (!attachment) return;
|
|
88
|
+
socket.serializeAttachment({
|
|
89
|
+
role: "expired",
|
|
90
|
+
connectedAt: attachment.connectedAt,
|
|
91
|
+
lastSeenAt: attachment.lastSeenAt,
|
|
92
|
+
} satisfies DaemonAttachment);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
private openSockets(role: DaemonRole): WebSocket[] {
|
|
96
|
+
return this.context.getWebSockets().filter((socket) => this.attachment(socket)?.role === role && socket.readyState === WebSocket.OPEN);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
private connectedAt(socket: WebSocket): number {
|
|
100
|
+
return Date.parse(this.attachment(socket)?.connectedAt ?? "") || 0;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function sanitizeProbeId(value: unknown): string | undefined {
|
|
105
|
+
if (typeof value !== "string" || !/^probe_[A-Za-z0-9_-]{8,240}$/.test(value)) return undefined;
|
|
106
|
+
return value;
|
|
107
|
+
}
|