machine-bridge-mcp 1.2.2 → 1.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -1
- package/README.md +2 -0
- package/SECURITY.md +1 -1
- package/browser-extension/manifest.json +2 -2
- package/docs/AGENT_CONTEXT.md +1 -1
- package/docs/ARCHITECTURE.md +14 -13
- package/docs/AUDIT.md +22 -0
- package/docs/ENGINEERING.md +1 -1
- package/docs/LOGGING.md +6 -6
- package/docs/OPERATIONS.md +7 -4
- package/docs/TESTING.md +3 -3
- package/docs/UPGRADING.md +3 -3
- package/package.json +1 -1
- package/src/local/capability-ranking.mjs +3 -2
- package/src/local/daemon-process.mjs +11 -4
- package/src/local/relay-connection.mjs +93 -17
- package/src/local/runtime.mjs +33 -12
- package/src/worker/daemon-liveness.ts +63 -0
- package/src/worker/daemon-sockets.ts +107 -0
- package/src/worker/index.ts +153 -97
- package/src/worker/observability.ts +2 -1
|
@@ -6,6 +6,7 @@ const DEFAULT_HEARTBEAT_INTERVAL_MS = 25_000;
|
|
|
6
6
|
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75_000;
|
|
7
7
|
const DEFAULT_CONNECT_TIMEOUT_MS = 15_000;
|
|
8
8
|
const DEFAULT_HANDSHAKE_TIMEOUT_MS = 10_000;
|
|
9
|
+
const DEFAULT_READINESS_TIMEOUT_MS = 15_000;
|
|
9
10
|
const DEFAULT_OUTAGE_WARN_AFTER_MS = 10_000;
|
|
10
11
|
const DEFAULT_OUTAGE_WARN_REPEAT_MS = 60_000;
|
|
11
12
|
const DEFAULT_OUTAGE_WARN_MAX_REPEAT_MS = 15 * 60_000;
|
|
@@ -43,6 +44,7 @@ export class RelayConnection {
|
|
|
43
44
|
this.heartbeatTimeoutMs = boundedPositiveInteger(options.heartbeatTimeoutMs, DEFAULT_HEARTBEAT_TIMEOUT_MS);
|
|
44
45
|
this.connectTimeoutMs = boundedPositiveInteger(options.connectTimeoutMs, DEFAULT_CONNECT_TIMEOUT_MS);
|
|
45
46
|
this.handshakeTimeoutMs = boundedPositiveInteger(options.handshakeTimeoutMs, DEFAULT_HANDSHAKE_TIMEOUT_MS);
|
|
47
|
+
this.readinessTimeoutMs = boundedPositiveInteger(options.readinessTimeoutMs, DEFAULT_READINESS_TIMEOUT_MS);
|
|
46
48
|
this.outageWarnAfterMs = boundedPositiveInteger(options.outageWarnAfterMs, DEFAULT_OUTAGE_WARN_AFTER_MS);
|
|
47
49
|
this.outageWarnRepeatMs = boundedPositiveInteger(options.outageWarnRepeatMs, DEFAULT_OUTAGE_WARN_REPEAT_MS);
|
|
48
50
|
this.outageWarnMaxRepeatMs = Math.max(
|
|
@@ -52,7 +54,9 @@ export class RelayConnection {
|
|
|
52
54
|
|
|
53
55
|
this.closed = true;
|
|
54
56
|
this.socket = null;
|
|
57
|
+
this.authenticated = false;
|
|
55
58
|
this.ready = false;
|
|
59
|
+
this.readinessProbeDelivered = false;
|
|
56
60
|
this.hasConnected = false;
|
|
57
61
|
this.connectedAt = 0;
|
|
58
62
|
this.lastInboundAt = 0;
|
|
@@ -61,6 +65,7 @@ export class RelayConnection {
|
|
|
61
65
|
this.connectTimer = null;
|
|
62
66
|
this.heartbeatTimer = null;
|
|
63
67
|
this.handshakeTimer = null;
|
|
68
|
+
this.readinessTimer = null;
|
|
64
69
|
this.outageWarnTimer = null;
|
|
65
70
|
this.outageStartedAt = 0;
|
|
66
71
|
this.outageAttempts = 0;
|
|
@@ -79,7 +84,9 @@ export class RelayConnection {
|
|
|
79
84
|
|
|
80
85
|
status() {
|
|
81
86
|
return {
|
|
87
|
+
authenticated: this.authenticated,
|
|
82
88
|
ready: this.ready,
|
|
89
|
+
readiness_probe_delivered: this.readinessProbeDelivered,
|
|
83
90
|
closed: this.closed,
|
|
84
91
|
network_route: this.networkRoute,
|
|
85
92
|
reconnect_attempt: this.reconnectAttempt,
|
|
@@ -89,7 +96,7 @@ export class RelayConnection {
|
|
|
89
96
|
}
|
|
90
97
|
|
|
91
98
|
currentSessionId() {
|
|
92
|
-
return this.
|
|
99
|
+
return this.authenticated ? this.activeSessionId : 0;
|
|
93
100
|
}
|
|
94
101
|
|
|
95
102
|
start() {
|
|
@@ -105,11 +112,14 @@ export class RelayConnection {
|
|
|
105
112
|
|
|
106
113
|
stop() {
|
|
107
114
|
this.closed = true;
|
|
115
|
+
this.authenticated = false;
|
|
108
116
|
this.ready = false;
|
|
117
|
+
this.readinessProbeDelivered = false;
|
|
109
118
|
this.activeSessionId = 0;
|
|
110
119
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
111
120
|
this.clearTimer("connectTimer", "clearTimeout");
|
|
112
121
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
122
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
113
123
|
this.clearTimer("reconnectTimer", "clearTimeout");
|
|
114
124
|
this.clearTimer("outageWarnTimer", "clearTimeout");
|
|
115
125
|
const socket = this.socket;
|
|
@@ -129,10 +139,11 @@ export class RelayConnection {
|
|
|
129
139
|
sendForSession(value, expectedSessionId) {
|
|
130
140
|
const sessionId = Number(expectedSessionId) || 0;
|
|
131
141
|
if (!sessionId || sessionId !== this.activeSessionId) return { ok: false, reason: "session_ended" };
|
|
132
|
-
if (!this.
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
142
|
+
if (!this.authenticated || !this.isSocketOpen(this.socket)) return { ok: false, reason: "transport_unavailable" };
|
|
143
|
+
if (!this.ready && value?.type !== "relay_probe_result") return { ok: false, reason: "transport_unavailable" };
|
|
144
|
+
if (!this.sendOnSocket(this.socket, value)) return { ok: false, reason: "send_failed" };
|
|
145
|
+
if (value?.type === "relay_probe_result") this.readinessProbeDelivered = true;
|
|
146
|
+
return { ok: true, reason: "sent" };
|
|
136
147
|
}
|
|
137
148
|
|
|
138
149
|
interrupt(category = "relay_transport_error") {
|
|
@@ -144,7 +155,7 @@ export class RelayConnection {
|
|
|
144
155
|
|
|
145
156
|
observeWelcome(message = {}) {
|
|
146
157
|
const socket = this.socket;
|
|
147
|
-
if (this.closed || this.
|
|
158
|
+
if (this.closed || this.authenticated || !this.isSocketOpen(socket)) return false;
|
|
148
159
|
const mismatch = welcomeMismatch(message, this.expectedServer, this.expectedVersion);
|
|
149
160
|
if (mismatch) {
|
|
150
161
|
this.logger.debug?.("remote relay welcome rejected", { reason: mismatch });
|
|
@@ -157,26 +168,54 @@ export class RelayConnection {
|
|
|
157
168
|
|
|
158
169
|
acknowledge(message = {}) {
|
|
159
170
|
const socket = this.socket;
|
|
160
|
-
if (this.closed || this.
|
|
171
|
+
if (this.closed || this.authenticated || !this.isSocketOpen(socket)) return false;
|
|
161
172
|
const mismatch = acknowledgementMismatch(message, this.expectedServer, this.expectedVersion);
|
|
162
173
|
if (mismatch) {
|
|
163
174
|
this.logger.debug?.("remote relay acknowledgement rejected", { reason: mismatch });
|
|
164
175
|
this.failPermanently("relay_protocol_mismatch");
|
|
165
176
|
return false;
|
|
166
177
|
}
|
|
167
|
-
this.
|
|
178
|
+
this.authenticated = true;
|
|
179
|
+
this.readinessProbeDelivered = false;
|
|
168
180
|
this.sessionGeneration += 1;
|
|
169
181
|
this.activeSessionId = this.sessionGeneration;
|
|
170
182
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
171
183
|
this.connectedAt = this.now();
|
|
172
184
|
this.lastInboundAt = this.connectedAt;
|
|
173
|
-
this.reconnectAttempt = 0;
|
|
174
185
|
this.startHeartbeat();
|
|
186
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
187
|
+
this.readinessTimer = this.scheduler.setTimeout(() => {
|
|
188
|
+
if (this.socket !== socket || this.closed || this.ready) return;
|
|
189
|
+
this.logger.debug?.("remote relay end-to-end readiness probe timed out", { timeout_ms: this.readinessTimeoutMs });
|
|
190
|
+
this.pendingCloseCategory = "relay_readiness_timeout";
|
|
191
|
+
terminateSocket(socket);
|
|
192
|
+
}, this.readinessTimeoutMs);
|
|
193
|
+
this.readinessTimer?.unref?.();
|
|
194
|
+
return true;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
confirmReady(message = {}) {
|
|
198
|
+
const socket = this.socket;
|
|
199
|
+
if (this.closed || !this.authenticated || this.ready || !this.isSocketOpen(socket)) return false;
|
|
200
|
+
const mismatch = readinessMismatch(message, this.expectedServer, this.expectedVersion);
|
|
201
|
+
if (!mismatch && !this.readinessProbeDelivered) {
|
|
202
|
+
this.logger.debug?.("remote relay declared readiness before the end-to-end probe result was delivered");
|
|
203
|
+
this.failPermanently("relay_protocol_mismatch");
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
if (mismatch) {
|
|
207
|
+
this.logger.debug?.("remote relay readiness acknowledgement rejected", { reason: mismatch });
|
|
208
|
+
this.failPermanently("relay_protocol_mismatch");
|
|
209
|
+
return false;
|
|
210
|
+
}
|
|
211
|
+
this.ready = true;
|
|
212
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
213
|
+
this.reconnectAttempt = 0;
|
|
175
214
|
|
|
176
215
|
if (!this.hasConnected) {
|
|
177
|
-
this.logger.info?.("remote relay connected");
|
|
216
|
+
this.logger.info?.("remote relay connected and end-to-end result delivery verified");
|
|
178
217
|
} else if (this.outageStartedAt > 0) {
|
|
179
|
-
const outageMs = Math.max(0, this.
|
|
218
|
+
const outageMs = Math.max(0, this.now() - this.outageStartedAt);
|
|
180
219
|
if (this.outageNoticeEmitted) {
|
|
181
220
|
this.logger.info?.(`remote relay connection restored after ${formatDuration(outageMs)} (${formatAttempts(this.outageAttempts)})`);
|
|
182
221
|
this.logger.debug?.("remote relay outage recovery details", {
|
|
@@ -204,13 +243,20 @@ export class RelayConnection {
|
|
|
204
243
|
handleServerError(message = {}) {
|
|
205
244
|
const errorCode = sanitizeProtocolErrorCode(message?.error);
|
|
206
245
|
this.logger.debug?.("remote relay reported a protocol error", { error_code: errorCode });
|
|
207
|
-
if (errorCode === "daemon_hello_timeout" && !this.
|
|
246
|
+
if (errorCode === "daemon_hello_timeout" && !this.authenticated) {
|
|
208
247
|
const socket = this.socket;
|
|
209
248
|
if (this.closed || !socket) return true;
|
|
210
249
|
this.pendingCloseCategory = "relay_handshake_timeout";
|
|
211
250
|
terminateSocket(socket);
|
|
212
251
|
return true;
|
|
213
252
|
}
|
|
253
|
+
if (errorCode === "daemon_ready_timeout" && this.authenticated && !this.ready) {
|
|
254
|
+
const socket = this.socket;
|
|
255
|
+
if (this.closed || !socket) return true;
|
|
256
|
+
this.pendingCloseCategory = "relay_readiness_timeout";
|
|
257
|
+
terminateSocket(socket);
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
214
260
|
this.failPermanently("relay_protocol_error");
|
|
215
261
|
return true;
|
|
216
262
|
}
|
|
@@ -272,8 +318,14 @@ export class RelayConnection {
|
|
|
272
318
|
socket.on("message", (data) => {
|
|
273
319
|
if (this.socket !== socket || this.closed) return;
|
|
274
320
|
this.lastInboundAt = this.now();
|
|
321
|
+
// Bind results to the generation that received this message.
|
|
322
|
+
const relayContext = {
|
|
323
|
+
sessionId: this.activeSessionId,
|
|
324
|
+
authenticated: this.authenticated === true,
|
|
325
|
+
ready: this.ready === true,
|
|
326
|
+
};
|
|
275
327
|
try {
|
|
276
|
-
const outcome = this.onMessage(data);
|
|
328
|
+
const outcome = this.onMessage(data, relayContext);
|
|
277
329
|
if (outcome && typeof outcome.catch === "function") {
|
|
278
330
|
outcome.catch((error) => this.logger.error?.("daemon message handler failed", { error_class: classifyOperationalError(error) }));
|
|
279
331
|
}
|
|
@@ -285,16 +337,20 @@ export class RelayConnection {
|
|
|
285
337
|
socket.on("close", (code, reason) => {
|
|
286
338
|
if (this.socket !== socket) return;
|
|
287
339
|
const wasReady = this.ready;
|
|
340
|
+
const wasAuthenticated = this.authenticated;
|
|
288
341
|
this.socket = null;
|
|
342
|
+
this.authenticated = false;
|
|
289
343
|
this.ready = false;
|
|
344
|
+
this.readinessProbeDelivered = false;
|
|
290
345
|
this.activeSessionId = 0;
|
|
291
346
|
this.clearTimer("connectTimer", "clearTimeout");
|
|
292
347
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
293
348
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
349
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
294
350
|
const reasonText = sanitizeCloseReason(reason);
|
|
295
351
|
const category = this.pendingCloseCategory || relayCloseCategory(code, reasonText);
|
|
296
352
|
this.pendingCloseCategory = "";
|
|
297
|
-
const connectedForMs =
|
|
353
|
+
const connectedForMs = wasAuthenticated && this.connectedAt > 0 ? Math.max(0, this.now() - this.connectedAt) : 0;
|
|
298
354
|
this.logger.debug?.("remote relay transport closed", {
|
|
299
355
|
close_code: Number(code) || 0,
|
|
300
356
|
close_reason: reasonText || "<none>",
|
|
@@ -318,7 +374,7 @@ export class RelayConnection {
|
|
|
318
374
|
this.closed = true;
|
|
319
375
|
this.clearTimer("reconnectTimer", "clearTimeout");
|
|
320
376
|
this.clearTimer("outageWarnTimer", "clearTimeout");
|
|
321
|
-
this.logger.warn?.("daemon connection was replaced by a newer
|
|
377
|
+
this.logger.warn?.("daemon connection was replaced by a newer verified instance");
|
|
322
378
|
queueMicrotask(() => {
|
|
323
379
|
try { this.onSuperseded(); } catch (error) {
|
|
324
380
|
this.logger.error?.("daemon superseded callback failed", { error_class: classifyOperationalError(error) });
|
|
@@ -348,12 +404,15 @@ export class RelayConnection {
|
|
|
348
404
|
if (this.closed) return;
|
|
349
405
|
const socket = this.socket;
|
|
350
406
|
this.closed = true;
|
|
407
|
+
this.authenticated = false;
|
|
351
408
|
this.ready = false;
|
|
409
|
+
this.readinessProbeDelivered = false;
|
|
352
410
|
this.activeSessionId = 0;
|
|
353
411
|
this.socket = null;
|
|
354
412
|
this.clearTimer("connectTimer", "clearTimeout");
|
|
355
413
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
356
414
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
415
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
357
416
|
this.clearTimer("reconnectTimer", "clearTimeout");
|
|
358
417
|
this.clearTimer("outageWarnTimer", "clearTimeout");
|
|
359
418
|
if (wasReady) {
|
|
@@ -413,7 +472,7 @@ export class RelayConnection {
|
|
|
413
472
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
414
473
|
this.heartbeatTimer = this.scheduler.setInterval(() => {
|
|
415
474
|
const socket = this.socket;
|
|
416
|
-
if (this.closed || !this.
|
|
475
|
+
if (this.closed || !this.authenticated || !this.isSocketOpen(socket)) return;
|
|
417
476
|
const silentForMs = Math.max(0, this.now() - this.lastInboundAt);
|
|
418
477
|
if (silentForMs >= this.heartbeatTimeoutMs) {
|
|
419
478
|
this.logger.debug?.("remote relay heartbeat timed out", { silent_for_ms: silentForMs });
|
|
@@ -505,6 +564,7 @@ export function relayCloseCategory(code, reason = "") {
|
|
|
505
564
|
const reasonText = String(reason || "");
|
|
506
565
|
if (isSupersededClose(numeric, reasonText)) return "superseded";
|
|
507
566
|
if (numeric === 1008 && reasonText === "daemon hello timeout") return "relay_handshake_timeout";
|
|
567
|
+
if (numeric === 1008 && reasonText === "daemon ready timeout") return "relay_readiness_timeout";
|
|
508
568
|
if (numeric === 1008 && ["stale daemon candidate", "expired daemon candidate"].includes(reasonText)) return "relay_restarting_or_unavailable";
|
|
509
569
|
if (numeric === 1008 && ["daemon hello required", "missing daemon attachment", "invalid daemon candidate timestamp"].includes(reasonText)) return "relay_protocol_error";
|
|
510
570
|
if (numeric === 1000) return "normal_close";
|
|
@@ -541,6 +601,7 @@ function relayCloseUserCause(category) {
|
|
|
541
601
|
relay_authentication_failed: "relay authentication failed",
|
|
542
602
|
relay_connect_timeout: "relay connection attempt timed out",
|
|
543
603
|
relay_handshake_timeout: "relay authentication acknowledgement timed out",
|
|
604
|
+
relay_readiness_timeout: "end-to-end relay readiness verification timed out",
|
|
544
605
|
relay_heartbeat_timeout: "relay stopped responding",
|
|
545
606
|
relay_transport_error: "relay transport error",
|
|
546
607
|
relay_protocol_error: "relay protocol error",
|
|
@@ -572,8 +633,23 @@ export function acknowledgementMismatch(message, expectedServer = "", expectedVe
|
|
|
572
633
|
return "";
|
|
573
634
|
}
|
|
574
635
|
|
|
636
|
+
export function readinessMismatch(message, expectedServer = "", expectedVersion = "") {
|
|
637
|
+
if (!message || typeof message !== "object" || Array.isArray(message)) return "invalid_readiness_acknowledgement";
|
|
638
|
+
if (message.type !== "ready_ack") return "unexpected_readiness_acknowledgement_type";
|
|
639
|
+
if (expectedServer && message.server !== expectedServer) return "server_identity_mismatch";
|
|
640
|
+
if (expectedVersion && message.version !== expectedVersion) return "server_version_mismatch";
|
|
641
|
+
if (typeof message.server !== "string" || !message.server || typeof message.version !== "string" || !message.version) return "incomplete_readiness_acknowledgement";
|
|
642
|
+
return "";
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
export function isRelayReadyContext(relayContext = {}, relay = null) {
|
|
646
|
+
if (relayContext?.ready === true) return true;
|
|
647
|
+
if (relayContext?.ready === false) return false;
|
|
648
|
+
return Number(relayContext?.sessionId) > 0 && relay?.status?.()?.ready === true;
|
|
649
|
+
}
|
|
650
|
+
|
|
575
651
|
export function isSupersededClose(code, reason) {
|
|
576
|
-
return Number(code) === 1012 && String(reason || "") === "replaced by
|
|
652
|
+
return Number(code) === 1012 && String(reason || "") === "replaced by verified daemon";
|
|
577
653
|
}
|
|
578
654
|
|
|
579
655
|
export function reconnectDelay(attempt, random = Math.random) {
|
package/src/local/runtime.mjs
CHANGED
|
@@ -2,7 +2,7 @@ import { mkdirSync, mkdtempSync, realpathSync, rmSync } from "node:fs";
|
|
|
2
2
|
import { lstat, realpath, stat } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
|
-
import { RelayConnection } from "./relay-connection.mjs";
|
|
5
|
+
import { isRelayReadyContext, RelayConnection } from "./relay-connection.mjs";
|
|
6
6
|
import { ProcessSessionManager } from "./process-sessions.mjs";
|
|
7
7
|
export { MAX_COMMAND_BYTES } from "./process-sessions.mjs";
|
|
8
8
|
import { MCP_SUPPORTED_PROTOCOL_VERSIONS, normalizePolicy, PolicyGate, SERVER_NAME } from "./tools.mjs";
|
|
@@ -302,10 +302,13 @@ export class LocalRuntime {
|
|
|
302
302
|
return;
|
|
303
303
|
}
|
|
304
304
|
if (this.handleRelayControlMessage(message)) return;
|
|
305
|
-
if (message.type
|
|
306
|
-
this.handleRelayProtocolViolation("
|
|
305
|
+
if (message.type === "relay_probe") {
|
|
306
|
+
if (isRelayReadyContext(relayContext, this.relay)) return this.handleRelayProtocolViolation("unexpected_relay_probe");
|
|
307
|
+
this.handleRelayProbe(message, relayContext);
|
|
307
308
|
return;
|
|
308
309
|
}
|
|
310
|
+
if (message.type !== "tool_call") return this.handleRelayProtocolViolation("unexpected_server_message_type");
|
|
311
|
+
if (!isRelayReadyContext(relayContext, this.relay)) return this.handleRelayProtocolViolation("tool_call_before_ready");
|
|
309
312
|
await this.handleRelayToolCall(message, relayContext);
|
|
310
313
|
}
|
|
311
314
|
|
|
@@ -318,6 +321,10 @@ export class LocalRuntime {
|
|
|
318
321
|
this.relay?.acknowledge(message);
|
|
319
322
|
return true;
|
|
320
323
|
}
|
|
324
|
+
if (message.type === "ready_ack") {
|
|
325
|
+
this.relay?.confirmReady(message);
|
|
326
|
+
return true;
|
|
327
|
+
}
|
|
321
328
|
if (message.type === "pong") return true;
|
|
322
329
|
if (message.type === "error") {
|
|
323
330
|
this.relay?.handleServerError(message);
|
|
@@ -338,6 +345,13 @@ export class LocalRuntime {
|
|
|
338
345
|
this.logger.error?.("remote relay protocol error; upgrade and redeploy both components, then restart the daemon");
|
|
339
346
|
}
|
|
340
347
|
|
|
348
|
+
handleRelayProbe(message, relayContext = {}) {
|
|
349
|
+
const id = typeof message?.id === "string" && /^probe_[A-Za-z0-9_-]{8,240}$/.test(message.id) ? message.id : "";
|
|
350
|
+
const relaySessionId = Number(relayContext.sessionId) || 0;
|
|
351
|
+
if (!id || !relaySessionId) return this.handleRelayProtocolViolation("invalid_relay_probe");
|
|
352
|
+
this.deliverRelayToolResult({ type: "relay_probe_result", id }, relaySessionId);
|
|
353
|
+
}
|
|
354
|
+
|
|
341
355
|
async handleRelayToolCall(message, relayContext = {}) {
|
|
342
356
|
const envelope = normalizeRelayToolCall(message);
|
|
343
357
|
const relaySessionId = Number(relayContext.sessionId) || 0;
|
|
@@ -387,15 +401,21 @@ export class LocalRuntime {
|
|
|
387
401
|
}
|
|
388
402
|
|
|
389
403
|
deliverRelayToolResult(response, relaySessionId = 0) {
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
:
|
|
404
|
+
const sessionId = Number(relaySessionId) || 0;
|
|
405
|
+
const outcome = this.relay?.sendForSession
|
|
406
|
+
? this.relay.sendForSession(response, sessionId)
|
|
407
|
+
: (this.send(response) ? { ok: true, reason: "sent" } : { ok: false, reason: "transport_unavailable" });
|
|
408
|
+
if (outcome?.ok) return true;
|
|
409
|
+
const reason = String(outcome?.reason || "transport_unavailable");
|
|
410
|
+
const missingSession = reason === "session_ended" && sessionId <= 0;
|
|
411
|
+
this.logger.event?.(missingSession ? "error" : "debug", "relay.tool_result.discarded", {
|
|
412
|
+
call_id: shortCallId(response?.id), reason, relay_session_id: sessionId,
|
|
413
|
+
active_session_id: Number(this.relay?.currentSessionId?.() || 0),
|
|
414
|
+
}, missingSession
|
|
415
|
+
? "Discarded a tool result because the relay session id was missing from the inbound tool_call context"
|
|
416
|
+
: reason === "send_failed"
|
|
417
|
+
? "Could not send a tool result because the relay transport failed"
|
|
418
|
+
: "Discarded a tool result because its relay session had ended");
|
|
399
419
|
if (reason === "send_failed") this.relay?.interrupt?.("relay_transport_error");
|
|
400
420
|
return false;
|
|
401
421
|
}
|
|
@@ -722,6 +742,7 @@ function createRelayConnection(runtime, { workerUrl, secret, expectedVersion, on
|
|
|
722
742
|
});
|
|
723
743
|
}
|
|
724
744
|
|
|
745
|
+
|
|
725
746
|
function handleRelayData(runtime, data, relayContext = {}) {
|
|
726
747
|
const raw = typeof data === "string" ? data : Buffer.from(data).toString("utf8");
|
|
727
748
|
if (Buffer.byteLength(raw) > MAX_WS_MESSAGE_BYTES) {
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker-side authenticated daemon liveness.
|
|
3
|
+
*
|
|
4
|
+
* Local daemons heartbeat about every 25s and treat 75s of silence as half-open.
|
|
5
|
+
* The Worker must apply a symmetric bound: role=daemon + readyState=OPEN is not
|
|
6
|
+
* enough after Durable Object hibernation or a half-closed transport, because
|
|
7
|
+
* those sockets still look connected while tool_call never returns.
|
|
8
|
+
*/
|
|
9
|
+
export const DAEMON_LIVENESS_TIMEOUT_MS = 90_000;
|
|
10
|
+
export const DAEMON_READY_TIMEOUT_MS = 15_000;
|
|
11
|
+
|
|
12
|
+
export type DaemonRole = "candidate" | "probing" | "expired" | "daemon";
|
|
13
|
+
|
|
14
|
+
export interface DaemonLivenessFields {
|
|
15
|
+
role?: DaemonRole | string;
|
|
16
|
+
connectedAt?: string;
|
|
17
|
+
lastSeenAt?: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function daemonLastSeenMs(attachment: DaemonLivenessFields | undefined | null): number {
|
|
21
|
+
const raw = attachment?.lastSeenAt || attachment?.connectedAt || "";
|
|
22
|
+
return Date.parse(raw);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function isLiveDaemonAttachment(
|
|
26
|
+
attachment: DaemonLivenessFields | undefined | null,
|
|
27
|
+
now = Date.now(),
|
|
28
|
+
): boolean {
|
|
29
|
+
if (!attachment || attachment.role !== "daemon") return false;
|
|
30
|
+
const lastSeen = daemonLastSeenMs(attachment);
|
|
31
|
+
if (!Number.isFinite(lastSeen)) return false;
|
|
32
|
+
const age = now - lastSeen;
|
|
33
|
+
return age >= 0 && age <= DAEMON_LIVENESS_TIMEOUT_MS;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function daemonLivenessDeadlineMs(attachment: DaemonLivenessFields | undefined | null): number {
|
|
37
|
+
const lastSeen = daemonLastSeenMs(attachment);
|
|
38
|
+
if (!Number.isFinite(lastSeen)) return Number.NaN;
|
|
39
|
+
return lastSeen + DAEMON_LIVENESS_TIMEOUT_MS;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function daemonReadyDeadlineMs(attachment: DaemonLivenessFields | undefined | null): number {
|
|
43
|
+
if (!attachment || attachment.role !== "probing") return Number.NaN;
|
|
44
|
+
const connectedAt = Date.parse(attachment.connectedAt || "");
|
|
45
|
+
if (!Number.isFinite(connectedAt)) return Number.NaN;
|
|
46
|
+
return connectedAt + DAEMON_READY_TIMEOUT_MS;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function withDaemonLastSeenAt<T extends DaemonLivenessFields>(
|
|
50
|
+
attachment: T,
|
|
51
|
+
lastSeenAt = new Date().toISOString(),
|
|
52
|
+
): T & { lastSeenAt: string } {
|
|
53
|
+
return { ...attachment, lastSeenAt };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export const DAEMON_HELLO_TIMEOUT_MS = 10_000;
|
|
57
|
+
|
|
58
|
+
export function isFreshDaemonCandidate(connectedAt: string, now = Date.now()): boolean {
|
|
59
|
+
const timestamp = Date.parse(connectedAt);
|
|
60
|
+
if (!Number.isFinite(timestamp)) return false;
|
|
61
|
+
const age = now - timestamp;
|
|
62
|
+
return age >= 0 && age <= DAEMON_HELLO_TIMEOUT_MS;
|
|
63
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { isLiveDaemonAttachment, withDaemonLastSeenAt, type DaemonRole } from "./daemon-liveness.ts";
|
|
2
|
+
import { sanitizeMetadataText } from "./http.ts";
|
|
3
|
+
import { sanitizeDaemonPolicy, sanitizeDaemonTools, type DaemonPolicy } from "./policy.ts";
|
|
4
|
+
|
|
5
|
+
export interface DaemonAttachment {
|
|
6
|
+
role: DaemonRole;
|
|
7
|
+
connectedAt: string;
|
|
8
|
+
lastSeenAt?: string;
|
|
9
|
+
probeId?: string;
|
|
10
|
+
policy?: DaemonPolicy;
|
|
11
|
+
tools?: string[];
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
interface WebSocketContext {
|
|
15
|
+
getWebSockets(): WebSocket[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export class DaemonSocketRegistry {
|
|
19
|
+
constructor(private readonly context: WebSocketContext) {}
|
|
20
|
+
|
|
21
|
+
attachment(socket: WebSocket): DaemonAttachment | undefined {
|
|
22
|
+
const raw = socket.deserializeAttachment();
|
|
23
|
+
if (!raw || typeof raw !== "object") return undefined;
|
|
24
|
+
const candidate = raw as Partial<DaemonAttachment>;
|
|
25
|
+
if (!["candidate", "probing", "expired", "daemon"].includes(String(candidate.role))) return undefined;
|
|
26
|
+
const policy = sanitizeDaemonPolicy(candidate.policy);
|
|
27
|
+
return {
|
|
28
|
+
role: candidate.role as DaemonRole,
|
|
29
|
+
connectedAt: sanitizeMetadataText(candidate.connectedAt, 64) ?? "",
|
|
30
|
+
lastSeenAt: sanitizeMetadataText(candidate.lastSeenAt, 64),
|
|
31
|
+
probeId: sanitizeProbeId(candidate.probeId),
|
|
32
|
+
policy,
|
|
33
|
+
tools: sanitizeDaemonTools(candidate.tools, policy),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
readyAttachment(socket: WebSocket): DaemonAttachment | undefined {
|
|
38
|
+
const attachment = this.attachment(socket);
|
|
39
|
+
return attachment?.role === "daemon" ? attachment : undefined;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
candidateSockets(): WebSocket[] { return this.openSockets("candidate"); }
|
|
43
|
+
probingSockets(): WebSocket[] { return this.openSockets("probing"); }
|
|
44
|
+
readyRoleSockets(): WebSocket[] { return this.openSockets("daemon").sort((left, right) => this.connectedAt(right) - this.connectedAt(left)); }
|
|
45
|
+
readySockets(now = Date.now()): WebSocket[] {
|
|
46
|
+
return this.readyRoleSockets().filter((socket) => isLiveDaemonAttachment(this.readyAttachment(socket), now));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
nonReadySockets(): WebSocket[] {
|
|
50
|
+
return this.context.getWebSockets().filter((socket) => this.attachment(socket)?.role !== "daemon" && socket.readyState === WebSocket.OPEN);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
beginCandidate(socket: WebSocket, connectedAt = new Date().toISOString()): void {
|
|
54
|
+
socket.serializeAttachment({ role: "candidate", connectedAt } satisfies DaemonAttachment);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
beginProbe(socket: WebSocket, values: { connectedAt: string; probeId: string; policy: DaemonPolicy; tools: string[] }): void {
|
|
58
|
+
socket.serializeAttachment({
|
|
59
|
+
role: "probing",
|
|
60
|
+
connectedAt: values.connectedAt,
|
|
61
|
+
lastSeenAt: values.connectedAt,
|
|
62
|
+
probeId: values.probeId,
|
|
63
|
+
policy: values.policy,
|
|
64
|
+
tools: values.tools,
|
|
65
|
+
} satisfies DaemonAttachment);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
promote(socket: WebSocket, lastSeenAt = new Date().toISOString()): DaemonAttachment | undefined {
|
|
69
|
+
const attachment = this.attachment(socket);
|
|
70
|
+
if (attachment?.role !== "probing") return undefined;
|
|
71
|
+
const ready = { ...attachment, role: "daemon" as const, lastSeenAt };
|
|
72
|
+
delete ready.probeId;
|
|
73
|
+
socket.serializeAttachment(ready satisfies DaemonAttachment);
|
|
74
|
+
return ready;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
touch(socket: WebSocket, lastSeenAt = new Date().toISOString()): DaemonAttachment | undefined {
|
|
78
|
+
const attachment = this.attachment(socket);
|
|
79
|
+
if (!attachment || (attachment.role !== "probing" && attachment.role !== "daemon")) return undefined;
|
|
80
|
+
const touched = withDaemonLastSeenAt(attachment, lastSeenAt);
|
|
81
|
+
socket.serializeAttachment(touched satisfies DaemonAttachment);
|
|
82
|
+
return touched;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
expire(socket: WebSocket): void {
|
|
86
|
+
const attachment = this.attachment(socket);
|
|
87
|
+
if (!attachment) return;
|
|
88
|
+
socket.serializeAttachment({
|
|
89
|
+
role: "expired",
|
|
90
|
+
connectedAt: attachment.connectedAt,
|
|
91
|
+
lastSeenAt: attachment.lastSeenAt,
|
|
92
|
+
} satisfies DaemonAttachment);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
private openSockets(role: DaemonRole): WebSocket[] {
|
|
96
|
+
return this.context.getWebSockets().filter((socket) => this.attachment(socket)?.role === role && socket.readyState === WebSocket.OPEN);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
private connectedAt(socket: WebSocket): number {
|
|
100
|
+
return Date.parse(this.attachment(socket)?.connectedAt ?? "") || 0;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function sanitizeProbeId(value: unknown): string | undefined {
|
|
105
|
+
if (typeof value !== "string" || !/^probe_[A-Za-z0-9_-]{8,240}$/.test(value)) return undefined;
|
|
106
|
+
return value;
|
|
107
|
+
}
|