machine-bridge-mcp 1.2.1 → 1.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -1
- package/README.md +2 -0
- package/SECURITY.md +1 -1
- package/browser-extension/manifest.json +2 -2
- package/docs/AGENT_CONTEXT.md +1 -1
- package/docs/ARCHITECTURE.md +14 -13
- package/docs/AUDIT.md +24 -0
- package/docs/ENGINEERING.md +1 -1
- package/docs/LOGGING.md +12 -8
- package/docs/OPERATIONS.md +10 -5
- package/docs/TESTING.md +3 -3
- package/docs/UPGRADING.md +5 -3
- package/package.json +1 -1
- package/src/local/call-registry.mjs +9 -3
- package/src/local/capability-ranking.mjs +3 -2
- package/src/local/log.mjs +10 -2
- package/src/local/relay-connection.mjs +99 -12
- package/src/local/runtime.mjs +86 -28
- package/src/local/tool-executor.mjs +3 -3
- package/src/worker/daemon-liveness.ts +63 -0
- package/src/worker/daemon-sockets.ts +107 -0
- package/src/worker/index.ts +210 -113
- package/src/worker/observability.ts +5 -2
- package/src/worker/pending-calls.ts +17 -0
- package/wrangler.jsonc +1 -1
|
@@ -6,6 +6,7 @@ const DEFAULT_HEARTBEAT_INTERVAL_MS = 25_000;
|
|
|
6
6
|
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75_000;
|
|
7
7
|
const DEFAULT_CONNECT_TIMEOUT_MS = 15_000;
|
|
8
8
|
const DEFAULT_HANDSHAKE_TIMEOUT_MS = 10_000;
|
|
9
|
+
const DEFAULT_READINESS_TIMEOUT_MS = 15_000;
|
|
9
10
|
const DEFAULT_OUTAGE_WARN_AFTER_MS = 10_000;
|
|
10
11
|
const DEFAULT_OUTAGE_WARN_REPEAT_MS = 60_000;
|
|
11
12
|
const DEFAULT_OUTAGE_WARN_MAX_REPEAT_MS = 15 * 60_000;
|
|
@@ -43,6 +44,7 @@ export class RelayConnection {
|
|
|
43
44
|
this.heartbeatTimeoutMs = boundedPositiveInteger(options.heartbeatTimeoutMs, DEFAULT_HEARTBEAT_TIMEOUT_MS);
|
|
44
45
|
this.connectTimeoutMs = boundedPositiveInteger(options.connectTimeoutMs, DEFAULT_CONNECT_TIMEOUT_MS);
|
|
45
46
|
this.handshakeTimeoutMs = boundedPositiveInteger(options.handshakeTimeoutMs, DEFAULT_HANDSHAKE_TIMEOUT_MS);
|
|
47
|
+
this.readinessTimeoutMs = boundedPositiveInteger(options.readinessTimeoutMs, DEFAULT_READINESS_TIMEOUT_MS);
|
|
46
48
|
this.outageWarnAfterMs = boundedPositiveInteger(options.outageWarnAfterMs, DEFAULT_OUTAGE_WARN_AFTER_MS);
|
|
47
49
|
this.outageWarnRepeatMs = boundedPositiveInteger(options.outageWarnRepeatMs, DEFAULT_OUTAGE_WARN_REPEAT_MS);
|
|
48
50
|
this.outageWarnMaxRepeatMs = Math.max(
|
|
@@ -52,7 +54,9 @@ export class RelayConnection {
|
|
|
52
54
|
|
|
53
55
|
this.closed = true;
|
|
54
56
|
this.socket = null;
|
|
57
|
+
this.authenticated = false;
|
|
55
58
|
this.ready = false;
|
|
59
|
+
this.readinessProbeDelivered = false;
|
|
56
60
|
this.hasConnected = false;
|
|
57
61
|
this.connectedAt = 0;
|
|
58
62
|
this.lastInboundAt = 0;
|
|
@@ -61,6 +65,7 @@ export class RelayConnection {
|
|
|
61
65
|
this.connectTimer = null;
|
|
62
66
|
this.heartbeatTimer = null;
|
|
63
67
|
this.handshakeTimer = null;
|
|
68
|
+
this.readinessTimer = null;
|
|
64
69
|
this.outageWarnTimer = null;
|
|
65
70
|
this.outageStartedAt = 0;
|
|
66
71
|
this.outageAttempts = 0;
|
|
@@ -73,18 +78,27 @@ export class RelayConnection {
|
|
|
73
78
|
this.connectedOnce = null;
|
|
74
79
|
this.connectedOnceResolve = null;
|
|
75
80
|
this.connectedOnceReject = null;
|
|
81
|
+
this.sessionGeneration = 0;
|
|
82
|
+
this.activeSessionId = 0;
|
|
76
83
|
}
|
|
77
84
|
|
|
78
85
|
status() {
|
|
79
86
|
return {
|
|
87
|
+
authenticated: this.authenticated,
|
|
80
88
|
ready: this.ready,
|
|
89
|
+
readiness_probe_delivered: this.readinessProbeDelivered,
|
|
81
90
|
closed: this.closed,
|
|
82
91
|
network_route: this.networkRoute,
|
|
83
92
|
reconnect_attempt: this.reconnectAttempt,
|
|
84
93
|
outage_active: this.outageStartedAt > 0,
|
|
94
|
+
session_generation: this.sessionGeneration,
|
|
85
95
|
};
|
|
86
96
|
}
|
|
87
97
|
|
|
98
|
+
currentSessionId() {
|
|
99
|
+
return this.authenticated ? this.activeSessionId : 0;
|
|
100
|
+
}
|
|
101
|
+
|
|
88
102
|
start() {
|
|
89
103
|
if (!this.closed && this.connectedOnce) return this.connectedOnce;
|
|
90
104
|
this.closed = false;
|
|
@@ -98,10 +112,14 @@ export class RelayConnection {
|
|
|
98
112
|
|
|
99
113
|
stop() {
|
|
100
114
|
this.closed = true;
|
|
115
|
+
this.authenticated = false;
|
|
101
116
|
this.ready = false;
|
|
117
|
+
this.readinessProbeDelivered = false;
|
|
118
|
+
this.activeSessionId = 0;
|
|
102
119
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
103
120
|
this.clearTimer("connectTimer", "clearTimeout");
|
|
104
121
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
122
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
105
123
|
this.clearTimer("reconnectTimer", "clearTimeout");
|
|
106
124
|
this.clearTimer("outageWarnTimer", "clearTimeout");
|
|
107
125
|
const socket = this.socket;
|
|
@@ -118,6 +136,16 @@ export class RelayConnection {
|
|
|
118
136
|
return this.sendOnSocket(this.socket, value);
|
|
119
137
|
}
|
|
120
138
|
|
|
139
|
+
sendForSession(value, expectedSessionId) {
|
|
140
|
+
const sessionId = Number(expectedSessionId) || 0;
|
|
141
|
+
if (!sessionId || sessionId !== this.activeSessionId) return { ok: false, reason: "session_ended" };
|
|
142
|
+
if (!this.authenticated || !this.isSocketOpen(this.socket)) return { ok: false, reason: "transport_unavailable" };
|
|
143
|
+
if (!this.ready && value?.type !== "relay_probe_result") return { ok: false, reason: "transport_unavailable" };
|
|
144
|
+
if (!this.sendOnSocket(this.socket, value)) return { ok: false, reason: "send_failed" };
|
|
145
|
+
if (value?.type === "relay_probe_result") this.readinessProbeDelivered = true;
|
|
146
|
+
return { ok: true, reason: "sent" };
|
|
147
|
+
}
|
|
148
|
+
|
|
121
149
|
interrupt(category = "relay_transport_error") {
|
|
122
150
|
if (this.closed || !this.socket) return false;
|
|
123
151
|
this.pendingCloseCategory = String(category || "relay_transport_error");
|
|
@@ -127,7 +155,7 @@ export class RelayConnection {
|
|
|
127
155
|
|
|
128
156
|
observeWelcome(message = {}) {
|
|
129
157
|
const socket = this.socket;
|
|
130
|
-
if (this.closed || this.
|
|
158
|
+
if (this.closed || this.authenticated || !this.isSocketOpen(socket)) return false;
|
|
131
159
|
const mismatch = welcomeMismatch(message, this.expectedServer, this.expectedVersion);
|
|
132
160
|
if (mismatch) {
|
|
133
161
|
this.logger.debug?.("remote relay welcome rejected", { reason: mismatch });
|
|
@@ -140,24 +168,54 @@ export class RelayConnection {
|
|
|
140
168
|
|
|
141
169
|
acknowledge(message = {}) {
|
|
142
170
|
const socket = this.socket;
|
|
143
|
-
if (this.closed || this.
|
|
171
|
+
if (this.closed || this.authenticated || !this.isSocketOpen(socket)) return false;
|
|
144
172
|
const mismatch = acknowledgementMismatch(message, this.expectedServer, this.expectedVersion);
|
|
145
173
|
if (mismatch) {
|
|
146
174
|
this.logger.debug?.("remote relay acknowledgement rejected", { reason: mismatch });
|
|
147
175
|
this.failPermanently("relay_protocol_mismatch");
|
|
148
176
|
return false;
|
|
149
177
|
}
|
|
150
|
-
this.
|
|
178
|
+
this.authenticated = true;
|
|
179
|
+
this.readinessProbeDelivered = false;
|
|
180
|
+
this.sessionGeneration += 1;
|
|
181
|
+
this.activeSessionId = this.sessionGeneration;
|
|
151
182
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
152
183
|
this.connectedAt = this.now();
|
|
153
184
|
this.lastInboundAt = this.connectedAt;
|
|
154
|
-
this.reconnectAttempt = 0;
|
|
155
185
|
this.startHeartbeat();
|
|
186
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
187
|
+
this.readinessTimer = this.scheduler.setTimeout(() => {
|
|
188
|
+
if (this.socket !== socket || this.closed || this.ready) return;
|
|
189
|
+
this.logger.debug?.("remote relay end-to-end readiness probe timed out", { timeout_ms: this.readinessTimeoutMs });
|
|
190
|
+
this.pendingCloseCategory = "relay_readiness_timeout";
|
|
191
|
+
terminateSocket(socket);
|
|
192
|
+
}, this.readinessTimeoutMs);
|
|
193
|
+
this.readinessTimer?.unref?.();
|
|
194
|
+
return true;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
confirmReady(message = {}) {
|
|
198
|
+
const socket = this.socket;
|
|
199
|
+
if (this.closed || !this.authenticated || this.ready || !this.isSocketOpen(socket)) return false;
|
|
200
|
+
const mismatch = readinessMismatch(message, this.expectedServer, this.expectedVersion);
|
|
201
|
+
if (!mismatch && !this.readinessProbeDelivered) {
|
|
202
|
+
this.logger.debug?.("remote relay declared readiness before the end-to-end probe result was delivered");
|
|
203
|
+
this.failPermanently("relay_protocol_mismatch");
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
if (mismatch) {
|
|
207
|
+
this.logger.debug?.("remote relay readiness acknowledgement rejected", { reason: mismatch });
|
|
208
|
+
this.failPermanently("relay_protocol_mismatch");
|
|
209
|
+
return false;
|
|
210
|
+
}
|
|
211
|
+
this.ready = true;
|
|
212
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
213
|
+
this.reconnectAttempt = 0;
|
|
156
214
|
|
|
157
215
|
if (!this.hasConnected) {
|
|
158
|
-
this.logger.info?.("remote relay connected");
|
|
216
|
+
this.logger.info?.("remote relay connected and end-to-end result delivery verified");
|
|
159
217
|
} else if (this.outageStartedAt > 0) {
|
|
160
|
-
const outageMs = Math.max(0, this.
|
|
218
|
+
const outageMs = Math.max(0, this.now() - this.outageStartedAt);
|
|
161
219
|
if (this.outageNoticeEmitted) {
|
|
162
220
|
this.logger.info?.(`remote relay connection restored after ${formatDuration(outageMs)} (${formatAttempts(this.outageAttempts)})`);
|
|
163
221
|
this.logger.debug?.("remote relay outage recovery details", {
|
|
@@ -185,13 +243,20 @@ export class RelayConnection {
|
|
|
185
243
|
handleServerError(message = {}) {
|
|
186
244
|
const errorCode = sanitizeProtocolErrorCode(message?.error);
|
|
187
245
|
this.logger.debug?.("remote relay reported a protocol error", { error_code: errorCode });
|
|
188
|
-
if (errorCode === "daemon_hello_timeout" && !this.
|
|
246
|
+
if (errorCode === "daemon_hello_timeout" && !this.authenticated) {
|
|
189
247
|
const socket = this.socket;
|
|
190
248
|
if (this.closed || !socket) return true;
|
|
191
249
|
this.pendingCloseCategory = "relay_handshake_timeout";
|
|
192
250
|
terminateSocket(socket);
|
|
193
251
|
return true;
|
|
194
252
|
}
|
|
253
|
+
if (errorCode === "daemon_ready_timeout" && this.authenticated && !this.ready) {
|
|
254
|
+
const socket = this.socket;
|
|
255
|
+
if (this.closed || !socket) return true;
|
|
256
|
+
this.pendingCloseCategory = "relay_readiness_timeout";
|
|
257
|
+
terminateSocket(socket);
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
195
260
|
this.failPermanently("relay_protocol_error");
|
|
196
261
|
return true;
|
|
197
262
|
}
|
|
@@ -253,8 +318,10 @@ export class RelayConnection {
|
|
|
253
318
|
socket.on("message", (data) => {
|
|
254
319
|
if (this.socket !== socket || this.closed) return;
|
|
255
320
|
this.lastInboundAt = this.now();
|
|
321
|
+
// Bind results to the generation that received this message.
|
|
322
|
+
const relayContext = { sessionId: this.activeSessionId, authenticated: this.authenticated, ready: this.ready };
|
|
256
323
|
try {
|
|
257
|
-
const outcome = this.onMessage(data);
|
|
324
|
+
const outcome = this.onMessage(data, relayContext);
|
|
258
325
|
if (outcome && typeof outcome.catch === "function") {
|
|
259
326
|
outcome.catch((error) => this.logger.error?.("daemon message handler failed", { error_class: classifyOperationalError(error) }));
|
|
260
327
|
}
|
|
@@ -266,15 +333,20 @@ export class RelayConnection {
|
|
|
266
333
|
socket.on("close", (code, reason) => {
|
|
267
334
|
if (this.socket !== socket) return;
|
|
268
335
|
const wasReady = this.ready;
|
|
336
|
+
const wasAuthenticated = this.authenticated;
|
|
269
337
|
this.socket = null;
|
|
338
|
+
this.authenticated = false;
|
|
270
339
|
this.ready = false;
|
|
340
|
+
this.readinessProbeDelivered = false;
|
|
341
|
+
this.activeSessionId = 0;
|
|
271
342
|
this.clearTimer("connectTimer", "clearTimeout");
|
|
272
343
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
273
344
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
345
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
274
346
|
const reasonText = sanitizeCloseReason(reason);
|
|
275
347
|
const category = this.pendingCloseCategory || relayCloseCategory(code, reasonText);
|
|
276
348
|
this.pendingCloseCategory = "";
|
|
277
|
-
const connectedForMs =
|
|
349
|
+
const connectedForMs = wasAuthenticated && this.connectedAt > 0 ? Math.max(0, this.now() - this.connectedAt) : 0;
|
|
278
350
|
this.logger.debug?.("remote relay transport closed", {
|
|
279
351
|
close_code: Number(code) || 0,
|
|
280
352
|
close_reason: reasonText || "<none>",
|
|
@@ -298,7 +370,7 @@ export class RelayConnection {
|
|
|
298
370
|
this.closed = true;
|
|
299
371
|
this.clearTimer("reconnectTimer", "clearTimeout");
|
|
300
372
|
this.clearTimer("outageWarnTimer", "clearTimeout");
|
|
301
|
-
this.logger.warn?.("daemon connection was replaced by a newer
|
|
373
|
+
this.logger.warn?.("daemon connection was replaced by a newer verified instance");
|
|
302
374
|
queueMicrotask(() => {
|
|
303
375
|
try { this.onSuperseded(); } catch (error) {
|
|
304
376
|
this.logger.error?.("daemon superseded callback failed", { error_class: classifyOperationalError(error) });
|
|
@@ -328,11 +400,15 @@ export class RelayConnection {
|
|
|
328
400
|
if (this.closed) return;
|
|
329
401
|
const socket = this.socket;
|
|
330
402
|
this.closed = true;
|
|
403
|
+
this.authenticated = false;
|
|
331
404
|
this.ready = false;
|
|
405
|
+
this.readinessProbeDelivered = false;
|
|
406
|
+
this.activeSessionId = 0;
|
|
332
407
|
this.socket = null;
|
|
333
408
|
this.clearTimer("connectTimer", "clearTimeout");
|
|
334
409
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
335
410
|
this.clearTimer("handshakeTimer", "clearTimeout");
|
|
411
|
+
this.clearTimer("readinessTimer", "clearTimeout");
|
|
336
412
|
this.clearTimer("reconnectTimer", "clearTimeout");
|
|
337
413
|
this.clearTimer("outageWarnTimer", "clearTimeout");
|
|
338
414
|
if (wasReady) {
|
|
@@ -392,7 +468,7 @@ export class RelayConnection {
|
|
|
392
468
|
this.clearTimer("heartbeatTimer", "clearInterval");
|
|
393
469
|
this.heartbeatTimer = this.scheduler.setInterval(() => {
|
|
394
470
|
const socket = this.socket;
|
|
395
|
-
if (this.closed || !this.
|
|
471
|
+
if (this.closed || !this.authenticated || !this.isSocketOpen(socket)) return;
|
|
396
472
|
const silentForMs = Math.max(0, this.now() - this.lastInboundAt);
|
|
397
473
|
if (silentForMs >= this.heartbeatTimeoutMs) {
|
|
398
474
|
this.logger.debug?.("remote relay heartbeat timed out", { silent_for_ms: silentForMs });
|
|
@@ -484,6 +560,7 @@ export function relayCloseCategory(code, reason = "") {
|
|
|
484
560
|
const reasonText = String(reason || "");
|
|
485
561
|
if (isSupersededClose(numeric, reasonText)) return "superseded";
|
|
486
562
|
if (numeric === 1008 && reasonText === "daemon hello timeout") return "relay_handshake_timeout";
|
|
563
|
+
if (numeric === 1008 && reasonText === "daemon ready timeout") return "relay_readiness_timeout";
|
|
487
564
|
if (numeric === 1008 && ["stale daemon candidate", "expired daemon candidate"].includes(reasonText)) return "relay_restarting_or_unavailable";
|
|
488
565
|
if (numeric === 1008 && ["daemon hello required", "missing daemon attachment", "invalid daemon candidate timestamp"].includes(reasonText)) return "relay_protocol_error";
|
|
489
566
|
if (numeric === 1000) return "normal_close";
|
|
@@ -520,6 +597,7 @@ function relayCloseUserCause(category) {
|
|
|
520
597
|
relay_authentication_failed: "relay authentication failed",
|
|
521
598
|
relay_connect_timeout: "relay connection attempt timed out",
|
|
522
599
|
relay_handshake_timeout: "relay authentication acknowledgement timed out",
|
|
600
|
+
relay_readiness_timeout: "end-to-end relay readiness verification timed out",
|
|
523
601
|
relay_heartbeat_timeout: "relay stopped responding",
|
|
524
602
|
relay_transport_error: "relay transport error",
|
|
525
603
|
relay_protocol_error: "relay protocol error",
|
|
@@ -551,8 +629,17 @@ export function acknowledgementMismatch(message, expectedServer = "", expectedVe
|
|
|
551
629
|
return "";
|
|
552
630
|
}
|
|
553
631
|
|
|
632
|
+
export function readinessMismatch(message, expectedServer = "", expectedVersion = "") {
|
|
633
|
+
if (!message || typeof message !== "object" || Array.isArray(message)) return "invalid_readiness_acknowledgement";
|
|
634
|
+
if (message.type !== "ready_ack") return "unexpected_readiness_acknowledgement_type";
|
|
635
|
+
if (expectedServer && message.server !== expectedServer) return "server_identity_mismatch";
|
|
636
|
+
if (expectedVersion && message.version !== expectedVersion) return "server_version_mismatch";
|
|
637
|
+
if (typeof message.server !== "string" || !message.server || typeof message.version !== "string" || !message.version) return "incomplete_readiness_acknowledgement";
|
|
638
|
+
return "";
|
|
639
|
+
}
|
|
640
|
+
|
|
554
641
|
export function isSupersededClose(code, reason) {
|
|
555
|
-
return Number(code) === 1012 && String(reason || "") === "replaced by
|
|
642
|
+
return Number(code) === 1012 && String(reason || "") === "replaced by verified daemon";
|
|
556
643
|
}
|
|
557
644
|
|
|
558
645
|
export function reconnectDelay(attempt, random = Math.random) {
|
package/src/local/runtime.mjs
CHANGED
|
@@ -113,6 +113,8 @@ export class LocalRuntime {
|
|
|
113
113
|
this.processTracker = new ProcessTracker();
|
|
114
114
|
this.lifecycle = new LifecycleController("local runtime");
|
|
115
115
|
this.observability = new RuntimeObservability();
|
|
116
|
+
this.activeRelayCalls = new Set();
|
|
117
|
+
this.suppressedRelayResults = new Map();
|
|
116
118
|
this.callRegistry = new CallRegistry({
|
|
117
119
|
maximum: MAX_CONCURRENT_TOOL_CALLS,
|
|
118
120
|
onCancel: (record) => {
|
|
@@ -121,7 +123,7 @@ export class LocalRuntime {
|
|
|
121
123
|
this.processTracker.terminateCall(record.id);
|
|
122
124
|
this.logger.event?.("debug", "tool.call.cancel_requested", {
|
|
123
125
|
call_id: shortCallId(record.id), tool: record.tool, origin: record.origin,
|
|
124
|
-
});
|
|
126
|
+
}, "Tool call cancellation requested");
|
|
125
127
|
},
|
|
126
128
|
onFinish: (record) => this.processTracker.releaseCall(record.id),
|
|
127
129
|
});
|
|
@@ -289,7 +291,7 @@ export class LocalRuntime {
|
|
|
289
291
|
return this.relay?.send(value) === true;
|
|
290
292
|
}
|
|
291
293
|
|
|
292
|
-
async handleMessage(raw) {
|
|
294
|
+
async handleMessage(raw, relayContext = {}) {
|
|
293
295
|
let message;
|
|
294
296
|
try { message = JSON.parse(raw); } catch {
|
|
295
297
|
this.handleRelayProtocolViolation("invalid_server_json");
|
|
@@ -300,11 +302,14 @@ export class LocalRuntime {
|
|
|
300
302
|
return;
|
|
301
303
|
}
|
|
302
304
|
if (this.handleRelayControlMessage(message)) return;
|
|
303
|
-
if (message.type
|
|
304
|
-
this.handleRelayProtocolViolation("
|
|
305
|
+
if (message.type === "relay_probe") {
|
|
306
|
+
if (relayContext.ready === true) return this.handleRelayProtocolViolation("unexpected_relay_probe");
|
|
307
|
+
this.handleRelayProbe(message, relayContext);
|
|
305
308
|
return;
|
|
306
309
|
}
|
|
307
|
-
|
|
310
|
+
if (message.type !== "tool_call") return this.handleRelayProtocolViolation("unexpected_server_message_type");
|
|
311
|
+
if (relayContext.ready !== true) return this.handleRelayProtocolViolation("tool_call_before_ready");
|
|
312
|
+
await this.handleRelayToolCall(message, relayContext);
|
|
308
313
|
}
|
|
309
314
|
|
|
310
315
|
handleRelayControlMessage(message) {
|
|
@@ -316,13 +321,17 @@ export class LocalRuntime {
|
|
|
316
321
|
this.relay?.acknowledge(message);
|
|
317
322
|
return true;
|
|
318
323
|
}
|
|
324
|
+
if (message.type === "ready_ack") {
|
|
325
|
+
this.relay?.confirmReady(message);
|
|
326
|
+
return true;
|
|
327
|
+
}
|
|
319
328
|
if (message.type === "pong") return true;
|
|
320
329
|
if (message.type === "error") {
|
|
321
330
|
this.relay?.handleServerError(message);
|
|
322
331
|
return true;
|
|
323
332
|
}
|
|
324
333
|
if (message.type === "cancel_call") {
|
|
325
|
-
if (typeof message.id === "string") this.
|
|
334
|
+
if (typeof message.id === "string") this.cancelRelayCall(message.id, "caller_cancelled");
|
|
326
335
|
return true;
|
|
327
336
|
}
|
|
328
337
|
return false;
|
|
@@ -336,37 +345,78 @@ export class LocalRuntime {
|
|
|
336
345
|
this.logger.error?.("remote relay protocol error; upgrade and redeploy both components, then restart the daemon");
|
|
337
346
|
}
|
|
338
347
|
|
|
339
|
-
|
|
348
|
+
handleRelayProbe(message, relayContext = {}) {
|
|
349
|
+
const id = typeof message?.id === "string" && /^probe_[A-Za-z0-9_-]{8,240}$/.test(message.id) ? message.id : "";
|
|
350
|
+
const relaySessionId = Number(relayContext.sessionId) || 0;
|
|
351
|
+
if (!id || !relaySessionId) return this.handleRelayProtocolViolation("invalid_relay_probe");
|
|
352
|
+
this.deliverRelayToolResult({ type: "relay_probe_result", id }, relaySessionId);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
async handleRelayToolCall(message, relayContext = {}) {
|
|
340
356
|
const envelope = normalizeRelayToolCall(message);
|
|
357
|
+
const relaySessionId = Number(relayContext.sessionId) || 0;
|
|
341
358
|
if (!envelope.ok) {
|
|
342
|
-
this.logger.
|
|
359
|
+
this.logger.warn?.("Received an invalid tool request from the relay; the request was rejected.");
|
|
360
|
+
this.logger.event?.("debug", "relay.tool_call.invalid", {
|
|
361
|
+
has_call_id: Boolean(envelope.id),
|
|
362
|
+
}, "Invalid relay tool request details");
|
|
343
363
|
if (envelope.id) this.deliverRelayToolResult({
|
|
344
364
|
type: "tool_result",
|
|
345
365
|
id: envelope.id,
|
|
346
366
|
ok: false,
|
|
347
367
|
error: { code: "invalid_request", message: "invalid tool_call envelope", retryable: false },
|
|
348
|
-
});
|
|
368
|
+
}, relaySessionId);
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
if (this.activeRelayCalls.has(envelope.id)) {
|
|
372
|
+
this.handleRelayProtocolViolation("duplicate_tool_call_id");
|
|
349
373
|
return;
|
|
350
374
|
}
|
|
351
|
-
|
|
375
|
+
this.activeRelayCalls.add(envelope.id);
|
|
352
376
|
try {
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
377
|
+
let response;
|
|
378
|
+
try {
|
|
379
|
+
const result = await this.executeTool(envelope.tool, envelope.arguments, {
|
|
380
|
+
callId: envelope.id,
|
|
381
|
+
origin: "relay",
|
|
382
|
+
timeoutMs: envelope.timeoutMs,
|
|
383
|
+
authorization: envelope.authorization,
|
|
384
|
+
});
|
|
385
|
+
response = { type: "tool_result", id: envelope.id, ok: true, result };
|
|
386
|
+
} catch (error) {
|
|
387
|
+
response = { type: "tool_result", id: envelope.id, ok: false, error: publicError(error) };
|
|
388
|
+
}
|
|
389
|
+
const suppressionReason = this.suppressedRelayResults.get(envelope.id) || "";
|
|
390
|
+
if (suppressionReason) {
|
|
391
|
+
this.logger.event?.("debug", "relay.tool_result.discarded", {
|
|
392
|
+
call_id: shortCallId(envelope.id), reason: suppressionReason,
|
|
393
|
+
}, "Discarded a tool result because the caller was no longer waiting");
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
this.deliverRelayToolResult(response, relaySessionId);
|
|
397
|
+
} finally {
|
|
398
|
+
this.activeRelayCalls.delete(envelope.id);
|
|
399
|
+
this.suppressedRelayResults.delete(envelope.id);
|
|
362
400
|
}
|
|
363
|
-
this.deliverRelayToolResult(response);
|
|
364
401
|
}
|
|
365
402
|
|
|
366
|
-
deliverRelayToolResult(response) {
|
|
367
|
-
|
|
368
|
-
this.
|
|
369
|
-
|
|
403
|
+
deliverRelayToolResult(response, relaySessionId = 0) {
|
|
404
|
+
const sessionId = Number(relaySessionId) || 0;
|
|
405
|
+
const outcome = this.relay?.sendForSession
|
|
406
|
+
? this.relay.sendForSession(response, sessionId)
|
|
407
|
+
: (this.send(response) ? { ok: true, reason: "sent" } : { ok: false, reason: "transport_unavailable" });
|
|
408
|
+
if (outcome?.ok) return true;
|
|
409
|
+
const reason = String(outcome?.reason || "transport_unavailable");
|
|
410
|
+
const missingSession = reason === "session_ended" && sessionId <= 0;
|
|
411
|
+
this.logger.event?.(missingSession ? "error" : "debug", "relay.tool_result.discarded", {
|
|
412
|
+
call_id: shortCallId(response?.id), reason, relay_session_id: sessionId,
|
|
413
|
+
active_session_id: Number(this.relay?.currentSessionId?.() || 0),
|
|
414
|
+
}, missingSession
|
|
415
|
+
? "Discarded a tool result because the relay session id was missing from the inbound tool_call context"
|
|
416
|
+
: reason === "send_failed"
|
|
417
|
+
? "Could not send a tool result because the relay transport failed"
|
|
418
|
+
: "Discarded a tool result because its relay session had ended");
|
|
419
|
+
if (reason === "send_failed") this.relay?.interrupt?.("relay_transport_error");
|
|
370
420
|
return false;
|
|
371
421
|
}
|
|
372
422
|
|
|
@@ -379,11 +429,19 @@ export class LocalRuntime {
|
|
|
379
429
|
return this.callRegistry.cancel(callId, reason);
|
|
380
430
|
}
|
|
381
431
|
|
|
432
|
+
cancelRelayCall(callId, suppressionReason = "caller_cancelled") {
|
|
433
|
+
const id = String(callId);
|
|
434
|
+
if (this.activeRelayCalls.has(id)) this.suppressedRelayResults.set(id, suppressionReason);
|
|
435
|
+
return this.cancelCall(id, "remote cancellation");
|
|
436
|
+
}
|
|
437
|
+
|
|
382
438
|
handleRelayDisconnect() {
|
|
439
|
+
for (const callId of this.activeRelayCalls) this.suppressedRelayResults.set(callId, "relay_disconnected");
|
|
383
440
|
const cancelled = this.callRegistry.cancelOrigin("relay", "remote relay disconnected");
|
|
384
441
|
this.terminateActiveProcesses("SIGTERM", true);
|
|
385
442
|
if (cancelled > 0) {
|
|
386
|
-
this.logger.event?.("debug", "relay.calls.cancelled_on_disconnect", { cancelled_calls: cancelled }
|
|
443
|
+
this.logger.event?.("debug", "relay.calls.cancelled_on_disconnect", { cancelled_calls: cancelled },
|
|
444
|
+
"Cancelled in-flight tool calls after the relay connection ended");
|
|
387
445
|
}
|
|
388
446
|
}
|
|
389
447
|
|
|
@@ -669,7 +727,7 @@ function createRelayConnection(runtime, { workerUrl, secret, expectedVersion, on
|
|
|
669
727
|
policy: runtime.policy,
|
|
670
728
|
protocol_versions: MCP_SUPPORTED_PROTOCOL_VERSIONS,
|
|
671
729
|
}),
|
|
672
|
-
onMessage: (data) => handleRelayData(runtime, data),
|
|
730
|
+
onMessage: (data, relayContext) => handleRelayData(runtime, data, relayContext),
|
|
673
731
|
onDisconnect: () => runtime.handleRelayDisconnect(),
|
|
674
732
|
onSuperseded: () => {
|
|
675
733
|
runtime.terminateActiveProcesses("SIGKILL");
|
|
@@ -684,13 +742,13 @@ function createRelayConnection(runtime, { workerUrl, secret, expectedVersion, on
|
|
|
684
742
|
});
|
|
685
743
|
}
|
|
686
744
|
|
|
687
|
-
function handleRelayData(runtime, data) {
|
|
745
|
+
function handleRelayData(runtime, data, relayContext = {}) {
|
|
688
746
|
const raw = typeof data === "string" ? data : Buffer.from(data).toString("utf8");
|
|
689
747
|
if (Buffer.byteLength(raw) > MAX_WS_MESSAGE_BYTES) {
|
|
690
748
|
runtime.handleRelayProtocolViolation("server_message_too_large");
|
|
691
749
|
return;
|
|
692
750
|
}
|
|
693
|
-
return runtime.handleMessage(raw);
|
|
751
|
+
return runtime.handleMessage(raw, relayContext);
|
|
694
752
|
}
|
|
695
753
|
|
|
696
754
|
function normalizeRelayToolCall(message) {
|
|
@@ -67,7 +67,7 @@ function observabilityMiddleware(observability, logger, safeMessage, slowMs) {
|
|
|
67
67
|
call_id: shortCallId(operation.context.callId),
|
|
68
68
|
tool: operation.tool,
|
|
69
69
|
origin: operation.context.origin,
|
|
70
|
-
});
|
|
70
|
+
}, "Tool call started");
|
|
71
71
|
try {
|
|
72
72
|
const result = await next(operation);
|
|
73
73
|
const durationMs = performance.now() - started;
|
|
@@ -75,7 +75,7 @@ function observabilityMiddleware(observability, logger, safeMessage, slowMs) {
|
|
|
75
75
|
observability.finish(operation.tool, { status: "completed", durationMs, slow });
|
|
76
76
|
logger.event?.("debug", slow ? "tool.call.slow" : "tool.call.completed", {
|
|
77
77
|
call_id: shortCallId(operation.context.callId), tool: operation.tool, origin: operation.context.origin, duration_ms: durationMs,
|
|
78
|
-
});
|
|
78
|
+
}, slow ? "Tool call completed slowly" : "Tool call completed");
|
|
79
79
|
return result;
|
|
80
80
|
} catch (error) {
|
|
81
81
|
const normalized = normalizeBridgeError(error, { safeMessage: () => safeMessage(error, operation.args) });
|
|
@@ -86,7 +86,7 @@ function observabilityMiddleware(observability, logger, safeMessage, slowMs) {
|
|
|
86
86
|
logger.event?.("debug", "tool.call.failed", {
|
|
87
87
|
call_id: shortCallId(operation.context.callId), tool: operation.tool, origin: operation.context.origin,
|
|
88
88
|
duration_ms: durationMs, error_code: code, retryable: normalized.retryable,
|
|
89
|
-
});
|
|
89
|
+
}, "Tool call failed");
|
|
90
90
|
throw normalized;
|
|
91
91
|
}
|
|
92
92
|
};
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker-side authenticated daemon liveness.
|
|
3
|
+
*
|
|
4
|
+
* Local daemons heartbeat about every 25s and treat 75s of silence as half-open.
|
|
5
|
+
* The Worker must apply a symmetric bound: role=daemon + readyState=OPEN is not
|
|
6
|
+
* enough after Durable Object hibernation or a half-closed transport, because
|
|
7
|
+
* those sockets still look connected while tool_call never returns.
|
|
8
|
+
*/
|
|
9
|
+
export const DAEMON_LIVENESS_TIMEOUT_MS = 90_000;
|
|
10
|
+
export const DAEMON_READY_TIMEOUT_MS = 15_000;
|
|
11
|
+
|
|
12
|
+
export type DaemonRole = "candidate" | "probing" | "expired" | "daemon";
|
|
13
|
+
|
|
14
|
+
export interface DaemonLivenessFields {
|
|
15
|
+
role?: DaemonRole | string;
|
|
16
|
+
connectedAt?: string;
|
|
17
|
+
lastSeenAt?: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function daemonLastSeenMs(attachment: DaemonLivenessFields | undefined | null): number {
|
|
21
|
+
const raw = attachment?.lastSeenAt || attachment?.connectedAt || "";
|
|
22
|
+
return Date.parse(raw);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function isLiveDaemonAttachment(
|
|
26
|
+
attachment: DaemonLivenessFields | undefined | null,
|
|
27
|
+
now = Date.now(),
|
|
28
|
+
): boolean {
|
|
29
|
+
if (!attachment || attachment.role !== "daemon") return false;
|
|
30
|
+
const lastSeen = daemonLastSeenMs(attachment);
|
|
31
|
+
if (!Number.isFinite(lastSeen)) return false;
|
|
32
|
+
const age = now - lastSeen;
|
|
33
|
+
return age >= 0 && age <= DAEMON_LIVENESS_TIMEOUT_MS;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function daemonLivenessDeadlineMs(attachment: DaemonLivenessFields | undefined | null): number {
|
|
37
|
+
const lastSeen = daemonLastSeenMs(attachment);
|
|
38
|
+
if (!Number.isFinite(lastSeen)) return Number.NaN;
|
|
39
|
+
return lastSeen + DAEMON_LIVENESS_TIMEOUT_MS;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function daemonReadyDeadlineMs(attachment: DaemonLivenessFields | undefined | null): number {
|
|
43
|
+
if (!attachment || attachment.role !== "probing") return Number.NaN;
|
|
44
|
+
const connectedAt = Date.parse(attachment.connectedAt || "");
|
|
45
|
+
if (!Number.isFinite(connectedAt)) return Number.NaN;
|
|
46
|
+
return connectedAt + DAEMON_READY_TIMEOUT_MS;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function withDaemonLastSeenAt<T extends DaemonLivenessFields>(
|
|
50
|
+
attachment: T,
|
|
51
|
+
lastSeenAt = new Date().toISOString(),
|
|
52
|
+
): T & { lastSeenAt: string } {
|
|
53
|
+
return { ...attachment, lastSeenAt };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export const DAEMON_HELLO_TIMEOUT_MS = 10_000;
|
|
57
|
+
|
|
58
|
+
export function isFreshDaemonCandidate(connectedAt: string, now = Date.now()): boolean {
|
|
59
|
+
const timestamp = Date.parse(connectedAt);
|
|
60
|
+
if (!Number.isFinite(timestamp)) return false;
|
|
61
|
+
const age = now - timestamp;
|
|
62
|
+
return age >= 0 && age <= DAEMON_HELLO_TIMEOUT_MS;
|
|
63
|
+
}
|