machine-bridge-mcp 3.0.0-beta.20 → 3.0.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +2 -0
- package/SECURITY.md +4 -4
- package/browser-extension/manifest.json +1 -1
- package/docs/ARCHITECTURE.md +11 -9
- package/docs/AUDIT.md +16 -0
- package/docs/ENGINEERING.md +1 -1
- package/docs/LOCAL_AUTHORIZATION.md +1 -1
- package/docs/LOGGING.md +3 -1
- package/docs/MULTI_ACCOUNT.md +1 -1
- package/docs/OPERATIONS.md +5 -3
- package/docs/OVERVIEW.md +1 -1
- package/docs/PRIVACY.md +1 -1
- package/docs/TESTING.md +9 -7
- package/docs/THREAT_MODEL.md +3 -2
- package/docs/UPGRADING.md +7 -1
- package/package.json +1 -1
- package/src/local/process-tree-ownership.mjs +17 -5
- package/src/local/process-tree.mjs +6 -2
- package/src/local/relay-connection.mjs +24 -11
- package/src/local/runtime-activation.mjs +16 -0
- package/src/local/runtime.mjs +3 -1
- package/src/local/worker-deployment.mjs +16 -6
- package/src/worker/daemon-socket-attachment.ts +52 -0
- package/src/worker/daemon-sockets.ts +17 -48
- package/src/worker/durable-stream-calls.ts +129 -0
- package/src/worker/durable-stream-result.ts +22 -0
- package/src/worker/index.ts +120 -88
- package/src/worker/mcp-pending-call-expiry.ts +20 -0
- package/src/worker/mcp-pending-call-inspection.ts +33 -0
- package/src/worker/mcp-pending-call-records.ts +58 -0
- package/src/worker/mcp-pending-call-storage.ts +18 -0
- package/src/worker/mcp-pending-call-store.ts +285 -0
- package/src/worker/mcp-resumption-index.ts +41 -0
- package/src/worker/mcp-resumption-records.ts +8 -2
- package/src/worker/mcp-resumption.ts +80 -87
- package/src/worker/mcp-stream-dispatch.ts +42 -54
- package/src/worker/pending-admission.ts +15 -0
- package/src/worker/pending-call-contract.ts +4 -7
- package/src/worker/pending-calls.ts +4 -10
- package/src/worker/runtime-alarm-storage.ts +35 -0
- package/src/worker/runtime-alarm.ts +29 -39
- package/src/worker/websocket-protocol.ts +4 -0
- package/src/worker/worker-entry.ts +50 -0
|
@@ -167,7 +167,7 @@ export class RelayConnection {
|
|
|
167
167
|
this.logger.debug?.("remote relay welcome received");
|
|
168
168
|
Promise.resolve(this.helloMessage(message)).then((hello) => {
|
|
169
169
|
if (this.socket !== socket || this.closed || this.authenticated) return;
|
|
170
|
-
|
|
170
|
+
this.sendOnSocket(socket, hello);
|
|
171
171
|
}).catch((error) => {
|
|
172
172
|
this.logger.debug?.("could not create daemon authentication proof", { error_class: classifyOperationalError(error) });
|
|
173
173
|
this.failPermanently("relay_authentication_failed");
|
|
@@ -256,18 +256,18 @@ export class RelayConnection {
|
|
|
256
256
|
|
|
257
257
|
handleServerError(message = {}) {
|
|
258
258
|
const errorCode = sanitizeProtocolErrorCode(message?.error);
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
if (
|
|
259
|
+
const reconnectCategory = relayServerErrorReconnectCategory(errorCode, {
|
|
260
|
+
authenticated: this.authenticated,
|
|
261
|
+
ready: this.ready,
|
|
262
|
+
});
|
|
263
|
+
this.logger.debug?.(
|
|
264
|
+
reconnectCategory ? "remote relay requested connection recovery" : "remote relay reported a protocol error",
|
|
265
|
+
{ error_code: errorCode, reconnect_category: reconnectCategory || "none" },
|
|
266
|
+
);
|
|
267
|
+
if (reconnectCategory) {
|
|
268
268
|
const socket = this.socket;
|
|
269
269
|
if (this.closed || !socket) return true;
|
|
270
|
-
this.pendingCloseCategory =
|
|
270
|
+
this.pendingCloseCategory = reconnectCategory;
|
|
271
271
|
terminateSocket(socket);
|
|
272
272
|
return true;
|
|
273
273
|
}
|
|
@@ -587,12 +587,25 @@ export class RelayConnection {
|
|
|
587
587
|
}
|
|
588
588
|
}
|
|
589
589
|
|
|
590
|
+
export function relayServerErrorReconnectCategory(errorCode, state = {}) {
|
|
591
|
+
const code = sanitizeProtocolErrorCode(errorCode);
|
|
592
|
+
const authenticated = state?.authenticated === true;
|
|
593
|
+
const ready = state?.ready === true;
|
|
594
|
+
if (code === "daemon_hello_timeout" && !authenticated) return "relay_handshake_timeout";
|
|
595
|
+
if (code === "daemon_ready_timeout" && authenticated && !ready) return "relay_readiness_timeout";
|
|
596
|
+
if (code === "daemon_transport_error") return "relay_transport_error";
|
|
597
|
+
if (code === "daemon_liveness_timeout") return "relay_heartbeat_timeout";
|
|
598
|
+
return "";
|
|
599
|
+
}
|
|
600
|
+
|
|
590
601
|
export function relayCloseCategory(code, reason = "") {
|
|
591
602
|
const numeric = Number(code);
|
|
592
603
|
const reasonText = String(reason || "");
|
|
593
604
|
if (isSupersededClose(numeric, reasonText)) return "superseded";
|
|
594
605
|
if (numeric === 1008 && reasonText === "daemon hello timeout") return "relay_handshake_timeout";
|
|
595
606
|
if (numeric === 1008 && reasonText === "daemon ready timeout") return "relay_readiness_timeout";
|
|
607
|
+
if ([1008, 1012].includes(numeric) && ["daemon pong failed", "daemon send failed"].includes(reasonText)) return "relay_transport_error";
|
|
608
|
+
if ([1008, 1012].includes(numeric) && reasonText === "daemon liveness timeout") return "relay_heartbeat_timeout";
|
|
596
609
|
if (numeric === 1008 && ["stale daemon candidate", "expired daemon candidate"].includes(reasonText)) return "relay_restarting_or_unavailable";
|
|
597
610
|
if (numeric === 1008 && ["daemon hello required", "missing daemon attachment", "invalid daemon candidate timestamp"].includes(reasonText)) return "relay_protocol_error";
|
|
598
611
|
if (numeric === 1000) return "normal_close";
|
|
@@ -19,11 +19,14 @@ export async function activatePersistentRuntime(options = {}) {
|
|
|
19
19
|
let daemonLock = null;
|
|
20
20
|
let candidateRuntime = null;
|
|
21
21
|
let candidateRelayVerified = false;
|
|
22
|
+
let providerStopped = false;
|
|
23
|
+
let serviceStarted = false;
|
|
22
24
|
try {
|
|
23
25
|
const providerStop = await options.stopAutostart();
|
|
24
26
|
if (providerStop?.ok === false && providerStop?.active === true) {
|
|
25
27
|
throw new Error("the existing autostart service could not be stopped before activation");
|
|
26
28
|
}
|
|
29
|
+
providerStopped = providerStop?.active_before === true && providerStop?.active === false;
|
|
27
30
|
|
|
28
31
|
daemonLock = await options.acquireDaemonLock();
|
|
29
32
|
if (!daemonLock?.acquired) {
|
|
@@ -60,6 +63,7 @@ export async function activatePersistentRuntime(options = {}) {
|
|
|
60
63
|
if (serviceStart?.ok === false) {
|
|
61
64
|
throw new Error(`autostart start failed (${serviceStart.provider || "unknown"})`);
|
|
62
65
|
}
|
|
66
|
+
serviceStarted = true;
|
|
63
67
|
|
|
64
68
|
const convergence = await waitForActivatedRuntime({
|
|
65
69
|
expectedVersion: options.expectedVersion,
|
|
@@ -85,6 +89,9 @@ export async function activatePersistentRuntime(options = {}) {
|
|
|
85
89
|
try { startupLock.release(); }
|
|
86
90
|
catch (failure) { cleanupErrors.push(failure); }
|
|
87
91
|
}
|
|
92
|
+
try {
|
|
93
|
+
await restoreStoppedProvider(options.startAutostart, { providerStopped, serviceStarted });
|
|
94
|
+
} catch (failure) { cleanupErrors.push(failure); }
|
|
88
95
|
if (cleanupErrors.length) {
|
|
89
96
|
throw new AggregateError(
|
|
90
97
|
[error, ...cleanupErrors],
|
|
@@ -95,6 +102,15 @@ export async function activatePersistentRuntime(options = {}) {
|
|
|
95
102
|
}
|
|
96
103
|
}
|
|
97
104
|
|
|
105
|
+
|
|
106
|
+
async function restoreStoppedProvider(startAutostart, { providerStopped, serviceStarted }) {
|
|
107
|
+
if (!providerStopped || serviceStarted) return;
|
|
108
|
+
const restored = await startAutostart();
|
|
109
|
+
if (restored?.ok === false) {
|
|
110
|
+
throw new Error(`previous autostart service could not be restored (${restored.provider || "unknown"})`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
98
114
|
export async function waitForActivatedRuntime({
|
|
99
115
|
inspectDaemon,
|
|
100
116
|
checkWorker,
|
package/src/local/runtime.mjs
CHANGED
|
@@ -359,7 +359,9 @@ export class LocalRuntime {
|
|
|
359
359
|
const relaySessionId = Number(relayContext.sessionId) || 0;
|
|
360
360
|
if (!id || !relaySessionId) return this.handleRelayProtocolViolation("invalid_relay_probe");
|
|
361
361
|
const outcome = this.relay?.sendForSession?.({ type: "relay_probe_result", id }, relaySessionId);
|
|
362
|
-
if (!outcome?.ok
|
|
362
|
+
if (!outcome?.ok && !["send_failed", "transport_unavailable", "session_ended"].includes(outcome?.reason)) {
|
|
363
|
+
this.handleRelayProtocolViolation("relay_probe_delivery_failed");
|
|
364
|
+
}
|
|
363
365
|
}
|
|
364
366
|
|
|
365
367
|
async handleRelayToolCall(message) {
|
|
@@ -12,6 +12,8 @@ import {
|
|
|
12
12
|
workerHealthUserReason,
|
|
13
13
|
} from "./worker-health.mjs";
|
|
14
14
|
|
|
15
|
+
const DEFAULT_DEPLOYMENT_HEALTH_ATTEMPTS = 20;
|
|
16
|
+
|
|
15
17
|
export async function ensureWorkerDeployment(state, args = {}, options = {}) {
|
|
16
18
|
const logger = options.logger || console;
|
|
17
19
|
const expectedVersion = options.expectedVersion || currentPackageVersion(options.packageRoot || packageRoot);
|
|
@@ -24,18 +26,24 @@ export async function ensureWorkerDeployment(state, args = {}, options = {}) {
|
|
|
24
26
|
const complete = hasCompleteWorkerState(state.worker);
|
|
25
27
|
|
|
26
28
|
if (!args.forceWorker && !args.rotateSecrets && complete && state.worker.deployHash === desiredHash) {
|
|
27
|
-
const
|
|
29
|
+
const recordedCurrentDeployment = state.worker.deployedVersion === expectedVersion;
|
|
30
|
+
const attempts = recordedCurrentDeployment
|
|
31
|
+
? (options.recordedDeploymentHealthAttempts ?? DEFAULT_DEPLOYMENT_HEALTH_ATTEMPTS)
|
|
32
|
+
: (options.existingHealthAttempts ?? 2);
|
|
33
|
+
const health = await retryHealthFn(state.worker.url, expectedVersion, attempts, healthOptions);
|
|
28
34
|
if (health.ok) {
|
|
29
35
|
logger.success?.("Worker unchanged and healthy", { url: state.worker.url });
|
|
30
36
|
logger.debug?.("Worker health route", { network_route: health.networkRoute || "unknown" });
|
|
31
37
|
return state.worker;
|
|
32
38
|
}
|
|
39
|
+
logger.debug?.("Worker health check detail", { health_error: health.error, network_route: health.networkRoute || "unknown" });
|
|
40
|
+
if (recordedCurrentDeployment) {
|
|
41
|
+
throw workerVerificationError(health.error, { deploymentSucceeded: false, recordedCurrentDeployment: true });
|
|
42
|
+
}
|
|
33
43
|
if (!workerHealthRequiresRedeploy(health.error)) {
|
|
34
|
-
logger.debug?.("Worker health check detail", { health_error: health.error, network_route: health.networkRoute || "unknown" });
|
|
35
44
|
throw workerVerificationError(health.error, { deploymentSucceeded: false });
|
|
36
45
|
}
|
|
37
46
|
logger.warn?.("Recorded Worker is stale; redeploying the same Worker", { reason: workerHealthUserReason(health.error) });
|
|
38
|
-
logger.debug?.("Worker health check detail", { health_error: health.error, network_route: health.networkRoute || "unknown" });
|
|
39
47
|
}
|
|
40
48
|
|
|
41
49
|
logger.info?.("Checking Cloudflare Wrangler login");
|
|
@@ -68,7 +76,7 @@ export async function ensureWorkerDeployment(state, args = {}, options = {}) {
|
|
|
68
76
|
state.worker.updatedAt = new Date().toISOString();
|
|
69
77
|
saveStateFn(state);
|
|
70
78
|
|
|
71
|
-
const health = await retryHealthFn(state.worker.url, expectedVersion, options.deploymentHealthAttempts
|
|
79
|
+
const health = await retryHealthFn(state.worker.url, expectedVersion, options.deploymentHealthAttempts ?? DEFAULT_DEPLOYMENT_HEALTH_ATTEMPTS, healthOptions);
|
|
72
80
|
if (!health.ok) {
|
|
73
81
|
logger.debug?.("Worker post-deployment health detail", { health_error: health.error, network_route: health.networkRoute || "unknown" });
|
|
74
82
|
throw workerVerificationError(health.error, { deploymentSucceeded: true });
|
|
@@ -143,12 +151,14 @@ function collectHashFiles(target, out) {
|
|
|
143
151
|
}
|
|
144
152
|
}
|
|
145
153
|
|
|
146
|
-
function workerVerificationError(reason, { deploymentSucceeded }) {
|
|
154
|
+
function workerVerificationError(reason, { deploymentSucceeded, recordedCurrentDeployment = false }) {
|
|
147
155
|
const readable = workerHealthUserReason(reason);
|
|
148
156
|
const guidance = workerVerificationGuidance(reason);
|
|
149
157
|
const message = deploymentSucceeded
|
|
150
158
|
? `Cloudflare reported the Worker deployment succeeded, but ${readable}. The deployment fingerprint was saved, so retrying will verify the same Worker instead of deploying again. ${guidance}`
|
|
151
|
-
:
|
|
159
|
+
: recordedCurrentDeployment
|
|
160
|
+
? `The current Worker deployment fingerprint is already recorded, but ${readable}. No deployment was attempted because duplicating the recorded deployment is unsafe; use --force-worker only after diagnosis. ${guidance}`
|
|
161
|
+
: `The recorded Worker could not be verified because ${readable}. No deployment was attempted. ${guidance}`;
|
|
152
162
|
const error = new Error(message);
|
|
153
163
|
error.code = "worker_health_unverified";
|
|
154
164
|
error.healthError = reason;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { sanitizeDaemonChallengeAttachment } from "./daemon-auth.ts";
|
|
2
|
+
import type { DaemonRole } from "./daemon-liveness.ts";
|
|
3
|
+
import { sanitizeMetadataText } from "./http.ts";
|
|
4
|
+
import { sanitizeDaemonPolicy, sanitizeDaemonTools, type DaemonPolicy } from "./policy.ts";
|
|
5
|
+
|
|
6
|
+
export interface DaemonAttachment {
|
|
7
|
+
role: DaemonRole;
|
|
8
|
+
connectedAt: string;
|
|
9
|
+
lastSeenAt?: string;
|
|
10
|
+
probeId?: string;
|
|
11
|
+
instanceId?: string;
|
|
12
|
+
connectionId?: string;
|
|
13
|
+
policy?: DaemonPolicy;
|
|
14
|
+
tools?: string[];
|
|
15
|
+
authChallenge?: string;
|
|
16
|
+
authIssuedAt?: number;
|
|
17
|
+
authExpiresAt?: number;
|
|
18
|
+
workerOrigin?: string;
|
|
19
|
+
authSessionPublicKeyJson?: string;
|
|
20
|
+
authSessionKeyId?: string;
|
|
21
|
+
authCertificateExpiresAt?: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function sanitizeDaemonAttachment(value: unknown): DaemonAttachment | undefined {
|
|
25
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
26
|
+
const candidate = value as Partial<DaemonAttachment>;
|
|
27
|
+
if (!["candidate", "probing", "expired", "daemon"].includes(String(candidate.role))) return undefined;
|
|
28
|
+
const policy = sanitizeDaemonPolicy(candidate.policy);
|
|
29
|
+
return {
|
|
30
|
+
role: candidate.role as DaemonRole,
|
|
31
|
+
connectedAt: sanitizeMetadataText(candidate.connectedAt, 64) ?? "",
|
|
32
|
+
lastSeenAt: sanitizeMetadataText(candidate.lastSeenAt, 64),
|
|
33
|
+
probeId: sanitizeProbeId(candidate.probeId),
|
|
34
|
+
instanceId: sanitizeDaemonInstanceId(candidate.instanceId),
|
|
35
|
+
connectionId: sanitizeConnectionId(candidate.connectionId),
|
|
36
|
+
policy,
|
|
37
|
+
tools: sanitizeDaemonTools(candidate.tools, policy),
|
|
38
|
+
...sanitizeDaemonChallengeAttachment(candidate as Record<string, unknown>),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function sanitizeProbeId(value: unknown): string | undefined {
|
|
43
|
+
return typeof value === "string" && /^probe_[A-Za-z0-9_-]{8,240}$/.test(value) ? value : undefined;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function sanitizeDaemonInstanceId(value: unknown): string | undefined {
|
|
47
|
+
return typeof value === "string" && /^daemon_[A-Za-z0-9_-]{16,96}$/.test(value) ? value : undefined;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function sanitizeConnectionId(value: unknown): string | undefined {
|
|
51
|
+
return typeof value === "string" && /^connection_[A-Za-z0-9_-]{43}$/.test(value) ? value : undefined;
|
|
52
|
+
}
|
|
@@ -1,24 +1,7 @@
|
|
|
1
1
|
import { isLiveDaemonAttachment, withDaemonLastSeenAt, type DaemonRole } from "./daemon-liveness.ts";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
export interface DaemonAttachment {
|
|
6
|
-
role: DaemonRole;
|
|
7
|
-
connectedAt: string;
|
|
8
|
-
lastSeenAt?: string;
|
|
9
|
-
probeId?: string;
|
|
10
|
-
instanceId?: string;
|
|
11
|
-
policy?: DaemonPolicy;
|
|
12
|
-
tools?: string[];
|
|
13
|
-
authChallenge?: string;
|
|
14
|
-
authIssuedAt?: number;
|
|
15
|
-
authExpiresAt?: number;
|
|
16
|
-
workerOrigin?: string;
|
|
17
|
-
authSessionPublicKeyJson?: string;
|
|
18
|
-
authSessionKeyId?: string;
|
|
19
|
-
authCertificateExpiresAt?: number;
|
|
20
|
-
}
|
|
21
|
-
|
|
2
|
+
import type { DaemonPolicy } from "./policy.ts";
|
|
3
|
+
import type { DaemonChallenge } from "./daemon-auth.ts";
|
|
4
|
+
import { sanitizeDaemonAttachment, type DaemonAttachment } from "./daemon-socket-attachment.ts";
|
|
22
5
|
interface WebSocketContext {
|
|
23
6
|
getWebSockets(): WebSocket[];
|
|
24
7
|
}
|
|
@@ -28,21 +11,7 @@ export class DaemonSocketRegistry {
|
|
|
28
11
|
constructor(context: WebSocketContext) { this.context = context; }
|
|
29
12
|
|
|
30
13
|
attachment(socket: WebSocket): DaemonAttachment | undefined {
|
|
31
|
-
|
|
32
|
-
if (!raw || typeof raw !== "object") return undefined;
|
|
33
|
-
const candidate = raw as Partial<DaemonAttachment>;
|
|
34
|
-
if (!["candidate", "probing", "expired", "daemon"].includes(String(candidate.role))) return undefined;
|
|
35
|
-
const policy = sanitizeDaemonPolicy(candidate.policy);
|
|
36
|
-
return {
|
|
37
|
-
role: candidate.role as DaemonRole,
|
|
38
|
-
connectedAt: sanitizeMetadataText(candidate.connectedAt, 64) ?? "",
|
|
39
|
-
lastSeenAt: sanitizeMetadataText(candidate.lastSeenAt, 64),
|
|
40
|
-
probeId: sanitizeProbeId(candidate.probeId),
|
|
41
|
-
instanceId: sanitizeDaemonInstanceId(candidate.instanceId),
|
|
42
|
-
policy,
|
|
43
|
-
tools: sanitizeDaemonTools(candidate.tools, policy),
|
|
44
|
-
...sanitizeDaemonChallengeAttachment(candidate as Record<string, unknown>),
|
|
45
|
-
};
|
|
14
|
+
return sanitizeDaemonAttachment(socket.deserializeAttachment());
|
|
46
15
|
}
|
|
47
16
|
|
|
48
17
|
readyAttachment(socket: WebSocket): DaemonAttachment | undefined {
|
|
@@ -68,11 +37,13 @@ export class DaemonSocketRegistry {
|
|
|
68
37
|
socket: WebSocket,
|
|
69
38
|
challenge: DaemonChallenge,
|
|
70
39
|
preflight: { sessionPublicKeyJson: string; sessionKeyId: string; certificateExpiresAt: number },
|
|
40
|
+
connectionId: string,
|
|
71
41
|
connectedAt = new Date().toISOString(),
|
|
72
42
|
): void {
|
|
73
43
|
socket.serializeAttachment({
|
|
74
44
|
role: "candidate",
|
|
75
45
|
connectedAt,
|
|
46
|
+
connectionId,
|
|
76
47
|
authChallenge: challenge.challenge,
|
|
77
48
|
authIssuedAt: challenge.issuedAt,
|
|
78
49
|
authExpiresAt: challenge.expiresAt,
|
|
@@ -83,10 +54,11 @@ export class DaemonSocketRegistry {
|
|
|
83
54
|
} satisfies DaemonAttachment);
|
|
84
55
|
}
|
|
85
56
|
|
|
86
|
-
beginProbe(socket: WebSocket, values: { connectedAt: string; probeId: string; instanceId: string; policy: DaemonPolicy; tools: string[] }): void {
|
|
57
|
+
beginProbe(socket: WebSocket, values: { connectedAt: string; probeId: string; instanceId: string; connectionId: string; policy: DaemonPolicy; tools: string[] }): void {
|
|
87
58
|
socket.serializeAttachment({
|
|
88
59
|
role: "probing", connectedAt: values.connectedAt, lastSeenAt: values.connectedAt,
|
|
89
|
-
probeId: values.probeId, instanceId: values.instanceId,
|
|
60
|
+
probeId: values.probeId, instanceId: values.instanceId, connectionId: values.connectionId,
|
|
61
|
+
policy: values.policy, tools: values.tools,
|
|
90
62
|
} satisfies DaemonAttachment);
|
|
91
63
|
}
|
|
92
64
|
|
|
@@ -111,10 +83,17 @@ export class DaemonSocketRegistry {
|
|
|
111
83
|
const attachment = this.attachment(socket);
|
|
112
84
|
if (!attachment) return;
|
|
113
85
|
socket.serializeAttachment({
|
|
114
|
-
role: "expired", connectedAt: attachment.connectedAt, lastSeenAt: attachment.lastSeenAt,
|
|
86
|
+
role: "expired", connectedAt: attachment.connectedAt, lastSeenAt: attachment.lastSeenAt,
|
|
87
|
+
instanceId: attachment.instanceId, connectionId: attachment.connectionId,
|
|
115
88
|
} satisfies DaemonAttachment);
|
|
116
89
|
}
|
|
117
90
|
|
|
91
|
+
socketForConnectionId(connectionId: string): WebSocket | undefined {
|
|
92
|
+
if (!connectionId) return undefined;
|
|
93
|
+
return this.context.getWebSockets().find((socket) =>
|
|
94
|
+
this.attachment(socket)?.connectionId === connectionId && socket.readyState === WebSocket.OPEN);
|
|
95
|
+
}
|
|
96
|
+
|
|
118
97
|
private openSockets(role: DaemonRole): WebSocket[] {
|
|
119
98
|
return this.context.getWebSockets().filter((socket) => this.attachment(socket)?.role === role && socket.readyState === WebSocket.OPEN);
|
|
120
99
|
}
|
|
@@ -123,13 +102,3 @@ export class DaemonSocketRegistry {
|
|
|
123
102
|
return Date.parse(this.attachment(socket)?.connectedAt ?? "") || 0;
|
|
124
103
|
}
|
|
125
104
|
}
|
|
126
|
-
|
|
127
|
-
function sanitizeProbeId(value: unknown): string | undefined {
|
|
128
|
-
if (typeof value !== "string" || !/^probe_[A-Za-z0-9_-]{8,240}$/.test(value)) return undefined;
|
|
129
|
-
return value;
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
function sanitizeDaemonInstanceId(value: unknown): string | undefined {
|
|
133
|
-
if (typeof value !== "string" || !/^daemon_[A-Za-z0-9_-]{16,96}$/.test(value)) return undefined;
|
|
134
|
-
return value;
|
|
135
|
-
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { daemonLastSeenMs } from "./daemon-liveness.ts";
|
|
2
|
+
import type { DaemonSocketRegistry } from "./daemon-sockets.ts";
|
|
3
|
+
import { publicWorkerToolError, WorkerToolError } from "./errors.ts";
|
|
4
|
+
import { workerErrorClass } from "./http.ts";
|
|
5
|
+
import type { PendingCallOutcome } from "./pending-call-contract.ts";
|
|
6
|
+
import type { McpPendingCallStore, PendingStreamCallView } from "./mcp-pending-call-store.ts";
|
|
7
|
+
import { streamTerminalMessage } from "./mcp-stream-dispatch.ts";
|
|
8
|
+
import { transformDurableStreamOutcome } from "./durable-stream-result.ts";
|
|
9
|
+
import type { WorkerObservability } from "./observability.ts";
|
|
10
|
+
import { sendWebSocketQuietly } from "./websocket-protocol.ts";
|
|
11
|
+
|
|
12
|
+
type InvalidateDaemonSocket = (
|
|
13
|
+
socket: WebSocket,
|
|
14
|
+
message: string,
|
|
15
|
+
closeReason: string,
|
|
16
|
+
errorCode?: string,
|
|
17
|
+
) => Promise<void>;
|
|
18
|
+
|
|
19
|
+
type TransientPendingSnapshot = {
|
|
20
|
+
active: number;
|
|
21
|
+
detached: number;
|
|
22
|
+
request_keys: number;
|
|
23
|
+
oldest_ms: number;
|
|
24
|
+
by_tool: Record<string, number>;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export class DurableStreamCallCoordinator {
|
|
28
|
+
private readonly calls: McpPendingCallStore;
|
|
29
|
+
private readonly daemonRegistry: DaemonSocketRegistry;
|
|
30
|
+
private readonly observability: WorkerObservability;
|
|
31
|
+
private readonly maximumPendingCalls: number;
|
|
32
|
+
private readonly invalidateDaemonSocket: InvalidateDaemonSocket;
|
|
33
|
+
|
|
34
|
+
constructor(
|
|
35
|
+
calls: McpPendingCallStore,
|
|
36
|
+
daemonRegistry: DaemonSocketRegistry,
|
|
37
|
+
observability: WorkerObservability,
|
|
38
|
+
maximumPendingCalls: number,
|
|
39
|
+
invalidateDaemonSocket: InvalidateDaemonSocket,
|
|
40
|
+
) {
|
|
41
|
+
this.calls = calls;
|
|
42
|
+
this.daemonRegistry = daemonRegistry;
|
|
43
|
+
this.observability = observability;
|
|
44
|
+
this.maximumPendingCalls = maximumPendingCalls;
|
|
45
|
+
this.invalidateDaemonSocket = invalidateDaemonSocket;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async snapshot(transient: TransientPendingSnapshot): Promise<Record<string, unknown>> {
|
|
49
|
+
const durable = await this.calls.snapshot(this.maximumPendingCalls);
|
|
50
|
+
const byTool = Object.assign(Object.create(null) as Record<string, number>, transient.by_tool);
|
|
51
|
+
for (const [tool, count] of Object.entries(durable.by_tool)) byTool[tool] = (byTool[tool] ?? 0) + count;
|
|
52
|
+
return {
|
|
53
|
+
active: transient.active + durable.active,
|
|
54
|
+
detached: transient.detached + durable.detached,
|
|
55
|
+
request_keys: transient.request_keys + durable.request_keys,
|
|
56
|
+
maximum: this.maximumPendingCalls,
|
|
57
|
+
oldest_ms: Math.max(transient.oldest_ms, durable.oldest_ms),
|
|
58
|
+
by_tool: byTool,
|
|
59
|
+
transient: transient.active,
|
|
60
|
+
durable_streams: durable.active,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async settle(
|
|
65
|
+
callId: string,
|
|
66
|
+
connectionId: string,
|
|
67
|
+
outcome: PendingCallOutcome,
|
|
68
|
+
knownCall?: PendingStreamCallView,
|
|
69
|
+
): Promise<boolean> {
|
|
70
|
+
const call = knownCall ?? await this.calls.get(callId);
|
|
71
|
+
if (!call || call.connection_id !== connectionId) return false;
|
|
72
|
+
const normalized = transformDurableStreamOutcome(call, outcome);
|
|
73
|
+
const code = normalized.ok ? "" : publicWorkerToolError(normalized.error).code;
|
|
74
|
+
try {
|
|
75
|
+
const completed = await this.calls.complete(
|
|
76
|
+
callId,
|
|
77
|
+
connectionId,
|
|
78
|
+
streamTerminalMessage(call.requestId, normalized),
|
|
79
|
+
);
|
|
80
|
+
if (!completed) return false;
|
|
81
|
+
} catch (error) {
|
|
82
|
+
this.observability.event("error", "mcp.stream.persist.failed", { error_class: workerErrorClass(error) });
|
|
83
|
+
}
|
|
84
|
+
this.observability.callFinished(call.tool, code);
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async expire(call: PendingStreamCallView): Promise<void> {
|
|
89
|
+
const socket = call.state === "attached" ? this.daemonRegistry.socketForConnectionId(call.connection_id) : undefined;
|
|
90
|
+
if (socket) {
|
|
91
|
+
sendWebSocketQuietly(socket, { type: "cancel_call", id: call.call_id });
|
|
92
|
+
const silentForMs = Date.now() - daemonLastSeenMs(this.daemonRegistry.readyAttachment(socket));
|
|
93
|
+
if (!Number.isFinite(silentForMs) || silentForMs > 45_000) {
|
|
94
|
+
void this.invalidateDaemonSocket(socket, "daemon became unresponsive", "daemon liveness timeout");
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
const error = call.state === "attached"
|
|
98
|
+
? new WorkerToolError("timeout", `daemon tool timed out: ${call.tool}`, true)
|
|
99
|
+
: new WorkerToolError("unavailable", "daemon disconnected; reconnect grace expired", true);
|
|
100
|
+
await this.settle(call.call_id, call.connection_id, { ok: false, error }, call);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async expireDue(now = Date.now()): Promise<number> {
|
|
104
|
+
const due = await this.calls.due(now);
|
|
105
|
+
for (const call of due) await this.expire(call);
|
|
106
|
+
return due.length;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async cancel(requestKey: string): Promise<boolean> {
|
|
110
|
+
const call = await this.calls.getByRequestKey(requestKey);
|
|
111
|
+
if (!call) return false;
|
|
112
|
+
const socket = call.state === "attached" ? this.daemonRegistry.socketForConnectionId(call.connection_id) : undefined;
|
|
113
|
+
if (socket) sendWebSocketQuietly(socket, { type: "cancel_call", id: call.call_id });
|
|
114
|
+
return this.settle(
|
|
115
|
+
call.call_id,
|
|
116
|
+
call.connection_id,
|
|
117
|
+
{ ok: false, error: new WorkerToolError("cancelled", "tool call cancelled by client") },
|
|
118
|
+
call,
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
detach(connectionId: string, graceMs: number): Promise<number> {
|
|
123
|
+
return this.calls.detach(connectionId, graceMs);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
rebind(daemonInstanceId: string, connectionId: string): Promise<string[]> {
|
|
127
|
+
return this.calls.rebind(daemonInstanceId, connectionId);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { decorateProjectOverview } from "./authority.ts";
|
|
2
|
+
import type { PendingCallOutcome } from "./pending-call-contract.ts";
|
|
3
|
+
import type { PendingStreamCallView } from "./mcp-pending-call-store.ts";
|
|
4
|
+
|
|
5
|
+
export function transformDurableStreamOutcome(
|
|
6
|
+
call: PendingStreamCallView,
|
|
7
|
+
outcome: PendingCallOutcome,
|
|
8
|
+
): PendingCallOutcome {
|
|
9
|
+
if (!outcome.ok || call.transform?.kind !== "project_overview") return outcome;
|
|
10
|
+
try {
|
|
11
|
+
return {
|
|
12
|
+
ok: true,
|
|
13
|
+
value: decorateProjectOverview(outcome.value, {
|
|
14
|
+
accountId: call.transform.account_id,
|
|
15
|
+
accountVersion: call.transform.account_version,
|
|
16
|
+
role: call.transform.role,
|
|
17
|
+
}),
|
|
18
|
+
};
|
|
19
|
+
} catch (error) {
|
|
20
|
+
return { ok: false, error: error instanceof Error ? error : new Error("stream result transformation failed") };
|
|
21
|
+
}
|
|
22
|
+
}
|