machine-bridge-mcp 1.2.9 → 1.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/browser-extension/manifest.json +2 -2
- package/docs/ARCHITECTURE.md +10 -7
- package/docs/AUDIT.md +10 -0
- package/docs/LOGGING.md +1 -1
- package/docs/MULTI_ACCOUNT.md +2 -2
- package/docs/OPERATIONS.md +2 -2
- package/package.json +1 -1
- package/src/local/relay-call-recovery.mjs +148 -0
- package/src/local/relay-connection.mjs +5 -0
- package/src/local/runtime-relay.mjs +16 -0
- package/src/local/runtime.mjs +59 -39
- package/src/worker/daemon-sockets.ts +10 -1
- package/src/worker/index.ts +45 -9
- package/src/worker/pending-call-contract.ts +26 -0
- package/src/worker/pending-calls.ts +41 -25
- package/tsconfig.local.json +1 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.2.10 - 2026-07-20
|
|
4
|
+
|
|
5
|
+
### Relay interruption recovery
|
|
6
|
+
|
|
7
|
+
- Preserve an in-flight MCP tool call across a brief daemon WebSocket interruption instead of converting every transient proxy or network reset into an immediate cancellation. The Worker now detaches pending calls for a bounded 30-second grace period and rebinds them only when the same local daemon process instance completes the full authenticated readiness probe on its replacement socket.
|
|
8
|
+
- Keep the local operation running during that grace period and queue a completed result until the relay is ready again. Before `ready_ack`, the Worker sends an authoritative `resume_calls` set so the runtime cancels work whose client disappeared during the outage; results are replayed only for same-instance calls that still have a receiver. A different daemon instance cannot claim them, and an unrecovered outage cancels ordinary calls and process trees when the grace period expires.
|
|
9
|
+
- Add deterministic registry/runtime regressions and a real Worker/OAuth/MCP fault-injection test that starts a tool call, forcibly drops the daemon WebSocket, reconnects the same daemon instance, and proves the original HTTP request completes. Expose the bounded `pending_calls.detached` count for diagnosis, require a validated ephemeral daemon instance identifier in the current-version hello contract, and update architecture, logging, operations, audit, and multi-account documentation.
|
|
10
|
+
|
|
3
11
|
## 1.2.9 - 2026-07-18
|
|
4
12
|
|
|
5
13
|
- Repair cross-platform release infrastructure found by PR CI: the layered check runner now invokes the pinned npm CLI through Node instead of spawning `npm.cmd`, and `release:accept` computes and locally validates the portable Git-content digest through a temporary index so CI can verify accepted package content across merge commits without mutating the maintainer index.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"manifest_version": 3,
|
|
3
3
|
"name": "Machine Bridge Browser",
|
|
4
|
-
"version": "1.2.
|
|
4
|
+
"version": "1.2.10",
|
|
5
5
|
"description": "Connects the current Chromium browser profile to the local Machine Bridge runtime for user-authorized automation.",
|
|
6
6
|
"permissions": [
|
|
7
7
|
"tabs",
|
|
@@ -30,5 +30,5 @@
|
|
|
30
30
|
"action": {
|
|
31
31
|
"default_title": "Machine Bridge Browser"
|
|
32
32
|
},
|
|
33
|
-
"version_name": "1.2.
|
|
33
|
+
"version_name": "1.2.10"
|
|
34
34
|
}
|
package/docs/ARCHITECTURE.md
CHANGED
|
@@ -35,13 +35,13 @@ A canonical workspace receives an independent profile, Worker name, secret set,
|
|
|
35
35
|
- `runtime-diagnostics.mjs` owns fixed local probes and their stable interpretation;
|
|
36
36
|
- `runtime-capabilities.mjs` composes agent, application, and browser capability results;
|
|
37
37
|
- `runtime-tool-handlers.mjs` owns catalog-to-handler registration;
|
|
38
|
-
- `runtime-relay.mjs` owns relay construction and inbound envelope normalization;
|
|
38
|
+
- `runtime-relay.mjs` owns relay construction and inbound envelope normalization, while `relay-call-recovery.mjs` owns the bounded disconnect grace, result queue, authoritative resumed-call reconciliation, replay, and expiry cleanup;
|
|
39
39
|
- `runtime-paths.mjs` owns runtime-directory creation, containment checks, and error-path redaction;
|
|
40
40
|
- `managed-job-lock.mjs`, `managed-job-runner.mjs`, `managed-job-storage.mjs`, and `managed-job-projection.mjs` separate transition ownership, detached runner identity, private persistence/diagnostics, and public result shaping from the managed-job lifecycle;
|
|
41
41
|
- `browser-request-registry.mjs`, `browser-broker-routes.mjs`, `browser-broker-server.mjs`, and `browser-bridge-http.mjs` separate direct request ownership, runtime-client proxy routing, authenticated loopback WebSocket upgrades/listening, and loopback HTTP handling from broker startup and extension handover;
|
|
42
42
|
- managed jobs, local resources, application automation, and browser automation remain separate managers.
|
|
43
43
|
|
|
44
|
-
Architecture tests cap the orchestration module and each extracted service independently and reject a return of low-level process, patch, diagnostic, or capability-scoring logic to `LocalRuntime`. `RelayConnection` owns remote WebSocket transport, `hello_ack` authentication, end-to-end `relay_probe`/`ready_ack` readiness, heartbeat liveness, reconnect backoff, outage logging, and a monotonically increasing in-memory
|
|
44
|
+
Architecture tests cap the orchestration module and each extracted service independently and reject a return of low-level process, patch, diagnostic, or capability-scoring logic to `LocalRuntime`. `RelayConnection` owns remote WebSocket transport, `hello_ack` authentication, end-to-end `relay_probe`/`ready_ack` readiness, heartbeat liveness, reconnect backoff, outage logging, and a monotonically increasing in-memory transport generation. The generation still protects the pre-ready probe and prevents arbitrary use of a stale socket. Ordinary tool calls additionally bind to an ephemeral identifier generated once per local daemon process. If a ready socket drops, the Worker detaches its pending calls for at most thirty seconds; only a replacement socket that presents the same daemon-process identifier and completes the full readiness probe can reclaim them. The local runtime keeps those calls alive and queues completed results during the same bounded interval, then replays them after readiness. A different process instance, an explicit cancellation, or grace expiry cannot receive those results. Stdio mode invokes `LocalRuntime` directly without that adapter.
|
|
45
45
|
|
|
46
46
|
`daemon-process.mjs` owns workspace-daemon inspection and takeover. It distinguishes platform service state from the lock-owning Node process, validates PID and process-start identity, canonicalizes workspace/state paths before comparison, parses bounded process command lines without executing them, and accepts lock-backed `--daemon-only` recovery processes that omit repeated path flags. Stop/takeover sends `SIGTERM` only to a verified same-workspace service daemon. If it remains alive after the grace period, the code revalidates PID, process-start identity, command line, entrypoint, daemon mode, workspace, and state root before sending `SIGKILL`; a foreground, replaced-PID, or otherwise unverifiable process remains untouched. CLI orchestration never treats a missing launchd/systemd job as proof that the process exited.
|
|
47
47
|
|
|
@@ -160,11 +160,12 @@ Remote OAuth binds each code, access token, and refresh token to a named Machine
|
|
|
160
160
|
7. The MCP client initializes against the sole current protocol version; an obsolete client must upgrade rather than enter a legacy execution path. The Worker returns a stateless HMAC-bound `MCP-Session-Id`, and later request/cancellation correlation is scoped by OAuth token, MCP session, JSON-RPC id type, and id value. Two clients may therefore reuse the same JSON-RPC id concurrently without collision. Sessionless POSTs remain independent and are not inserted into a token-global cancellation index. When the daemon advertises `session_bootstrap`, the Worker requests bounded local instructions and appends them to the initialization result; failure degrades to static instructions.
|
|
161
161
|
8. A new daemon first authenticates as a bounded `probing` socket. The Worker sends a random `relay_probe`; the local runtime returns it through the normal session-bound result-delivery path; only the matching result produces `ready_ack`, promotion to the active daemon, and safe replacement of an incumbent connection.
|
|
162
162
|
9. `tools/list` is derived only from the active end-to-end-verified daemon; without one, only `server_info` is advertised.
|
|
163
|
-
10. `tools/call` receives a random relay call ID and is bound to the current daemon socket, authenticated client request key, incoming HTTP abort signal
|
|
163
|
+
10. `tools/call` receives a random relay call ID and is bound to the current daemon socket, that daemon process's ephemeral instance identifier, the authenticated client request key, and the incoming HTTP abort signal.
|
|
164
164
|
11. The runtime validates policy and arguments, executes the tool, and returns a bounded result.
|
|
165
|
-
12.
|
|
166
|
-
13. A matching cancellation notification or incoming HTTP client disconnect removes the pending indexes
|
|
167
|
-
14.
|
|
165
|
+
12. If the socket remains ready, the Durable Object accepts the result only from that socket. If it drops, the Worker detaches the pending call for at most thirty seconds and accepts completion only after a replacement socket with the same daemon-process identifier has passed the end-to-end readiness probe. The local runtime preserves the operation and queues a completion over the same interval.
|
|
166
|
+
13. A matching cancellation notification or incoming HTTP client disconnect removes the pending indexes and sends best-effort cancellation to a connected daemon. Local completion that races with cancellation is discarded. On every readiness handover, the Worker first sends an authoritative bounded `resume_calls` set; the runtime cancels active calls and queued results absent from that set before accepting `ready_ack`. A request cancelled while disconnected therefore cannot be revived by a fast reconnect.
|
|
167
|
+
14. If same-instance readiness does not return before the grace deadline, the Worker rejects the detached request and the local runtime cancels ordinary calls, terminates their process trees, and discards queued results. A newly started daemon has a different instance identifier and cannot inherit prior calls.
|
|
168
|
+
15. `start_job` is different: after durable acceptance, the detached runner is no longer bound to the relay call or socket. Later cancellation uses `cancel_job` or the local CLI.
|
|
168
169
|
|
|
169
170
|
Duplicate in-flight JSON-RPC IDs are rejected only within the same authenticated MCP session. The request key includes OAuth token identity, the HMAC-bound MCP session, JSON-RPC id type, and id value, so separate initialized clients may safely reuse the same numeric id.
|
|
170
171
|
|
|
@@ -234,7 +235,9 @@ A connection-attempt deadline terminates sockets stuck in `CONNECTING`. After op
|
|
|
234
235
|
|
|
235
236
|
Reconnect uses bounded exponential backoff with jitter. Brief self-healing interruptions are debug-only. An unresolved outage is promoted to a rate-limited warning after a grace period, and recovery produces one summary. Raw close codes and reason strings remain debug-only.
|
|
236
237
|
|
|
237
|
-
The Worker stores socket transitions in `DaemonSocketRegistry`: `candidate` before hello, `probing` after authentication, `daemon` only after the end-to-end result probe, and `expired` after terminal failure. Durable Object alarms enforce separate hello, readiness, and steady-state liveness deadlines across hibernation. A healthy incumbent remains active while a replacement is probed; a malformed, silent, or
|
|
238
|
+
The Worker stores socket transitions in `DaemonSocketRegistry`: `candidate` before hello, `probing` after authentication, `daemon` only after the end-to-end result probe, and `expired` after terminal failure. Durable Object alarms enforce separate hello, readiness, and steady-state liveness deadlines across hibernation. A healthy incumbent remains active while a replacement is probed; a malformed, silent, incompatible, or identity-mismatched replacement is closed without displacing it. Only a verified candidate receives `ready_ack` and then replaces the old socket. Ready daemons stay live only while inbound traffic refreshes `lastSeenAt`; silent half-open or hibernation-restored sockets are reclaimed instead of advertising `daemon.connected` while tool calls time out.
|
|
239
|
+
|
|
240
|
+
Each daemon process generates a random bounded `instance_id` at startup and includes it in every reconnect hello. Pending calls normally retain their assigned socket. On an unexpected socket loss, only those records are detached and a thirty-second timer bounds recovery. A verified socket with the same instance ID rebinds them; another process or socket cannot resolve them. The local runtime mirrors that state machine by preserving active calls and completed-result envelopes until relay readiness returns. Before `ready_ack`, the Worker sends the exact IDs that still have remote waiters; the runtime cancels everything else and only then replays retained results through the verified socket. Grace expiry restores the terminal behavior: reject remote waiters, cancel local ordinary calls, terminate process trees, and discard undeliverable results. This does not make calls durable across daemon restart or machine failure; managed jobs remain the separate durable mechanism.
|
|
238
241
|
|
|
239
242
|
## Persistence
|
|
240
243
|
|
package/docs/AUDIT.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Security and privacy audit notes
|
|
2
2
|
|
|
3
|
+
## 2026-07-20 version 1.2.10 transient-relay recovery audit
|
|
4
|
+
|
|
5
|
+
Repeated ChatGPT web executions were terminating even though the local daemon and Worker were healthy immediately afterward. The causal defect was not the tool implementation or its configured timeout. A brief WebSocket loss—made more likely when the daemon relay traverses an environment-selected local proxy—was promoted into a terminal request failure at both ends: the Worker rejected every pending call assigned to the closed socket, while the local runtime cancelled all relay-origin calls and killed their ordinary child processes. Automatic WebSocket reconnection restored future traffic but could not recover the request already abandoned by both state machines.
|
|
6
|
+
|
|
7
|
+
The correction introduces a bounded continuation identity rather than weakening cancellation globally. Every local daemon process creates one random ephemeral `instance_id` and presents it in the current-version hello. On socket loss, the Worker detaches only calls owned by that socket and retains their OAuth/session request indexes for thirty seconds. A replacement socket may rebind them only after presenting the same process identifier and completing the existing end-to-end readiness probe. The local runtime mirrors this interval: active calls continue, completed result envelopes are retained in memory, and readiness triggers replay. A different process instance cannot claim the pending IDs; daemon restart remains terminal rather than pretending to provide durable execution.
|
|
8
|
+
|
|
9
|
+
The recovery path remains bounded and fail closed. Explicit MCP cancellation and HTTP client abandonment still remove the remote receiver immediately. A connected daemon receives cancellation as before. If cancellation occurs while disconnected, the next same-instance handshake carries an authoritative `resume_calls` set before `ready_ack`; the local runtime cancels any active or completed call absent from that set, so a fast reconnect cannot revive a request whose HTTP receiver disappeared. Timeout remains authoritative. At thirty seconds without same-instance readiness, the Worker rejects detached requests and the runtime cancels calls, discards queued results, and enters the existing persistent-outage reporting path. No tool arguments, commands, paths, outputs, proxy endpoint, or result content are added to logs or durable storage.
|
|
10
|
+
|
|
11
|
+
Regression evidence covers three layers: deterministic pending-registry detach/rebind/expiry tests; local runtime queue/replay, resumed-call reconciliation, cancellation, and grace behavior; and real workerd OAuth/MCP/WebSocket fault injection that both completes a detached request after same-instance reconnect and proves a request cancelled while detached is excluded from the next `resume_calls` set. Malformed or missing daemon-instance identifiers are rejected as protocol errors. The version remains a current-only Worker/daemon protocol unit and requires normal candidate convergence before live use.
|
|
12
|
+
|
|
3
13
|
## 2026-07-18 version 1.2.9 interactive release-handoff correction
|
|
4
14
|
|
|
5
15
|
The release-integrity mechanism correctly bound acceptance to exact package bytes, but the operational role split was wrong. Documentation and repository guards required the owner to run the acceptance command personally and prohibited the coding agent from recording it. That contradicted the established working process: the owner should perform the one action the agent cannot perform honestly—start the exact candidate on the maintainer machine—while the agent should connect to that candidate, verify its version, readiness, and representative functionality, then record acceptance and complete source release operations.
|
package/docs/LOGGING.md
CHANGED
|
@@ -79,7 +79,7 @@ With `--verbose`, the same incident additionally includes bounded structured fie
|
|
|
79
79
|
|
|
80
80
|
All per-tool starts, successes, failures, cancellations, timing, and expected late-result disposal are debug-only. The MCP response already reports the outcome to the caller; duplicating routine tool traffic at default levels creates noise and can reveal activity patterns.
|
|
81
81
|
|
|
82
|
-
A completed local result is
|
|
82
|
+
A completed local result is normally sent on the ready relay connection. If that socket disappears, the runtime queues the bounded result envelope during the thirty-second same-daemon reconnect window rather than logging a terminal delivery failure. Debug output records only a shortened call ID and queue/reconnect counts. After the same daemon process completes readiness, replay emits one recovery event; a different process cannot inherit the result. Explicit caller cancellation suppresses eventual delivery. If the relay does not recover before the grace deadline, ordinary calls are cancelled, queued results are discarded, and the existing outage state machine determines whether the persistent failure warrants a warning. Tool arguments, commands, and result content are never logged.
|
|
83
83
|
|
|
84
84
|
Debug per-tool fields may include tool name, duration, coarse outcome class, and a shortened random call identifier. The identifier is for correlating adjacent local events and is not a stable audit identifier.
|
|
85
85
|
|
package/docs/MULTI_ACCOUNT.md
CHANGED
|
@@ -93,9 +93,9 @@ The first start of a new deployment creates an `owner` account automatically and
|
|
|
93
93
|
|
|
94
94
|
## Concurrency and revocation
|
|
95
95
|
|
|
96
|
-
Pending calls remain bound to the OAuth access token and JSON-RPC request ID. Duplicate in-flight IDs under one token are rejected. Account revocation blocks new requests immediately; calls already relayed remain subject to ordinary cancellation, deadlines,
|
|
96
|
+
Pending calls remain bound to the OAuth access token and JSON-RPC request ID. Duplicate in-flight IDs under one token are rejected. Account revocation blocks new requests immediately; calls already relayed remain subject to ordinary cancellation, deadlines, local role validation, and the bounded same-daemon reconnect state machine. A replacement process cannot inherit a detached call.
|
|
97
97
|
|
|
98
|
-
|
|
98
|
+
A relay interruption keeps ordinary relay-owned calls alive only inside the bounded same-daemon reconnect window. Reconciliation or grace expiry cancels calls that no longer have a remote receiver and terminates their child process trees. Process promises settle on cancellation even when a child does not emit `close`; process ownership remains tracked until actual exit.
|
|
99
99
|
|
|
100
100
|
## Audit and privacy
|
|
101
101
|
|
package/docs/OPERATIONS.md
CHANGED
|
@@ -51,7 +51,7 @@ A successful diagnostic result applies only to that probe. An MCP host can still
|
|
|
51
51
|
|
|
52
52
|
Machine Bridge supports concurrent calls: the Worker admits up to 32 pending daemon calls and the local runtime admits up to 16 active tool calls. These are capacity limits, not a single global execution queue. Each successful MCP initialization receives a signed `MCP-Session-Id`; JSON-RPC ids and cancellation are scoped to that session, so separate chat windows may reuse the same numeric ids safely even when they share one OAuth account and token.
|
|
53
53
|
|
|
54
|
-
`server_info.worker.pending_calls` reports `active`, `request_keys`, `maximum`, `oldest_ms`, and `by_tool`. `worker.sockets_live` separately reports `authenticated`, `probing`, `ready`, and `candidates`; only `ready` sockets contribute to `daemon.connected` and tool advertisement. A nonzero `active` count means work is in flight, not that the bridge is locked. Calls for simple reads and probes should continue while another independent process call runs. Explicit MCP cancellation, an incoming HTTP client disconnect,
|
|
54
|
+
`server_info.worker.pending_calls` reports `active`, `detached`, `request_keys`, `maximum`, `oldest_ms`, and `by_tool`. `worker.sockets_live` separately reports `authenticated`, `probing`, `ready`, and `candidates`; only `ready` sockets contribute to `daemon.connected` and tool advertisement. A nonzero `active` count means work is in flight, not that the bridge is locked. `detached > 0` means a daemon socket was lost and those requests are inside the bounded thirty-second same-instance reconnect window. Calls for simple reads and probes should continue while another independent process call runs. Explicit MCP cancellation, an incoming HTTP client disconnect, and timeout remove the pending record and its session request key. A daemon-socket closure detaches only calls assigned to that socket; the same daemon process can reclaim them after completing readiness, while another process cannot. Grace expiry rejects the request and cancels the local ordinary operation. Refreshing a chat page is not the recovery mechanism and should not be required.
|
|
55
55
|
|
|
56
56
|
`server_info.worker.observability.calls.unmatched_results` counts results that reached the Worker after their pending record was already removed. A small increase can accompany cancellation or timeout races, especially during mixed-version upgrade convergence; sustained growth together with old pending calls indicates incompatible components or a lifecycle defect. The counter contains no tool arguments or result data.
|
|
57
57
|
|
|
@@ -147,7 +147,7 @@ Uninstall acquires a state-root `maintenance.lock` that blocks new profile/state
|
|
|
147
147
|
|
|
148
148
|
### Lifecycle and pending-call diagnosis
|
|
149
149
|
|
|
150
|
-
`server_info.runtime.lifecycle` reports `ready`, `starting`, `running`, `failed`, `stopping`, or `stopped`. `server_info.observability.in_flight_calls` and `server_info.runtime.processes` distinguish a blocked call from a surviving process. `server_info.runtime.execution_guardrails` reports the enforced local concurrency/timeout/stdin/output/session limits and explicitly states that CPU quota, memory quota, and network isolation are not enforced in process. Worker `server_info.worker.pending_calls` reports
|
|
150
|
+
`server_info.runtime.lifecycle` reports `ready`, `starting`, `running`, `failed`, `stopping`, or `stopped`. `server_info.observability.in_flight_calls` and `server_info.runtime.processes` distinguish a blocked call from a surviving process. `server_info.runtime.execution_guardrails` reports the enforced local concurrency/timeout/stdin/output/session limits and explicitly states that CPU quota, memory quota, and network isolation are not enforced in process. Worker `server_info.worker.pending_calls` reports the internal-call index, client request-key index, and detached-call count. All three must return to zero after a terminal result, explicit cancellation, client disconnect, timeout, or reconnect-grace expiry. During a brief daemon interruption, `active` and `request_keys` may remain nonzero while `detached` identifies the recoverable subset; after same-instance readiness, `detached` returns to zero without losing those requests. Nonzero request-key counts after active calls reach zero indicate a lifecycle defect rather than normal load. `worker.observability.calls.unmatched_results` is the bounded counter for late results that no longer have a receiver.
|
|
151
151
|
|
|
152
152
|
Stable errors include `policy_denied`, `invalid_request`, `timeout`, `cancelled`, `network_error`, `unavailable`, `limit_exceeded`, and `integrity_error`, with retryability metadata. Diagnose by code first; free-form messages are guidance, not an API contract.
|
|
153
153
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "machine-bridge-mcp",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.10",
|
|
4
4
|
"description": "Cross-client MCP bridge for local agent context, structured browser and application automation, files, Git, processes, resources, and durable jobs over stdio or OAuth relay.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/** @typedef {{id?: unknown, [key: string]: unknown}} RelayResult */
|
|
4
|
+
/** @typedef {{event?: (level: string, name: string, fields: Record<string, unknown>, message: string) => void, warn?: (message: string) => void}} RecoveryLogger */
|
|
5
|
+
/** @typedef {{setTimeout: (callback: () => void, delay: number) => any, clearTimeout: (handle: any) => void}} RecoveryScheduler */
|
|
6
|
+
/**
|
|
7
|
+
* @typedef {{
|
|
8
|
+
* logger?: RecoveryLogger,
|
|
9
|
+
* send?: (value: RelayResult) => boolean,
|
|
10
|
+
* isRecoverable?: () => boolean,
|
|
11
|
+
* activeCallIds?: () => Iterable<string>,
|
|
12
|
+
* suppressCall?: (callId: string, reason: string) => void,
|
|
13
|
+
* cancelOrigin?: (reason: string) => number,
|
|
14
|
+
* terminate?: () => void,
|
|
15
|
+
* graceMs?: unknown,
|
|
16
|
+
* scheduler?: RecoveryScheduler,
|
|
17
|
+
* }} RelayCallRecoveryOptions
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const DEFAULT_RECONNECT_GRACE_MS = 30_000;
|
|
21
|
+
|
|
22
|
+
export class RelayCallRecovery {
|
|
23
|
+
/** @param {RelayCallRecoveryOptions} [options] */
|
|
24
|
+
constructor(options = {}) {
|
|
25
|
+
/** @type {RecoveryLogger} */
|
|
26
|
+
this.logger = options.logger || { warn: (message) => console.warn(message) };
|
|
27
|
+
this.send = typeof options.send === "function" ? options.send : () => false;
|
|
28
|
+
this.isRecoverable = typeof options.isRecoverable === "function" ? options.isRecoverable : () => false;
|
|
29
|
+
this.activeCallIds = typeof options.activeCallIds === "function" ? options.activeCallIds : () => [];
|
|
30
|
+
this.suppressCall = typeof options.suppressCall === "function" ? options.suppressCall : () => {};
|
|
31
|
+
this.cancelOrigin = typeof options.cancelOrigin === "function" ? options.cancelOrigin : () => 0;
|
|
32
|
+
this.terminate = typeof options.terminate === "function" ? options.terminate : () => {};
|
|
33
|
+
this.graceMs = positiveInteger(options.graceMs, DEFAULT_RECONNECT_GRACE_MS);
|
|
34
|
+
this.scheduler = options.scheduler || { setTimeout, clearTimeout };
|
|
35
|
+
/** @type {Map<string, RelayResult>} */
|
|
36
|
+
this.pendingResults = new Map();
|
|
37
|
+
/** @type {any} */
|
|
38
|
+
this.reconnectTimer = null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** @param {RelayResult} response */
|
|
42
|
+
deliver(response) {
|
|
43
|
+
const callId = String(response?.id || "");
|
|
44
|
+
if (this.send(response)) {
|
|
45
|
+
if (callId) this.pendingResults.delete(callId);
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
if (callId && this.isRecoverable()) {
|
|
49
|
+
this.pendingResults.set(callId, response);
|
|
50
|
+
this.scheduleExpiry();
|
|
51
|
+
this.logger.event?.("debug", "relay.tool_result.queued", {
|
|
52
|
+
call_id: shortCallId(callId), queued_results: this.pendingResults.size,
|
|
53
|
+
}, "Queued a completed tool result while the relay reconnects");
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
this.logger.event?.("debug", "relay.tool_result.discarded", {
|
|
57
|
+
call_id: shortCallId(callId), reason: "transport_unavailable",
|
|
58
|
+
}, "Discarded a tool result because the relay is no longer recoverable");
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** @param {unknown} callId */
|
|
63
|
+
discard(callId) {
|
|
64
|
+
return this.pendingResults.delete(String(callId));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** @param {Iterable<string>} resumedCallIds @param {(callId: string) => boolean} cancelCall */
|
|
68
|
+
reconcile(resumedCallIds, cancelCall) {
|
|
69
|
+
const resumed = new Set(resumedCallIds);
|
|
70
|
+
let cancelled = 0;
|
|
71
|
+
let discarded = 0;
|
|
72
|
+
for (const callId of this.activeCallIds()) {
|
|
73
|
+
if (!resumed.has(callId) && cancelCall(callId)) cancelled += 1;
|
|
74
|
+
}
|
|
75
|
+
for (const callId of [...this.pendingResults.keys()]) {
|
|
76
|
+
if (!resumed.has(callId) && this.pendingResults.delete(callId)) discarded += 1;
|
|
77
|
+
}
|
|
78
|
+
if (cancelled > 0 || discarded > 0) {
|
|
79
|
+
this.logger.event?.("debug", "relay.calls.reconciled", { cancelled_calls: cancelled, discarded_results: discarded },
|
|
80
|
+
"Cancelled relay work that no longer had a waiting client after reconnect");
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
disconnected() {
|
|
85
|
+
const activeCalls = [...this.activeCallIds()].length;
|
|
86
|
+
if (activeCalls === 0 && this.pendingResults.size === 0) return;
|
|
87
|
+
this.scheduleExpiry();
|
|
88
|
+
this.logger.event?.("debug", "relay.calls.awaiting_reconnect", {
|
|
89
|
+
active_calls: activeCalls, queued_results: this.pendingResults.size, grace_ms: this.graceMs,
|
|
90
|
+
}, "Keeping in-flight tool calls alive during a brief relay interruption");
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
ready() {
|
|
94
|
+
this.clearTimer();
|
|
95
|
+
let delivered = 0;
|
|
96
|
+
for (const [callId, response] of [...this.pendingResults]) {
|
|
97
|
+
if (!this.send(response)) {
|
|
98
|
+
this.scheduleExpiry();
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
this.pendingResults.delete(callId);
|
|
102
|
+
delivered += 1;
|
|
103
|
+
}
|
|
104
|
+
if (delivered > 0) {
|
|
105
|
+
this.logger.event?.("info", "relay.tool_results.replayed", { delivered_results: delivered },
|
|
106
|
+
"Delivered completed tool results after the relay reconnected");
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
stop() {
|
|
111
|
+
this.clearTimer();
|
|
112
|
+
this.pendingResults.clear();
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
scheduleExpiry() {
|
|
116
|
+
if (this.reconnectTimer) return;
|
|
117
|
+
this.reconnectTimer = this.scheduler.setTimeout(() => {
|
|
118
|
+
this.reconnectTimer = null;
|
|
119
|
+
for (const callId of this.activeCallIds()) this.suppressCall(callId, "relay_reconnect_timeout");
|
|
120
|
+
const cancelled = this.cancelOrigin("remote relay reconnect grace expired");
|
|
121
|
+
const discarded = this.pendingResults.size;
|
|
122
|
+
this.pendingResults.clear();
|
|
123
|
+
this.terminate();
|
|
124
|
+
if (cancelled > 0 || discarded > 0) {
|
|
125
|
+
this.logger.warn?.(`remote relay did not recover within ${this.graceMs / 1000} seconds; cancelled ${cancelled} call(s) and discarded ${discarded} queued result(s)`);
|
|
126
|
+
}
|
|
127
|
+
}, this.graceMs);
|
|
128
|
+
this.reconnectTimer?.unref?.();
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
clearTimer() {
|
|
132
|
+
if (!this.reconnectTimer) return;
|
|
133
|
+
this.scheduler.clearTimeout(this.reconnectTimer);
|
|
134
|
+
this.reconnectTimer = null;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** @param {unknown} value @param {number} fallback */
|
|
139
|
+
function positiveInteger(value, fallback) {
|
|
140
|
+
const number = Number(value);
|
|
141
|
+
return Number.isFinite(number) && number > 0 ? Math.floor(number) : fallback;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** @param {unknown} value */
|
|
145
|
+
function shortCallId(value) {
|
|
146
|
+
const text = String(value || "");
|
|
147
|
+
return text.length <= 18 ? text : `${text.slice(0, 10)}...${text.slice(-5)}`;
|
|
148
|
+
}
|
|
@@ -31,6 +31,7 @@ export class RelayConnection {
|
|
|
31
31
|
this.expectedVersion = String(options.expectedVersion || "");
|
|
32
32
|
this.onMessage = typeof options.onMessage === "function" ? options.onMessage : () => {};
|
|
33
33
|
this.onDisconnect = typeof options.onDisconnect === "function" ? options.onDisconnect : () => {};
|
|
34
|
+
this.onReady = typeof options.onReady === "function" ? options.onReady : () => {};
|
|
34
35
|
this.onSuperseded = typeof options.onSuperseded === "function" ? options.onSuperseded : () => {};
|
|
35
36
|
this.onFatal = typeof options.onFatal === "function" ? options.onFatal : () => {};
|
|
36
37
|
this.WebSocketClass = options.WebSocketClass || WebSocket;
|
|
@@ -230,8 +231,12 @@ export class RelayConnection {
|
|
|
230
231
|
}
|
|
231
232
|
}
|
|
232
233
|
|
|
234
|
+
const reconnected = this.hasConnected;
|
|
233
235
|
this.hasConnected = true;
|
|
234
236
|
this.resetOutage();
|
|
237
|
+
try { this.onReady({ reconnected, sessionId: this.activeSessionId }); } catch (error) {
|
|
238
|
+
this.logger.error?.("relay ready callback failed", { error_class: classifyOperationalError(error) });
|
|
239
|
+
}
|
|
235
240
|
if (this.connectedOnceResolve) {
|
|
236
241
|
this.connectedOnceResolve(true);
|
|
237
242
|
this.connectedOnceResolve = null;
|
|
@@ -18,12 +18,14 @@ export function createRuntimeRelayConnection(runtime, { workerUrl, secret, expec
|
|
|
18
18
|
expectedVersion: String(expectedVersion || ""),
|
|
19
19
|
helloMessage: () => ({
|
|
20
20
|
type: "hello",
|
|
21
|
+
instance_id: runtime.relayInstanceId,
|
|
21
22
|
tools: runtime.tools(),
|
|
22
23
|
policy: runtime.policy,
|
|
23
24
|
protocol_versions: MCP_SUPPORTED_PROTOCOL_VERSIONS,
|
|
24
25
|
}),
|
|
25
26
|
onMessage: (data, relayContext) => handleRelayData(runtime, data, relayContext),
|
|
26
27
|
onDisconnect: () => runtime.handleRelayDisconnect(),
|
|
28
|
+
onReady: () => runtime.handleRelayReady(),
|
|
27
29
|
onSuperseded: () => {
|
|
28
30
|
runtime.terminateActiveProcesses("SIGKILL");
|
|
29
31
|
runtime.processSessionManager.clear();
|
|
@@ -37,6 +39,20 @@ export function createRuntimeRelayConnection(runtime, { workerUrl, secret, expec
|
|
|
37
39
|
});
|
|
38
40
|
}
|
|
39
41
|
|
|
42
|
+
export function normalizeRelayResumeCalls(message) {
|
|
43
|
+
if (!Array.isArray(message?.ids) || message.ids.length > 32) return { ok: false, ids: [] };
|
|
44
|
+
const ids = [];
|
|
45
|
+
const seen = new Set();
|
|
46
|
+
for (const value of message.ids) {
|
|
47
|
+
if (typeof value !== "string" || !/^call_[A-Za-z0-9_-]{8,240}$/.test(value) || seen.has(value)) {
|
|
48
|
+
return { ok: false, ids: [] };
|
|
49
|
+
}
|
|
50
|
+
seen.add(value);
|
|
51
|
+
ids.push(value);
|
|
52
|
+
}
|
|
53
|
+
return { ok: true, ids };
|
|
54
|
+
}
|
|
55
|
+
|
|
40
56
|
export function normalizeRelayToolCall(message) {
|
|
41
57
|
const id = typeof message.id === "string" && message.id.length <= 256 ? message.id : "";
|
|
42
58
|
const tool = typeof message.tool === "string" && message.tool.length <= 128 ? message.tool : "";
|
package/src/local/runtime.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { randomBytes } from "node:crypto";
|
|
1
2
|
import { realpathSync, rmSync } from "node:fs";
|
|
2
3
|
import { lstat, realpath, stat } from "node:fs/promises";
|
|
3
4
|
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
|
|
@@ -30,7 +31,8 @@ import { AccountAccessGate } from "./account-access.mjs";
|
|
|
30
31
|
import { buildProjectOverview, buildRuntimeInfo } from "./runtime-reporting.mjs";
|
|
31
32
|
import { diagnoseRuntime as runRuntimeDiagnostics } from "./runtime-diagnostics.mjs";
|
|
32
33
|
import { bindRuntimeToolHandlers, runtimeToolHandlerNames as registeredRuntimeToolHandlerNames } from "./runtime-tool-handlers.mjs";
|
|
33
|
-
import { createRuntimeRelayConnection, normalizeRelayToolCall } from "./runtime-relay.mjs";
|
|
34
|
+
import { createRuntimeRelayConnection, normalizeRelayResumeCalls, normalizeRelayToolCall } from "./runtime-relay.mjs";
|
|
35
|
+
import { RelayCallRecovery } from "./relay-call-recovery.mjs";
|
|
34
36
|
import { assertContainedPath, createRuntimeDir, redactRuntimeErrorMessage, stateRootFromProfileStatePath } from "./runtime-paths.mjs";
|
|
35
37
|
import {
|
|
36
38
|
resolveTaskCapabilities as resolveRuntimeTaskCapabilities,
|
|
@@ -59,8 +61,10 @@ export class LocalRuntime {
|
|
|
59
61
|
this.processTracker = new ProcessTracker();
|
|
60
62
|
this.lifecycle = new LifecycleController("local runtime");
|
|
61
63
|
this.observability = new RuntimeObservability();
|
|
64
|
+
this.relayInstanceId = `daemon_${randomBytes(18).toString("base64url")}`;
|
|
62
65
|
this.activeRelayCalls = new Set();
|
|
63
66
|
this.suppressedRelayResults = new Map();
|
|
67
|
+
this.relayResumeSessionId = 0;
|
|
64
68
|
this.callRegistry = new CallRegistry({
|
|
65
69
|
maximum: MAX_CONCURRENT_TOOL_CALLS,
|
|
66
70
|
onCancel: (record) => {
|
|
@@ -171,10 +175,16 @@ export class LocalRuntime {
|
|
|
171
175
|
slowMs: SLOW_TOOL_CALL_MS,
|
|
172
176
|
});
|
|
173
177
|
this.relay = createRuntimeRelayConnection(this, {
|
|
174
|
-
workerUrl: remoteWorkerUrl,
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
+
workerUrl: remoteWorkerUrl, secret: remoteSecret, expectedVersion: expectedRelayVersion, onFatal,
|
|
179
|
+
});
|
|
180
|
+
this.relayCallRecovery = new RelayCallRecovery({
|
|
181
|
+
logger: this.logger,
|
|
182
|
+
send: (value) => this.send(value),
|
|
183
|
+
isRecoverable: () => Boolean(this.relay && !this.relay.status?.().closed),
|
|
184
|
+
activeCallIds: () => this.activeRelayCalls,
|
|
185
|
+
suppressCall: (callId, reason) => this.suppressedRelayResults.set(callId, reason),
|
|
186
|
+
cancelOrigin: (reason) => this.callRegistry.cancelOrigin("relay", reason),
|
|
187
|
+
terminate: () => this.terminateActiveProcesses("SIGTERM", true),
|
|
178
188
|
});
|
|
179
189
|
}
|
|
180
190
|
|
|
@@ -219,6 +229,7 @@ export class LocalRuntime {
|
|
|
219
229
|
if (!this.lifecycle.beginStop()) return;
|
|
220
230
|
try {
|
|
221
231
|
this.relay?.stop();
|
|
232
|
+
this.relayCallRecovery.stop();
|
|
222
233
|
this.callRegistry.cancelAll("runtime stopped");
|
|
223
234
|
this.terminateActiveProcesses("SIGKILL");
|
|
224
235
|
this.processSessionManager.clear();
|
|
@@ -243,7 +254,7 @@ export class LocalRuntime {
|
|
|
243
254
|
this.handleRelayProtocolViolation("invalid_server_message");
|
|
244
255
|
return;
|
|
245
256
|
}
|
|
246
|
-
if (this.handleRelayControlMessage(message)) return;
|
|
257
|
+
if (this.handleRelayControlMessage(message, relayContext)) return;
|
|
247
258
|
if (message.type === "relay_probe") {
|
|
248
259
|
if (isRelayReadyContext(relayContext, this.relay)) return this.handleRelayProtocolViolation("unexpected_relay_probe");
|
|
249
260
|
this.handleRelayProbe(message, relayContext);
|
|
@@ -251,20 +262,38 @@ export class LocalRuntime {
|
|
|
251
262
|
}
|
|
252
263
|
if (message.type !== "tool_call") return this.handleRelayProtocolViolation("unexpected_server_message_type");
|
|
253
264
|
if (!isRelayReadyContext(relayContext, this.relay)) return this.handleRelayProtocolViolation("tool_call_before_ready");
|
|
254
|
-
await this.handleRelayToolCall(message
|
|
265
|
+
await this.handleRelayToolCall(message);
|
|
255
266
|
}
|
|
256
267
|
|
|
257
|
-
handleRelayControlMessage(message) {
|
|
268
|
+
handleRelayControlMessage(message, relayContext = {}) {
|
|
258
269
|
if (message.type === "welcome") {
|
|
259
270
|
this.relay?.observeWelcome(message);
|
|
260
271
|
return true;
|
|
261
272
|
}
|
|
262
273
|
if (message.type === "hello_ack") {
|
|
274
|
+
this.relayResumeSessionId = 0;
|
|
263
275
|
this.relay?.acknowledge(message);
|
|
264
276
|
return true;
|
|
265
277
|
}
|
|
278
|
+
if (message.type === "resume_calls") {
|
|
279
|
+
const sessionId = Number(relayContext.sessionId) || 0;
|
|
280
|
+
const resume = normalizeRelayResumeCalls(message);
|
|
281
|
+
if (!resume.ok || !sessionId || relayContext.authenticated !== true || relayContext.ready === true) {
|
|
282
|
+
this.handleRelayProtocolViolation("invalid_resume_calls");
|
|
283
|
+
return true;
|
|
284
|
+
}
|
|
285
|
+
this.reconcileRelayCalls(resume.ids);
|
|
286
|
+
this.relayResumeSessionId = sessionId;
|
|
287
|
+
return true;
|
|
288
|
+
}
|
|
266
289
|
if (message.type === "ready_ack") {
|
|
290
|
+
const sessionId = Number(relayContext.sessionId) || 0;
|
|
291
|
+
if (!sessionId || sessionId !== this.relayResumeSessionId) {
|
|
292
|
+
this.handleRelayProtocolViolation("resume_calls_required");
|
|
293
|
+
return true;
|
|
294
|
+
}
|
|
267
295
|
this.relay?.confirmReady(message);
|
|
296
|
+
this.relayResumeSessionId = 0;
|
|
268
297
|
return true;
|
|
269
298
|
}
|
|
270
299
|
if (message.type === "pong") return true;
|
|
@@ -291,12 +320,12 @@ export class LocalRuntime {
|
|
|
291
320
|
const id = typeof message?.id === "string" && /^probe_[A-Za-z0-9_-]{8,240}$/.test(message.id) ? message.id : "";
|
|
292
321
|
const relaySessionId = Number(relayContext.sessionId) || 0;
|
|
293
322
|
if (!id || !relaySessionId) return this.handleRelayProtocolViolation("invalid_relay_probe");
|
|
294
|
-
this.
|
|
323
|
+
const outcome = this.relay?.sendForSession?.({ type: "relay_probe_result", id }, relaySessionId);
|
|
324
|
+
if (!outcome?.ok) this.handleRelayProtocolViolation("relay_probe_delivery_failed");
|
|
295
325
|
}
|
|
296
326
|
|
|
297
|
-
async handleRelayToolCall(message
|
|
327
|
+
async handleRelayToolCall(message) {
|
|
298
328
|
const envelope = normalizeRelayToolCall(message);
|
|
299
|
-
const relaySessionId = Number(relayContext.sessionId) || 0;
|
|
300
329
|
if (!envelope.ok) {
|
|
301
330
|
this.logger.warn?.("Received an invalid tool request from the relay; the request was rejected.");
|
|
302
331
|
this.logger.event?.("debug", "relay.tool_call.invalid", {
|
|
@@ -307,7 +336,7 @@ export class LocalRuntime {
|
|
|
307
336
|
id: envelope.id,
|
|
308
337
|
ok: false,
|
|
309
338
|
error: { code: "invalid_request", message: "invalid tool_call envelope", retryable: false },
|
|
310
|
-
}
|
|
339
|
+
});
|
|
311
340
|
return;
|
|
312
341
|
}
|
|
313
342
|
if (this.activeRelayCalls.has(envelope.id)) {
|
|
@@ -335,31 +364,15 @@ export class LocalRuntime {
|
|
|
335
364
|
}, "Discarded a tool result because the caller was no longer waiting");
|
|
336
365
|
return;
|
|
337
366
|
}
|
|
338
|
-
this.deliverRelayToolResult(response
|
|
367
|
+
this.deliverRelayToolResult(response);
|
|
339
368
|
} finally {
|
|
340
369
|
this.activeRelayCalls.delete(envelope.id);
|
|
341
370
|
this.suppressedRelayResults.delete(envelope.id);
|
|
342
371
|
}
|
|
343
372
|
}
|
|
344
373
|
|
|
345
|
-
deliverRelayToolResult(response
|
|
346
|
-
|
|
347
|
-
const outcome = this.relay?.sendForSession
|
|
348
|
-
? this.relay.sendForSession(response, sessionId)
|
|
349
|
-
: (this.send(response) ? { ok: true, reason: "sent" } : { ok: false, reason: "transport_unavailable" });
|
|
350
|
-
if (outcome?.ok) return true;
|
|
351
|
-
const reason = String(outcome?.reason || "transport_unavailable");
|
|
352
|
-
const missingSession = reason === "session_ended" && sessionId <= 0;
|
|
353
|
-
this.logger.event?.(missingSession ? "error" : "debug", "relay.tool_result.discarded", {
|
|
354
|
-
call_id: shortCallId(response?.id), reason, relay_session_id: sessionId,
|
|
355
|
-
active_session_id: Number(this.relay?.currentSessionId?.() || 0),
|
|
356
|
-
}, missingSession
|
|
357
|
-
? "Discarded a tool result because the relay session id was missing from the inbound tool_call context"
|
|
358
|
-
: reason === "send_failed"
|
|
359
|
-
? "Could not send a tool result because the relay transport failed"
|
|
360
|
-
: "Discarded a tool result because its relay session had ended");
|
|
361
|
-
if (reason === "send_failed") this.relay?.interrupt?.("relay_transport_error");
|
|
362
|
-
return false;
|
|
374
|
+
deliverRelayToolResult(response) {
|
|
375
|
+
return this.relayCallRecovery.deliver(response);
|
|
363
376
|
}
|
|
364
377
|
|
|
365
378
|
finishCall(callId) {
|
|
@@ -373,18 +386,25 @@ export class LocalRuntime {
|
|
|
373
386
|
|
|
374
387
|
cancelRelayCall(callId, suppressionReason = "caller_cancelled") {
|
|
375
388
|
const id = String(callId);
|
|
389
|
+
const discardedResult = this.relayCallRecovery.discard(id);
|
|
376
390
|
if (this.activeRelayCalls.has(id)) this.suppressedRelayResults.set(id, suppressionReason);
|
|
377
|
-
return this.cancelCall(id, "remote cancellation");
|
|
391
|
+
return this.cancelCall(id, "remote cancellation") || discardedResult;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
reconcileRelayCalls(resumedCallIds) {
|
|
395
|
+
this.relayCallRecovery.reconcile(
|
|
396
|
+
resumedCallIds,
|
|
397
|
+
(callId) => this.cancelRelayCall(callId, "caller_no_longer_waiting"),
|
|
398
|
+
);
|
|
378
399
|
}
|
|
379
400
|
|
|
380
401
|
handleRelayDisconnect() {
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
}
|
|
402
|
+
this.relayResumeSessionId = 0;
|
|
403
|
+
this.relayCallRecovery.disconnected();
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
handleRelayReady() {
|
|
407
|
+
this.relayCallRecovery.ready();
|
|
388
408
|
}
|
|
389
409
|
|
|
390
410
|
async executeTool(tool, args, context = {}) {
|
|
@@ -7,6 +7,7 @@ export interface DaemonAttachment {
|
|
|
7
7
|
connectedAt: string;
|
|
8
8
|
lastSeenAt?: string;
|
|
9
9
|
probeId?: string;
|
|
10
|
+
instanceId?: string;
|
|
10
11
|
policy?: DaemonPolicy;
|
|
11
12
|
tools?: string[];
|
|
12
13
|
}
|
|
@@ -29,6 +30,7 @@ export class DaemonSocketRegistry {
|
|
|
29
30
|
connectedAt: sanitizeMetadataText(candidate.connectedAt, 64) ?? "",
|
|
30
31
|
lastSeenAt: sanitizeMetadataText(candidate.lastSeenAt, 64),
|
|
31
32
|
probeId: sanitizeProbeId(candidate.probeId),
|
|
33
|
+
instanceId: sanitizeDaemonInstanceId(candidate.instanceId),
|
|
32
34
|
policy,
|
|
33
35
|
tools: sanitizeDaemonTools(candidate.tools, policy),
|
|
34
36
|
};
|
|
@@ -54,12 +56,13 @@ export class DaemonSocketRegistry {
|
|
|
54
56
|
socket.serializeAttachment({ role: "candidate", connectedAt } satisfies DaemonAttachment);
|
|
55
57
|
}
|
|
56
58
|
|
|
57
|
-
beginProbe(socket: WebSocket, values: { connectedAt: string; probeId: string; policy: DaemonPolicy; tools: string[] }): void {
|
|
59
|
+
beginProbe(socket: WebSocket, values: { connectedAt: string; probeId: string; instanceId: string; policy: DaemonPolicy; tools: string[] }): void {
|
|
58
60
|
socket.serializeAttachment({
|
|
59
61
|
role: "probing",
|
|
60
62
|
connectedAt: values.connectedAt,
|
|
61
63
|
lastSeenAt: values.connectedAt,
|
|
62
64
|
probeId: values.probeId,
|
|
65
|
+
instanceId: values.instanceId,
|
|
63
66
|
policy: values.policy,
|
|
64
67
|
tools: values.tools,
|
|
65
68
|
} satisfies DaemonAttachment);
|
|
@@ -89,6 +92,7 @@ export class DaemonSocketRegistry {
|
|
|
89
92
|
role: "expired",
|
|
90
93
|
connectedAt: attachment.connectedAt,
|
|
91
94
|
lastSeenAt: attachment.lastSeenAt,
|
|
95
|
+
instanceId: attachment.instanceId,
|
|
92
96
|
} satisfies DaemonAttachment);
|
|
93
97
|
}
|
|
94
98
|
|
|
@@ -105,3 +109,8 @@ function sanitizeProbeId(value: unknown): string | undefined {
|
|
|
105
109
|
if (typeof value !== "string" || !/^probe_[A-Za-z0-9_-]{8,240}$/.test(value)) return undefined;
|
|
106
110
|
return value;
|
|
107
111
|
}
|
|
112
|
+
|
|
113
|
+
function sanitizeDaemonInstanceId(value: unknown): string | undefined {
|
|
114
|
+
if (typeof value !== "string" || !/^daemon_[A-Za-z0-9_-]{16,96}$/.test(value)) return undefined;
|
|
115
|
+
return value;
|
|
116
|
+
}
|
package/src/worker/index.ts
CHANGED
|
@@ -34,13 +34,14 @@ import {
|
|
|
34
34
|
} from "./websocket-protocol.ts";
|
|
35
35
|
|
|
36
36
|
const SERVER_NAME = String(serverMetadata.name);
|
|
37
|
-
const SERVER_VERSION = "1.2.
|
|
37
|
+
const SERVER_VERSION = "1.2.10";
|
|
38
38
|
const MCP_PROTOCOL_VERSION = String(serverMetadata.protocolVersion);
|
|
39
39
|
const MCP_SUPPORTED_PROTOCOL_VERSIONS = serverMetadata.supportedProtocolVersions.map((value) => String(value));
|
|
40
40
|
const DEFAULT_MAX_BODY_BYTES = 8 * 1024 * 1024;
|
|
41
41
|
const MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
42
42
|
const MAX_PENDING_CALLS = 32;
|
|
43
43
|
const MAX_DAEMON_MESSAGE_BYTES = 8 * 1024 * 1024;
|
|
44
|
+
const DAEMON_RECONNECT_GRACE_MS = 30_000;
|
|
44
45
|
|
|
45
46
|
interface BridgeEnv extends OAuthControllerEnv {
|
|
46
47
|
BRIDGE: DurableObjectNamespace<BridgeRoom>;
|
|
@@ -179,11 +180,17 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
179
180
|
return;
|
|
180
181
|
}
|
|
181
182
|
const daemonPolicy = sanitizeDaemonPolicy(body.policy);
|
|
183
|
+
const instanceId = daemonInstanceId(body.instance_id);
|
|
184
|
+
if (!instanceId) {
|
|
185
|
+
rejectDaemonMessage(ws, "invalid_daemon_instance", 1002, "daemon instance id required");
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
182
188
|
const authenticatedAt = new Date().toISOString();
|
|
183
189
|
const probeId = randomToken("probe");
|
|
184
190
|
this.daemonRegistry.beginProbe(ws, {
|
|
185
191
|
connectedAt: authenticatedAt,
|
|
186
192
|
probeId,
|
|
193
|
+
instanceId,
|
|
187
194
|
policy: daemonPolicy,
|
|
188
195
|
tools: sanitizeDaemonTools(body.tools, daemonPolicy),
|
|
189
196
|
});
|
|
@@ -218,11 +225,17 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
218
225
|
return;
|
|
219
226
|
}
|
|
220
227
|
const readyAt = new Date().toISOString();
|
|
221
|
-
|
|
228
|
+
const readyAttachment = this.daemonRegistry.promote(ws, readyAt);
|
|
229
|
+
if (!readyAttachment) {
|
|
222
230
|
rejectDaemonMessage(ws, "invalid_relay_readiness_state", 1002, "invalid daemon readiness state");
|
|
223
231
|
return;
|
|
224
232
|
}
|
|
233
|
+
const reboundCallIds = this.pending.rebindInstance(readyAttachment.instanceId ?? "", ws);
|
|
234
|
+
if (reboundCallIds.length > 0) {
|
|
235
|
+
this.observability.event("info", "daemon.calls.rebound", { rebound_calls: reboundCallIds.length });
|
|
236
|
+
}
|
|
225
237
|
try {
|
|
238
|
+
ws.send(JSON.stringify({ type: "resume_calls", ids: reboundCallIds }));
|
|
226
239
|
ws.send(JSON.stringify({ type: "ready_ack", server: SERVER_NAME, version: SERVER_VERSION }));
|
|
227
240
|
} catch {
|
|
228
241
|
this.invalidateDaemonSocket(ws, "daemon readiness acknowledgement failed", "daemon ready timeout", "daemon_ready_timeout");
|
|
@@ -263,7 +276,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
263
276
|
|
|
264
277
|
private async cleanupDaemonSocket(ws: WebSocket, message: string): Promise<void> {
|
|
265
278
|
this.observability.socketDisconnected();
|
|
266
|
-
this.
|
|
279
|
+
this.detachDaemonSocketCalls(ws, message);
|
|
267
280
|
await this.scheduleSocketAlarms();
|
|
268
281
|
}
|
|
269
282
|
|
|
@@ -428,6 +441,9 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
428
441
|
this.reclaimStaleDaemonSockets();
|
|
429
442
|
const socket = this.daemonRegistry.readySockets()[0];
|
|
430
443
|
if (!socket) throw new WorkerToolError("unavailable", "local daemon is not connected; keep the CLI start command running", true);
|
|
444
|
+
const daemonAttachment = this.daemonRegistry.readyAttachment(socket);
|
|
445
|
+
const daemonInstanceId = daemonAttachment?.instanceId ?? "";
|
|
446
|
+
if (!daemonInstanceId) throw new WorkerToolError("unavailable", "local daemon connection is missing its instance identity", true);
|
|
431
447
|
const id = randomToken("call");
|
|
432
448
|
const timeoutMs = daemonToolTimeoutMs(name, args);
|
|
433
449
|
let result: Promise<unknown>;
|
|
@@ -435,20 +451,23 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
435
451
|
result = this.pending.register({
|
|
436
452
|
id,
|
|
437
453
|
socket,
|
|
454
|
+
daemonInstanceId,
|
|
438
455
|
clientRequestKey: requestKey,
|
|
439
456
|
tool: name,
|
|
440
457
|
timeoutMs,
|
|
441
458
|
onTimeout: (record) => {
|
|
442
|
-
sendWebSocketQuietly(record.socket, { type: "cancel_call", id: record.id });
|
|
443
|
-
const silentForMs =
|
|
444
|
-
|
|
459
|
+
if (record.socket) sendWebSocketQuietly(record.socket, { type: "cancel_call", id: record.id });
|
|
460
|
+
const silentForMs = record.socket
|
|
461
|
+
? Date.now() - daemonLastSeenMs(this.daemonRegistry.readyAttachment(record.socket))
|
|
462
|
+
: 0;
|
|
463
|
+
if (record.socket && (!Number.isFinite(silentForMs) || silentForMs > 45_000)) {
|
|
445
464
|
this.invalidateDaemonSocket(record.socket, "daemon became unresponsive", "daemon liveness timeout");
|
|
446
465
|
}
|
|
447
466
|
return new WorkerToolError("timeout", `daemon tool timed out: ${name}`, true);
|
|
448
467
|
},
|
|
449
468
|
signal,
|
|
450
469
|
onAbort: (record) => {
|
|
451
|
-
sendWebSocketQuietly(record.socket, { type: "cancel_call", id: record.id });
|
|
470
|
+
if (record.socket) sendWebSocketQuietly(record.socket, { type: "cancel_call", id: record.id });
|
|
452
471
|
return new WorkerToolError("cancelled", "MCP client stopped waiting for the tool result");
|
|
453
472
|
},
|
|
454
473
|
});
|
|
@@ -482,7 +501,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
482
501
|
private cancelClientRequest(requestKey?: string): void {
|
|
483
502
|
if (!requestKey) return;
|
|
484
503
|
this.pending.cancelRequest(requestKey, (record) => {
|
|
485
|
-
sendWebSocketQuietly(record.socket, { type: "cancel_call", id: record.id });
|
|
504
|
+
if (record.socket) sendWebSocketQuietly(record.socket, { type: "cancel_call", id: record.id });
|
|
486
505
|
return new WorkerToolError("cancelled", "tool call cancelled by client");
|
|
487
506
|
});
|
|
488
507
|
}
|
|
@@ -542,12 +561,24 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
542
561
|
closeReason: string,
|
|
543
562
|
errorCode = "daemon_liveness_timeout",
|
|
544
563
|
): void {
|
|
564
|
+
this.detachDaemonSocketCalls(ws, message);
|
|
545
565
|
this.daemonRegistry.expire(ws);
|
|
546
|
-
this.pending.rejectSocket(ws, () => new WorkerToolError("unavailable", message, true));
|
|
547
566
|
sendWebSocketQuietly(ws, { type: "error", error: errorCode });
|
|
548
567
|
closeWebSocketQuietly(ws, 1008, closeReason);
|
|
549
568
|
}
|
|
550
569
|
|
|
570
|
+
private detachDaemonSocketCalls(ws: WebSocket, message: string): number {
|
|
571
|
+
const attachment = this.daemonRegistry.attachment(ws);
|
|
572
|
+
if (!attachment?.instanceId) {
|
|
573
|
+
return this.pending.rejectSocket(ws, () => new WorkerToolError("unavailable", message, true));
|
|
574
|
+
}
|
|
575
|
+
return this.pending.detachSocket(
|
|
576
|
+
ws,
|
|
577
|
+
DAEMON_RECONNECT_GRACE_MS,
|
|
578
|
+
() => new WorkerToolError("unavailable", `${message}; reconnect grace expired`, true),
|
|
579
|
+
);
|
|
580
|
+
}
|
|
581
|
+
|
|
551
582
|
private reclaimStaleDaemonSockets(now = Date.now()): void {
|
|
552
583
|
for (const socket of this.daemonRegistry.readyRoleSockets()) {
|
|
553
584
|
const deadline = daemonLivenessDeadlineMs(this.daemonRegistry.readyAttachment(socket));
|
|
@@ -697,3 +728,8 @@ export default {
|
|
|
697
728
|
return stub.fetch(request);
|
|
698
729
|
},
|
|
699
730
|
} satisfies ExportedHandler<BridgeEnv>;
|
|
731
|
+
|
|
732
|
+
function daemonInstanceId(value: unknown): string {
|
|
733
|
+
if (typeof value !== "string" || !/^daemon_[A-Za-z0-9_-]{16,96}$/.test(value)) return "";
|
|
734
|
+
return value;
|
|
735
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export interface PendingCallRecord {
|
|
2
|
+
id: string;
|
|
3
|
+
socket?: WebSocket;
|
|
4
|
+
daemonInstanceId?: string;
|
|
5
|
+
reconnectTimeout?: ReturnType<typeof setTimeout>;
|
|
6
|
+
clientRequestKey?: string;
|
|
7
|
+
tool: string;
|
|
8
|
+
startedAt: number;
|
|
9
|
+
timeout: ReturnType<typeof setTimeout>;
|
|
10
|
+
resolve: (value: unknown) => void;
|
|
11
|
+
reject: (error: Error) => void;
|
|
12
|
+
signal?: AbortSignal;
|
|
13
|
+
abortHandler?: () => void;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface RegisterPendingCall {
|
|
17
|
+
id: string;
|
|
18
|
+
socket: WebSocket;
|
|
19
|
+
daemonInstanceId?: string;
|
|
20
|
+
clientRequestKey?: string;
|
|
21
|
+
tool: string;
|
|
22
|
+
timeoutMs: number;
|
|
23
|
+
onTimeout: (record: PendingCallRecord) => Error;
|
|
24
|
+
signal?: AbortSignal;
|
|
25
|
+
onAbort?: (record: PendingCallRecord) => Error;
|
|
26
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { PendingCallRecord, RegisterPendingCall } from "./pending-call-contract.ts";
|
|
2
|
+
|
|
1
3
|
export class PendingCallRegistrationError extends Error {
|
|
2
4
|
readonly code: "conflict" | "limit_exceeded";
|
|
3
5
|
readonly retryable: boolean;
|
|
@@ -10,30 +12,6 @@ export class PendingCallRegistrationError extends Error {
|
|
|
10
12
|
}
|
|
11
13
|
}
|
|
12
14
|
|
|
13
|
-
export interface PendingCallRecord {
|
|
14
|
-
id: string;
|
|
15
|
-
socket: WebSocket;
|
|
16
|
-
clientRequestKey?: string;
|
|
17
|
-
tool: string;
|
|
18
|
-
startedAt: number;
|
|
19
|
-
timeout: ReturnType<typeof setTimeout>;
|
|
20
|
-
resolve: (value: unknown) => void;
|
|
21
|
-
reject: (error: Error) => void;
|
|
22
|
-
signal?: AbortSignal;
|
|
23
|
-
abortHandler?: () => void;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
interface RegisterPendingCall {
|
|
27
|
-
id: string;
|
|
28
|
-
socket: WebSocket;
|
|
29
|
-
clientRequestKey?: string;
|
|
30
|
-
tool: string;
|
|
31
|
-
timeoutMs: number;
|
|
32
|
-
onTimeout: (record: PendingCallRecord) => Error;
|
|
33
|
-
signal?: AbortSignal;
|
|
34
|
-
onAbort?: (record: PendingCallRecord) => Error;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
15
|
export class PendingCallRegistry {
|
|
38
16
|
private readonly maximum: number;
|
|
39
17
|
private readonly byId = new Map<string, PendingCallRecord>();
|
|
@@ -78,6 +56,7 @@ export class PendingCallRegistry {
|
|
|
78
56
|
const record: PendingCallRecord = {
|
|
79
57
|
id: input.id,
|
|
80
58
|
socket: input.socket,
|
|
59
|
+
daemonInstanceId: input.daemonInstanceId,
|
|
81
60
|
clientRequestKey: input.clientRequestKey,
|
|
82
61
|
tool: String(input.tool || "unknown"),
|
|
83
62
|
startedAt,
|
|
@@ -126,16 +105,52 @@ export class PendingCallRegistry {
|
|
|
126
105
|
return ids.length;
|
|
127
106
|
}
|
|
128
107
|
|
|
129
|
-
|
|
108
|
+
detachSocket(
|
|
109
|
+
socket: WebSocket,
|
|
110
|
+
graceMs: number,
|
|
111
|
+
createError: (record: PendingCallRecord) => Error,
|
|
112
|
+
): number {
|
|
113
|
+
const records = [...this.byId.values()].filter((record) => record.socket === socket);
|
|
114
|
+
const delay = Math.max(1, Math.floor(Number(graceMs) || 1));
|
|
115
|
+
for (const record of records) {
|
|
116
|
+
record.socket = undefined;
|
|
117
|
+
if (record.reconnectTimeout) clearTimeout(record.reconnectTimeout);
|
|
118
|
+
record.reconnectTimeout = setTimeout(() => {
|
|
119
|
+
const current = this.byId.get(record.id);
|
|
120
|
+
if (!current || current.socket) return;
|
|
121
|
+
const expired = this.take(record.id);
|
|
122
|
+
if (expired) expired.reject(createError(expired));
|
|
123
|
+
}, delay);
|
|
124
|
+
}
|
|
125
|
+
return records.length;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
rebindInstance(daemonInstanceId: string, socket: WebSocket): string[] {
|
|
129
|
+
if (!daemonInstanceId) return [];
|
|
130
|
+
const rebound: string[] = [];
|
|
131
|
+
for (const record of this.byId.values()) {
|
|
132
|
+
if (record.socket || record.daemonInstanceId !== daemonInstanceId) continue;
|
|
133
|
+
if (record.reconnectTimeout) clearTimeout(record.reconnectTimeout);
|
|
134
|
+
record.reconnectTimeout = undefined;
|
|
135
|
+
record.socket = socket;
|
|
136
|
+
rebound.push(record.id);
|
|
137
|
+
}
|
|
138
|
+
return rebound;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
snapshot(): { active: number; detached: number; request_keys: number; maximum: number; oldest_ms: number; by_tool: Record<string, number> } {
|
|
130
142
|
const now = performance.now();
|
|
131
143
|
const byTool: Record<string, number> = {};
|
|
144
|
+
let detached = 0;
|
|
132
145
|
let oldestMs = 0;
|
|
133
146
|
for (const record of this.byId.values()) {
|
|
134
147
|
byTool[record.tool] = (byTool[record.tool] || 0) + 1;
|
|
148
|
+
if (!record.socket) detached += 1;
|
|
135
149
|
oldestMs = Math.max(oldestMs, now - record.startedAt);
|
|
136
150
|
}
|
|
137
151
|
return {
|
|
138
152
|
active: this.byId.size,
|
|
153
|
+
detached,
|
|
139
154
|
request_keys: this.byRequestKey.size,
|
|
140
155
|
maximum: this.maximum,
|
|
141
156
|
oldest_ms: oldestMs,
|
|
@@ -147,6 +162,7 @@ export class PendingCallRegistry {
|
|
|
147
162
|
const record = this.byId.get(id);
|
|
148
163
|
if (!record) return undefined;
|
|
149
164
|
clearTimeout(record.timeout);
|
|
165
|
+
if (record.reconnectTimeout) clearTimeout(record.reconnectTimeout);
|
|
150
166
|
if (record.signal && record.abortHandler) record.signal.removeEventListener("abort", record.abortHandler);
|
|
151
167
|
this.byId.delete(id);
|
|
152
168
|
if (record.clientRequestKey && this.byRequestKey.get(record.clientRequestKey) === id) {
|
package/tsconfig.local.json
CHANGED