machine-bridge-mcp 3.0.0-beta.15 → 3.0.0-beta.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/browser-extension/manifest.json +1 -1
- package/docs/ARCHITECTURE.md +3 -3
- package/docs/AUDIT.md +14 -0
- package/docs/OPERATIONS.md +7 -1
- package/docs/TESTING.md +1 -1
- package/package.json +1 -1
- package/src/local/managed-job-plan.mjs +2 -2
- package/src/worker/index.ts +39 -77
- package/src/worker/pending-call-contract.ts +2 -0
- package/src/worker/pending-call-deadlines.ts +15 -1
- package/src/worker/pending-calls.ts +28 -11
- package/src/worker/runtime-alarm.ts +119 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 3.0.0-beta.16 - 2026-07-25
|
|
4
|
+
|
|
5
|
+
### Pending-call recovery and verified handover
|
|
6
|
+
|
|
7
|
+
- Separate the upstream MCP host/connector shard-mapper incident from Machine Bridge evidence. The exact temporary-keyspace error never appeared in Worker or daemon diagnostics and did not increment Worker server-error counters, so it is documented as an external boundary failure with unknown platform ownership rather than misclassified as a local daemon, OAuth, Git, or Cloudflare defect.
|
|
8
|
+
- Close the Machine Bridge failure-amplification path discovered after recovery. Pending calls now retain monotonic operation and reconnect deadlines, schedule the earliest deadline through the Durable Object alarm, and run a compensating overdue sweep on every HTTP/WebSocket event. In-memory timers remain the fast path; a transient alarm-storage error is observable without converting already-dispatched work into a false terminal failure.
|
|
9
|
+
- Make verified same-instance daemon handover atomic with respect to in-flight calls. Both attached and detached records move to the replacement before the incumbent closes, the complete `resume_calls` set is sent, remaining operation timeout is preserved, and failed replacement acknowledgement restores ownership to a still-open incumbent.
|
|
10
|
+
- Add deterministic disabled-timer deadline tests, direct runtime-alarm scheduling/failure tests, and a real Wrangler/workerd race regression that connects a same-instance replacement while the incumbent still owns an active call. The call remains active rather than detached and completes through the verified replacement.
|
|
11
|
+
- Use null-prototype dictionaries for managed-job `env` and `env_resources`, so valid variable names such as `__proto__`, `constructor`, `toString`, and `valueOf` remain ordinary own data instead of mutating JavaScript object prototypes. Add behavior coverage without weakening duplicate-variable rejection.
|
|
12
|
+
- Correct architecture and operations documentation that still described Durable Object `waitUntil` ownership or direct rejection during socket replacement. Document the three deadline enforcement paths, stale-pending diagnosis, host/connector internal-storage error triage, and the exact test evidence.
|
|
13
|
+
|
|
3
14
|
## 3.0.0-beta.15 - 2026-07-25
|
|
4
15
|
|
|
5
16
|
### Event-driven streamed-call settlement
|
|
@@ -30,6 +30,6 @@
|
|
|
30
30
|
"action": {
|
|
31
31
|
"default_title": "Machine Bridge Browser"
|
|
32
32
|
},
|
|
33
|
-
"version_name": "3.0.0-beta.
|
|
33
|
+
"version_name": "3.0.0-beta.16",
|
|
34
34
|
"key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxryYkpZhq8+VAQLHcGS9BAHQcyKX8RHGIpIwvtIVRU/rcOcE0bNdnM0aZJ/h6xWQsGDHlhvjT2+1aJaAn/9k8473BRWajzVXld961CdHYVFVHoce2hHiSJ0xydWrHMMZhAm0mN0UzjEpgZ0tMw209efcZHIvSwuxhteZMRy4kyiVjwFlOf5oXFCxRuCJnPj3AK9CmCf4XgEBuPIJ0TZmjGHOOdBvJmbCNnAWXYEo5/mf7MfCGhV4IJ1hNuhpoNQfOFKMUcw9/v/IpT62XpfXdGYTfGYCmCjC+gntK1spbkr2P4/2+sYMQtLpse71mpSNGXfcf3abU55Vpn+gncSxRQIDAQAB"
|
|
35
35
|
}
|
package/docs/ARCHITECTURE.md
CHANGED
|
@@ -111,7 +111,7 @@ All requests for a deployed Worker route to one named Durable Object. It owns:
|
|
|
111
111
|
- OAuth clients, authorization codes, hashed access-token records, an independently versioned hashed refresh-token store, and throttling metadata;
|
|
112
112
|
- one active end-to-end-verified daemon WebSocket plus bounded candidate and probing sockets;
|
|
113
113
|
- policy/tool metadata attached to the active socket;
|
|
114
|
-
- a bounded in-memory map of pending daemon calls;
|
|
114
|
+
- a bounded in-memory map of pending daemon calls, with monotonic operation/reconnect deadlines projected onto Durable Object alarms and rechecked at every event boundary;
|
|
115
115
|
- bounded resumable MCP delivery metadata and terminal responses for recently disconnected SSE clients.
|
|
116
116
|
|
|
117
117
|
`BridgeRoom` owns Durable Object routing, MCP authorization/dispatch, daemon WebSocket lifecycle, pending relay-call composition, cancellation, and resumable state. `mcp-stream-proxy.ts` owns the outer-Worker transport adapter: it strips public internal-control headers, obtains a bounded authenticated descriptor, creates the client-facing SSE stream, and polls terminal state through short immediate service-binding requests. `mcp-access.ts` owns shared Bearer/DPoP authorization for POST and recovery GET. `mcp-resumption-http.ts` owns recovery routing and signed session/protocol binding and returns descriptors rather than a long-lived response. `mcp-resumption.ts` owns stream admission, transaction ordering, immediate pending/terminal polls, expiry, replay, and lifecycle state; `mcp-resumption-records.ts` owns the compact metadata index, terminal-message bounds, serialization, and SHA-256 integrity metadata. `mcp-stream.ts` owns SSE sequence-zero/sequence-one framing and heartbeats. `mcp-jsonrpc.ts` owns JSON-RPC shape validation, result/error framing, MCP tool-result projection, session-instruction bounds, and protocol-header validation. `websocket-protocol.ts` owns record validation plus best-effort send/close/rejection helpers. `OAuthController` owns OAuth-store pruning, registration throttling, authorization submission, account-admin routing, token exchange, access-token verification, and the serialization queue for OAuth mutations. Worker-internal TypeScript imports use explicit `.ts` specifiers and JSON import attributes, so the same modules are directly executable under the pinned Node runtime and bundled by Wrangler.
|
|
@@ -172,7 +172,7 @@ Remote OAuth binds each code, access token, and refresh token to a named Machine
|
|
|
172
172
|
7. The MCP client initializes against the sole current protocol version; an obsolete client must upgrade rather than enter a legacy execution path. The Worker returns a stateless HMAC-bound `MCP-Session-Id`, and later request/cancellation correlation is scoped by OAuth token, MCP session, JSON-RPC id type, and id value. Two clients may therefore reuse the same JSON-RPC id concurrently without collision. Sessionless POSTs remain independent and are not inserted into a token-global cancellation index. When the daemon advertises `session_bootstrap`, the Worker requests bounded local instructions and appends them to the initialization result; failure degrades to static instructions.
|
|
173
173
|
8. A new daemon first authenticates as a bounded `probing` socket. The Worker sends a random `relay_probe`; the local runtime returns it through the normal session-bound result-delivery path; only the matching result produces `ready_ack`, promotion to the active daemon, and safe replacement of an incumbent connection.
|
|
174
174
|
9. `tools/list` is derived only from the active end-to-end-verified daemon; without one, only `server_info` is advertised.
|
|
175
|
-
10. `tools/call` receives a random relay call ID and is bound to the current daemon socket, that daemon process's ephemeral instance identifier, and the authenticated client request key. When the client accepts `text/event-stream`,
|
|
175
|
+
10. `tools/call` receives a random relay call ID and is bound to the current daemon socket, that daemon process's ephemeral instance identifier, and the authenticated client request key. When the client accepts `text/event-stream`, `BridgeRoom` commits recovery state, registers an event-settled pending call, sends the daemon envelope, and returns a bounded descriptor immediately; the outer Worker owns the SSE priming frame, keepalives, and terminal polling. No unresolved terminal Promise or Durable Object `waitUntil` owns the dispatch. JSON-only clients retain the single terminal response.
|
|
176
176
|
11. The runtime validates policy and arguments, executes the tool, and returns a bounded result. Closing or losing the HTTP response stream only makes that stream unwritable; it is not an MCP cancellation and does not remove the pending request.
|
|
177
177
|
12. If the socket remains ready, the Durable Object accepts the result only from that socket. If it drops, the Worker detaches the pending call for at most two minutes and accepts completion only after a replacement socket with the same daemon-process identifier has passed the end-to-end readiness probe. The local runtime preserves the operation and queues a completion over the same shared interval. The Worker pauses the record's remaining normal deadline while detached and resumes it after same-instance rebinding, so connected calls are not granted an unconditional recovery extension.
|
|
178
178
|
13. Only a matching session-scoped `notifications/cancelled` request removes the pending indexes and sends best-effort cancellation to a connected daemon. Local completion that races with explicit cancellation is discarded. On every readiness handover, the Worker first sends an authoritative bounded `resume_calls` set; the runtime cancels active calls and queued results absent from that set before accepting `ready_ack`. A request explicitly cancelled while disconnected therefore cannot be revived by a fast reconnect.
|
|
@@ -255,7 +255,7 @@ Reconnect uses bounded exponential backoff with jitter. Brief self-healing inter
|
|
|
255
255
|
|
|
256
256
|
The Worker stores socket transitions in `DaemonSocketRegistry`: `candidate` before hello, `probing` after authentication, `daemon` only after the end-to-end result probe, and `expired` after terminal failure. Durable Object alarms enforce separate hello, readiness, and steady-state liveness deadlines across hibernation. A healthy incumbent remains active while a replacement is probed; a malformed, silent, incompatible, or identity-mismatched replacement is closed without displacing it. Only a verified candidate receives `ready_ack` and then replaces the old socket. Ready daemons stay live only while inbound traffic refreshes `lastSeenAt`; silent half-open or hibernation-restored sockets are reclaimed instead of advertising `daemon.connected` while tool calls time out.
|
|
257
257
|
|
|
258
|
-
Each daemon process generates a random bounded `instance_id` at startup and includes it in every reconnect hello. Pending calls normally retain their assigned socket. On an unexpected socket loss, only those records are detached and the shared two-minute relay contract bounds recovery.
|
|
258
|
+
Each daemon process generates a random bounded `instance_id` at startup and includes it in every reconnect hello. Pending calls normally retain their assigned socket. On an unexpected socket loss, only those records are detached and the shared two-minute relay contract bounds recovery. During verified same-instance handover, the Worker transfers both already-detached calls and still-attached calls from the incumbent socket to the replacement before closing the incumbent; this prevents the asynchronous close event from creating a detached call after the only rebind pass. If replacement acknowledgement fails, ownership is restored to the still-open same-instance incumbent. Another process cannot inherit or resolve these calls. The local runtime mirrors that state machine by preserving active calls and completed-result envelopes until relay readiness returns. Before `ready_ack`, the Worker sends the exact IDs that still have remote waiters; the runtime cancels everything else and only then replays retained results through the verified socket. Grace expiry restores the terminal behavior: reject remote waiters, cancel local ordinary calls, terminate process trees, and discard undeliverable results. The shared execution envelope remains independent of reconnect grace. Worker operation countdown is paused while detached or transferred and resumed with its remaining budget, so handover cannot reset the normal timeout. In-memory timers are only the fast path: the earliest monotonic pending deadline is also scheduled as a Durable Object alarm, and every HTTP or WebSocket event performs a compensating overdue scan. A transient alarm-storage failure is observable but does not turn an already-dispatched operation into a false terminal failure; the next event scan remains the bounded recovery path. This does not make calls durable across daemon restart or machine failure; managed jobs remain the separate durable mechanism.
|
|
259
259
|
|
|
260
260
|
## Persistence
|
|
261
261
|
|
package/docs/AUDIT.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Security and privacy audit notes
|
|
2
2
|
|
|
3
|
+
## 2026-07-25 version 3.0.0-beta.16 pending-call recovery and boundary audit
|
|
4
|
+
|
|
5
|
+
The reported incident exposed two separate failure domains. The host/connector layer returned `No shard mapper found` for a temporary high-replication backfill keyspace and then rejected even minimal MCP calls. The exact text did not exist in Machine Bridge source, deployed Worker events, daemon logs, or local command output; Worker HTTP server-error counters remained unchanged. The original shard-routing failure therefore occurred before or outside the deployed Worker/daemon boundary. The evidence is insufficient to identify the owner of that upstream temporary store, so this audit does not misattribute it to Cloudflare, the local daemon, OAuth, or Git. The host path later recovered without credential rotation or state deletion.
|
|
6
|
+
|
|
7
|
+
Independent inspection after recovery found a real Machine Bridge amplification defect: two calls remained active and detached beyond the shared two-minute reconnect grace. Pending operation and reconnect expiry used only in-memory `setTimeout` callbacks. Event-driven streamed calls intentionally return from the initiating Durable Object event, so a timer alone was not a valid cross-event lifecycle guarantee. The Worker now records monotonic operation/reconnect deadlines, exposes the earliest remaining duration to one combined Durable Object alarm, and performs an overdue scan at every HTTP and WebSocket event boundary. Timer callbacks remain the low-latency path. Alarm-storage failure emits only a bounded coarse event and does not falsely fail work that was already dispatched; the next event scan is the compensating path.
|
|
8
|
+
|
|
9
|
+
A second race existed during verified daemon replacement. The previous implementation rebound only already-detached calls, then closed the incumbent socket. Calls still attached to that socket were detached later by its asynchronous close callback, after the only rebind pass, and could remain orphaned. Same-instance promotion now transfers both attached and detached calls to the candidate before closing the incumbent, sends the complete authoritative `resume_calls` set, preserves the remaining normal timeout, and restores ownership to a still-open incumbent if replacement acknowledgement fails. Different daemon instances remain unable to inherit calls.
|
|
10
|
+
|
|
11
|
+
The broader review found a separate prototype-sensitive input defect in managed-job environment maps. POSIX/Windows-compatible variable validation permits names such as `__proto__`, `constructor`, `toString`, and `valueOf`, but validation accumulated them into ordinary JavaScript objects. Assigning `__proto__` invoked the legacy prototype setter rather than creating an own environment entry. Both plain and resource-backed environment maps now use null-prototype dictionaries, with stage/inspection regression coverage for all prototype-shaped keys and the existing duplicate-variable denial.
|
|
12
|
+
|
|
13
|
+
Documentation had also drifted: the remote lifecycle still claimed streamed work used Durable Object `waitUntil`, despite beta.15 explicitly removing and architecture tests forbidding that design. Architecture, operations, testing, changelog, and this audit now describe event-settled dispatch, alarm-plus-sweep deadline enforcement, atomic same-instance handover, upstream-host diagnosis, and prototype-safe environment maps. Complete and production dependency audits report zero vulnerabilities; privacy history, critical-module coverage, Worker dry-run, deterministic lifecycle tests, managed-job integration, type checks, lint, and real Wrangler OAuth/MCP integration pass before candidate preparation.
|
|
14
|
+
|
|
15
|
+
These source changes do not claim to repair the upstream shard mapper or prove its platform owner. They prevent a transient external failure or socket replacement from leaving permanent pending-call occupancy inside Machine Bridge. No Worker deployment, daemon/service replacement, global installation, credential rotation, push, tag, npm publication, or GitHub Release is performed by the source audit itself. Exact beta.16 candidate activation and owner-machine verification remain mandatory.
|
|
16
|
+
|
|
3
17
|
## 2026-07-25 version 3.0.0-beta.15 event-lifecycle audit
|
|
4
18
|
|
|
5
19
|
The exact beta.14 candidate was activated by the repository owner. The Worker, launchd daemon, private runtime path, activation record, candidate checksum, `status`, and `doctor` all converged on `3.0.0-beta.14`; the globally installed beta.12 package remained the explicit rollback baseline. A random temporary owner account and OAuth client then exercised the production protocol and were removed after the run. Sequence zero, token/session isolation, disconnect recovery, terminal sequence one, and empty acknowledgement replay all passed.
|
package/docs/OPERATIONS.md
CHANGED
|
@@ -51,12 +51,18 @@ A successful diagnostic result applies only to that probe. An MCP host can still
|
|
|
51
51
|
|
|
52
52
|
Machine Bridge supports concurrent calls: the Worker admits up to 32 pending daemon calls and the local runtime admits up to 16 active tool calls. These are capacity limits, not a single global execution queue. Each successful MCP initialization receives a signed `MCP-Session-Id`; JSON-RPC ids and cancellation are scoped to that session, so separate chat windows may reuse the same numeric ids safely even when they share one OAuth account and token.
|
|
53
53
|
|
|
54
|
-
`server_info.worker.pending_calls` reports `active`, `detached`, `request_keys`, `maximum`, `oldest_ms`, and `by_tool`. `worker.sockets_live` separately reports `authenticated`, `probing`, `ready`, and `candidates`; only `ready` sockets contribute to `daemon.connected` and tool advertisement. A nonzero `active` count means work is in flight, not that the bridge is locked. `detached > 0` means a daemon socket was lost and those requests are inside the bounded two-minute same-instance reconnect window. Calls for simple reads and probes should continue while another independent process call runs. Only explicit session-scoped MCP cancellation, timeout, or reconnect-grace expiry removes the pending record and its request key; an HTTP response disconnect is not cancellation. A daemon-socket closure detaches only calls assigned to that socket; the same daemon process can reclaim them after completing readiness, while another process cannot. Grace expiry rejects the request and cancels the local ordinary operation. Refreshing a chat page is not the recovery mechanism and should not be required.
|
|
54
|
+
`server_info.worker.pending_calls` reports `active`, `detached`, `request_keys`, `maximum`, `oldest_ms`, and `by_tool`. `worker.sockets_live` separately reports `authenticated`, `probing`, `ready`, and `candidates`; only `ready` sockets contribute to `daemon.connected` and tool advertisement. A nonzero `active` count means work is in flight, not that the bridge is locked. `detached > 0` means a daemon socket was lost and those requests are inside the bounded two-minute same-instance reconnect window. Calls for simple reads and probes should continue while another independent process call runs. Only explicit session-scoped MCP cancellation, timeout, or reconnect-grace expiry removes the pending record and its request key; an HTTP response disconnect is not cancellation. A daemon-socket closure detaches only calls assigned to that socket; the same daemon process can reclaim them after completing readiness, while another process cannot. A verified same-instance replacement transfers both detached and still-attached calls before the incumbent closes. Normal and reconnect deadlines have three enforcement paths: monotonic in-event timers, a Durable Object alarm, and an overdue sweep at the next HTTP/WebSocket event. Therefore `detached > 0` with `oldest_ms` materially beyond the two-minute grace is a lifecycle defect rather than normal recovery. Grace expiry rejects the request and cancels the local ordinary operation. Refreshing a chat page is not the recovery mechanism and should not be required.
|
|
55
55
|
|
|
56
56
|
For Streamable HTTP clients such as ChatGPT that advertise `text/event-stream`, the outer Worker returns an immediate sequence-zero SSE event identifier and a keepalive comment every ten seconds until the terminal sequence-one JSON-RPC result. `BridgeRoom` never owns the long-lived public stream or an unresolved terminal Promise. Stream initiation commits recovery state, registers the daemon call, sends it, and returns a descriptor; a later WebSocket result, explicit cancellation, timeout, send failure, or reconnect-grace expiry writes the terminal result. Short pending/terminal polls therefore coexist with concurrent `server_info`, recovery, and session-scoped `notifications/cancelled` requests while SSE remains open. Caller-supplied internal stream headers are removed at the public boundary. If the client or an intermediary closes the stream, Machine Bridge keeps the bounded operation alive; only `notifications/cancelled` carries cancellation semantics. A compatible host resumes the original stream with authenticated `GET /mcp`, the original `MCP-Session-Id`, and `Last-Event-ID`; it must not repeat the POST. Recovery records are token/session-bound, retained for at most two minutes, limited to 64 streams, and persist at most 1.5 MiB of terminal JSON. Error `-32002` means the online result exceeded the replay budget; `-32003` means the Worker restarted before it could persist a terminal result and the operation may already have produced side effects; reconcile state before retrying. Error `-32005` means stored replay data failed integrity validation.
|
|
57
57
|
|
|
58
58
|
`server_info.worker.observability.calls.unmatched_results` counts results that reached the Worker after their pending record was already removed. A small increase can accompany cancellation or timeout races, especially during mixed-version upgrade convergence; sustained growth together with old pending calls indicates incompatible components or a lifecycle defect. The counter contains no tool arguments or result data.
|
|
59
59
|
|
|
60
|
+
### MCP host or connector internal-storage errors
|
|
61
|
+
|
|
62
|
+
An error naming an internal shard mapper, temporary keyspace, backfill store, or connector database is not automatically a Machine Bridge Worker or daemon error. During the beta.15 incident, the exact error text was absent from repository source, Worker events, daemon logs, and local process output; Worker HTTP `server_error` counters also did not increase, while the host temporarily failed even `server_info`. That evidence places the original failure before or outside the deployed Worker/daemon boundary, but it does not identify which upstream platform component owned the temporary store. Do not rotate OAuth/device credentials, delete local state, or restart a healthy daemon solely because of such a message.
|
|
63
|
+
|
|
64
|
+
After the host path recovers, run `server_info`, `machine-mcp doctor`, and `machine-mcp service status`. Compare Worker `requests.server_error`, pending-call age, ready socket count, daemon PID/start time, and local logs. If the upstream text never appears locally and Worker server errors remain unchanged, report the host/connector incident separately. If pending calls remain older than their operation or reconnect deadline, that is a Machine Bridge lifecycle issue and should be investigated independently rather than attributed to the upstream shard error.
|
|
65
|
+
|
|
60
66
|
### Relay interruption messages
|
|
61
67
|
|
|
62
68
|
A reconnect warning is evidence of a transport outage, not proof that the daemon process exited. Compare daemon PID/process start, `connected_at`, `last_seen_at`, `runtime.relay.last_disconnected_at`, close category/code, and outage count. A system VPN/TUN may remain shown as connected while its internal route is unavailable; Machine Bridge reports that route only as `system-network-stack` with application-proxy scope. The reconnect schedule now tops out at fifteen seconds.
|
package/docs/TESTING.md
CHANGED
|
@@ -159,4 +159,4 @@ The stdio integration test also sends an oversized line, verifies bounded reject
|
|
|
159
159
|
|
|
160
160
|
`npm run mcp-resumption:test` directly exercises stream cursor parsing, OAuth-token/MCP-session isolation, immediate pending/terminal polls, active and completed replay, Worker-restart ambiguity, result-size fallback, SHA-256 tamper detection, transient persistence failure, expiry, capacity, and completed-record eviction.
|
|
161
161
|
|
|
162
|
-
`npm run worker-runtime-infrastructure:test` verifies outer-Worker stream ownership, stripping of caller-supplied internal headers, bounded descriptor/poll adaptation, sequence-zero/sequence-one framing, poll-error closure, and the shared two-minute/64-stream/1.5-MiB contract. It also models the production event-lifecycle boundary: event-mode registration must return without a terminal Promise or early settlement, while later success, daemon rejection, explicit cancellation, timeout, send failure, result transformation, persistence failure, and same-instance reconnect each produce one terminal result and remove pending indexes. `npm run worker:integration-test` performs the real Wrangler path: keep SSE open while a concurrent `server_info` succeeds and explicit cancellation reaches the matching daemon call; disconnect after sequence zero; reject another session; recover with GET plus `Last-Event-ID`; and prove a sequence-one acknowledgement is not delivered twice. Static architecture checks forbid a stream-initiation `dispatchJsonRpc` Promise, `resumption.attach`, Durable Object `waitUntil`, or Promise-valued recovery state. The parser accumulates complete SSE events and does not assume network chunk boundaries. CORS coverage requires both `DPoP` and `Last-Event-ID`.
|
|
162
|
+
`npm run worker-runtime-infrastructure:test` verifies outer-Worker stream ownership, stripping of caller-supplied internal headers, bounded descriptor/poll adaptation, sequence-zero/sequence-one framing, poll-error closure, and the shared two-minute/64-stream/1.5-MiB contract. It also models the production event-lifecycle boundary: event-mode registration must return without a terminal Promise or early settlement, while later success, daemon rejection, explicit cancellation, timeout, send failure, result transformation, persistence failure, and same-instance reconnect each produce one terminal result and remove pending indexes. Deadline tests deliberately use a scheduler that never fires callbacks, advance the monotonic clock, and prove that event-boundary sweeps expire both attached operation deadlines and detached reconnect deadlines without leaking request keys. A direct runtime-alarm coordinator test verifies earliest-pending scheduling, alarm removal when no deadline remains, event-entry expiry before rescheduling, and bounded reporting when Durable Object alarm storage fails. The same suite also proves that direct same-instance handover transfers an attached call and preserves its remaining timeout budget. `npm run worker:integration-test` performs the real Wrangler path: keep SSE open while a concurrent `server_info` succeeds and explicit cancellation reaches the matching daemon call; connect a verified same-instance replacement while the incumbent still owns an in-flight call and prove transfer occurs before incumbent close; disconnect after sequence zero; reject another session; recover with GET plus `Last-Event-ID`; and prove a sequence-one acknowledgement is not delivered twice. Managed-job integration treats `__proto__`, `constructor`, `toString`, and `valueOf` environment/resource-map keys as ordinary own data while retaining duplicate-key rejection. Static architecture checks forbid a stream-initiation `dispatchJsonRpc` Promise, `resumption.attach`, Durable Object `waitUntil`, or Promise-valued recovery state. The parser accumulates complete SSE events and does not assume network chunk boundaries. CORS coverage requires both `DPoP` and `Last-Event-ID`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "machine-bridge-mcp",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.16",
|
|
4
4
|
"description": "Cross-client MCP bridge for local agent context, structured browser and application automation, files, Git, processes, resources, and durable jobs over stdio or OAuth relay.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -125,7 +125,7 @@ function validateEnv(value, label) {
|
|
|
125
125
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`);
|
|
126
126
|
const entries = Object.entries(value);
|
|
127
127
|
if (entries.length > 64) throw new Error(`${label} has too many entries`);
|
|
128
|
-
const out =
|
|
128
|
+
const out = Object.create(null);
|
|
129
129
|
for (const [key, raw] of entries) {
|
|
130
130
|
if (!/^[A-Za-z_][A-Za-z0-9_]{0,127}$/.test(key)) throw new Error(`${label} contains invalid variable name: ${key}`);
|
|
131
131
|
out[key] = boundedString(raw, 16 * 1024, `${label}.${key}`);
|
|
@@ -138,7 +138,7 @@ function validateEnvResources(value, label) {
|
|
|
138
138
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`);
|
|
139
139
|
const entries = Object.entries(value);
|
|
140
140
|
if (entries.length > 32) throw new Error(`${label} has too many entries`);
|
|
141
|
-
const out =
|
|
141
|
+
const out = Object.create(null);
|
|
142
142
|
for (const [key, raw] of entries) {
|
|
143
143
|
if (!/^[A-Za-z_][A-Za-z0-9_]{0,127}$/.test(key)) throw new Error(`${label} contains invalid variable name: ${key}`);
|
|
144
144
|
out[key] = validateResourceName(raw);
|
package/src/worker/index.ts
CHANGED
|
@@ -3,15 +3,14 @@ import serverMetadata from "../shared/server-metadata.json" with { type: "json"
|
|
|
3
3
|
import relayContract from "../shared/relay-contract.json" with { type: "json" };
|
|
4
4
|
import { PendingCallRegistrationError, PendingCallRegistry } from "./pending-calls.ts";
|
|
5
5
|
import {
|
|
6
|
-
DAEMON_HELLO_TIMEOUT_MS,
|
|
7
6
|
DAEMON_LIVENESS_TIMEOUT_MS,
|
|
8
7
|
DAEMON_READY_TIMEOUT_MS,
|
|
9
8
|
daemonLastSeenMs,
|
|
10
9
|
daemonLivenessDeadlineMs,
|
|
11
|
-
daemonReadyDeadlineMs,
|
|
12
10
|
isFreshDaemonCandidate,
|
|
13
11
|
} from "./daemon-liveness.ts";
|
|
14
12
|
import { DaemonSocketRegistry } from "./daemon-sockets.ts";
|
|
13
|
+
import { processRuntimeAlarm, scheduleRuntimeAlarm } from "./runtime-alarm.ts";
|
|
15
14
|
import { consumeDaemonPreflightNonce, createDaemonChallenge, verifyDaemonAuthentication, verifyDaemonPreflight } from "./daemon-auth.ts";
|
|
16
15
|
import { mcpClientRequestKey, resolveMcpSession } from "./mcp-session.ts";
|
|
17
16
|
import { acceptsEventStream } from "./mcp-stream.ts";
|
|
@@ -45,7 +44,7 @@ import {
|
|
|
45
44
|
} from "./websocket-protocol.ts";
|
|
46
45
|
|
|
47
46
|
const SERVER_NAME = String(serverMetadata.name);
|
|
48
|
-
const SERVER_VERSION = "3.0.0-beta.
|
|
47
|
+
const SERVER_VERSION = "3.0.0-beta.16";
|
|
49
48
|
const MCP_PROTOCOL_VERSION = String(serverMetadata.protocolVersion);
|
|
50
49
|
const MCP_SUPPORTED_PROTOCOL_VERSIONS = serverMetadata.supportedProtocolVersions.map((value) => String(value));
|
|
51
50
|
const DEFAULT_MAX_BODY_BYTES = 8 * 1024 * 1024;
|
|
@@ -79,6 +78,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
79
78
|
}
|
|
80
79
|
|
|
81
80
|
async fetch(request: Request): Promise<Response> {
|
|
81
|
+
await this.pending.expireDue();
|
|
82
82
|
const base = baseUrl(request);
|
|
83
83
|
const extraOrigins = this.env.MBM_ALLOWED_ORIGINS ?? "";
|
|
84
84
|
let response: Response;
|
|
@@ -146,6 +146,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
async webSocketMessage(ws: WebSocket, message: string | ArrayBuffer): Promise<void> {
|
|
149
|
+
await this.pending.expireDue();
|
|
149
150
|
const size = typeof message === "string" ? new TextEncoder().encode(message).byteLength : message.byteLength;
|
|
150
151
|
if (size > MAX_DAEMON_MESSAGE_BYTES) {
|
|
151
152
|
closeWebSocketQuietly(ws, 1009, "message too large");
|
|
@@ -189,7 +190,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
189
190
|
}
|
|
190
191
|
if (!isFreshDaemonCandidate(socketAttachment.connectedAt)) {
|
|
191
192
|
closeWebSocketQuietly(ws, 1008, "stale daemon candidate");
|
|
192
|
-
await this.
|
|
193
|
+
await this.scheduleRuntimeAlarm();
|
|
193
194
|
return;
|
|
194
195
|
}
|
|
195
196
|
const instanceId = daemonInstanceId(body.instance_id);
|
|
@@ -245,10 +246,10 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
245
246
|
} catch {
|
|
246
247
|
this.daemonRegistry.expire(ws);
|
|
247
248
|
closeWebSocketQuietly(ws, 1011, "daemon readiness probe failed");
|
|
248
|
-
await this.
|
|
249
|
+
await this.scheduleRuntimeAlarm();
|
|
249
250
|
return;
|
|
250
251
|
}
|
|
251
|
-
await this.
|
|
252
|
+
await this.scheduleRuntimeAlarm();
|
|
252
253
|
return;
|
|
253
254
|
}
|
|
254
255
|
|
|
@@ -276,7 +277,10 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
276
277
|
rejectDaemonMessage(ws, "invalid_relay_readiness_state", 1002, "invalid daemon readiness state");
|
|
277
278
|
return;
|
|
278
279
|
}
|
|
279
|
-
const
|
|
280
|
+
const daemonInstanceId = readyAttachment.instanceId ?? "";
|
|
281
|
+
const previousSockets = this.daemonRegistry.readyRoleSockets().filter((socket) => socket !== ws);
|
|
282
|
+
const fallbackSocket = previousSockets.find((socket) => this.daemonRegistry.readyAttachment(socket)?.instanceId === daemonInstanceId);
|
|
283
|
+
const reboundCallIds = this.pending.rebindInstance(daemonInstanceId, ws);
|
|
280
284
|
if (reboundCallIds.length > 0) {
|
|
281
285
|
this.observability.event("info", "daemon.calls.rebound", { rebound_calls: reboundCallIds.length });
|
|
282
286
|
}
|
|
@@ -285,12 +289,17 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
285
289
|
ws.send(JSON.stringify({ type: "ready_ack", server: SERVER_NAME, version: SERVER_VERSION }));
|
|
286
290
|
} catch {
|
|
287
291
|
await this.invalidateDaemonSocket(ws, "daemon readiness acknowledgement failed", "daemon ready timeout", "daemon_ready_timeout");
|
|
292
|
+
if (fallbackSocket?.readyState === WebSocket.OPEN) this.pending.rebindInstance(daemonInstanceId, fallbackSocket);
|
|
293
|
+
await this.scheduleRuntimeAlarm();
|
|
288
294
|
return;
|
|
289
295
|
}
|
|
290
296
|
this.observability.socketReady();
|
|
291
|
-
for (const previous of
|
|
297
|
+
for (const previous of previousSockets) {
|
|
298
|
+
await this.detachDaemonSocketCalls(previous, "daemon connection replaced after verified handover");
|
|
299
|
+
this.daemonRegistry.expire(previous);
|
|
292
300
|
closeWebSocketQuietly(previous, 1012, "replaced by verified daemon");
|
|
293
301
|
}
|
|
302
|
+
await this.scheduleRuntimeAlarm();
|
|
294
303
|
return;
|
|
295
304
|
}
|
|
296
305
|
|
|
@@ -323,7 +332,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
323
332
|
private async cleanupDaemonSocket(ws: WebSocket, message: string): Promise<void> {
|
|
324
333
|
this.observability.socketDisconnected();
|
|
325
334
|
await this.detachDaemonSocketCalls(ws, message);
|
|
326
|
-
await this.
|
|
335
|
+
await this.scheduleRuntimeAlarm();
|
|
327
336
|
}
|
|
328
337
|
private async handleMcp(request: Request, base: string): Promise<Response> {
|
|
329
338
|
if (request.method !== "GET" && request.method !== "POST") {
|
|
@@ -418,6 +427,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
418
427
|
accountVersion: access.authorized.accountVersion, role: access.authorized.role })
|
|
419
428
|
: undefined,
|
|
420
429
|
});
|
|
430
|
+
await this.scheduleRuntimeAlarm();
|
|
421
431
|
}
|
|
422
432
|
} catch (error) {
|
|
423
433
|
await persistImmediateStreamOutcome({
|
|
@@ -544,6 +554,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
544
554
|
}
|
|
545
555
|
throw error;
|
|
546
556
|
}
|
|
557
|
+
await this.scheduleRuntimeAlarm();
|
|
547
558
|
this.observability.callStarted(name);
|
|
548
559
|
try {
|
|
549
560
|
socket.send(JSON.stringify({
|
|
@@ -610,7 +621,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
610
621
|
this.ctx.acceptWebSocket(server);
|
|
611
622
|
this.observability.socketCandidate();
|
|
612
623
|
this.daemonRegistry.beginCandidate(server, challenge, preflight);
|
|
613
|
-
await this.
|
|
624
|
+
await this.scheduleRuntimeAlarm();
|
|
614
625
|
const welcomed = trySendWebSocket(server, {
|
|
615
626
|
type: "welcome",
|
|
616
627
|
server: SERVER_NAME,
|
|
@@ -626,7 +637,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
626
637
|
if (!welcomed) {
|
|
627
638
|
this.daemonRegistry.expire(server);
|
|
628
639
|
closeWebSocketQuietly(server, 1011, "daemon welcome failed");
|
|
629
|
-
await this.
|
|
640
|
+
await this.scheduleRuntimeAlarm();
|
|
630
641
|
}
|
|
631
642
|
return new Response(null, { status: 101, webSocket: client });
|
|
632
643
|
}
|
|
@@ -657,7 +668,7 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
657
668
|
|
|
658
669
|
private async touchDaemonSocket(ws: WebSocket): Promise<void> {
|
|
659
670
|
if (!this.daemonRegistry.touch(ws)) return;
|
|
660
|
-
await this.
|
|
671
|
+
await this.scheduleRuntimeAlarm();
|
|
661
672
|
}
|
|
662
673
|
|
|
663
674
|
private async invalidateDaemonSocket(
|
|
@@ -694,73 +705,24 @@ export class BridgeRoom extends DurableObject<BridgeEnv> {
|
|
|
694
705
|
}
|
|
695
706
|
|
|
696
707
|
async alarm(): Promise<void> {
|
|
697
|
-
|
|
698
|
-
let nextDeadline = Number.POSITIVE_INFINITY;
|
|
699
|
-
for (const socket of this.daemonRegistry.candidateSockets()) {
|
|
700
|
-
const attachment = this.daemonRegistry.attachment(socket);
|
|
701
|
-
const connectedAt = Date.parse(attachment?.connectedAt ?? "");
|
|
702
|
-
const deadline = connectedAt + DAEMON_HELLO_TIMEOUT_MS;
|
|
703
|
-
if (!Number.isFinite(connectedAt) || deadline <= now) {
|
|
704
|
-
this.daemonRegistry.expire(socket);
|
|
705
|
-
sendWebSocketQuietly(socket, { type: "error", error: "daemon_hello_timeout" });
|
|
706
|
-
closeWebSocketQuietly(socket, 1008, "daemon hello timeout");
|
|
707
|
-
continue;
|
|
708
|
-
}
|
|
709
|
-
nextDeadline = Math.min(nextDeadline, deadline);
|
|
710
|
-
}
|
|
711
|
-
for (const socket of this.daemonRegistry.probingSockets()) {
|
|
712
|
-
const attachment = this.daemonRegistry.attachment(socket);
|
|
713
|
-
const readyDeadline = daemonReadyDeadlineMs(attachment);
|
|
714
|
-
const liveDeadline = daemonLivenessDeadlineMs(attachment);
|
|
715
|
-
if (!Number.isFinite(readyDeadline) || !Number.isFinite(liveDeadline) || Math.min(readyDeadline, liveDeadline) <= now) {
|
|
716
|
-
await this.invalidateDaemonSocket(socket, "daemon did not complete end-to-end readiness verification", "daemon ready timeout", "daemon_ready_timeout");
|
|
717
|
-
continue;
|
|
718
|
-
}
|
|
719
|
-
nextDeadline = Math.min(nextDeadline, readyDeadline, liveDeadline);
|
|
720
|
-
}
|
|
721
|
-
for (const socket of this.daemonRegistry.readyRoleSockets()) {
|
|
722
|
-
const deadline = daemonLivenessDeadlineMs(this.daemonRegistry.readyAttachment(socket));
|
|
723
|
-
if (!Number.isFinite(deadline) || deadline <= now) {
|
|
724
|
-
await this.invalidateDaemonSocket(socket, "daemon became unresponsive", "daemon liveness timeout");
|
|
725
|
-
continue;
|
|
726
|
-
}
|
|
727
|
-
nextDeadline = Math.min(nextDeadline, deadline);
|
|
728
|
-
}
|
|
729
|
-
if (Number.isFinite(nextDeadline)) await this.ctx.storage.setAlarm(nextDeadline);
|
|
730
|
-
else await this.ctx.storage.deleteAlarm();
|
|
708
|
+
await processRuntimeAlarm(this.runtimeAlarmContext());
|
|
731
709
|
}
|
|
732
710
|
|
|
733
|
-
private async
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
if (!Number.isFinite(readyDeadline) || !Number.isFinite(liveDeadline)) {
|
|
749
|
-
await this.invalidateDaemonSocket(socket, "daemon readiness state is invalid", "daemon ready timeout", "daemon_ready_timeout");
|
|
750
|
-
continue;
|
|
751
|
-
}
|
|
752
|
-
nextDeadline = Math.min(nextDeadline, readyDeadline, liveDeadline);
|
|
753
|
-
}
|
|
754
|
-
for (const socket of this.daemonRegistry.readyRoleSockets()) {
|
|
755
|
-
const deadline = daemonLivenessDeadlineMs(this.daemonRegistry.readyAttachment(socket));
|
|
756
|
-
if (!Number.isFinite(deadline)) {
|
|
757
|
-
await this.invalidateDaemonSocket(socket, "daemon became unresponsive", "invalid daemon liveness timestamp");
|
|
758
|
-
continue;
|
|
759
|
-
}
|
|
760
|
-
nextDeadline = Math.min(nextDeadline, deadline);
|
|
761
|
-
}
|
|
762
|
-
if (Number.isFinite(nextDeadline)) await this.ctx.storage.setAlarm(Math.max(Date.now(), nextDeadline));
|
|
763
|
-
else await this.ctx.storage.deleteAlarm();
|
|
711
|
+
private async scheduleRuntimeAlarm(): Promise<void> {
|
|
712
|
+
await scheduleRuntimeAlarm(this.runtimeAlarmContext());
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
private runtimeAlarmContext() {
|
|
716
|
+
return {
|
|
717
|
+
storage: this.ctx.storage,
|
|
718
|
+
pending: this.pending,
|
|
719
|
+
daemonRegistry: this.daemonRegistry,
|
|
720
|
+
invalidateDaemonSocket: (socket: WebSocket, message: string, closeReason: string, errorCode?: string) =>
|
|
721
|
+
this.invalidateDaemonSocket(socket, message, closeReason, errorCode),
|
|
722
|
+
onScheduleError: (error: unknown) => this.observability.event(
|
|
723
|
+
"error", "runtime.alarm.schedule.failed", { error_class: workerErrorClass(error) },
|
|
724
|
+
),
|
|
725
|
+
};
|
|
764
726
|
}
|
|
765
727
|
|
|
766
728
|
private daemonStatus(detail: boolean): Record<string, unknown> {
|
|
@@ -11,6 +11,8 @@ export interface PendingCallRecord {
|
|
|
11
11
|
socket?: WebSocket;
|
|
12
12
|
daemonInstanceId?: string;
|
|
13
13
|
reconnectTimeout?: ReturnType<typeof setTimeout>;
|
|
14
|
+
reconnectDeadlineAt?: number;
|
|
15
|
+
onReconnectTimeout?: (record: PendingCallRecord) => Error;
|
|
14
16
|
clientRequestKey?: string;
|
|
15
17
|
tool: string;
|
|
16
18
|
startedAt: number;
|
|
@@ -22,6 +22,16 @@ export class PendingCallDeadlines {
|
|
|
22
22
|
return this.clock();
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
nextDelayMs(record: PendingCallRecord): number {
|
|
26
|
+
const deadline = record.socket ? record.deadlineAt : record.reconnectDeadlineAt;
|
|
27
|
+
return Number.isFinite(deadline) ? Math.max(0, Number(deadline) - this.clock()) : Number.POSITIVE_INFINITY;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
isDue(record: PendingCallRecord, now = this.clock()): boolean {
|
|
31
|
+
const deadline = record.socket ? record.deadlineAt : record.reconnectDeadlineAt;
|
|
32
|
+
return Number.isFinite(deadline) && Number(deadline) <= now;
|
|
33
|
+
}
|
|
34
|
+
|
|
25
35
|
armOperation(record: PendingCallRecord, delayMs: number, expire: (id: string) => void): void {
|
|
26
36
|
if (record.timeout) this.scheduler.clearTimeout(record.timeout);
|
|
27
37
|
const delay = positiveDelay(delayMs);
|
|
@@ -38,17 +48,21 @@ export class PendingCallDeadlines {
|
|
|
38
48
|
|
|
39
49
|
armReconnect(record: PendingCallRecord, delayMs: number, expire: (id: string) => void): void {
|
|
40
50
|
if (record.reconnectTimeout) this.scheduler.clearTimeout(record.reconnectTimeout);
|
|
41
|
-
|
|
51
|
+
const delay = positiveDelay(delayMs);
|
|
52
|
+
record.reconnectDeadlineAt = this.clock() + delay;
|
|
53
|
+
record.reconnectTimeout = this.scheduler.setTimeout(() => expire(record.id), delay);
|
|
42
54
|
}
|
|
43
55
|
|
|
44
56
|
clearReconnect(record: PendingCallRecord): void {
|
|
45
57
|
if (record.reconnectTimeout) this.scheduler.clearTimeout(record.reconnectTimeout);
|
|
46
58
|
record.reconnectTimeout = undefined;
|
|
59
|
+
record.reconnectDeadlineAt = undefined;
|
|
47
60
|
}
|
|
48
61
|
|
|
49
62
|
clear(record: PendingCallRecord): void {
|
|
50
63
|
if (record.timeout) this.scheduler.clearTimeout(record.timeout);
|
|
51
64
|
if (record.reconnectTimeout) this.scheduler.clearTimeout(record.reconnectTimeout);
|
|
65
|
+
record.reconnectDeadlineAt = undefined;
|
|
52
66
|
}
|
|
53
67
|
}
|
|
54
68
|
|
|
@@ -25,6 +25,17 @@ export class PendingCallRegistry {
|
|
|
25
25
|
get size(): number { return this.byId.size; }
|
|
26
26
|
hasRequestKey(requestKey?: string): boolean { return Boolean(requestKey && this.byRequestKey.has(requestKey)); }
|
|
27
27
|
|
|
28
|
+
nextDeadlineDelayMs(): number {
|
|
29
|
+
return Math.min(Number.POSITIVE_INFINITY, ...[...this.byId.values()].map((record) => this.deadlines.nextDelayMs(record)));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async expireDue(now = this.deadlines.now()): Promise<number> {
|
|
33
|
+
const due = [...this.byId.values()].filter((record) => this.deadlines.isDue(record, now));
|
|
34
|
+
let expired = 0;
|
|
35
|
+
for (const record of due) expired += Number(await this.expireRecord(record));
|
|
36
|
+
return expired;
|
|
37
|
+
}
|
|
38
|
+
|
|
28
39
|
register(input: RegisterPendingCall): Promise<unknown> {
|
|
29
40
|
this.assertCanRegister(input);
|
|
30
41
|
let resolveResult!: (value: unknown) => void;
|
|
@@ -66,8 +77,9 @@ export class PendingCallRegistry {
|
|
|
66
77
|
const delay = Math.max(1, Math.floor(Number(graceMs) || 1));
|
|
67
78
|
for (const record of records) {
|
|
68
79
|
record.socket = undefined;
|
|
80
|
+
record.onReconnectTimeout = createError;
|
|
69
81
|
this.deadlines.pauseOperation(record);
|
|
70
|
-
this.deadlines.armReconnect(record, delay, (id) => { void this.
|
|
82
|
+
this.deadlines.armReconnect(record, delay, (id) => { void this.expireReconnect(id); });
|
|
71
83
|
}
|
|
72
84
|
return records.length;
|
|
73
85
|
}
|
|
@@ -76,8 +88,10 @@ export class PendingCallRegistry {
|
|
|
76
88
|
if (!daemonInstanceId) return [];
|
|
77
89
|
const rebound: string[] = [];
|
|
78
90
|
for (const record of this.byId.values()) {
|
|
79
|
-
if (record.
|
|
80
|
-
this.deadlines.
|
|
91
|
+
if (record.daemonInstanceId !== daemonInstanceId || record.socket === socket) continue;
|
|
92
|
+
if (record.socket) this.deadlines.pauseOperation(record);
|
|
93
|
+
else this.deadlines.clearReconnect(record);
|
|
94
|
+
record.onReconnectTimeout = undefined;
|
|
81
95
|
record.socket = socket;
|
|
82
96
|
this.deadlines.armOperation(record, record.remainingTimeoutMs, (id) => { void this.expireOperation(id); });
|
|
83
97
|
rebound.push(record.id);
|
|
@@ -122,17 +136,21 @@ export class PendingCallRegistry {
|
|
|
122
136
|
else if (input.signal && abortHandler) input.signal.addEventListener("abort", abortHandler, { once: true });
|
|
123
137
|
}
|
|
124
138
|
|
|
125
|
-
private expireOperation(id: string): Promise<boolean> {
|
|
126
|
-
|
|
127
|
-
|
|
139
|
+
private expireOperation(id: string): Promise<boolean> { return this.expireRecord(this.byId.get(id)); }
|
|
140
|
+
private expireReconnect(id: string): Promise<boolean> {
|
|
141
|
+
const record = this.byId.get(id); return record && !record.socket ? this.expireRecord(record) : Promise.resolve(false);
|
|
142
|
+
}
|
|
143
|
+
private expireRecord(record?: PendingCallRecord): Promise<boolean> {
|
|
144
|
+
if (!record) return Promise.resolve(false);
|
|
145
|
+
return record.socket
|
|
146
|
+
? this.fail(record.id, record.onTimeout, "pending daemon call timed out")
|
|
147
|
+
: this.fail(record.id, record.onReconnectTimeout, "pending daemon reconnect expired");
|
|
128
148
|
}
|
|
129
|
-
|
|
130
149
|
private expire(id: string, createError: ((record: PendingCallRecord) => Error) | undefined, fallback: string): Promise<boolean> {
|
|
131
150
|
const record = this.byId.get(id);
|
|
132
|
-
|
|
133
|
-
|
|
151
|
+
const applicable = record && (fallback.includes("reconnect") ? !record.socket : true);
|
|
152
|
+
return applicable ? this.fail(id, createError, fallback) : Promise.resolve(false);
|
|
134
153
|
}
|
|
135
|
-
|
|
136
154
|
private async fail(id: string, createError: ((record: PendingCallRecord) => Error) | undefined, fallback: string): Promise<boolean> {
|
|
137
155
|
const record = this.byId.get(id);
|
|
138
156
|
if (!record) return false;
|
|
@@ -140,7 +158,6 @@ export class PendingCallRegistry {
|
|
|
140
158
|
try { error = createError?.(record) ?? new Error(fallback); } catch { error = new Error(fallback); }
|
|
141
159
|
return this.finish(id, { ok: false, error });
|
|
142
160
|
}
|
|
143
|
-
|
|
144
161
|
private async finish(id: string, outcome: PendingCallOutcome): Promise<boolean> {
|
|
145
162
|
const record = this.take(id);
|
|
146
163
|
if (!record) return false;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DAEMON_HELLO_TIMEOUT_MS,
|
|
3
|
+
daemonLivenessDeadlineMs,
|
|
4
|
+
daemonReadyDeadlineMs,
|
|
5
|
+
} from "./daemon-liveness.ts";
|
|
6
|
+
import type { DaemonSocketRegistry } from "./daemon-sockets.ts";
|
|
7
|
+
import type { PendingCallRegistry } from "./pending-calls.ts";
|
|
8
|
+
import { closeWebSocketQuietly, sendWebSocketQuietly } from "./websocket-protocol.ts";
|
|
9
|
+
|
|
10
|
+
interface AlarmStorage {
|
|
11
|
+
setAlarm(scheduledTime: number | Date): Promise<void>;
|
|
12
|
+
deleteAlarm(): Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
type InvalidateDaemonSocket = (
|
|
16
|
+
socket: WebSocket,
|
|
17
|
+
message: string,
|
|
18
|
+
closeReason: string,
|
|
19
|
+
errorCode?: string,
|
|
20
|
+
) => Promise<void>;
|
|
21
|
+
|
|
22
|
+
interface RuntimeAlarmContext {
|
|
23
|
+
storage: AlarmStorage;
|
|
24
|
+
pending: PendingCallRegistry;
|
|
25
|
+
daemonRegistry: DaemonSocketRegistry;
|
|
26
|
+
invalidateDaemonSocket: InvalidateDaemonSocket;
|
|
27
|
+
onScheduleError: (error: unknown) => void;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function processRuntimeAlarm(context: RuntimeAlarmContext, now = Date.now()): Promise<void> {
|
|
31
|
+
await context.pending.expireDue();
|
|
32
|
+
let nextDeadline = pendingAlarmDeadline(context.pending, now);
|
|
33
|
+
for (const socket of context.daemonRegistry.candidateSockets()) {
|
|
34
|
+
const attachment = context.daemonRegistry.attachment(socket);
|
|
35
|
+
const connectedAt = Date.parse(attachment?.connectedAt ?? "");
|
|
36
|
+
const deadline = connectedAt + DAEMON_HELLO_TIMEOUT_MS;
|
|
37
|
+
if (!Number.isFinite(connectedAt) || deadline <= now) {
|
|
38
|
+
context.daemonRegistry.expire(socket);
|
|
39
|
+
sendWebSocketQuietly(socket, { type: "error", error: "daemon_hello_timeout" });
|
|
40
|
+
closeWebSocketQuietly(socket, 1008, "daemon hello timeout");
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
nextDeadline = Math.min(nextDeadline, deadline);
|
|
44
|
+
}
|
|
45
|
+
for (const socket of context.daemonRegistry.probingSockets()) {
|
|
46
|
+
const attachment = context.daemonRegistry.attachment(socket);
|
|
47
|
+
const readyDeadline = daemonReadyDeadlineMs(attachment);
|
|
48
|
+
const liveDeadline = daemonLivenessDeadlineMs(attachment);
|
|
49
|
+
if (!Number.isFinite(readyDeadline) || !Number.isFinite(liveDeadline) || Math.min(readyDeadline, liveDeadline) <= now) {
|
|
50
|
+
await context.invalidateDaemonSocket(
|
|
51
|
+
socket,
|
|
52
|
+
"daemon did not complete end-to-end readiness verification",
|
|
53
|
+
"daemon ready timeout",
|
|
54
|
+
"daemon_ready_timeout",
|
|
55
|
+
);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
nextDeadline = Math.min(nextDeadline, readyDeadline, liveDeadline);
|
|
59
|
+
}
|
|
60
|
+
for (const socket of context.daemonRegistry.readyRoleSockets()) {
|
|
61
|
+
const deadline = daemonLivenessDeadlineMs(context.daemonRegistry.readyAttachment(socket));
|
|
62
|
+
if (!Number.isFinite(deadline) || deadline <= now) {
|
|
63
|
+
await context.invalidateDaemonSocket(socket, "daemon became unresponsive", "daemon liveness timeout");
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
nextDeadline = Math.min(nextDeadline, deadline);
|
|
67
|
+
}
|
|
68
|
+
await writeRuntimeAlarm(context, nextDeadline, now);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export async function scheduleRuntimeAlarm(context: RuntimeAlarmContext, now = Date.now()): Promise<void> {
|
|
72
|
+
let nextDeadline = pendingAlarmDeadline(context.pending, now);
|
|
73
|
+
for (const socket of context.daemonRegistry.candidateSockets()) {
|
|
74
|
+
const connectedAt = Date.parse(context.daemonRegistry.attachment(socket)?.connectedAt ?? "");
|
|
75
|
+
if (!Number.isFinite(connectedAt)) {
|
|
76
|
+
closeWebSocketQuietly(socket, 1008, "invalid daemon candidate timestamp");
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
nextDeadline = Math.min(nextDeadline, connectedAt + DAEMON_HELLO_TIMEOUT_MS);
|
|
80
|
+
}
|
|
81
|
+
for (const socket of context.daemonRegistry.probingSockets()) {
|
|
82
|
+
const attachment = context.daemonRegistry.attachment(socket);
|
|
83
|
+
const readyDeadline = daemonReadyDeadlineMs(attachment);
|
|
84
|
+
const liveDeadline = daemonLivenessDeadlineMs(attachment);
|
|
85
|
+
if (!Number.isFinite(readyDeadline) || !Number.isFinite(liveDeadline)) {
|
|
86
|
+
await context.invalidateDaemonSocket(
|
|
87
|
+
socket,
|
|
88
|
+
"daemon readiness state is invalid",
|
|
89
|
+
"daemon ready timeout",
|
|
90
|
+
"daemon_ready_timeout",
|
|
91
|
+
);
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
nextDeadline = Math.min(nextDeadline, readyDeadline, liveDeadline);
|
|
95
|
+
}
|
|
96
|
+
for (const socket of context.daemonRegistry.readyRoleSockets()) {
|
|
97
|
+
const deadline = daemonLivenessDeadlineMs(context.daemonRegistry.readyAttachment(socket));
|
|
98
|
+
if (!Number.isFinite(deadline)) {
|
|
99
|
+
await context.invalidateDaemonSocket(socket, "daemon became unresponsive", "invalid daemon liveness timestamp");
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
nextDeadline = Math.min(nextDeadline, deadline);
|
|
103
|
+
}
|
|
104
|
+
await writeRuntimeAlarm(context, nextDeadline, now);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function pendingAlarmDeadline(pending: PendingCallRegistry, now: number): number {
|
|
108
|
+
const delay = pending.nextDeadlineDelayMs();
|
|
109
|
+
return Number.isFinite(delay) ? now + Math.max(1, Math.ceil(delay)) : Number.POSITIVE_INFINITY;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function writeRuntimeAlarm(context: RuntimeAlarmContext, nextDeadline: number, now: number): Promise<void> {
|
|
113
|
+
try {
|
|
114
|
+
if (Number.isFinite(nextDeadline)) await context.storage.setAlarm(Math.max(now, nextDeadline));
|
|
115
|
+
else await context.storage.deleteAlarm();
|
|
116
|
+
} catch (error) {
|
|
117
|
+
context.onScheduleError(error);
|
|
118
|
+
}
|
|
119
|
+
}
|