@intx/hub-sessions 0.1.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/LICENSE +176 -0
  2. package/README.md +82 -1
  3. package/dist/agent-repo.d.ts +93 -0
  4. package/dist/agent-repo.js +109 -0
  5. package/dist/agent-state-kind.d.ts +12 -0
  6. package/dist/agent-state-kind.js +189 -0
  7. package/dist/asset-service.d.ts +104 -0
  8. package/dist/asset-service.js +267 -0
  9. package/dist/committed-source-tree.d.ts +10 -0
  10. package/dist/committed-source-tree.js +35 -0
  11. package/dist/credential-push.d.ts +33 -0
  12. package/dist/credential-push.js +109 -0
  13. package/dist/event-collector-registry.d.ts +20 -0
  14. package/dist/event-collector-registry.js +115 -0
  15. package/dist/event-collector.d.ts +39 -0
  16. package/dist/event-collector.js +365 -0
  17. package/dist/hub-session-lookups.d.ts +135 -0
  18. package/dist/hub-session-lookups.js +663 -0
  19. package/dist/hub-session-orchestrator.d.ts +25 -0
  20. package/dist/hub-session-orchestrator.js +87 -0
  21. package/dist/index.d.ts +27 -0
  22. package/dist/index.js +24 -0
  23. package/dist/package-registry-kind.d.ts +70 -0
  24. package/dist/package-registry-kind.js +260 -0
  25. package/dist/repo-store/index.d.ts +4 -0
  26. package/dist/repo-store/index.js +3 -0
  27. package/dist/repo-store/store.d.ts +41 -0
  28. package/dist/repo-store/store.js +1829 -0
  29. package/dist/repo-store/subscribe-kind.d.ts +56 -0
  30. package/dist/repo-store/subscribe-kind.js +144 -0
  31. package/dist/repo-store/types.d.ts +571 -0
  32. package/dist/repo-store/types.js +42 -0
  33. package/dist/session-service.d.ts +416 -0
  34. package/dist/session-service.js +1191 -0
  35. package/dist/sidecar-allocation/contracts.d.ts +78 -0
  36. package/dist/sidecar-allocation/contracts.js +21 -0
  37. package/dist/sidecar-allocation/index.d.ts +4 -0
  38. package/dist/sidecar-allocation/index.js +3 -0
  39. package/dist/sidecar-allocation/placement-policy.d.ts +11 -0
  40. package/dist/sidecar-allocation/placement-policy.js +21 -0
  41. package/dist/sidecar-allocation/plugin-registry.d.ts +11 -0
  42. package/dist/sidecar-allocation/plugin-registry.js +37 -0
  43. package/dist/sidecar-allocation/reconciler.d.ts +42 -0
  44. package/dist/sidecar-allocation/reconciler.js +431 -0
  45. package/dist/skill-kind.d.ts +41 -0
  46. package/dist/skill-kind.js +292 -0
  47. package/dist/substrate.d.ts +8 -0
  48. package/dist/substrate.js +21 -0
  49. package/dist/workflow-allocation-service.d.ts +58 -0
  50. package/dist/workflow-allocation-service.js +239 -0
  51. package/dist/workflow-closure-resolution.d.ts +106 -0
  52. package/dist/workflow-closure-resolution.js +123 -0
  53. package/dist/workflow-definition-ensure.d.ts +24 -0
  54. package/dist/workflow-definition-ensure.js +75 -0
  55. package/dist/workflow-dispatch-service.d.ts +40 -0
  56. package/dist/workflow-dispatch-service.js +146 -0
  57. package/dist/workflow-dispatch-settlement.d.ts +29 -0
  58. package/dist/workflow-dispatch-settlement.js +140 -0
  59. package/dist/workflow-kind.d.ts +37 -0
  60. package/dist/workflow-kind.js +310 -0
  61. package/dist/workflow-probe-gate.d.ts +214 -0
  62. package/dist/workflow-probe-gate.js +207 -0
  63. package/dist/workflow-run-event-log.d.ts +21 -0
  64. package/dist/workflow-run-event-log.js +51 -0
  65. package/dist/workflow-run-kind.d.ts +440 -0
  66. package/dist/workflow-run-kind.js +2916 -0
  67. package/dist/workflow-run-reader.d.ts +47 -0
  68. package/dist/workflow-run-reader.js +153 -0
  69. package/dist/workflow-run-restore.d.ts +15 -0
  70. package/dist/workflow-run-restore.js +26 -0
  71. package/dist/workflow-source-closure.d.ts +35 -0
  72. package/dist/workflow-source-closure.js +342 -0
  73. package/dist/ws/index.d.ts +3 -0
  74. package/dist/ws/index.js +3 -0
  75. package/dist/ws/sidecar-events.d.ts +222 -0
  76. package/dist/ws/sidecar-events.js +72 -0
  77. package/dist/ws/sidecar-handler.d.ts +305 -0
  78. package/dist/ws/sidecar-handler.js +2537 -0
  79. package/dist/ws/sidecar-token-authenticator.d.ts +17 -0
  80. package/dist/ws/sidecar-token-authenticator.js +81 -0
  81. package/package.json +35 -12
  82. package/src/agent-repo.test.ts +0 -310
  83. package/src/agent-repo.ts +0 -165
  84. package/src/agent-state-kind.test.ts +0 -247
  85. package/src/agent-state-kind.ts +0 -204
  86. package/src/asset-service.test.ts +0 -540
  87. package/src/asset-service.ts +0 -378
  88. package/src/available-skills-stanza.test.ts +0 -87
  89. package/src/available-skills-stanza.ts +0 -47
  90. package/src/credential-push.ts +0 -65
  91. package/src/event-collector-registry.test.ts +0 -73
  92. package/src/event-collector-registry.ts +0 -171
  93. package/src/event-collector.test.ts +0 -1387
  94. package/src/event-collector.ts +0 -424
  95. package/src/hub-session-lookups.ts +0 -206
  96. package/src/hub-session-orchestrator.test.ts +0 -510
  97. package/src/hub-session-orchestrator.ts +0 -213
  98. package/src/index.ts +0 -78
  99. package/src/repo-store/index.ts +0 -15
  100. package/src/repo-store/store.test.ts +0 -1169
  101. package/src/repo-store/store.ts +0 -428
  102. package/src/repo-store/types.ts +0 -253
  103. package/src/session-service.test.ts +0 -895
  104. package/src/session-service.ts +0 -464
  105. package/src/skill-kind.test.ts +0 -599
  106. package/src/skill-kind.ts +0 -350
  107. package/src/ws/index.ts +0 -18
  108. package/src/ws/sidecar-events.test.ts +0 -96
  109. package/src/ws/sidecar-events.ts +0 -231
  110. package/src/ws/sidecar-handler.test.ts +0 -2217
  111. package/src/ws/sidecar-handler.ts +0 -1574
  112. package/tsconfig.json +0 -4
  113. package/tsconfig.tsbuildinfo +0 -1
@@ -0,0 +1,2537 @@
1
+ // Hub-side websocket handler for sidecar connections.
2
+ //
3
+ // Accepts websocket upgrades, processes register frames, maintains a routing
4
+ // table of agentAddress → sidecar connection, and dispatches frames between
5
+ // sidecars and the hub's internal systems.
6
+ import { getLogger } from "@intx/log";
7
+ import { verifyEd25519 } from "@intx/crypto";
8
+ import { chunkPack, createPackReceiver } from "@intx/pack-transport";
9
+ import { base64Decode, deriveMessageId, deriveWorkflowRunId, hexDecode, hexEncode, isRunAddress, } from "@intx/types";
10
+ import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy";
11
+ import { type } from "arktype";
12
+ import { SidecarFrame, } from "@intx/types/sidecar";
13
+ import { createSidecarEmitter, } from "./sidecar-events.js";
14
+ const logger = getLogger(["hub", "ws", "sidecar"]);
15
+ /**
16
+ * Whether this connection owns `address` for routing/lifecycle purposes --
17
+ * as a challenged session address OR a hub-minted workflow-substrate address.
18
+ * The two sets are kept physically distinct (they differ on the challenge /
19
+ * re-add path), but ownership readers -- pack-transfer authorization,
20
+ * in-flight cancellation, disconnect teardown -- must see the union, or a
21
+ * reconnected workflow deployment (which lives only in `workflowAddresses`)
22
+ * is silently treated as unowned even though its mail routes.
23
+ */
24
+ function connOwnsAddress(conn, address) {
25
+ return (conn.agentAddresses.has(address) || conn.workflowAddresses.has(address));
26
+ }
27
+ /**
28
+ * Bind pack writes to the repository implied by the authenticated address.
29
+ * An allocated credential is narrower still: it may only write its one
30
+ * deployment's workflow-run repository and never a standalone agent-state
31
+ * repository.
32
+ */
33
+ function connCanPushRepo(conn, agentAddress, repoId) {
34
+ if (conn.identity.kind === "allocated" &&
35
+ agentAddress !== conn.identity.workflowRunAddress) {
36
+ return false;
37
+ }
38
+ if (repoId.kind === "agent-state") {
39
+ return conn.identity.kind === "shared" && repoId.id === agentAddress;
40
+ }
41
+ return (repoId.kind === "workflow-run" &&
42
+ repoId.id === deriveWorkflowRunRepoId(agentAddress));
43
+ }
44
+ /** The deduped set of every address this connection owns (session + workflow). */
45
+ function ownedAddresses(conn) {
46
+ return new Set([...conn.agentAddresses, ...conn.workflowAddresses]);
47
+ }
48
+ const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
49
+ const DEFAULT_CHALLENGE_TIMEOUT_MS = 30_000;
50
+ // A probe fetches a workflow's dependency closure from a registry and
51
+ // evaluates it on the sidecar, so it runs longer than a routine request; its
52
+ // default timeout is correspondingly wider than DEFAULT_REQUEST_TIMEOUT_MS.
53
+ export const DEFAULT_PROBE_TIMEOUT_MS = 60_000;
54
+ const DEFAULT_DISCONNECT_QUEUE_MAX_SIZE = 100;
55
+ const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000;
56
+ const DEFAULT_PING_TIMEOUT_MS = 60_000;
57
+ const DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS = 10_000;
58
+ const DEFAULT_MAIL_ACK_MAX_RETRIES = 5;
59
+ export function createSidecarRouter(config) {
60
+ const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, challengeTimeoutMs = DEFAULT_CHALLENGE_TIMEOUT_MS, probeTimeoutMs = DEFAULT_PROBE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, validateSidecarIdentity = async () => true, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, lookups = {}, } = config;
61
+ // Receiver-dispatch surface. Wire-layer callsites emit events here;
62
+ // host code subscribes via `router.events`.
63
+ const events = createSidecarEmitter();
64
+ // ws handle → registered connection
65
+ const connections = new Map();
66
+ const allocatedConnections = new Map();
67
+ const allocationFences = new Map();
68
+ const allocationWaiters = new Map();
69
+ // agentAddress → ws handle (routing table)
70
+ const addressIndex = new Map();
71
+ // requestId → pending promise
72
+ const pending = new Map();
73
+ const pendingDeploys = new Map();
74
+ const pendingChallenges = new Map();
75
+ const disconnectedAgents = new Map();
76
+ const pendingMail = new Map();
77
+ // agentAddress → retention TTL timer for un-acked pending mail held across a
78
+ // disconnect. On close the per-entry retry timers are cleared (the socket is
79
+ // gone) but the entries are RETAINED so a verified reconnect can redeliver
80
+ // them; this timer bounds that retention so a sidecar that never reconnects
81
+ // does not leak entries. Cleared when the address reconnects (redelivery) or
82
+ // its last pending entry is acked.
83
+ const pendingMailRetention = new Map();
84
+ // agentAddress → set of subscriber callbacks for agent events
85
+ const agentSubscribers = new Map();
86
+ // agentAddress → cached connector-thread state, populated by
87
+ // connector.state.changed frames. Hub-side mail composition reads this
88
+ // to set threading headers on user-originated mail. Absent entries mean
89
+ // "no state reported yet" (e.g. mid-reconnect); callers must treat that
90
+ // identically to a null entry (no active thread).
91
+ const connectorStates = new Map();
92
+ // ws handle → liveness timer (reset on each ping from the sidecar)
93
+ const livenessTimers = new Map();
94
+ // Per-ws serialization chain for QUEUE-class frames (see `frameBypassesQueue`
95
+ // for the split and the invariant behind it). A frame that establishes or
96
+ // reads routing waits for earlier queued frames on the same ws to complete,
97
+ // so it observes their finished effects -- most importantly an async
98
+ // register's routing write, which would otherwise land after a following
99
+ // connector.state.changed / mail / pack frame and silently drop it. It holds
100
+ // a single in-flight promise per ws (replaced each queued frame), cleared on
101
+ // close.
102
+ const messageChains = new Map();
103
+ const pendingPacks = new Map();
104
+ let packCounter = 0;
105
+ const pendingUndeploys = new Map();
106
+ const pendingProbes = new Map();
107
+ // Receives agent-state packs pushed from sidecars. The wire frames
108
+ // (`repo.pack.push` / `repo.pack.done`) are shared with the
109
+ // workflow-run flow; dispatch on `repoId.kind` picks which receiver
110
+ // observes the chunks. The two receivers maintain independent
111
+ // in-flight pack state and independent cancel-by-agent semantics so a
112
+ // pending workflow-run transfer cannot disturb a concurrent agent-
113
+ // state transfer for the same agent and vice versa.
114
+ const agentStatePackReceiver = createPackReceiver();
115
+ const workflowRunPackReceiver = createPackReceiver();
116
+ let requestCounter = 0;
117
+ // Surface disconnect-queue mail that is being dropped rather than delivered.
118
+ // Every dropped frame reaches the same channel routing failures already use
119
+ // (`mail.outbound.undelivered`, logged by the orchestrator), plus a warn that
120
+ // names the recipient and the drop count so a size-cap eviction or a TTL
121
+ // expiry of a still-full queue is visible instead of silent. A queued frame
122
+ // is always a `mail.inbound` carrying the sender's rawMessage; a frame of any
123
+ // other shape has no rawMessage to relay and is surfaced by the warn alone.
124
+ function surfaceDroppedFrames(agentAddress, frames, reason) {
125
+ if (frames.length === 0)
126
+ return;
127
+ logger.warn `Dropping ${String(frames.length)} queued message(s) for ${agentAddress}: ${reason}`;
128
+ for (const frame of frames) {
129
+ if (frame.type !== "mail.inbound")
130
+ continue;
131
+ events.emit("mail.outbound.undelivered", {
132
+ rawMessage: frame.rawMessage,
133
+ recipients: [agentAddress],
134
+ });
135
+ }
136
+ }
137
+ function enqueueForDisconnected(agentAddress, frame) {
138
+ const entry = disconnectedAgents.get(agentAddress);
139
+ if (entry === undefined)
140
+ return false;
141
+ if (entry.queue.length >= disconnectQueueMaxSize) {
142
+ const evicted = entry.queue.shift();
143
+ if (evicted !== undefined) {
144
+ surfaceDroppedFrames(agentAddress, [evicted], "disconnect queue full");
145
+ }
146
+ }
147
+ entry.queue.push(frame);
148
+ return true;
149
+ }
150
+ function flushDisconnectedQueue(agentAddress, conn) {
151
+ const entry = disconnectedAgents.get(agentAddress);
152
+ if (entry === undefined)
153
+ return;
154
+ clearTimeout(entry.timer);
155
+ disconnectedAgents.delete(agentAddress);
156
+ for (const frame of entry.queue) {
157
+ conn.send(frame);
158
+ }
159
+ if (entry.queue.length > 0) {
160
+ logger.info `Flushed ${String(entry.queue.length)} queued message(s) to ${agentAddress}`;
161
+ }
162
+ }
163
+ // Track a connected-window `mail.inbound` for redelivery until the sidecar
164
+ // acks its durable inbox write. Replaces any prior entry for the same
165
+ // (agentAddress, messageId) -- clearing its timer first so no timer leaks --
166
+ // which keeps a re-sent delivery from arming a second concurrent retry loop.
167
+ function trackPendingMail(agentAddress, messageId, frame, runGrants, allocatedTarget) {
168
+ let byId = pendingMail.get(agentAddress);
169
+ if (byId === undefined) {
170
+ byId = new Map();
171
+ pendingMail.set(agentAddress, byId);
172
+ }
173
+ const existing = byId.get(messageId);
174
+ if (existing !== undefined)
175
+ clearTimeout(existing.timer);
176
+ byId.set(messageId, {
177
+ agentAddress,
178
+ messageId,
179
+ frame,
180
+ attempts: 0,
181
+ timer: setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs),
182
+ ...(runGrants !== undefined ? { runGrants } : {}),
183
+ ...(allocatedTarget !== undefined ? { allocatedTarget } : {}),
184
+ });
185
+ }
186
+ // Replay a mail-triggered run's grants ahead of a redelivery of its trigger
187
+ // mail. Same-connection FIFO lands the `run.grants` frame before the mail, so
188
+ // the redelivered run resolves its onRunStart barrier instead of failing
189
+ // closed on missing grants.
190
+ function replayRunGrantsAhead(conn, entry) {
191
+ if (entry.runGrants === undefined)
192
+ return;
193
+ conn.send({
194
+ type: "run.grants",
195
+ agentAddress: entry.agentAddress,
196
+ runId: entry.runGrants.runId,
197
+ stepGrants: entry.runGrants.stepGrants,
198
+ });
199
+ }
200
+ function deletePendingMail(byId, agentAddress, messageId) {
201
+ byId.delete(messageId);
202
+ if (byId.size === 0) {
203
+ pendingMail.delete(agentAddress);
204
+ // The retention TTL guards a non-empty pending set; drop it once the set
205
+ // is empty so it never outlives the entries it was bounding.
206
+ const retention = pendingMailRetention.get(agentAddress);
207
+ if (retention !== undefined) {
208
+ clearTimeout(retention);
209
+ pendingMailRetention.delete(agentAddress);
210
+ }
211
+ }
212
+ }
213
+ function retryPendingMail(agentAddress, messageId) {
214
+ const byId = pendingMail.get(agentAddress);
215
+ if (byId === undefined)
216
+ return;
217
+ const entry = byId.get(messageId);
218
+ if (entry === undefined)
219
+ return;
220
+ if (entry.attempts >= mailAckMaxRetries) {
221
+ // The sidecar never acked within the retry budget. The ack is withheld
222
+ // precisely because the sidecar's durable inbox write failed, so the
223
+ // mail was NOT delivered: surface it as undelivered so the host can relay
224
+ // it onto an external transport, then drop the pending entry so its timer
225
+ // does not leak.
226
+ if (entry.frame.type === "mail.inbound") {
227
+ events.emit("mail.outbound.undelivered", {
228
+ rawMessage: entry.frame.rawMessage,
229
+ recipients: [agentAddress],
230
+ });
231
+ }
232
+ deletePendingMail(byId, agentAddress, messageId);
233
+ logger.warn `Gave up redelivering mail ${messageId} to ${agentAddress} after ${String(entry.attempts)} un-acked attempt(s)`;
234
+ return;
235
+ }
236
+ // Redeliver over the address's CURRENT owner: a verified reconnect may have
237
+ // moved the address to a new connection since the original delivery.
238
+ const ws = addressIndex.get(agentAddress);
239
+ const conn = ws !== undefined ? connections.get(ws) : undefined;
240
+ const allocated = entry.allocatedTarget === undefined
241
+ ? undefined
242
+ : allocatedConnections.get(entry.allocatedTarget.allocationId);
243
+ const targetStillOwnsAddress = entry.allocatedTarget === undefined ||
244
+ (allocated !== undefined &&
245
+ allocated.identity.generation === entry.allocatedTarget.generation &&
246
+ allocated.ws === ws);
247
+ if (conn === undefined || !targetStillOwnsAddress) {
248
+ // No live connection to recover into. Connected-window redelivery only
249
+ // applies while the address is routable; a disconnected address is not
250
+ // retried here.
251
+ deletePendingMail(byId, agentAddress, messageId);
252
+ logger.warn `Dropping un-acked mail ${messageId} for ${agentAddress}: no live connection to redeliver over`;
253
+ return;
254
+ }
255
+ entry.attempts += 1;
256
+ replayRunGrantsAhead(conn, entry);
257
+ conn.send(entry.frame);
258
+ entry.timer = setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs);
259
+ }
260
+ function resolvePendingMail(agentAddress, messageId) {
261
+ const byId = pendingMail.get(agentAddress);
262
+ if (byId === undefined)
263
+ return;
264
+ const entry = byId.get(messageId);
265
+ if (entry === undefined)
266
+ return;
267
+ clearTimeout(entry.timer);
268
+ deletePendingMail(byId, agentAddress, messageId);
269
+ }
270
+ // Hold an address's un-acked pending mail across a disconnect. The per-entry
271
+ // retry timers are cleared -- retrying over the dead socket is pointless --
272
+ // but the entries are KEPT so a verified reconnect can redeliver them. A
273
+ // retention TTL (the disconnect-queue horizon) bounds the hold so a sidecar
274
+ // that never reconnects does not leak; on expiry the still-un-acked entries
275
+ // are surfaced as `mail.outbound.undelivered` so the host can relay them,
276
+ // since a withheld ack means the sidecar's durable write never landed.
277
+ function retainPendingMailForAddress(agentAddress) {
278
+ const byId = pendingMail.get(agentAddress);
279
+ if (byId === undefined)
280
+ return;
281
+ for (const entry of byId.values())
282
+ clearTimeout(entry.timer);
283
+ const existing = pendingMailRetention.get(agentAddress);
284
+ if (existing !== undefined)
285
+ clearTimeout(existing);
286
+ const timer = setTimeout(() => {
287
+ pendingMailRetention.delete(agentAddress);
288
+ const expired = pendingMail.get(agentAddress);
289
+ pendingMail.delete(agentAddress);
290
+ if (expired !== undefined && expired.size > 0) {
291
+ for (const entry of expired.values()) {
292
+ if (entry.frame.type !== "mail.inbound")
293
+ continue;
294
+ events.emit("mail.outbound.undelivered", {
295
+ rawMessage: entry.frame.rawMessage,
296
+ recipients: [agentAddress],
297
+ });
298
+ }
299
+ logger.warn `Dropping ${String(expired.size)} un-acked message(s) for ${agentAddress}: pending-mail retention TTL expired`;
300
+ }
301
+ }, disconnectQueueTTLMs);
302
+ pendingMailRetention.set(agentAddress, timer);
303
+ }
304
+ // Redeliver an address's retained un-acked pending mail on a verified
305
+ // reconnect. Replays identical bytes (same messageId) over the new
306
+ // connection, so the sidecar's inbox dedups a message it already wrote
307
+ // (effectively-once) and processes one it had dropped (no loss). Re-arms the
308
+ // connected-window retry over the new connection with a fresh per-generation
309
+ // budget, so a redelivery that is itself dropped before its ack is retried.
310
+ function redeliverPendingMail(agentAddress, conn) {
311
+ const retention = pendingMailRetention.get(agentAddress);
312
+ if (retention !== undefined) {
313
+ clearTimeout(retention);
314
+ pendingMailRetention.delete(agentAddress);
315
+ }
316
+ const byId = pendingMail.get(agentAddress);
317
+ if (byId === undefined)
318
+ return;
319
+ for (const entry of [...byId.values()]) {
320
+ if (entry.allocatedTarget !== undefined &&
321
+ (conn.identity.kind !== "allocated" ||
322
+ conn.identity.allocationId !== entry.allocatedTarget.allocationId ||
323
+ conn.identity.generation !== entry.allocatedTarget.generation)) {
324
+ // The Hub-owned dispatch row survives generation replacement and will
325
+ // be requeued by the allocation-ready callback. Do not leak or replay
326
+ // this generation-local retry entry onto a different worker.
327
+ clearTimeout(entry.timer);
328
+ deletePendingMail(byId, agentAddress, entry.messageId);
329
+ continue;
330
+ }
331
+ replayRunGrantsAhead(conn, entry);
332
+ conn.send(entry.frame);
333
+ entry.attempts = 0;
334
+ entry.timer = setTimeout(() => retryPendingMail(agentAddress, entry.messageId), mailAckRetryIntervalMs);
335
+ }
336
+ if (byId.size > 0) {
337
+ logger.info `Redelivered ${String(byId.size)} un-acked message(s) to ${agentAddress} on reconnect`;
338
+ }
339
+ }
340
+ function resetLivenessTimer(ws) {
341
+ const existing = livenessTimers.get(ws);
342
+ if (existing !== undefined)
343
+ clearTimeout(existing);
344
+ const timer = setTimeout(() => {
345
+ livenessTimers.delete(ws);
346
+ logger.warn `Sidecar ping timeout, closing connection`;
347
+ ws.close();
348
+ }, pingTimeoutMs);
349
+ livenessTimers.set(ws, timer);
350
+ }
351
+ function handlePing(ws) {
352
+ resetLivenessTimer(ws);
353
+ // Always respond with pong, even before register/reconnect completes.
354
+ // The sidecar's ping timer starts on open, which may fire before the
355
+ // async registration handshake finishes.
356
+ ws.send(JSON.stringify({ type: "pong" }));
357
+ }
358
+ function handleOpen(ws) {
359
+ // Connection is not usable until a register frame arrives.
360
+ // Start the liveness timer immediately — a sidecar that connects
361
+ // but never sends a ping will be reaped.
362
+ resetLivenessTimer(ws);
363
+ }
364
+ function handleMessage(ws, data) {
365
+ let raw;
366
+ try {
367
+ raw = JSON.parse(data);
368
+ }
369
+ catch {
370
+ logger.warn `Unparseable frame from sidecar connection`;
371
+ return;
372
+ }
373
+ const validated = SidecarFrame(raw);
374
+ if (validated instanceof type.errors) {
375
+ logger.warn `Invalid sidecar frame: ${validated.summary}`;
376
+ return;
377
+ }
378
+ const frame = validated;
379
+ // Bypass frames (liveness + terminal responses to outbound requests)
380
+ // dispatch immediately: they resolve the very promises a queued handler
381
+ // may be blocked on, so queuing them would deadlock the round-trip.
382
+ if (frameBypassesQueue(frame)) {
383
+ // Guard so a bypass handler's failure -- a synchronous throw or an async
384
+ // ack handler's rejection -- is logged rather than floating out of the
385
+ // immediate dispatch. The async wrapper turns a synchronous throw into a
386
+ // rejection too, matching the queue path's .then/.catch coverage.
387
+ void (async () => dispatchFrame(ws, frame))().catch((err) => {
388
+ logger.warn `Frame handler failed for ${frame.type}: ${err instanceof Error ? err.message : String(err)}`;
389
+ });
390
+ return;
391
+ }
392
+ // Everything else serializes per ws so a frame that establishes or reads
393
+ // routing observes earlier queued frames' completed effects.
394
+ const prev = messageChains.get(ws) ?? Promise.resolve();
395
+ const next = prev
396
+ .then(() => dispatchFrame(ws, frame))
397
+ .catch((err) => {
398
+ logger.warn `Frame handler failed for ${frame.type}: ${err instanceof Error ? err.message : String(err)}`;
399
+ });
400
+ messageChains.set(ws, next);
401
+ }
402
+ function assertNever(x) {
403
+ throw new Error(`Unclassified sidecar frame type: ${JSON.stringify(x)}`);
404
+ }
405
+ // Whether `frame` bypasses the per-ws serialization chain. Invariant: a frame
406
+ // bypasses IFF it is liveness (ping) OR a terminal response to an
407
+ // already-issued outbound request -- correlated purely by
408
+ // requestId/transferId/agentAddress in the pending maps, touching no routing
409
+ // state. Such a frame has no ordering obligation against new inbound frames
410
+ // (a response cannot resolve "too early" for a request that already went
411
+ // out), and it is exactly what in-flight queued handlers block on, so it MUST
412
+ // run out of band or the challenge round-trip deadlocks (challenge.response
413
+ // -> agent.reconnected reaction -> sendSourcesUpdate awaits a later
414
+ // session.ack). Every other frame establishes or reads routing, or carries an
415
+ // inbound payload whose order matters, so it queues. The exhaustive switch +
416
+ // assertNever makes adding a SidecarFrame variant without classifying it a
417
+ // compile error, not a latent deadlock or a silent bypass hole.
418
+ function frameBypassesQueue(frame) {
419
+ switch (frame.type) {
420
+ case "ping":
421
+ case "session.ack":
422
+ case "session.error":
423
+ case "agent.deploy.ack":
424
+ case "agent.error":
425
+ case "agent.undeploy.ack":
426
+ case "repo.pack.ack":
427
+ case "repo.pack.reject":
428
+ case "workflow.probe.result":
429
+ case "workflow.probe.error":
430
+ return true;
431
+ case "register":
432
+ case "reconnect":
433
+ case "challenge.response":
434
+ case "mail.outbound":
435
+ case "agent.event":
436
+ case "connector.state.changed":
437
+ case "mail.inbound.ack":
438
+ case "signal.correlation.register":
439
+ case "repo.pack.push":
440
+ case "repo.pack.done":
441
+ return false;
442
+ default:
443
+ return assertNever(frame);
444
+ }
445
+ }
446
+ // Runs one frame's handler. Returns the handler's promise for async handlers
447
+ // so the per-ws chain can await bounded completion; sync handlers return
448
+ // void. Never awaits a promise that resolves on a LATER same-ws frame -- the
449
+ // only such await (the challenge round-trip's session.ack) is reached via a
450
+ // bypass frame, which does not queue.
451
+ function dispatchFrame(ws, frame) {
452
+ switch (frame.type) {
453
+ case "register": {
454
+ const agentAddresses = frame.agentAddresses;
455
+ return authenticateHandshake(ws, frame, (identity) => handleRegister(ws, identity, agentAddresses));
456
+ }
457
+ case "reconnect": {
458
+ const agentAddresses = frame.agentAddresses;
459
+ const deployRefs = frame.deployRefs ?? {};
460
+ return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses, deployRefs));
461
+ }
462
+ case "challenge.response":
463
+ return handleChallengeResponse(ws, frame.responses);
464
+ case "agent.deploy.ack":
465
+ return handleDeployAck(ws, frame);
466
+ case "agent.error":
467
+ rejectDeployPendingFromFrame(ws, frame.agentAddress, frame.error);
468
+ rejectUndeployPending(ws, frame.agentAddress, frame.error);
469
+ return;
470
+ case "agent.undeploy.ack":
471
+ resolveUndeployPending(ws, frame.agentAddress);
472
+ return;
473
+ case "ping":
474
+ handlePing(ws);
475
+ return;
476
+ case "mail.outbound":
477
+ if (frame.delivered !== true) {
478
+ return handleMailOutbound(frame.rawMessage, frame.recipients);
479
+ }
480
+ if (lookups.persistMail && frame.senderAddress) {
481
+ return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
482
+ }
483
+ if (!frame.senderAddress) {
484
+ logger.warn `Dropping delivered mail.outbound frame with no senderAddress`;
485
+ }
486
+ else {
487
+ logger.warn `Dropping delivered mail.outbound frame: no persistMail lookup configured`;
488
+ }
489
+ return;
490
+ case "agent.event":
491
+ events.emit("agent.event", {
492
+ agentAddress: frame.agentAddress,
493
+ sessionId: frame.sessionId,
494
+ event: frame.event,
495
+ });
496
+ dispatchToSubscribers(frame.agentAddress, frame.event);
497
+ return;
498
+ case "connector.state.changed":
499
+ // Gate the cache write on the sending sidecar actually owning
500
+ // the named agent. A misbehaving sidecar that knows another
501
+ // agent's address could otherwise poison the cached state.
502
+ if (addressIndex.get(frame.agentAddress) !== ws) {
503
+ logger.warn `Dropping connector.state.changed for ${frame.agentAddress}: not registered to this sidecar`;
504
+ return;
505
+ }
506
+ connectorStates.set(frame.agentAddress, frame.connectorState);
507
+ events.emit("connector.state.changed", {
508
+ agentAddress: frame.agentAddress,
509
+ connectorState: frame.connectorState,
510
+ });
511
+ return;
512
+ case "mail.inbound.ack": {
513
+ // Terminal receipt for a connected-window `mail.inbound`: the sidecar
514
+ // has durably written the message to its inbox. Gate on ownership --
515
+ // like connector.state.changed and signal.correlation.register -- so a
516
+ // sidecar cannot clear another sidecar's pending mail. The messageId is
517
+ // a hub-minted id only the owning sidecar ever received on the frame,
518
+ // so the ownership check is defense-in-depth, not the sole guard.
519
+ const conn = connections.get(ws);
520
+ if (conn === undefined)
521
+ return;
522
+ if (!connOwnsAddress(conn, frame.agentAddress)) {
523
+ logger.warn `Dropping mail.inbound.ack for ${frame.agentAddress}: not registered to this sidecar`;
524
+ return;
525
+ }
526
+ resolvePendingMail(frame.agentAddress, frame.messageId);
527
+ events.emit("mail.inbound.acknowledged", {
528
+ agentAddress: frame.agentAddress,
529
+ messageId: frame.messageId,
530
+ ...(conn.identity.kind === "allocated"
531
+ ? {
532
+ allocated: {
533
+ allocationId: conn.identity.allocationId,
534
+ anchorRunId: conn.identity.anchorRunId,
535
+ generation: conn.identity.generation,
536
+ },
537
+ }
538
+ : {}),
539
+ });
540
+ return;
541
+ }
542
+ case "signal.correlation.register":
543
+ return handleSignalCorrelationRegister(ws, frame);
544
+ case "session.ack":
545
+ resolvePending(frame.requestId);
546
+ return;
547
+ case "session.error":
548
+ rejectPending(frame.requestId, frame.error);
549
+ return;
550
+ case "repo.pack.ack":
551
+ resolvePackPending(ws, frame);
552
+ return;
553
+ case "repo.pack.reject":
554
+ rejectPackPending(ws, frame);
555
+ return;
556
+ case "repo.pack.push":
557
+ handlePackPush(ws, frame);
558
+ return;
559
+ case "repo.pack.done":
560
+ return handlePackDone(ws, frame);
561
+ case "workflow.probe.result":
562
+ resolveProbe(frame.requestId, {
563
+ projection: frame.projection,
564
+ grants: frame.grants,
565
+ grantWalkSnapshot: frame.grantWalkSnapshot,
566
+ wireHash: frame.wireHash,
567
+ });
568
+ return;
569
+ case "workflow.probe.error":
570
+ rejectProbe(frame.requestId, frame.error);
571
+ return;
572
+ default:
573
+ return assertNever(frame);
574
+ }
575
+ }
576
+ // Authenticate a register/reconnect handshake exactly once, then run the
577
+ // frame's handler with the verified identity. The claimed `sidecarId` on
578
+ // the frame is an unauthenticated hint: it is logged if it disagrees with
579
+ // the verified id but never trusted -- routing keys off the verified id.
580
+ // Fails closed by closing the connection when the authenticator rejects
581
+ // (returns null) or throws (e.g. a database failure), so a handshake never
582
+ // proceeds on unverified credentials.
583
+ async function authenticateHandshake(ws, frame, run) {
584
+ let identity;
585
+ try {
586
+ identity = await authenticateSidecar({
587
+ sidecarId: frame.sidecarId,
588
+ token: frame.token,
589
+ });
590
+ }
591
+ catch (err) {
592
+ logger.error `Rejected ${frame.type} from claimed sidecar ${frame.sidecarId}: authenticator failed: ${err instanceof Error ? err.message : String(err)}`;
593
+ ws.close();
594
+ return;
595
+ }
596
+ if (identity === null) {
597
+ logger.warn `Rejected ${frame.type} from claimed sidecar ${frame.sidecarId}: invalid token`;
598
+ ws.close();
599
+ return;
600
+ }
601
+ if (identity.sidecarId !== frame.sidecarId) {
602
+ logger.warn `Sidecar ${frame.type} claimed id ${frame.sidecarId} but token verifies as ${identity.sidecarId}; keying off the verified id`;
603
+ }
604
+ if (!(await validateSidecarIdentity(identity, "registration"))) {
605
+ logger.warn `Rejected ${frame.type} from sidecar ${identity.sidecarId}: credential identity is no longer current`;
606
+ ws.close();
607
+ return;
608
+ }
609
+ await run(identity);
610
+ }
611
+ async function notifyAllocationWaiters(allocationId) {
612
+ const waiters = allocationWaiters.get(allocationId);
613
+ const current = allocatedConnections.get(allocationId);
614
+ if (waiters === undefined || current === undefined)
615
+ return;
616
+ if (!(await validateSidecarIdentity(current.identity, "readiness")))
617
+ return;
618
+ for (const waiter of [...waiters]) {
619
+ if (waiter.generation !== current.identity.generation)
620
+ continue;
621
+ clearTimeout(waiter.timer);
622
+ waiters.delete(waiter);
623
+ waiter.resolve();
624
+ }
625
+ if (waiters.size === 0)
626
+ allocationWaiters.delete(allocationId);
627
+ }
628
+ async function handleAllocatedRegister(ws, identity, agentAddresses) {
629
+ if (allocationFences.get(identity.allocationId) !== identity.generation) {
630
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} generation ${String(identity.generation)} is not fenced as current`;
631
+ ws.close();
632
+ return;
633
+ }
634
+ if (agentAddresses.length > 0 &&
635
+ !(await validateSidecarIdentity(identity, "routing"))) {
636
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} is not ready to reclaim routes`;
637
+ ws.close();
638
+ return;
639
+ }
640
+ const existingOnSocket = connections.get(ws);
641
+ const newlyRoutedAddresses = new Set();
642
+ for (const address of agentAddresses) {
643
+ const alreadyOwned = existingOnSocket?.identity.kind === "allocated" &&
644
+ existingOnSocket.identity.allocationId === identity.allocationId &&
645
+ addressIndex.get(address) === ws &&
646
+ connOwnsAddress(existingOnSocket, address);
647
+ if (!alreadyOwned && address !== identity.workflowRunAddress) {
648
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} claimed unrelated address ${address}`;
649
+ ws.close();
650
+ return;
651
+ }
652
+ if (!alreadyOwned)
653
+ newlyRoutedAddresses.add(address);
654
+ }
655
+ if (allocationFences.get(identity.allocationId) !== identity.generation) {
656
+ ws.close();
657
+ return;
658
+ }
659
+ const current = allocatedConnections.get(identity.allocationId);
660
+ if (current !== undefined && current.ws !== ws) {
661
+ // A current-generation takeover is a reconnect, not a capacity loss.
662
+ // Remove the old socket from the allocation index before closing it so
663
+ // handleClose does not emit a false allocated-disconnect event.
664
+ allocatedConnections.delete(identity.allocationId);
665
+ handleClose(current.ws);
666
+ current.ws.close();
667
+ }
668
+ const conn = existingOnSocket ?? {
669
+ sidecarId: identity.sidecarId,
670
+ identity,
671
+ agentAddresses: new Set(),
672
+ workflowAddresses: new Set(),
673
+ send(frame) {
674
+ ws.send(JSON.stringify(frame));
675
+ },
676
+ };
677
+ if (conn.identity.kind !== "allocated" ||
678
+ conn.identity.allocationId !== identity.allocationId ||
679
+ conn.identity.generation !== identity.generation) {
680
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: socket identity changed during registration`;
681
+ ws.close();
682
+ return;
683
+ }
684
+ connections.set(ws, conn);
685
+ for (const address of agentAddresses) {
686
+ conn.workflowAddresses.add(address);
687
+ addressIndex.set(address, ws);
688
+ }
689
+ allocatedConnections.set(identity.allocationId, { ws, identity });
690
+ for (const address of newlyRoutedAddresses) {
691
+ redeliverPendingMail(address, conn);
692
+ }
693
+ logger.info `Allocated sidecar ${identity.sidecarId} registered for allocation ${identity.allocationId} generation ${String(identity.generation)}`;
694
+ await notifyAllocationWaiters(identity.allocationId);
695
+ events.emit("sidecar.allocated.connected", {
696
+ allocationId: identity.allocationId,
697
+ generation: identity.generation,
698
+ });
699
+ }
700
+ async function handleRegister(ws, identity, agentAddresses) {
701
+ if (identity.kind === "allocated") {
702
+ await handleAllocatedRegister(ws, identity, agentAddresses);
703
+ return;
704
+ }
705
+ const sidecarId = identity.sidecarId;
706
+ // Key-existence gate. A register frame is token-authenticated but carries
707
+ // no per-address ownership proof, so it may route an address ONLY if that
708
+ // address has no stored key yet -- a genuine keyless first-deploy (the
709
+ // token-bounded first-deploy trust model). An address that already has a
710
+ // key must prove ownership through the challenged reconnect path; routing
711
+ // it here on token auth alone is the register-frame sibling of the
712
+ // reconnect hijack. The keyless-only set is computed up front, BEFORE the
713
+ // ghost-cleanup and every routing mutation below, so a rejected address
714
+ // touches nothing: no eviction of a live owner, hence no downgrade from
715
+ // hijack to denial-of-service on the victim.
716
+ const lookupKey = lookups.lookupPublicKey;
717
+ const routableAddresses = [];
718
+ for (const addr of agentAddresses) {
719
+ if (lookupKey === undefined) {
720
+ // Fail closed: without the ownership lookup a keyed address cannot be
721
+ // told apart from a first-deploy, so route nothing and surface the
722
+ // misconfiguration. Empty first-connect registers never reach here.
723
+ logger.error `Cannot gate register routing for ${addr}: lookupPublicKey is not configured; refusing to route (challenged reconnect required)`;
724
+ continue;
725
+ }
726
+ let existingKey;
727
+ try {
728
+ existingKey = await lookupKey(addr);
729
+ }
730
+ catch (err) {
731
+ // Fail closed on a lookup error (e.g. a transient DB failure): route
732
+ // nothing for this address and surface the failure, rather than let
733
+ // the rejection float out of this void-dispatched handler and take
734
+ // down the hub.
735
+ logger.error `Key lookup failed for ${addr} during register: ${err instanceof Error ? err.message : String(err)}; failing closed (challenged reconnect required)`;
736
+ continue;
737
+ }
738
+ if (existingKey !== null) {
739
+ logger.warn `Refusing to route ${addr} via register: address already has a stored key; ownership must be proven via challenged reconnect`;
740
+ continue;
741
+ }
742
+ routableAddresses.push(addr);
743
+ }
744
+ // Additive re-register: inherit every address this ws already owns and ADD
745
+ // the frame's keyless first-deploys. Register never drops an owned route --
746
+ // an address proved via challenged reconnect stays routed, and removal
747
+ // happens via undeploy/disconnect, not register-omission.
748
+ //
749
+ // (Under the retired full-set model a register frame carried the sidecar's
750
+ // complete live set, so an omitted address meant "removed". Now the frame
751
+ // carries only keyless first-deploys the sidecar is adding -- keyed
752
+ // addresses arrive via reconnect -- so omission is meaningless, and a
753
+ // replace-on-register would wrongly drop an earlier first-deploy when a
754
+ // later one is registered, as well as any reconnect-verified keyed route.)
755
+ const existing = connections.get(ws);
756
+ const inheritedAgent = new Set(existing?.agentAddresses);
757
+ const inheritedWorkflow = new Set(existing?.workflowAddresses);
758
+ // Clean up ghost entries from OTHER connections for the newly-claimed
759
+ // addresses only. An inherited address is already owned by this ws
760
+ // (prevWs === ws), so it needs no eviction and its in-flight deploy must
761
+ // not be cancelled.
762
+ for (const addr of routableAddresses) {
763
+ const prevWs = addressIndex.get(addr);
764
+ if (prevWs !== undefined && prevWs !== ws) {
765
+ const prevConn = connections.get(prevWs);
766
+ if (prevConn !== undefined) {
767
+ prevConn.agentAddresses.delete(addr);
768
+ }
769
+ // The new owner is about to take over; the prior owner's
770
+ // cached state must not survive into the new owner's window
771
+ // before its bootstrap frame arrives.
772
+ connectorStates.delete(addr);
773
+ }
774
+ // Cancel any in-flight deploy for this address since the
775
+ // reconnecting sidecar is taking ownership.
776
+ const staleDeployReq = pendingDeploys.get(addr);
777
+ if (staleDeployReq !== undefined) {
778
+ clearTimeout(staleDeployReq.timer);
779
+ pendingDeploys.delete(addr);
780
+ staleDeployReq.reject(`Sidecar ${sidecarId} reconnected and claimed address "${addr}"`);
781
+ }
782
+ }
783
+ // New keyless first-deploys join the inherited session set; the workflow
784
+ // set is inherited unchanged (register never adds to it -- a
785
+ // workflow-derived address routes only through the challenged reconnect).
786
+ for (const addr of routableAddresses) {
787
+ inheritedAgent.add(addr);
788
+ }
789
+ const addrSet = inheritedAgent;
790
+ const workflowSet = inheritedWorkflow;
791
+ const conn = {
792
+ sidecarId,
793
+ identity,
794
+ agentAddresses: addrSet,
795
+ workflowAddresses: workflowSet,
796
+ send(frame) {
797
+ ws.send(JSON.stringify(frame));
798
+ },
799
+ };
800
+ connections.set(ws, conn);
801
+ // Only the newly-claimed addresses need a routing write + queue discard;
802
+ // inherited addresses already point at this ws in addressIndex.
803
+ for (const addr of routableAddresses) {
804
+ addressIndex.set(addr, ws);
805
+ // Discard any disconnect queue — register has no identity
806
+ // verification, so flushing to an unverified connection is unsafe.
807
+ // Use reconnect with challenge/response to preserve queued messages.
808
+ const staleQueue = disconnectedAgents.get(addr);
809
+ if (staleQueue !== undefined) {
810
+ clearTimeout(staleQueue.timer);
811
+ if (staleQueue.queue.length > 0) {
812
+ logger.warn `Discarding ${String(staleQueue.queue.length)} queued message(s) for ${addr} on unverified register`;
813
+ }
814
+ disconnectedAgents.delete(addr);
815
+ }
816
+ }
817
+ // Only keyless first-deploy addresses reach `addressIndex` here; an
818
+ // address that already has a stored key was filtered out by the gate
819
+ // above and must re-enter routing through the challenged reconnect path.
820
+ // Because the gate runs before the ghost-cleanup, a rejected (keyed)
821
+ // address never evicts its prior owner -- register cannot reclaim or
822
+ // disrupt a victim's route on token auth alone.
823
+ logger.info `Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`;
824
+ }
825
+ async function handleReconnect(ws, identity, agentAddresses, deployRefs = {}) {
826
+ if (identity.kind === "allocated") {
827
+ await handleAllocatedRegister(ws, identity, agentAddresses);
828
+ return;
829
+ }
830
+ const sidecarId = identity.sidecarId;
831
+ const lookupKey = lookups.lookupPublicKey;
832
+ if (lookupKey === undefined) {
833
+ logger.error `Received reconnect frame but no lookupPublicKey is configured`;
834
+ ws.close();
835
+ return;
836
+ }
837
+ // Cancel any existing pending challenge for this ws before doing
838
+ // async work, so concurrent reconnect frames don't race.
839
+ const existingChallenge = pendingChallenges.get(ws);
840
+ if (existingChallenge !== undefined) {
841
+ clearTimeout(existingChallenge.timer);
842
+ pendingChallenges.delete(ws);
843
+ }
844
+ // Capture the addresses this live connection already owns. The
845
+ // internal register below clears them, but ones the reconnect is not
846
+ // re-challenging are still valid — an agent provisioned during the
847
+ // sidecar's restore window is routed into addressIndex by its deploy
848
+ // and is not part of the disk-restored set this reconnect carries.
849
+ // Without preserving them, the reconnect silently drops a
850
+ // freshly-deployed agent from routing.
851
+ const previouslyOwned = new Set(connections.get(ws)?.agentAddresses);
852
+ // Register the sidecar connection immediately (with no addresses) so it
853
+ // can receive frames while the ownership challenge is pending. Every
854
+ // reconnect address -- session and workflow-derived alike -- enters
855
+ // routing only through the verified path below, never unchallenged here.
856
+ // Empty address list, so the key-existence gate has nothing to await.
857
+ // The identity was already verified at the dispatch boundary, so the
858
+ // internal register does not re-authenticate.
859
+ await handleRegister(ws, identity, []);
860
+ const conn = connections.get(ws);
861
+ if (conn === undefined)
862
+ return;
863
+ // Re-add the still-owned addresses the register cleared but this
864
+ // reconnect is not re-challenging. The challenged addresses below
865
+ // re-enter addressIndex through the verified path instead.
866
+ const claimedAddresses = new Set(agentAddresses);
867
+ for (const addr of previouslyOwned) {
868
+ if (claimedAddresses.has(addr))
869
+ continue;
870
+ conn.agentAddresses.add(addr);
871
+ addressIndex.set(addr, ws);
872
+ }
873
+ // Look up stored public keys for all claimed addresses. Fail closed on a
874
+ // lookup error (e.g. a transient DB failure): treat the address as
875
+ // unverifiable so it fails its challenge and stays unrouted, rather than
876
+ // letting the rejection float out of this void-dispatched handler as an
877
+ // unhandled rejection that could take down the hub.
878
+ const keyLookups = await Promise.all(agentAddresses.map(async (addr) => {
879
+ try {
880
+ return { address: addr, publicKeyHex: await lookupKey(addr) };
881
+ }
882
+ catch (err) {
883
+ logger.error `Key lookup failed for ${addr} during reconnect: ${err instanceof Error ? err.message : String(err)}; failing closed`;
884
+ return { address: addr, publicKeyHex: null };
885
+ }
886
+ }));
887
+ // If the connection was closed or superseded while we were awaiting
888
+ // key lookups, bail out.
889
+ if (!connections.has(ws))
890
+ return;
891
+ const challenges = new Map();
892
+ const challengeEntries = [];
893
+ for (const { address, publicKeyHex } of keyLookups) {
894
+ if (publicKeyHex === null) {
895
+ conn.send({
896
+ type: "challenge.failed",
897
+ address,
898
+ reason: "Unknown run address",
899
+ });
900
+ continue;
901
+ }
902
+ let publicKey;
903
+ try {
904
+ publicKey = hexDecode(publicKeyHex);
905
+ }
906
+ catch {
907
+ conn.send({
908
+ type: "challenge.failed",
909
+ address,
910
+ reason: "Stored public key is corrupt",
911
+ });
912
+ logger.error `Corrupt stored public key for ${address}`;
913
+ continue;
914
+ }
915
+ const nonce = crypto.getRandomValues(new Uint8Array(32));
916
+ challenges.set(address, { nonce, publicKey });
917
+ challengeEntries.push({ address, nonce: hexEncode(nonce) });
918
+ }
919
+ if (challenges.size === 0)
920
+ return;
921
+ // If another reconnect completed while we were building the
922
+ // challenge, it will have written its own entry. Don't overwrite.
923
+ if (pendingChallenges.has(ws))
924
+ return;
925
+ const timer = setTimeout(() => {
926
+ pendingChallenges.delete(ws);
927
+ logger.warn `Challenge timed out for sidecar ${sidecarId}`;
928
+ }, challengeTimeoutMs);
929
+ pendingChallenges.set(ws, {
930
+ sidecarId,
931
+ challenges,
932
+ deployRefs,
933
+ timer,
934
+ });
935
+ conn.send({ type: "challenge", challenges: challengeEntries });
936
+ }
937
+ async function handleChallengeResponse(ws, responses) {
938
+ const challenge = pendingChallenges.get(ws);
939
+ if (challenge === undefined) {
940
+ logger.warn `Received challenge.response with no pending challenge`;
941
+ return;
942
+ }
943
+ clearTimeout(challenge.timer);
944
+ pendingChallenges.delete(ws);
945
+ const conn = connections.get(ws);
946
+ if (conn === undefined)
947
+ return;
948
+ const verified = [];
949
+ const responded = new Set();
950
+ for (const { address, signature } of responses) {
951
+ responded.add(address);
952
+ const entry = challenge.challenges.get(address);
953
+ if (entry === undefined) {
954
+ conn.send({
955
+ type: "challenge.failed",
956
+ address,
957
+ reason: "Address was not challenged",
958
+ });
959
+ continue;
960
+ }
961
+ let valid = false;
962
+ try {
963
+ const nonceBytes = entry.nonce;
964
+ const addressBytes = new TextEncoder().encode(address);
965
+ const payload = new Uint8Array(nonceBytes.length + addressBytes.length);
966
+ payload.set(nonceBytes);
967
+ payload.set(addressBytes, nonceBytes.length);
968
+ const sigBytes = hexDecode(signature);
969
+ valid = await verifyEd25519(payload, sigBytes, entry.publicKey);
970
+ }
971
+ catch (err) {
972
+ logger.warn `Challenge failed for ${address}: ${err instanceof Error ? err.message : String(err)}`;
973
+ }
974
+ if (valid) {
975
+ verified.push(address);
976
+ }
977
+ else {
978
+ conn.send({
979
+ type: "challenge.failed",
980
+ address,
981
+ reason: "Signature verification failed",
982
+ });
983
+ }
984
+ }
985
+ // Notify about challenged addresses that were omitted from the response.
986
+ for (const address of challenge.challenges.keys()) {
987
+ if (!responded.has(address)) {
988
+ conn.send({
989
+ type: "challenge.failed",
990
+ address,
991
+ reason: "No response provided for challenged address",
992
+ });
993
+ logger.warn `Challenge failed for ${address}: no response provided`;
994
+ }
995
+ }
996
+ // Add verified addresses to the routing table immediately so that
997
+ // `agent.reconnected` subscribers can use sendRequest-based methods
998
+ // (e.g. sendSourcesUpdate). Addresses that fail governance are
999
+ // rolled back from the routing table afterward.
1000
+ for (const addr of verified) {
1001
+ // If a different ws still owns this address (live takeover via
1002
+ // verified reconnect), evict its cached connector state before
1003
+ // routing flips. The new owner's harness will bootstrap via
1004
+ // restore-fires-callback.
1005
+ const prevWs = addressIndex.get(addr);
1006
+ if (prevWs !== undefined && prevWs !== ws) {
1007
+ connectorStates.delete(addr);
1008
+ // Evict the reclaimed address from the superseded connection's owned
1009
+ // set. handleClose's cancelByAgent sweep iterates a connection's owned
1010
+ // union WITHOUT an ownership guard, so if the stale connection still
1011
+ // listed this address it would cancel THIS connection's in-flight pack
1012
+ // transfer for it when it finally closes. Delete from both sets: a
1013
+ // workflow-derived address lives on the workflow set, a launched agent
1014
+ // on the session set, and delete is a no-op for the absent one.
1015
+ const prevConn = connections.get(prevWs);
1016
+ if (prevConn !== undefined) {
1017
+ prevConn.workflowAddresses.delete(addr);
1018
+ prevConn.agentAddresses.delete(addr);
1019
+ }
1020
+ }
1021
+ // A run address goes on the workflow set (no disconnect queue -- its
1022
+ // in-flight state is reconstructed sidecar-locally on the next reconnect),
1023
+ // so handleClose reclaims it correctly. The routing pointer is the same
1024
+ // either way; only now it is written behind a passed challenge.
1025
+ //
1026
+ // The `else` (session set, queued for reconnect) is the retired
1027
+ // launched-agent path: launched agents no longer exist (the folded-launch
1028
+ // route was removed), so no current producer reaches it. It is left in
1029
+ // place for the reconnect-subsystem teardown to remove as its own
1030
+ // reviewable change, not folded into this collapse.
1031
+ if (isRunAddress(addr)) {
1032
+ conn.workflowAddresses.add(addr);
1033
+ }
1034
+ else {
1035
+ conn.agentAddresses.add(addr);
1036
+ }
1037
+ addressIndex.set(addr, ws);
1038
+ }
1039
+ const ready = [];
1040
+ const failed = [];
1041
+ for (const addr of verified) {
1042
+ // A workflow run needs routing + queue flush only, which the passed
1043
+ // challenge has now made safe: its in-flight state is reconstructed
1044
+ // sidecar-locally, so it does not enrol in the `agent.reconnected` session
1045
+ // reaction (event-collector restore). Skip the reaction for it.
1046
+ //
1047
+ // The reaction below was the retired launched-agent path (restore a
1048
+ // launched agent's collector). Launched agents no longer exist, so no
1049
+ // current producer reaches it; the `agent.reconnected` listener is gone,
1050
+ // so `listenerCount` is 0 and the emit is inert. Left for the
1051
+ // reconnect-subsystem teardown to remove.
1052
+ if (isRunAddress(addr)) {
1053
+ ready.push(addr);
1054
+ continue;
1055
+ }
1056
+ if (events.listenerCount("agent.reconnected") === 0) {
1057
+ ready.push(addr);
1058
+ continue;
1059
+ }
1060
+ try {
1061
+ await events.emitAndAwait("agent.reconnected", { agentAddress: addr });
1062
+ ready.push(addr);
1063
+ }
1064
+ catch (err) {
1065
+ logger.error `Failed to handle reconnection for ${addr}: ${err instanceof Error ? err.message : String(err)}`;
1066
+ failed.push(addr);
1067
+ }
1068
+ }
1069
+ // If a second reconnect arrived during the callback loop, our conn
1070
+ // is orphaned — handleRegister already rebuilt the connection and
1071
+ // cleared addressIndex. Bail out; the new reconnect flow will
1072
+ // re-verify these addresses from scratch.
1073
+ if (connections.get(ws) !== conn) {
1074
+ logger.warn `Challenge response processing aborted: connection superseded by new reconnect`;
1075
+ return;
1076
+ }
1077
+ // Roll back failed addresses from the routing table. Only the session set
1078
+ // is touched: a workflow-derived address can never be in `failed` -- it
1079
+ // early-`continue`s to `ready` above, before the reaction that populates
1080
+ // `failed` -- so it is never on the workflow set at this point.
1081
+ for (const addr of failed) {
1082
+ conn.agentAddresses.delete(addr);
1083
+ addressIndex.delete(addr);
1084
+ }
1085
+ // Flush queued messages and redeliver retained un-acked mail, only for
1086
+ // ready addresses. The disconnect queue carries mail that arrived WHILE
1087
+ // disconnected; pending-mail redelivery carries mail that was delivered
1088
+ // over the prior live connection but never acked (the connected-window
1089
+ // drop). Both replay over the verified new connection.
1090
+ for (const addr of ready) {
1091
+ flushDisconnectedQueue(addr, conn);
1092
+ redeliverPendingMail(addr, conn);
1093
+ }
1094
+ // Re-deploy agents whose deploy ref is stale or absent. Fire-and-forget
1095
+ // so reconnect completion is not blocked on pack transfer. The
1096
+ // wire layer owns the staleness comparison; the event fires only
1097
+ // when staleness is confirmed.
1098
+ const checkDeployRef = lookups.lookupDeployRef;
1099
+ if (checkDeployRef !== undefined) {
1100
+ for (const addr of ready) {
1101
+ // Workflow deployments are pinned-forever: a deployment keeps its
1102
+ // deploy-time definition until an explicit undeploy/redeploy, so the
1103
+ // deploy-ref freshness catch-up is deliberately NOT run for a run
1104
+ // address. A definition edited on the hub while the sidecar was
1105
+ // disconnected does not reconcile on reconnect; it affects only newly
1106
+ // created deployments. The deployment's in-flight run state is
1107
+ // reconstructed sidecar-locally at restore, not re-fetched here. Do NOT
1108
+ // add a reconcile path for these addresses -- see the "Workflow
1109
+ // Definition Versioning: Pinned-Forever" note under "Reconnect
1110
+ // Sequencing" in docs/IMPLEMENTATION.md. (Every routable address is a
1111
+ // run address now; the fall-through was the retired launched-agent
1112
+ // catch-up path.)
1113
+ if (isRunAddress(addr))
1114
+ continue;
1115
+ void (async () => {
1116
+ try {
1117
+ const hubRef = await checkDeployRef(addr);
1118
+ if (hubRef === null)
1119
+ return;
1120
+ const sidecarRef = challenge.deployRefs[addr];
1121
+ if (sidecarRef === hubRef)
1122
+ return;
1123
+ logger.info `Re-deploying ${addr}: sidecar ref ${sidecarRef ?? "(none)"} != hub ref ${hubRef.slice(0, 8)}`;
1124
+ await events.emitAndAwait("deploy.ref.stale", {
1125
+ agentAddress: addr,
1126
+ });
1127
+ }
1128
+ catch (err) {
1129
+ logger.error `Failed to re-deploy ${addr} after reconnect: ${err instanceof Error ? err.message : String(err)}`;
1130
+ }
1131
+ })();
1132
+ }
1133
+ }
1134
+ // Failed addresses: reset their queue TTL so messages survive until
1135
+ // the next reconnect attempt, and notify the sidecar.
1136
+ for (const addr of failed) {
1137
+ const entry = disconnectedAgents.get(addr);
1138
+ if (entry !== undefined) {
1139
+ clearTimeout(entry.timer);
1140
+ entry.timer = setTimeout(() => {
1141
+ const expired = disconnectedAgents.get(addr);
1142
+ disconnectedAgents.delete(addr);
1143
+ if (expired !== undefined) {
1144
+ surfaceDroppedFrames(addr, expired.queue, "disconnect queue TTL expired");
1145
+ }
1146
+ }, disconnectQueueTTLMs);
1147
+ }
1148
+ conn.send({
1149
+ type: "challenge.failed",
1150
+ address: addr,
1151
+ reason: "Reconnection rejected by governance",
1152
+ });
1153
+ }
1154
+ logger.info `Sidecar ${challenge.sidecarId} reconnected with ${String(ready.length)} verified agent(s)${failed.length > 0 ? `, ${String(failed.length)} rejected` : ""}`;
1155
+ }
1156
+ async function handleMailOutbound(rawMessage, recipients) {
1157
+ // A mail addressed to more than one workflow deployment would birth a
1158
+ // run per recipient from a single inbound mail. The stable runId
1159
+ // removed the Message-ID collision that originally forced this guard --
1160
+ // each recipient now derives its own per-deployment runId (its mail
1161
+ // address), so it is no longer a runId-collision guard. It stays a
1162
+ // deliberate one-workflow-recipient-per-mail restriction because the
1163
+ // fan-out is not verified end-to-end: per-recipient grants
1164
+ // materialization, consumed-tracking, and reply-addressing all assume a
1165
+ // single workflow recipient today. Lifting it means proving those three
1166
+ // hold per recipient, not just relaxing this check -- so fail loudly
1167
+ // rather than materialize a partial set. The guard only applies when a
1168
+ // materializer is wired -- absent one, no run is born from the mail, so
1169
+ // there is nothing to restrict.
1170
+ if (lookups.materializeMailTriggeredRunGrants !== undefined) {
1171
+ // A workflow recipient is one this hub owns: its address parses as a run
1172
+ // address. An external/federated address does not, and is not ours to
1173
+ // materialize a run for.
1174
+ const workflowRecipients = recipients.filter(isRunAddress);
1175
+ if (workflowRecipients.length > 1) {
1176
+ throw new Error(`mail addressed to multiple workflow-derived recipients (${workflowRecipients.join(", ")}); materializing a run for more than one workflow deployment from a single mail is unsupported`);
1177
+ }
1178
+ }
1179
+ // Route to locally connected sidecars first, then try disconnect queues.
1180
+ const unrouted = [];
1181
+ for (const recipient of recipients) {
1182
+ // Each recipient is isolated: a materialization failure or a
1183
+ // fail-closed rejection for one must not drop the mail for its
1184
+ // co-recipients. The catch fails THIS recipient closed (its run never
1185
+ // starts under-authorized) and continues to the rest.
1186
+ try {
1187
+ const outcome = await deliverMailToRecipient(recipient, rawMessage);
1188
+ if (outcome === "unrouted")
1189
+ unrouted.push(recipient);
1190
+ }
1191
+ catch (err) {
1192
+ logger.error `Failed to deliver mail to ${recipient}: ${err instanceof Error ? err.message : String(err)}`;
1193
+ }
1194
+ }
1195
+ // Anything not routed locally is emitted as a notification. The
1196
+ // host decides whether to relay onto an external transport, log,
1197
+ // or drop. The wire layer takes no stance.
1198
+ if (unrouted.length > 0) {
1199
+ events.emit("mail.outbound.undelivered", {
1200
+ rawMessage,
1201
+ recipients: unrouted,
1202
+ });
1203
+ }
1204
+ }
1205
+ // Deliver an inbound mail to one recipient, materializing a
1206
+ // mail-triggered run's grants first when the recipient is a workflow
1207
+ // deployment. Returns:
1208
+ // - `routed`: the mail reached a live connection or disconnect queue.
1209
+ // - `unrouted`: the mail was locally undeliverable and should be
1210
+ // relayed externally by the host.
1211
+ // - `failed-closed`: the run's grants could not be materialized safely,
1212
+ // so the mail is deliberately DROPPED for this recipient (not relayed)
1213
+ // to keep its run from starting under-authorized.
1214
+ //
1215
+ // A workflow deployment is the only recipient whose inbound mail can first
1216
+ // fire its stable run. Its grants are reserved, and the `run.grants` frame is
1217
+ // sent BEFORE the mail. Same-address FIFO guarantees it lands ahead of the
1218
+ // mail that dispatches the run, so the run's `onRunStart` barrier resolves its
1219
+ // grants rather than failing closed. Reservation happens before routing so
1220
+ // concurrent first deliveries cannot send different snapshots; a routing
1221
+ // failure leaves a grants-only, still-unfired run.
1222
+ async function deliverMailToRecipient(recipient, rawMessage) {
1223
+ if (lookups.materializeMailTriggeredRunGrants !== undefined &&
1224
+ isRunAddress(recipient)) {
1225
+ const runId = deriveWorkflowRunId(recipient);
1226
+ const result = await lookups.materializeMailTriggeredRunGrants({
1227
+ agentAddress: recipient,
1228
+ runId,
1229
+ });
1230
+ if (result.outcome === "rejected") {
1231
+ // The run's grants could not be materialized with sufficient
1232
+ // authority or it is already terminal. Fail the mail closed for this
1233
+ // recipient: routing or external relay would bypass that decision.
1234
+ logger.error `Refusing mail-triggered run ${runId} for ${recipient}: grant materialization rejected (${result.code}): ${result.message}`;
1235
+ return "failed-closed";
1236
+ }
1237
+ if (result.outcome === "materialized") {
1238
+ // Send the run's grants ahead of the mail. A `false` here means the
1239
+ // deployment is unroutable. Do not route the mail that would dispatch
1240
+ // it; the grants-only reservation remains the canonical snapshot for a
1241
+ // later first-delivery attempt.
1242
+ if (!sendRunGrants(recipient, runId, result.stepGrants)) {
1243
+ logger.error `Deployment ${recipient} is not routable for run ${runId}; retaining the unfired run's grant reservation for retry`;
1244
+ return "unrouted";
1245
+ }
1246
+ // Route through the messageId handshake `routeMail` -- NOT a
1247
+ // fire-and-forget send. This branch COMMITS a run, so a mail dropped in
1248
+ // the connected window (a socket that half-dies before the sidecar's
1249
+ // durable-write ack) would otherwise leave the run row "running"
1250
+ // forever with no body and no error. `routeMail` tracks the delivery
1251
+ // and redelivers identical bytes on reconnect, bringing the mail-relay
1252
+ // run-trigger to parity with the HTTP-trigger path. The messageId is the
1253
+ // mail's own id (derived over the same bytes the sidecar derives), so a
1254
+ // redelivery replays identically and the downstream RunStarted /
1255
+ // stable-runId dedup makes it effectively-once.
1256
+ const messageId = await deriveMessageId(base64Decode(rawMessage));
1257
+ const outcome = routeMail(recipient, rawMessage, messageId, { runId, stepGrants: result.stepGrants })
1258
+ ? "routed"
1259
+ : "unrouted";
1260
+ return outcome;
1261
+ }
1262
+ // `skip`: the address named no deployed workflow deployment. Forward
1263
+ // the mail without grants -- the run, if any, is not ours to
1264
+ // authorize. No run is committed here, so no ack handshake is needed.
1265
+ }
1266
+ return routeMail(recipient, rawMessage) ? "routed" : "unrouted";
1267
+ }
1268
+ async function handleMailPersist(persist, rawMessage, senderAddress, recipients) {
1269
+ let results;
1270
+ let raw;
1271
+ try {
1272
+ raw = Uint8Array.from(atob(rawMessage), (c) => c.charCodeAt(0));
1273
+ results = await persist({
1274
+ senderAddress,
1275
+ recipients,
1276
+ raw,
1277
+ });
1278
+ }
1279
+ catch (err) {
1280
+ logger.error `Failed to persist mail from ${senderAddress}: ${err instanceof Error ? err.message : String(err)}`;
1281
+ return;
1282
+ }
1283
+ for (const result of results) {
1284
+ events.emit("mail.persisted", {
1285
+ id: result.id,
1286
+ raw,
1287
+ createdAt: result.createdAt,
1288
+ direction: result.direction,
1289
+ runId: result.runId,
1290
+ address: result.address,
1291
+ });
1292
+ }
1293
+ }
1294
+ async function handleSignalCorrelationRegister(ws, frame) {
1295
+ // Gate the co-write on the sending sidecar actually owning the named
1296
+ // deployment address, mirroring the connector.state.changed and pack
1297
+ // handlers. A workflow deployment routes on the keyless workflow set, so
1298
+ // ownership is the union check, not addressIndex identity alone. Without
1299
+ // it a misbehaving sidecar that knows another deployment's address could
1300
+ // register a spurious correlation against it.
1301
+ const conn = connections.get(ws);
1302
+ if (conn === undefined)
1303
+ return;
1304
+ if (!connOwnsAddress(conn, frame.agentAddress)) {
1305
+ logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: not registered to this sidecar`;
1306
+ return;
1307
+ }
1308
+ const register = lookups.registerSignalCorrelation;
1309
+ if (register === undefined) {
1310
+ logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: no registerSignalCorrelation lookup configured`;
1311
+ return;
1312
+ }
1313
+ try {
1314
+ await register({
1315
+ correlationId: frame.correlationId,
1316
+ runId: frame.runId,
1317
+ anchorRunId: frame.anchorRunId,
1318
+ agentAddress: frame.agentAddress,
1319
+ kind: frame.kind,
1320
+ approvalSnapshot: frame.snapshot,
1321
+ });
1322
+ // The co-write resolves only when a row exists -- freshly inserted or
1323
+ // already present (both stores are idempotent on the correlationId). Ack
1324
+ // so the sidecar's link stops retrying a register whose frame may have
1325
+ // been lost on an open socket. A thrown co-write (undeployed deployment,
1326
+ // id mismatch) means no row, so no ack: the sidecar keeps retrying and
1327
+ // the reconnect re-emit remains the ultimate backstop.
1328
+ conn.send({
1329
+ type: "signal.correlation.register.ack",
1330
+ agentAddress: frame.agentAddress,
1331
+ correlationId: frame.correlationId,
1332
+ });
1333
+ }
1334
+ catch (err) {
1335
+ logger.error `Failed to register signal correlation ${frame.correlationId} for ${frame.agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
1336
+ }
1337
+ }
1338
+ function handleClose(ws) {
1339
+ const conn = connections.get(ws);
1340
+ if (conn === undefined)
1341
+ return;
1342
+ let allocated;
1343
+ for (const addr of conn.agentAddresses) {
1344
+ // Only remove routing and pending state if this connection still
1345
+ // owns the address. A reconnected sidecar may have already claimed it.
1346
+ if (addressIndex.get(addr) === ws) {
1347
+ addressIndex.delete(addr);
1348
+ // Drop cached connector state for the same reason: a takeover
1349
+ // sidecar's state lives in connectorStates under the same key,
1350
+ // and only this owner's close should evict it. The next
1351
+ // reconnect re-bootstraps via the router's
1352
+ // restore-fires-callback path.
1353
+ connectorStates.delete(addr);
1354
+ // Retain this address's un-acked pending mail across the disconnect:
1355
+ // its in-flight retry timers target a dead socket (cleared), but the
1356
+ // entries are held so a verified reconnect redelivers them, closing the
1357
+ // connected-window drop rather than losing the mail. Bounded by a
1358
+ // retention TTL.
1359
+ retainPendingMailForAddress(addr);
1360
+ // Create a queue entry so messages can accumulate while the
1361
+ // sidecar is disconnected. Skip if the agent is being undeployed --
1362
+ // there is no point queuing messages for an agent being torn down.
1363
+ if (!pendingUndeploys.has(addr)) {
1364
+ const timer = setTimeout(() => {
1365
+ const expired = disconnectedAgents.get(addr);
1366
+ disconnectedAgents.delete(addr);
1367
+ if (expired !== undefined) {
1368
+ surfaceDroppedFrames(addr, expired.queue, "disconnect queue TTL expired");
1369
+ }
1370
+ }, disconnectQueueTTLMs);
1371
+ disconnectedAgents.set(addr, { queue: [], timer });
1372
+ }
1373
+ }
1374
+ }
1375
+ // Remove this connection's workflow-substrate routes. No disconnect queue
1376
+ // is created: these addresses re-register (with the complete live set)
1377
+ // when the sidecar reconnects, and their in-flight run state is
1378
+ // reconstructed sidecar-locally, not from a hub-side queue. The ownership
1379
+ // guard mirrors the session loop above so a takeover by a newer ws is not
1380
+ // clobbered by the prior owner's close.
1381
+ for (const addr of conn.workflowAddresses) {
1382
+ if (addressIndex.get(addr) === ws) {
1383
+ addressIndex.delete(addr);
1384
+ // Retain un-acked workflow trigger mail across the disconnect for the
1385
+ // same reason as the session loop above -- a challenged reconnect
1386
+ // redelivers it. This is un-acked TRIGGER mail, distinct from the
1387
+ // deployment's in-flight run state (reconstructed sidecar-locally); the
1388
+ // "no disconnect queue" note above is about that run state, not this.
1389
+ retainPendingMailForAddress(addr);
1390
+ }
1391
+ }
1392
+ if (conn.identity.kind === "allocated") {
1393
+ const current = allocatedConnections.get(conn.identity.allocationId);
1394
+ if (current?.ws === ws) {
1395
+ allocatedConnections.delete(conn.identity.allocationId);
1396
+ allocated = {
1397
+ allocationId: conn.identity.allocationId,
1398
+ generation: conn.identity.generation,
1399
+ };
1400
+ }
1401
+ }
1402
+ connections.delete(ws);
1403
+ // Cancel the liveness timer for this connection.
1404
+ const livenessTimer = livenessTimers.get(ws);
1405
+ if (livenessTimer !== undefined) {
1406
+ clearTimeout(livenessTimer);
1407
+ livenessTimers.delete(ws);
1408
+ }
1409
+ // Drop the per-ws serialization chain; no more frames will queue on it.
1410
+ messageChains.delete(ws);
1411
+ // Cancel any pending challenge for this connection.
1412
+ const challengeReq = pendingChallenges.get(ws);
1413
+ if (challengeReq !== undefined) {
1414
+ clearTimeout(challengeReq.timer);
1415
+ pendingChallenges.delete(ws);
1416
+ }
1417
+ // Reject any in-flight requests that were sent to this sidecar.
1418
+ for (const [requestId, req] of pending) {
1419
+ if (req.ws !== ws)
1420
+ continue;
1421
+ clearTimeout(req.timer);
1422
+ pending.delete(requestId);
1423
+ req.reject(`Sidecar ${conn.sidecarId} disconnected`);
1424
+ }
1425
+ // Reject every deploy issued on this socket, including allocated
1426
+ // workflow deployments stored in `workflowAddresses` rather than
1427
+ // `agentAddresses`.
1428
+ for (const [agentAddress, req] of pendingDeploys) {
1429
+ if (req.ws !== ws)
1430
+ continue;
1431
+ clearTimeout(req.timer);
1432
+ pendingDeploys.delete(agentAddress);
1433
+ req.reject(`Sidecar ${conn.sidecarId} disconnected`);
1434
+ }
1435
+ // Reject any in-flight pack transfers for this sidecar.
1436
+ for (const [transferId, pack] of pendingPacks) {
1437
+ if (pack.ws !== ws)
1438
+ continue;
1439
+ clearTimeout(pack.timer);
1440
+ pendingPacks.delete(transferId);
1441
+ pack.reject(`Sidecar ${conn.sidecarId} disconnected`);
1442
+ }
1443
+ // Reject any in-flight undeploys for this sidecar.
1444
+ for (const [addr, req] of pendingUndeploys) {
1445
+ if (req.ws !== ws)
1446
+ continue;
1447
+ clearTimeout(req.timer);
1448
+ pendingUndeploys.delete(addr);
1449
+ req.reject(`Sidecar ${conn.sidecarId} disconnected`);
1450
+ }
1451
+ // Reject any in-flight probes sent to this sidecar. A probe never enters
1452
+ // the address maps, so this ws-keyed sweep is its ONLY disconnect cleanup:
1453
+ // without it a probe whose sidecar drops mid-flight would hang until its
1454
+ // own timeout instead of failing fast on the disconnect.
1455
+ for (const [requestId, probe] of pendingProbes) {
1456
+ if (probe.ws !== ws)
1457
+ continue;
1458
+ clearTimeout(probe.timer);
1459
+ pendingProbes.delete(requestId);
1460
+ probe.reject(`Sidecar ${conn.sidecarId} disconnected`);
1461
+ }
1462
+ // Cancel any in-flight inbound pack transfers from this sidecar
1463
+ // across both receivers. The two receivers track their own in-
1464
+ // flight transferIds, so a pending workflow-run transfer for an
1465
+ // agent that just disconnected won't outlive the connection just
1466
+ // because the agent-state receiver has nothing to cancel. Iterate the
1467
+ // owned union so a reconnected workflow deployment's transfer is
1468
+ // cancelled too; the deduped set avoids a double-cancel for an address
1469
+ // that is in both sets. A reclaimed address is not present here -- the
1470
+ // verified reconnect path that took it over evicts it from this
1471
+ // (superseded) connection's owned set -- so a stale close does not cancel
1472
+ // the new owner's work.
1473
+ const owned = ownedAddresses(conn);
1474
+ for (const addr of owned) {
1475
+ agentStatePackReceiver.cancelByAgent(addr);
1476
+ workflowRunPackReceiver.cancelByAgent(addr);
1477
+ }
1478
+ events.emit("sidecar.disconnect", {
1479
+ ownedAddresses: [...owned],
1480
+ ...(allocated !== undefined ? { allocated } : {}),
1481
+ });
1482
+ logger.info `Sidecar ${conn.sidecarId} disconnected`;
1483
+ }
1484
+ function nextRequestId() {
1485
+ return `req-${++requestCounter}`;
1486
+ }
1487
+ function sendRequest(agentAddress, buildFrame) {
1488
+ const ws = addressIndex.get(agentAddress);
1489
+ if (ws === undefined) {
1490
+ return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
1491
+ }
1492
+ const conn = connections.get(ws);
1493
+ if (conn === undefined) {
1494
+ return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
1495
+ }
1496
+ const requestId = nextRequestId();
1497
+ const frame = buildFrame(requestId);
1498
+ return new Promise((resolve, reject) => {
1499
+ const timer = setTimeout(() => {
1500
+ pending.delete(requestId);
1501
+ reject(new Error(`Request ${requestId} timed out after ${requestTimeoutMs}ms`));
1502
+ }, requestTimeoutMs);
1503
+ pending.set(requestId, {
1504
+ requestId,
1505
+ ws,
1506
+ resolve,
1507
+ reject(error) {
1508
+ reject(new Error(error));
1509
+ },
1510
+ timer,
1511
+ });
1512
+ conn.send(frame);
1513
+ });
1514
+ }
1515
+ function resolvePending(requestId) {
1516
+ const req = pending.get(requestId);
1517
+ if (req === undefined)
1518
+ return;
1519
+ clearTimeout(req.timer);
1520
+ pending.delete(requestId);
1521
+ req.resolve();
1522
+ }
1523
+ function rejectPending(requestId, error) {
1524
+ const req = pending.get(requestId);
1525
+ if (req === undefined)
1526
+ return;
1527
+ clearTimeout(req.timer);
1528
+ pending.delete(requestId);
1529
+ req.reject(error);
1530
+ }
1531
+ function packResponseMatches(entry, ws, frame) {
1532
+ return (entry.ws === ws &&
1533
+ entry.agentAddress === frame.agentAddress &&
1534
+ entry.repoId.kind === frame.repoId.kind &&
1535
+ entry.repoId.id === frame.repoId.id);
1536
+ }
1537
+ function resolvePackPending(ws, frame) {
1538
+ const entry = pendingPacks.get(frame.transferId);
1539
+ if (entry === undefined)
1540
+ return;
1541
+ if (!packResponseMatches(entry, ws, frame)) {
1542
+ logger.warn `Ignoring repo.pack.ack for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
1543
+ return;
1544
+ }
1545
+ clearTimeout(entry.timer);
1546
+ pendingPacks.delete(frame.transferId);
1547
+ entry.resolve();
1548
+ }
1549
+ function rejectPackPending(ws, frame) {
1550
+ const entry = pendingPacks.get(frame.transferId);
1551
+ if (entry === undefined)
1552
+ return;
1553
+ if (!packResponseMatches(entry, ws, frame)) {
1554
+ logger.warn `Ignoring repo.pack.reject for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
1555
+ return;
1556
+ }
1557
+ clearTimeout(entry.timer);
1558
+ pendingPacks.delete(frame.transferId);
1559
+ // Surface the receiver's specific cause when it carried one, so the awaiting
1560
+ // push sees "corrupt: <detail>" rather than only the coarse reason.
1561
+ entry.reject(frame.detail !== undefined
1562
+ ? `${frame.reason}: ${frame.detail}`
1563
+ : frame.reason);
1564
+ }
1565
+ function resolveUndeployPending(ws, agentAddress) {
1566
+ const req = pendingUndeploys.get(agentAddress);
1567
+ if (req === undefined) {
1568
+ logger.warn `Received agent.undeploy.ack for "${agentAddress}" with no pending undeploy`;
1569
+ return;
1570
+ }
1571
+ if (req.ws !== ws)
1572
+ return;
1573
+ clearTimeout(req.timer);
1574
+ pendingUndeploys.delete(agentAddress);
1575
+ req.resolve();
1576
+ }
1577
+ function rejectUndeployPending(ws, agentAddress, error) {
1578
+ const req = pendingUndeploys.get(agentAddress);
1579
+ if (req === undefined)
1580
+ return;
1581
+ if (req.ws !== ws)
1582
+ return;
1583
+ clearTimeout(req.timer);
1584
+ pendingUndeploys.delete(agentAddress);
1585
+ req.reject(error);
1586
+ }
1587
+ function resolveProbe(requestId, result) {
1588
+ const req = pendingProbes.get(requestId);
1589
+ if (req === undefined)
1590
+ return;
1591
+ clearTimeout(req.timer);
1592
+ pendingProbes.delete(requestId);
1593
+ req.resolve(result);
1594
+ }
1595
+ function rejectProbe(requestId, error) {
1596
+ const req = pendingProbes.get(requestId);
1597
+ if (req === undefined)
1598
+ return;
1599
+ clearTimeout(req.timer);
1600
+ pendingProbes.delete(requestId);
1601
+ req.reject(error);
1602
+ }
1603
+ // Routing rule: pick the receiver dedicated to the repoId.kind the
1604
+ // frame carries. The receivers' in-flight state is independent, so a
1605
+ // workflow-run transferId can never collide with or evict an
1606
+ // agent-state transferId for the same agentAddress.
1607
+ function pickPackReceiver(repoId) {
1608
+ switch (repoId.kind) {
1609
+ case "agent-state":
1610
+ return { receiver: agentStatePackReceiver };
1611
+ case "workflow-run":
1612
+ return { receiver: workflowRunPackReceiver };
1613
+ // The remaining kinds in `RepoKind` (`skill`, `package-registry`,
1614
+ // `workflow`) have no sidecar->hub pack flow today. A frame
1615
+ // arriving with those kinds is malformed at this layer.
1616
+ default:
1617
+ return null;
1618
+ }
1619
+ }
1620
+ function pickReceivePackLookup(repoId) {
1621
+ switch (repoId.kind) {
1622
+ case "agent-state":
1623
+ // The agent-state lookup ignores the `source` argument the workflow-run
1624
+ // lookup takes; the two are otherwise the same contract.
1625
+ return lookups.receiveAgentStatePack;
1626
+ case "workflow-run":
1627
+ return lookups.receiveWorkflowRunPack;
1628
+ default:
1629
+ return undefined;
1630
+ }
1631
+ }
1632
+ function handlePackPush(ws, frame) {
1633
+ const conn = connections.get(ws);
1634
+ if (conn === undefined)
1635
+ return;
1636
+ if (!connOwnsAddress(conn, frame.agentAddress)) {
1637
+ logger.warn `Received repo.pack.push for unrouted agent ${frame.agentAddress}`;
1638
+ return;
1639
+ }
1640
+ if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
1641
+ logger.warn `Rejected repo.pack.push outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
1642
+ conn.send({
1643
+ type: "repo.pack.reject",
1644
+ agentAddress: frame.agentAddress,
1645
+ repoId: frame.repoId,
1646
+ transferId: frame.transferId,
1647
+ reason: "path_violation",
1648
+ });
1649
+ return;
1650
+ }
1651
+ const picked = pickPackReceiver(frame.repoId);
1652
+ if (picked === null) {
1653
+ logger.warn `Received repo.pack.push with unsupported repoId.kind ${frame.repoId.kind}`;
1654
+ conn.send({
1655
+ type: "repo.pack.reject",
1656
+ agentAddress: frame.agentAddress,
1657
+ repoId: frame.repoId,
1658
+ transferId: frame.transferId,
1659
+ reason: "corrupt",
1660
+ });
1661
+ return;
1662
+ }
1663
+ const reason = picked.receiver.handlePush(frame);
1664
+ if (reason !== null) {
1665
+ conn.send({
1666
+ type: "repo.pack.reject",
1667
+ agentAddress: frame.agentAddress,
1668
+ repoId: frame.repoId,
1669
+ transferId: frame.transferId,
1670
+ reason,
1671
+ });
1672
+ }
1673
+ }
1674
+ async function handlePackDone(ws, frame) {
1675
+ const conn = connections.get(ws);
1676
+ if (conn === undefined)
1677
+ return;
1678
+ if (!connOwnsAddress(conn, frame.agentAddress)) {
1679
+ logger.warn `Received repo.pack.done for unrouted agent ${frame.agentAddress}`;
1680
+ return;
1681
+ }
1682
+ if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
1683
+ logger.warn `Rejected repo.pack.done outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
1684
+ conn.send({
1685
+ type: "repo.pack.reject",
1686
+ agentAddress: frame.agentAddress,
1687
+ repoId: frame.repoId,
1688
+ transferId: frame.transferId,
1689
+ reason: "path_violation",
1690
+ });
1691
+ return;
1692
+ }
1693
+ const picked = pickPackReceiver(frame.repoId);
1694
+ if (picked === null) {
1695
+ logger.warn `Received repo.pack.done with unsupported repoId.kind ${frame.repoId.kind}`;
1696
+ conn.send({
1697
+ type: "repo.pack.reject",
1698
+ agentAddress: frame.agentAddress,
1699
+ repoId: frame.repoId,
1700
+ transferId: frame.transferId,
1701
+ reason: "corrupt",
1702
+ });
1703
+ return;
1704
+ }
1705
+ const result = picked.receiver.handleDone(frame);
1706
+ if (result === null) {
1707
+ conn.send({
1708
+ type: "repo.pack.reject",
1709
+ agentAddress: frame.agentAddress,
1710
+ repoId: frame.repoId,
1711
+ transferId: frame.transferId,
1712
+ reason: "corrupt",
1713
+ });
1714
+ return;
1715
+ }
1716
+ const receivePackLookup = pickReceivePackLookup(frame.repoId);
1717
+ if (receivePackLookup === undefined) {
1718
+ conn.send({
1719
+ type: "repo.pack.ack",
1720
+ agentAddress: frame.agentAddress,
1721
+ repoId: frame.repoId,
1722
+ transferId: frame.transferId,
1723
+ });
1724
+ return;
1725
+ }
1726
+ const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha, conn.identity.kind === "allocated"
1727
+ ? {
1728
+ kind: "allocated",
1729
+ agentAddress: frame.agentAddress,
1730
+ allocationId: conn.identity.allocationId,
1731
+ anchorRunId: conn.identity.anchorRunId,
1732
+ generation: conn.identity.generation,
1733
+ }
1734
+ : { kind: "shared", agentAddress: frame.agentAddress });
1735
+ // Connection may have closed during async verification.
1736
+ const currentConn = connections.get(ws);
1737
+ if (currentConn === undefined)
1738
+ return;
1739
+ if (verdict.accepted) {
1740
+ currentConn.send({
1741
+ type: "repo.pack.ack",
1742
+ agentAddress: frame.agentAddress,
1743
+ repoId: frame.repoId,
1744
+ transferId: frame.transferId,
1745
+ });
1746
+ }
1747
+ else {
1748
+ currentConn.send({
1749
+ type: "repo.pack.reject",
1750
+ agentAddress: frame.agentAddress,
1751
+ repoId: frame.repoId,
1752
+ transferId: frame.transferId,
1753
+ reason: verdict.reason,
1754
+ });
1755
+ }
1756
+ }
1757
+ /**
1758
+ * Bind a per-step workflow-substrate address to a sidecar for the staging
1759
+ * window of a multi-step deploy, so `sendPack` can route the step's deploy
1760
+ * and asset packs before the deployment-level frame spawns the child.
1761
+ *
1762
+ * The address is hub-minted and workflow-derived (no per-address key), so
1763
+ * it enters the keyless `workflowAddresses` set -- never the challenged
1764
+ * `agentAddresses` set -- and is torn down by `unbindStepRoute` once the
1765
+ * step's packs land. `handleClose` reclaims it if the sidecar drops
1766
+ * mid-stage. Per-step addresses are not runtime-routed (mail, signals, and
1767
+ * drains use the deployment address), so the binding is transient: it is
1768
+ * never persisted into the reconnect set and never resurrected on
1769
+ * reconnect.
1770
+ */
1771
+ function fenceAllocation(allocationId, generation) {
1772
+ const existing = allocationFences.get(allocationId);
1773
+ if (existing !== undefined && generation < existing) {
1774
+ throw new Error(`Cannot move allocation ${allocationId} fence backward from ${String(existing)} to ${String(generation)}`);
1775
+ }
1776
+ allocationFences.set(allocationId, generation);
1777
+ const current = allocatedConnections.get(allocationId);
1778
+ if (current !== undefined && current.identity.generation !== generation) {
1779
+ handleClose(current.ws);
1780
+ current.ws.close();
1781
+ }
1782
+ const waiters = allocationWaiters.get(allocationId);
1783
+ if (waiters === undefined)
1784
+ return;
1785
+ for (const waiter of [...waiters]) {
1786
+ if (waiter.generation === generation)
1787
+ continue;
1788
+ clearTimeout(waiter.timer);
1789
+ waiters.delete(waiter);
1790
+ waiter.reject(new Error(`Allocation ${allocationId} advanced to generation ${String(generation)}`));
1791
+ }
1792
+ if (waiters.size === 0)
1793
+ allocationWaiters.delete(allocationId);
1794
+ }
1795
+ async function getAllocatedConnection(target, use) {
1796
+ if (allocationFences.get(target.allocationId) !== target.generation) {
1797
+ throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
1798
+ }
1799
+ const current = allocatedConnections.get(target.allocationId);
1800
+ if (current === undefined ||
1801
+ current.identity.generation !== target.generation) {
1802
+ throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId} generation ${String(target.generation)}`);
1803
+ }
1804
+ if (!(await validateSidecarIdentity(current.identity, use))) {
1805
+ if (allocatedConnections.get(target.allocationId) === current) {
1806
+ handleClose(current.ws);
1807
+ current.ws.close();
1808
+ }
1809
+ throw new Error(`Allocated sidecar identity is no longer current for allocation ${target.allocationId}`);
1810
+ }
1811
+ if (allocatedConnections.get(target.allocationId) !== current) {
1812
+ throw new Error(`Allocated sidecar connection changed for allocation ${target.allocationId}`);
1813
+ }
1814
+ const conn = connections.get(current.ws);
1815
+ if (conn === undefined ||
1816
+ conn.identity.kind !== "allocated" ||
1817
+ conn.identity.allocationId !== target.allocationId ||
1818
+ conn.identity.generation !== target.generation) {
1819
+ throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId}`);
1820
+ }
1821
+ return { ws: current.ws, conn };
1822
+ }
1823
+ async function isAllocatedSidecarReady(target) {
1824
+ try {
1825
+ await getAllocatedConnection(target, "readiness");
1826
+ return true;
1827
+ }
1828
+ catch {
1829
+ return false;
1830
+ }
1831
+ }
1832
+ async function isAllocatedWorkflowActive(target) {
1833
+ try {
1834
+ const { conn } = await getAllocatedConnection(target, "readiness");
1835
+ if (conn.identity.kind !== "allocated")
1836
+ return false;
1837
+ return conn.workflowAddresses.has(conn.identity.workflowRunAddress);
1838
+ }
1839
+ catch {
1840
+ return false;
1841
+ }
1842
+ }
1843
+ async function waitForAllocatedSidecar(target, timeoutMs) {
1844
+ if (await isAllocatedSidecarReady(target))
1845
+ return;
1846
+ if (allocationFences.get(target.allocationId) !== target.generation) {
1847
+ throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
1848
+ }
1849
+ if (timeoutMs <= 0) {
1850
+ throw new Error(`Timed out waiting for allocated sidecar ${target.allocationId}`);
1851
+ }
1852
+ await new Promise((resolve, reject) => {
1853
+ const waiter = {
1854
+ generation: target.generation,
1855
+ resolve,
1856
+ reject,
1857
+ timer: setTimeout(() => {
1858
+ const current = allocationWaiters.get(target.allocationId);
1859
+ current?.delete(waiter);
1860
+ if (current?.size === 0) {
1861
+ allocationWaiters.delete(target.allocationId);
1862
+ }
1863
+ reject(new Error(`Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`));
1864
+ }, timeoutMs),
1865
+ };
1866
+ let waiters = allocationWaiters.get(target.allocationId);
1867
+ if (waiters === undefined) {
1868
+ waiters = new Set();
1869
+ allocationWaiters.set(target.allocationId, waiters);
1870
+ }
1871
+ waiters.add(waiter);
1872
+ void notifyAllocationWaiters(target.allocationId);
1873
+ });
1874
+ }
1875
+ function bindStepRoute(stepAddress) {
1876
+ const ws = addressIndex.get(stepAddress) ?? findSidecarForNewAgent(stepAddress);
1877
+ if (ws === undefined) {
1878
+ throw new Error(`No sidecar available to stage workflow step "${stepAddress}"`);
1879
+ }
1880
+ const conn = connections.get(ws);
1881
+ if (conn === undefined) {
1882
+ throw new Error(`No sidecar connected to stage workflow step "${stepAddress}"`);
1883
+ }
1884
+ if (conn.identity.kind !== "shared") {
1885
+ throw new Error(`Allocated sidecar ${conn.sidecarId} cannot receive ordinary workflow step ${stepAddress}`);
1886
+ }
1887
+ conn.workflowAddresses.add(stepAddress);
1888
+ addressIndex.set(stepAddress, ws);
1889
+ }
1890
+ async function bindAllocatedStepRoute(target, stepAddress) {
1891
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
1892
+ const existing = addressIndex.get(stepAddress);
1893
+ if (existing !== undefined && existing !== ws) {
1894
+ throw new Error(`Workflow step ${stepAddress} is already routed to another sidecar`);
1895
+ }
1896
+ conn.workflowAddresses.add(stepAddress);
1897
+ addressIndex.set(stepAddress, ws);
1898
+ }
1899
+ /**
1900
+ * Remove a per-step route bound by `bindStepRoute` once the step's packs
1901
+ * have landed. Idempotent: an address that was never bound (or already
1902
+ * unbound, e.g. by a mid-stage `handleClose`) is a no-op.
1903
+ */
1904
+ function unbindStepRoute(stepAddress) {
1905
+ const ws = addressIndex.get(stepAddress);
1906
+ if (ws === undefined)
1907
+ return;
1908
+ const conn = connections.get(ws);
1909
+ if (conn !== undefined) {
1910
+ conn.workflowAddresses.delete(stepAddress);
1911
+ }
1912
+ addressIndex.delete(stepAddress);
1913
+ }
1914
+ function unbindAllocatedStepRoute(target, stepAddress) {
1915
+ const current = allocatedConnections.get(target.allocationId);
1916
+ if (current === undefined ||
1917
+ current.identity.generation !== target.generation) {
1918
+ return;
1919
+ }
1920
+ if (addressIndex.get(stepAddress) !== current.ws)
1921
+ return;
1922
+ connections.get(current.ws)?.workflowAddresses.delete(stepAddress);
1923
+ addressIndex.delete(stepAddress);
1924
+ }
1925
+ // Pack transfers may take longer than session requests due to data volume.
1926
+ const PACK_TIMEOUT_MS = requestTimeoutMs * 4;
1927
+ function sendPack(agentAddress, pack, ref, commitSha, options) {
1928
+ const ws = addressIndex.get(agentAddress);
1929
+ if (ws === undefined) {
1930
+ return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
1931
+ }
1932
+ const conn = connections.get(ws);
1933
+ if (conn === undefined) {
1934
+ return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
1935
+ }
1936
+ if (conn.identity.kind !== "shared") {
1937
+ return Promise.reject(new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound pack routing for "${agentAddress}"`));
1938
+ }
1939
+ return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
1940
+ }
1941
+ function sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options) {
1942
+ const transferId = `pack-${++packCounter}`;
1943
+ // For the agent-state flow the destination agent and the source repo
1944
+ // are the same entity, so `repoId.id === agentAddress`. Asset packs
1945
+ // override this with the SOURCE asset's id so audit can correlate
1946
+ // the pack back to its hub-side origin.
1947
+ const repoId = options?.repoId ?? {
1948
+ kind: "agent-state",
1949
+ id: agentAddress,
1950
+ };
1951
+ const mountPath = options?.mountPath;
1952
+ // Register pending entry before sending frames so that a synchronous
1953
+ // repo.pack.ack (e.g. in tests or loopback transports) resolves correctly.
1954
+ return new Promise((resolve, reject) => {
1955
+ const timer = setTimeout(() => {
1956
+ pendingPacks.delete(transferId);
1957
+ reject(new Error(`Pack transfer ${transferId} timed out after ${PACK_TIMEOUT_MS}ms`));
1958
+ }, PACK_TIMEOUT_MS);
1959
+ pendingPacks.set(transferId, {
1960
+ transferId,
1961
+ ws,
1962
+ agentAddress,
1963
+ repoId,
1964
+ resolve,
1965
+ reject(error) {
1966
+ reject(new Error(`Pack rejected: ${error}`));
1967
+ },
1968
+ timer,
1969
+ });
1970
+ // Send chunks
1971
+ for (const chunk of chunkPack(pack)) {
1972
+ conn.send({
1973
+ type: "repo.pack.push",
1974
+ agentAddress,
1975
+ repoId,
1976
+ transferId,
1977
+ seq: chunk.seq,
1978
+ data: chunk.data,
1979
+ });
1980
+ }
1981
+ // Send done
1982
+ conn.send({
1983
+ type: "repo.pack.done",
1984
+ agentAddress,
1985
+ repoId,
1986
+ transferId,
1987
+ ref,
1988
+ commitSha,
1989
+ ...(mountPath !== undefined ? { mountPath } : {}),
1990
+ });
1991
+ });
1992
+ }
1993
+ async function sendPackToAllocation(target, agentAddress, pack, ref, commitSha, options) {
1994
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
1995
+ if (addressIndex.get(agentAddress) !== ws) {
1996
+ throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
1997
+ }
1998
+ return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
1999
+ }
2000
+ async function sendWorkflowRunPackToAllocation(target, agentAddress, pack, ref, commitSha) {
2001
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2002
+ if (conn.identity.kind !== "allocated") {
2003
+ throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
2004
+ }
2005
+ if (agentAddress !== conn.identity.workflowRunAddress) {
2006
+ throw new Error(`Allocation ${target.allocationId} cannot restore unrelated address ${agentAddress}`);
2007
+ }
2008
+ if (conn.workflowAddresses.has(agentAddress)) {
2009
+ throw new Error(`Allocation ${target.allocationId} already hosts active workflow ${agentAddress}; refusing to overwrite its run history`);
2010
+ }
2011
+ return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, {
2012
+ repoId: {
2013
+ kind: "workflow-run",
2014
+ id: deriveWorkflowRunRepoId(agentAddress),
2015
+ },
2016
+ });
2017
+ }
2018
+ function routeMail(agentAddress, rawMessage, messageId, runGrants) {
2019
+ // Carry the hub-minted messageId on the frame so the sidecar's durable-
2020
+ // receipt ack (`mail.inbound.ack`) keys on the same id the hub tracks, and
2021
+ // a redelivery replays identical bytes for the downstream RunStarted dedup.
2022
+ // Optional: the workflow-trigger and session-conversation callers supply
2023
+ // it (they participate in the ack/retry handshake); a caller without a
2024
+ // hub-minted id omits it and the delivery is not tracked for redelivery.
2025
+ const frame = {
2026
+ type: "mail.inbound",
2027
+ agentAddress,
2028
+ rawMessage,
2029
+ ...(messageId !== undefined ? { messageId } : {}),
2030
+ };
2031
+ const ws = addressIndex.get(agentAddress);
2032
+ if (ws !== undefined) {
2033
+ const conn = connections.get(ws);
2034
+ if (conn !== undefined) {
2035
+ conn.send(frame);
2036
+ // Track the delivery for redelivery until the sidecar acks its durable
2037
+ // inbox write. Only mail carrying a hub-minted messageId participates
2038
+ // in the ack handshake; relayed agent-to-agent mail omits it and is
2039
+ // delivered fire-and-forget as before. A mail that triggered a workflow
2040
+ // run carries the run's grants so redelivery can replay them ahead of
2041
+ // the mail.
2042
+ if (messageId !== undefined) {
2043
+ trackPendingMail(agentAddress, messageId, frame, runGrants);
2044
+ }
2045
+ return true;
2046
+ }
2047
+ }
2048
+ // If the agent recently disconnected, queue for delivery on reconnect.
2049
+ return enqueueForDisconnected(agentAddress, frame);
2050
+ }
2051
+ function sendRunGrants(agentAddress, runId, stepGrants) {
2052
+ const frame = {
2053
+ type: "run.grants",
2054
+ agentAddress,
2055
+ runId,
2056
+ stepGrants,
2057
+ };
2058
+ const ws = addressIndex.get(agentAddress);
2059
+ if (ws !== undefined) {
2060
+ const conn = connections.get(ws);
2061
+ if (conn !== undefined) {
2062
+ conn.send(frame);
2063
+ return true;
2064
+ }
2065
+ }
2066
+ // Mirror routeMail: if the address has a live disconnect queue, ride it
2067
+ // so a run.grants issued in the window between deploy and the first
2068
+ // reconnect survives the same way the dispatching trigger mail does.
2069
+ // A queue exists only while the deployment address is still on
2070
+ // agentAddresses (pre-first-reconnect); after a challenged reconnect it
2071
+ // moves to workflowAddresses, which handleClose leaves unqueued because
2072
+ // that generation's in-flight run state is reconstructed sidecar-locally.
2073
+ // Returning without enqueueing there is correct; enqueueing is what keeps
2074
+ // grants and mail from diverging in the pre-reconnect window.
2075
+ return enqueueForDisconnected(agentAddress, frame);
2076
+ }
2077
+ async function sendWorkflowRunDispatchToAllocation(target, agentAddress, runId, stepGrants, rawMessage, messageId) {
2078
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2079
+ if (addressIndex.get(agentAddress) !== ws) {
2080
+ throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
2081
+ }
2082
+ const runGrants = { runId, stepGrants };
2083
+ conn.send({
2084
+ type: "run.grants",
2085
+ agentAddress,
2086
+ runId,
2087
+ stepGrants,
2088
+ });
2089
+ const frame = {
2090
+ type: "mail.inbound",
2091
+ agentAddress,
2092
+ rawMessage,
2093
+ messageId,
2094
+ };
2095
+ conn.send(frame);
2096
+ trackPendingMail(agentAddress, messageId, frame, runGrants, target);
2097
+ }
2098
+ async function handleDeployAck(ws, frame) {
2099
+ const req = pendingDeploys.get(frame.agentAddress);
2100
+ if (req === undefined) {
2101
+ logger.warn `Received agent.deploy.ack for "${frame.agentAddress}" with no pending deploy`;
2102
+ return;
2103
+ }
2104
+ if (req.ws !== ws)
2105
+ return;
2106
+ if (events.listenerCount("agent.deploy.ack") > 0) {
2107
+ try {
2108
+ const identity = connections.get(ws)?.identity;
2109
+ await events.emitAndAwait("agent.deploy.ack", {
2110
+ agentAddress: frame.agentAddress,
2111
+ publicKey: frame.publicKey,
2112
+ ...(identity?.kind === "allocated"
2113
+ ? {
2114
+ allocated: {
2115
+ allocationId: identity.allocationId,
2116
+ anchorRunId: identity.anchorRunId,
2117
+ generation: identity.generation,
2118
+ },
2119
+ }
2120
+ : {}),
2121
+ });
2122
+ }
2123
+ catch (err) {
2124
+ rejectDeployPending(req, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
2125
+ return;
2126
+ }
2127
+ }
2128
+ resolveDeployPending(req, frame.publicKey);
2129
+ }
2130
+ function resolveDeployPending(req, publicKey) {
2131
+ if (pendingDeploys.get(req.agentAddress) !== req)
2132
+ return;
2133
+ clearTimeout(req.timer);
2134
+ pendingDeploys.delete(req.agentAddress);
2135
+ req.resolve(publicKey);
2136
+ }
2137
+ function rejectDeployPending(req, error) {
2138
+ if (pendingDeploys.get(req.agentAddress) !== req)
2139
+ return;
2140
+ clearTimeout(req.timer);
2141
+ pendingDeploys.delete(req.agentAddress);
2142
+ req.reject(error);
2143
+ }
2144
+ function rejectDeployPendingFromFrame(ws, agentAddress, error) {
2145
+ const req = pendingDeploys.get(agentAddress);
2146
+ if (req === undefined || req.ws !== ws)
2147
+ return;
2148
+ rejectDeployPending(req, error);
2149
+ }
2150
+ async function sendAgentDeploy(agentAddress, harnessConfig, workflow) {
2151
+ if (hubPublicKeyHex === undefined) {
2152
+ throw new Error("Hub signing key is required for agent deployment");
2153
+ }
2154
+ const ws = addressIndex.get(agentAddress) ?? findSidecarForNewAgent(agentAddress);
2155
+ if (ws === undefined) {
2156
+ throw new Error(`No sidecar available for agent "${agentAddress}"`);
2157
+ }
2158
+ const conn = connections.get(ws);
2159
+ if (conn === undefined) {
2160
+ throw new Error(`No sidecar connected for agent "${agentAddress}"`);
2161
+ }
2162
+ if (conn.identity.kind !== "shared") {
2163
+ throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound deploy routing`);
2164
+ }
2165
+ return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
2166
+ }
2167
+ function sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow) {
2168
+ if (hubPublicKeyHex === undefined) {
2169
+ throw new Error("Hub signing key is required for agent deployment");
2170
+ }
2171
+ if (pendingDeploys.has(agentAddress)) {
2172
+ throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
2173
+ }
2174
+ const addressSet = conn.identity.kind === "allocated"
2175
+ ? conn.workflowAddresses
2176
+ : conn.agentAddresses;
2177
+ addressSet.add(agentAddress);
2178
+ addressIndex.set(agentAddress, ws);
2179
+ return new Promise((resolve, reject) => {
2180
+ const timer = setTimeout(() => {
2181
+ pendingDeploys.delete(agentAddress);
2182
+ if (addressIndex.get(agentAddress) === ws) {
2183
+ addressSet.delete(agentAddress);
2184
+ addressIndex.delete(agentAddress);
2185
+ }
2186
+ reject(new Error(`Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
2187
+ }, requestTimeoutMs);
2188
+ pendingDeploys.set(agentAddress, {
2189
+ agentAddress,
2190
+ ws,
2191
+ resolve(publicKey) {
2192
+ resolve({ publicKey });
2193
+ },
2194
+ reject(error) {
2195
+ if (addressIndex.get(agentAddress) === ws) {
2196
+ addressSet.delete(agentAddress);
2197
+ addressIndex.delete(agentAddress);
2198
+ }
2199
+ reject(new Error(error));
2200
+ },
2201
+ timer,
2202
+ });
2203
+ conn.send({
2204
+ type: "agent.deploy",
2205
+ agentAddress,
2206
+ agentId: harnessConfig.agentId,
2207
+ config: harnessConfig,
2208
+ hubPublicKey: hubPublicKeyHex,
2209
+ ...(workflow !== undefined ? { workflow } : {}),
2210
+ });
2211
+ });
2212
+ }
2213
+ async function sendAgentDeployToAllocation(target, agentAddress, harnessConfig, workflow) {
2214
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2215
+ if (conn.identity.kind !== "allocated") {
2216
+ throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
2217
+ }
2218
+ if (agentAddress !== conn.identity.workflowRunAddress) {
2219
+ throw new Error(`Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`);
2220
+ }
2221
+ const existing = addressIndex.get(agentAddress);
2222
+ if (existing !== undefined && existing !== ws) {
2223
+ throw new Error(`Deployment ${agentAddress} is already routed to another sidecar`);
2224
+ }
2225
+ return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
2226
+ }
2227
+ /**
2228
+ * Provision one step of a multi-step deploy on the sidecar WITHOUT
2229
+ * spawning: the sidecar initializes the step's agent-state repo and
2230
+ * records the hub key, so the follow-up deploy pack applies into a repo
2231
+ * and verifies against the recorded key -- but no supervisor or child is
2232
+ * constructed. The deployment-level workflow frame, sent once after every
2233
+ * step is provisioned, spawns the child.
2234
+ *
2235
+ * The step address must already be bound via `bindStepRoute`, which
2236
+ * resolves and records the sidecar; this reuses that route rather than
2237
+ * touching `agentAddresses`. Waits for the sidecar's `agent.deploy.ack`
2238
+ * so the caller can safely deliver the deploy pack afterward. On failure
2239
+ * the caller owns tearing the route down via `unbindStepRoute`.
2240
+ */
2241
+ function sendProvisionStep(agentAddress, harnessConfig) {
2242
+ const ws = addressIndex.get(agentAddress);
2243
+ if (ws === undefined) {
2244
+ throw new Error(`Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`);
2245
+ }
2246
+ const conn = connections.get(ws);
2247
+ if (conn === undefined) {
2248
+ throw new Error(`No sidecar connected for agent "${agentAddress}"`);
2249
+ }
2250
+ if (conn.identity.kind !== "shared") {
2251
+ throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound step provisioning`);
2252
+ }
2253
+ return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
2254
+ }
2255
+ function sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig) {
2256
+ if (hubPublicKeyHex === undefined) {
2257
+ throw new Error("Hub signing key is required for step provisioning");
2258
+ }
2259
+ if (pendingDeploys.has(agentAddress)) {
2260
+ throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
2261
+ }
2262
+ const hubKey = hubPublicKeyHex;
2263
+ return new Promise((resolve, reject) => {
2264
+ const timer = setTimeout(() => {
2265
+ pendingDeploys.delete(agentAddress);
2266
+ reject(new Error(`Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
2267
+ }, requestTimeoutMs);
2268
+ // The sidecar's `agent.deploy.ack` resolves this through
2269
+ // `resolveDeployPending` -> `req.resolve()`. The per-step address is
2270
+ // workflow-derived and records no hub-side key, so the ack's public
2271
+ // key is not needed and this resolves void.
2272
+ pendingDeploys.set(agentAddress, {
2273
+ agentAddress,
2274
+ ws,
2275
+ resolve(_publicKey) {
2276
+ resolve();
2277
+ },
2278
+ reject(error) {
2279
+ reject(new Error(error));
2280
+ },
2281
+ timer,
2282
+ });
2283
+ conn.send({
2284
+ type: "agent.deploy",
2285
+ agentAddress,
2286
+ agentId: harnessConfig.agentId,
2287
+ config: harnessConfig,
2288
+ hubPublicKey: hubKey,
2289
+ provisionStep: true,
2290
+ });
2291
+ });
2292
+ }
2293
+ async function sendProvisionStepToAllocation(target, agentAddress, harnessConfig) {
2294
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2295
+ if (addressIndex.get(agentAddress) !== ws) {
2296
+ throw new Error(`Step route ${agentAddress} is not bound to allocation ${target.allocationId}`);
2297
+ }
2298
+ return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
2299
+ }
2300
+ function findSidecarForNewAgent(_agentAddress) {
2301
+ for (const [ws, conn] of connections) {
2302
+ if (conn.identity.kind === "shared")
2303
+ return ws;
2304
+ }
2305
+ return undefined;
2306
+ }
2307
+ function sendProbe(args) {
2308
+ // Select any connected sidecar, exactly as the pre-deploy window does.
2309
+ // A probe runs in the sidecar's pre-deploy state, so it targets a
2310
+ // connection, not an address -- `findSidecarForNewAgent` ignores its
2311
+ // argument and returns the first connection. Throw immediately on an empty
2312
+ // registry rather than register a probe that could only time out.
2313
+ const ws = findSidecarForNewAgent("");
2314
+ if (ws === undefined) {
2315
+ return Promise.reject(new Error("No sidecar available to probe workflow"));
2316
+ }
2317
+ const conn = connections.get(ws);
2318
+ if (conn === undefined) {
2319
+ return Promise.reject(new Error("No sidecar connected to probe workflow"));
2320
+ }
2321
+ const requestId = nextRequestId();
2322
+ return new Promise((resolve, reject) => {
2323
+ const timer = setTimeout(() => {
2324
+ pendingProbes.delete(requestId);
2325
+ reject(new Error(`Probe ${requestId} timed out after ${probeTimeoutMs}ms`));
2326
+ }, probeTimeoutMs);
2327
+ pendingProbes.set(requestId, {
2328
+ requestId,
2329
+ ws,
2330
+ resolve,
2331
+ reject(error) {
2332
+ reject(new Error(error));
2333
+ },
2334
+ timer,
2335
+ });
2336
+ conn.send({
2337
+ type: "workflow.probe.request",
2338
+ requestId,
2339
+ source: args.source,
2340
+ closure: args.closure,
2341
+ entry: args.entry,
2342
+ ...(args.assets !== undefined ? { assets: args.assets } : {}),
2343
+ });
2344
+ });
2345
+ }
2346
+ function sendAgentUndeploy(agentAddress, reason) {
2347
+ const ws = addressIndex.get(agentAddress);
2348
+ if (ws === undefined) {
2349
+ return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
2350
+ }
2351
+ const conn = connections.get(ws);
2352
+ if (conn === undefined) {
2353
+ return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
2354
+ }
2355
+ return new Promise((resolve, reject) => {
2356
+ const timer = setTimeout(() => {
2357
+ pendingUndeploys.delete(agentAddress);
2358
+ removeAgentAddress(ws, agentAddress);
2359
+ reject(new Error(`Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
2360
+ }, requestTimeoutMs);
2361
+ pendingUndeploys.set(agentAddress, {
2362
+ agentAddress,
2363
+ ws,
2364
+ resolve() {
2365
+ removeAgentAddress(ws, agentAddress);
2366
+ resolve();
2367
+ },
2368
+ reject(error) {
2369
+ removeAgentAddress(ws, agentAddress);
2370
+ reject(new Error(error));
2371
+ },
2372
+ timer,
2373
+ });
2374
+ conn.send({
2375
+ type: "agent.undeploy",
2376
+ agentAddress,
2377
+ reason,
2378
+ });
2379
+ });
2380
+ }
2381
+ function removeAgentAddress(ws, agentAddress) {
2382
+ addressIndex.delete(agentAddress);
2383
+ const conn = connections.get(ws);
2384
+ if (conn !== undefined) {
2385
+ conn.agentAddresses.delete(agentAddress);
2386
+ }
2387
+ }
2388
+ function dispatchToSubscribers(agentAddress, event) {
2389
+ const subs = agentSubscribers.get(agentAddress);
2390
+ if (subs === undefined)
2391
+ return;
2392
+ for (const cb of [...subs]) {
2393
+ try {
2394
+ cb(event);
2395
+ }
2396
+ catch (err) {
2397
+ logger.warn `Agent subscriber threw: ${err instanceof Error ? err.message : String(err)}`;
2398
+ }
2399
+ }
2400
+ }
2401
+ function subscribeAgent(agentAddress, callback) {
2402
+ let subs = agentSubscribers.get(agentAddress);
2403
+ if (subs === undefined) {
2404
+ subs = new Set();
2405
+ agentSubscribers.set(agentAddress, subs);
2406
+ }
2407
+ subs.add(callback);
2408
+ return () => {
2409
+ const current = agentSubscribers.get(agentAddress);
2410
+ if (current === undefined)
2411
+ return;
2412
+ current.delete(callback);
2413
+ if (current.size === 0) {
2414
+ agentSubscribers.delete(agentAddress);
2415
+ }
2416
+ };
2417
+ }
2418
+ function getConnectedSidecars() {
2419
+ return Array.from(connections.values()).map((c) => c.sidecarId);
2420
+ }
2421
+ function getRoutableAddresses() {
2422
+ return Array.from(addressIndex.keys());
2423
+ }
2424
+ function getConnectorState(agentAddress) {
2425
+ return connectorStates.get(agentAddress) ?? null;
2426
+ }
2427
+ async function sendSourcesUpdate(agentAddress, sources, defaultSource) {
2428
+ await sendRequest(agentAddress, (requestId) => ({
2429
+ type: "sources.update",
2430
+ requestId,
2431
+ agentAddress,
2432
+ sources,
2433
+ defaultSource,
2434
+ }));
2435
+ }
2436
+ async function sendCredentialsUpdate(agentAddress, delivery) {
2437
+ await sendRequest(agentAddress, (requestId) => ({
2438
+ type: "credentials.update",
2439
+ requestId,
2440
+ agentAddress,
2441
+ delivery,
2442
+ }));
2443
+ }
2444
+ function sendSyncRequest(agentAddress) {
2445
+ const ws = addressIndex.get(agentAddress);
2446
+ if (ws === undefined) {
2447
+ throw new Error(`No sidecar connected for agent "${agentAddress}"`);
2448
+ }
2449
+ const conn = connections.get(ws);
2450
+ if (conn === undefined) {
2451
+ throw new Error(`No sidecar connected for agent "${agentAddress}"`);
2452
+ }
2453
+ const transferId = `sync-${++packCounter}`;
2454
+ conn.send({
2455
+ type: "sync.request",
2456
+ agentAddress,
2457
+ transferId,
2458
+ });
2459
+ }
2460
+ function sendSignalDeliver(opts) {
2461
+ const ws = addressIndex.get(opts.agentAddress);
2462
+ if (ws === undefined) {
2463
+ throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
2464
+ }
2465
+ const conn = connections.get(ws);
2466
+ if (conn === undefined) {
2467
+ throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
2468
+ }
2469
+ conn.send({
2470
+ type: "signal.deliver",
2471
+ agentAddress: opts.agentAddress,
2472
+ runId: opts.runId,
2473
+ signalName: opts.signalName,
2474
+ signalId: opts.signalId,
2475
+ payload: opts.payload,
2476
+ });
2477
+ }
2478
+ async function sendSignalDeliverToAllocation(target, opts) {
2479
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2480
+ if (addressIndex.get(opts.agentAddress) !== ws) {
2481
+ throw new Error(`Address ${opts.agentAddress} is not routed on allocation ${target.allocationId}`);
2482
+ }
2483
+ conn.send({ type: "signal.deliver", ...opts });
2484
+ }
2485
+ function sendDrain(opts) {
2486
+ const ws = addressIndex.get(opts.agentAddress);
2487
+ if (ws === undefined) {
2488
+ throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
2489
+ }
2490
+ const conn = connections.get(ws);
2491
+ if (conn === undefined) {
2492
+ throw new Error(`No sidecar connected for deployment "${opts.agentAddress}"`);
2493
+ }
2494
+ conn.send({
2495
+ type: "drain.deliver",
2496
+ agentAddress: opts.agentAddress,
2497
+ deadlineMs: opts.deadlineMs,
2498
+ });
2499
+ }
2500
+ return {
2501
+ handleOpen,
2502
+ handleMessage,
2503
+ handleClose,
2504
+ routeMail,
2505
+ sendRunGrants,
2506
+ sendAgentDeploy,
2507
+ sendProbe,
2508
+ sendAgentUndeploy,
2509
+ sendSourcesUpdate,
2510
+ sendCredentialsUpdate,
2511
+ sendPack,
2512
+ sendPackToAllocation,
2513
+ sendWorkflowRunPackToAllocation,
2514
+ fenceAllocation,
2515
+ waitForAllocatedSidecar,
2516
+ isAllocatedSidecarReady,
2517
+ isAllocatedWorkflowActive,
2518
+ sendAgentDeployToAllocation,
2519
+ bindStepRoute,
2520
+ bindAllocatedStepRoute,
2521
+ unbindStepRoute,
2522
+ unbindAllocatedStepRoute,
2523
+ sendProvisionStep,
2524
+ sendProvisionStepToAllocation,
2525
+ sendWorkflowRunDispatchToAllocation,
2526
+ sendSyncRequest,
2527
+ sendSignalDeliver,
2528
+ sendSignalDeliverToAllocation,
2529
+ sendDrain,
2530
+ subscribeAgent,
2531
+ dispatchAgentEvent: dispatchToSubscribers,
2532
+ getConnectedSidecars,
2533
+ getRoutableAddresses,
2534
+ getConnectorState,
2535
+ events,
2536
+ };
2537
+ }