@intx/hub-sessions 0.2.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +3 -5
  2. package/dist/agent-repo.d.ts +23 -7
  3. package/dist/agent-repo.js +19 -6
  4. package/dist/agent-state-kind.js +18 -63
  5. package/dist/asset-service.d.ts +1 -20
  6. package/dist/asset-service.js +19 -97
  7. package/dist/committed-source-tree.d.ts +10 -0
  8. package/dist/committed-source-tree.js +35 -0
  9. package/dist/credential-push.d.ts +52 -7
  10. package/dist/credential-push.js +178 -22
  11. package/dist/event-collector-registry.d.ts +3 -2
  12. package/dist/event-collector-registry.js +42 -13
  13. package/dist/event-collector.d.ts +12 -2
  14. package/dist/event-collector.js +45 -4
  15. package/dist/hub-session-lookups.d.ts +125 -7
  16. package/dist/hub-session-lookups.js +541 -86
  17. package/dist/hub-session-orchestrator.d.ts +2 -3
  18. package/dist/hub-session-orchestrator.js +22 -56
  19. package/dist/index.d.ts +19 -9
  20. package/dist/index.js +16 -7
  21. package/dist/reconciliation-scheduler.d.ts +14 -0
  22. package/dist/reconciliation-scheduler.js +55 -0
  23. package/dist/repo-store/index.d.ts +2 -1
  24. package/dist/repo-store/index.js +1 -0
  25. package/dist/repo-store/store.d.ts +1 -1
  26. package/dist/repo-store/store.js +138 -1
  27. package/dist/repo-store/subscribe-kind.d.ts +6 -3
  28. package/dist/repo-store/subscribe-kind.js +42 -77
  29. package/dist/repo-store/types.d.ts +94 -6
  30. package/dist/repo-store/user-principal-gate.d.ts +26 -0
  31. package/dist/repo-store/user-principal-gate.js +78 -0
  32. package/dist/session-service.d.ts +252 -126
  33. package/dist/session-service.js +851 -624
  34. package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
  35. package/dist/sidecar-allocation/capability-policy.js +124 -0
  36. package/dist/sidecar-allocation/contracts.d.ts +101 -0
  37. package/dist/sidecar-allocation/contracts.js +26 -0
  38. package/dist/sidecar-allocation/index.d.ts +5 -0
  39. package/dist/sidecar-allocation/index.js +4 -0
  40. package/dist/sidecar-allocation/operation.d.ts +10 -0
  41. package/dist/sidecar-allocation/operation.js +54 -0
  42. package/dist/sidecar-allocation/plugin-registry.d.ts +24 -0
  43. package/dist/sidecar-allocation/plugin-registry.js +61 -0
  44. package/dist/sidecar-allocation/reconciler.d.ts +54 -0
  45. package/dist/sidecar-allocation/reconciler.js +825 -0
  46. package/dist/skill-kind.js +12 -62
  47. package/dist/substrate.d.ts +3 -3
  48. package/dist/substrate.js +1 -1
  49. package/dist/workflow-allocation-service.d.ts +64 -0
  50. package/dist/workflow-allocation-service.js +554 -0
  51. package/dist/workflow-closure-resolution.d.ts +106 -0
  52. package/dist/workflow-closure-resolution.js +123 -0
  53. package/dist/workflow-definition-ensure.d.ts +24 -0
  54. package/dist/workflow-definition-ensure.js +75 -0
  55. package/dist/workflow-dispatch-service.d.ts +42 -0
  56. package/dist/workflow-dispatch-service.js +209 -0
  57. package/dist/workflow-dispatch-settlement.d.ts +29 -0
  58. package/dist/workflow-dispatch-settlement.js +140 -0
  59. package/dist/workflow-kind.d.ts +29 -1
  60. package/dist/workflow-kind.js +143 -139
  61. package/dist/workflow-probe-gate.d.ts +286 -0
  62. package/dist/workflow-probe-gate.js +382 -0
  63. package/dist/workflow-run-kind.d.ts +239 -32
  64. package/dist/workflow-run-kind.js +874 -188
  65. package/dist/workflow-run-reader.d.ts +1 -1
  66. package/dist/workflow-run-reader.js +3 -7
  67. package/dist/workflow-run-restore.d.ts +16 -0
  68. package/dist/workflow-run-restore.js +30 -0
  69. package/dist/workflow-source-closure.d.ts +35 -0
  70. package/dist/workflow-source-closure.js +342 -0
  71. package/dist/workflow-source-pins.d.ts +8 -0
  72. package/dist/workflow-source-pins.js +14 -0
  73. package/dist/ws/index.d.ts +3 -3
  74. package/dist/ws/index.js +2 -2
  75. package/dist/ws/pending-tracker.d.ts +93 -0
  76. package/dist/ws/pending-tracker.js +132 -0
  77. package/dist/ws/sidecar-events.d.ts +139 -37
  78. package/dist/ws/sidecar-events.js +2 -2
  79. package/dist/ws/sidecar-handler.d.ts +218 -60
  80. package/dist/ws/sidecar-handler.js +1713 -732
  81. package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
  82. package/dist/ws/sidecar-handler.test-helpers.js +95 -0
  83. package/dist/ws/sidecar-token-authenticator.d.ts +3 -1
  84. package/dist/ws/sidecar-token-authenticator.js +78 -7
  85. package/package.json +14 -13
  86. package/dist/available-skills-stanza.d.ts +0 -21
  87. package/dist/available-skills-stanza.js +0 -32
@@ -4,19 +4,37 @@
4
4
  // table of agentAddress → sidecar connection, and dispatches frames between
5
5
  // sidecars and the hub's internal systems.
6
6
  import { getLogger } from "@intx/log";
7
- import { verifyEd25519 } from "@intx/crypto";
8
7
  import { chunkPack, createPackReceiver } from "@intx/pack-transport";
9
- import { hexDecode, hexEncode } from "@intx/types";
10
- import { isWorkflowDerivedAddress } from "@intx/workflow-deploy";
8
+ import { base64Decode, deriveMessageId, deriveWorkflowRunId, isRunAddress, } from "@intx/types";
9
+ import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy";
11
10
  import { type } from "arktype";
12
- import { SidecarFrame, } from "@intx/types/sidecar";
11
+ import { MAX_MAIL_OUTBOUND_BODY_BYTES, SidecarFrame, } from "@intx/types/sidecar";
13
12
  import { createSidecarEmitter, } from "./sidecar-events.js";
13
+ import { PendingTracker, } from "./pending-tracker.js";
14
14
  const logger = getLogger(["hub", "ws", "sidecar"]);
15
+ function deployFrameFailure(message, frameSent, cause) {
16
+ return Object.assign(new Error(message, { cause }), { frameSent });
17
+ }
18
+ export function isDeployFrameFailure(err) {
19
+ return (err instanceof Error &&
20
+ "frameSent" in err &&
21
+ typeof err.frameSent === "boolean");
22
+ }
23
+ /**
24
+ * Identity validation failed or remained pending at the connection deadline.
25
+ * Readiness is unknown: the worker may be healthy behind the lookup, so
26
+ * callers must retry rather than treat this as a missed connection deadline.
27
+ */
28
+ export class SidecarIdentityValidationError extends Error {
29
+ constructor(allocationId, generation, cause) {
30
+ super(`Cannot validate sidecar identity for allocation ${allocationId} generation ${String(generation)}`, { cause });
31
+ this.name = "SidecarIdentityValidationError";
32
+ }
33
+ }
15
34
  /**
16
- * Whether this connection owns `address` for routing/lifecycle purposes --
17
- * as a challenged session address OR a hub-minted workflow-substrate address.
18
- * The two sets are kept physically distinct (they differ on the challenge /
19
- * re-add path), but ownership readers -- pack-transfer authorization,
35
+ * Whether this connection owns `address` for routing/lifecycle purposes.
36
+ * The legacy and workflow sets remain physically distinct, but ownership
37
+ * readers -- pack-transfer authorization,
20
38
  * in-flight cancellation, disconnect teardown -- must see the union, or a
21
39
  * reconnected workflow deployment (which lives only in `workflowAddresses`)
22
40
  * is silently treated as unowned even though its mail routes.
@@ -24,29 +42,88 @@ const logger = getLogger(["hub", "ws", "sidecar"]);
24
42
  function connOwnsAddress(conn, address) {
25
43
  return (conn.agentAddresses.has(address) || conn.workflowAddresses.has(address));
26
44
  }
45
+ /**
46
+ * Bind pack writes to the repository implied by the authenticated address.
47
+ * An allocated credential is narrower still: it may only write its one
48
+ * deployment's workflow-run repository and never a standalone agent-state
49
+ * repository.
50
+ */
51
+ function connCanPushRepo(conn, agentAddress, repoId) {
52
+ if (conn.identity.kind !== "allocated")
53
+ return false;
54
+ if (agentAddress !== conn.identity.workflowRunAddress) {
55
+ return false;
56
+ }
57
+ return (repoId.kind === "workflow-run" &&
58
+ repoId.id === deriveWorkflowRunRepoId(agentAddress));
59
+ }
27
60
  /** The deduped set of every address this connection owns (session + workflow). */
28
61
  function ownedAddresses(conn) {
29
62
  return new Set([...conn.agentAddresses, ...conn.workflowAddresses]);
30
63
  }
31
64
  const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
32
- const DEFAULT_CHALLENGE_TIMEOUT_MS = 30_000;
65
+ // A probe fetches a workflow's dependency closure from a registry and
66
+ // evaluates it on the sidecar, so it runs longer than a routine request; its
67
+ // default timeout is correspondingly wider than DEFAULT_REQUEST_TIMEOUT_MS.
68
+ export const DEFAULT_PROBE_TIMEOUT_MS = 60_000;
33
69
  const DEFAULT_DISCONNECT_QUEUE_MAX_SIZE = 100;
34
70
  const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000;
35
71
  const DEFAULT_PING_TIMEOUT_MS = 60_000;
72
+ const DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS = 10_000;
73
+ const DEFAULT_MAIL_ACK_MAX_RETRIES = 5;
74
+ // The hub re-resolves and re-pushes a key for each rotatable sender a sidecar
75
+ // reports on (re)connect. A legitimate sidecar caches keys for tens, maybe low
76
+ // hundreds of distinct user senders, so this cap sits well above ten times that
77
+ // ceiling: it NEVER truncates a real report -- dropping a genuine sender would
78
+ // leave its key stale, the exact failure this refresh exists to prevent. It
79
+ // bounds only a hostile or buggy sidecar, since a compromised authenticated
80
+ // sidecar could otherwise report an unbounded set and drive that many sequential
81
+ // DB resolves on every reconnect. The cap lives in the handler, not on the
82
+ // arktype frame schema, on purpose: rejecting an over-cap frame at parse would
83
+ // fail the whole reconnect (a hard outage) rather than degrade gracefully to a
84
+ // bounded refresh.
85
+ export const MAX_RESYNC_SENDER_ADDRESSES = 2048;
36
86
  export function createSidecarRouter(config) {
37
- const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, challengeTimeoutMs = DEFAULT_CHALLENGE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, lookups = {}, } = config;
87
+ const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, probeTimeoutMs = DEFAULT_PROBE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, validateSidecarIdentity, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, scheduleTimeout = (handler, ms) => {
88
+ const handle = setTimeout(handler, ms);
89
+ return () => {
90
+ clearTimeout(handle);
91
+ };
92
+ }, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, lookups = {}, } = config;
38
93
  // Receiver-dispatch surface. Wire-layer callsites emit events here;
39
94
  // host code subscribes via `router.events`.
40
95
  const events = createSidecarEmitter();
41
96
  // ws handle → registered connection
42
97
  const connections = new Map();
98
+ const allocatedConnections = new Map();
99
+ const allocationFences = new Map();
100
+ const allocationWaiters = new Map();
43
101
  // agentAddress → ws handle (routing table)
44
102
  const addressIndex = new Map();
45
- // requestId → pending promise
46
- const pending = new Map();
47
- const pendingDeploys = new Map();
48
- const pendingChallenges = new Map();
103
+ // requestId → pending promise (resolved by session.ack, rejected by
104
+ // session.error). `PendingTracker` owns the register/timeout/settle/sweep
105
+ // lifecycle shared by all five pending round-trips below; each entry's
106
+ // resolve/reject closures carry the per-round-trip cleanup.
107
+ const pendingRequests = new PendingTracker();
108
+ // agentAddress → pending deploy promise (matched by agent.deploy.ack/agent.error)
109
+ const pendingDeploys = new PendingTracker();
110
+ // Run addresses whose ALLOCATED deploy is mid-flight -- key-record has been
111
+ // started but not yet committed. pendingDeploys clears at the deploy ack, but an
112
+ // allocated run's key is recorded LATER by session-service's anchor-key update,
113
+ // so pendingDeploys alone under-covers the allocated pre-ack window. session-
114
+ // service brackets this marker across its deploy try/catch: set before the
115
+ // deploy emit, cleared by noteSenderDeploySettled on record or failure.
116
+ const allocatedKeyRecordInFlight = new Map();
49
117
  const disconnectedAgents = new Map();
118
+ const pendingMail = new Map();
119
+ // agentAddress → retention TTL timer for un-acked pending mail held across a
120
+ // disconnect. On close the per-entry retry timers are cleared (the socket is
121
+ // gone) but the entries are RETAINED so a verified reconnect can redeliver
122
+ // them; this timer bounds that retention so a sidecar that never reconnects
123
+ // does not leak entries. Cleared when the address reconnects (redelivery) or
124
+ // its last pending entry is acked.
125
+ const pendingMailRetention = new Map();
126
+ const deferredSenderMail = new Map();
50
127
  // agentAddress → set of subscriber callbacks for agent events
51
128
  const agentSubscribers = new Map();
52
129
  // agentAddress → cached connector-thread state, populated by
@@ -66,9 +143,17 @@ export function createSidecarRouter(config) {
66
143
  // a single in-flight promise per ws (replaced each queued frame), cleared on
67
144
  // close.
68
145
  const messageChains = new Map();
69
- const pendingPacks = new Map();
146
+ const pendingPacks = new PendingTracker();
70
147
  let packCounter = 0;
71
- const pendingUndeploys = new Map();
148
+ // agentAddress pending undeploy (resolved by agent.undeploy.ack)
149
+ const pendingUndeploys = new PendingTracker();
150
+ // requestId → pending workflow probe (resolved by workflow.probe.result,
151
+ // rejected by workflow.probe.error). Result-carrying, unlike the other
152
+ // trackers (which resolve void): a probe returns the sidecar's inert
153
+ // projection + grant set + wire hash. Keyed on requestId alone -- the
154
+ // probe runs in the sidecar's pre-deploy state and enters no address map,
155
+ // so `handleClose`'s ws-keyed sweep is its ONLY disconnect cleanup.
156
+ const pendingProbes = new PendingTracker();
72
157
  // Receives agent-state packs pushed from sidecars. The wire frames
73
158
  // (`repo.pack.push` / `repo.pack.done`) are shared with the
74
159
  // workflow-run flow; dispatch on `repoId.kind` picks which receiver
@@ -112,29 +197,303 @@ export function createSidecarRouter(config) {
112
197
  entry.queue.push(frame);
113
198
  return true;
114
199
  }
115
- function flushDisconnectedQueue(agentAddress, conn) {
116
- const entry = disconnectedAgents.get(agentAddress);
200
+ // Arm a redelivery-retry timer for a tracked pending mail. Wraps the async
201
+ // `retryPendingMail` so a rejection -- a socket write that throws once the
202
+ // sidecar is gone -- is logged rather than floating out of the timer as an
203
+ // unhandled rejection.
204
+ function scheduleMailRetry(agentAddress, messageId) {
205
+ return scheduleTimeout(() => {
206
+ void retryPendingMail(agentAddress, messageId).catch((err) => {
207
+ logger.warn `Redelivery retry for mail ${messageId} to ${agentAddress} failed: ${err instanceof Error ? err.message : String(err)}`;
208
+ });
209
+ }, mailAckRetryIntervalMs);
210
+ }
211
+ // Track a connected-window `mail.inbound` for redelivery until the sidecar
212
+ // acks its durable inbox write. Replaces any prior entry for the same
213
+ // (agentAddress, messageId) -- clearing its timer first so no timer leaks --
214
+ // which keeps a re-sent delivery from arming a second concurrent retry loop.
215
+ function trackPendingMail(agentAddress, messageId, frame, runGrants, allocatedTarget) {
216
+ let byId = pendingMail.get(agentAddress);
217
+ if (byId === undefined) {
218
+ byId = new Map();
219
+ pendingMail.set(agentAddress, byId);
220
+ }
221
+ const existing = byId.get(messageId);
222
+ if (existing !== undefined)
223
+ existing.cancelRetry();
224
+ byId.set(messageId, {
225
+ agentAddress,
226
+ messageId,
227
+ frame,
228
+ attempts: 0,
229
+ cancelRetry: scheduleMailRetry(agentAddress, messageId),
230
+ ...(runGrants !== undefined ? { runGrants } : {}),
231
+ ...(allocatedTarget !== undefined ? { allocatedTarget } : {}),
232
+ });
233
+ }
234
+ // Resolve the frame that must precede a redelivery of a trigger mail on the
235
+ // FIFO socket, re-resolving a keyless run sender's key so it still
236
+ // co-delivers. Returns:
237
+ // - a `run.grants` frame when the entry carries run grants (the redelivered
238
+ // run resolves its onRunStart barrier instead of failing closed on
239
+ // missing grants);
240
+ // - a bare `sender.key.refresh` frame when the entry carries NO run grants
241
+ // but its run sender's key was never co-delivered, so the recipient still
242
+ // caches the key ahead of the mail;
243
+ // - `undefined` when nothing must precede the mail.
244
+ //
245
+ // The re-resolve is KIND-GATED to run-address senders only. A run's
246
+ // deployment key is immutable once acked, so the re-resolved key equals the
247
+ // signing-time key -- safe. A user (non-run) sender's key may have rotated
248
+ // since it signed, so re-resolving would check the fixed signed bytes against
249
+ // a newer key and turn a valid message into a false `invalid`; such a sender
250
+ // lacking a captured key stays keyless (an honest `unknown`). An entry that
251
+ // captured `senderIdentities` at track time replays that snapshot as-is: it
252
+ // holds the signing-time key and is never re-resolved.
253
+ //
254
+ // Awaits any key resolve so the caller sends the returned frame and the mail
255
+ // back-to-back with no await between them, keeping the co-delivered key ahead
256
+ // of the mail on the FIFO socket.
257
+ async function resolveReplayLeadFrame(entry) {
258
+ const authenticatedSender = entry.frame.type === "mail.inbound"
259
+ ? entry.frame.authenticatedSender
260
+ : undefined;
261
+ const senderIsRun = authenticatedSender !== undefined && isRunAddress(authenticatedSender);
262
+ if (entry.runGrants === undefined) {
263
+ if (authenticatedSender === undefined || !senderIsRun)
264
+ return undefined;
265
+ const key = await reresolveRunSenderKey(authenticatedSender);
266
+ if (key === null)
267
+ return undefined;
268
+ return {
269
+ type: "sender.key.refresh",
270
+ address: authenticatedSender,
271
+ publicKey: key,
272
+ };
273
+ }
274
+ let senderIdentities = entry.runGrants.senderIdentities;
275
+ if (senderIdentities === undefined &&
276
+ authenticatedSender !== undefined &&
277
+ senderIsRun) {
278
+ const key = await reresolveRunSenderKey(authenticatedSender);
279
+ senderIdentities = senderIdentitiesFromKey(authenticatedSender, key);
280
+ }
281
+ return {
282
+ type: "run.grants",
283
+ agentAddress: entry.agentAddress,
284
+ runId: entry.runGrants.runId,
285
+ stepGrants: entry.runGrants.stepGrants,
286
+ ...(senderIdentities !== undefined ? { senderIdentities } : {}),
287
+ };
288
+ }
289
+ // Best-effort re-resolve of a run sender's hub-held key at replay time. Only
290
+ // called for a run-address sender, whose deployment key is immutable once
291
+ // acked, so the current key equals the signing-time key. Returns null when no
292
+ // resolver is wired or the sender has no durable key.
293
+ //
294
+ // This relies on `lookups.resolveSenderKey` being the BEST-EFFORT,
295
+ // NEVER-THROWS resolver (the contract at sidecar-events.ts:274-279, wired to
296
+ // resolveFrameSenderKey, which swallows faults to null). That contract is
297
+ // load-bearing here: `redeliverPendingMail` clears the retention TTL up-front
298
+ // and re-arms each entry's per-entry timer only on a successful send, so a
299
+ // resolver that THREW would abort the redeliver loop and strand the
300
+ // not-yet-processed entries with no timer and no TTL until a process restart.
301
+ // A strict/throwing resolver must NOT be wired here. Do not add a try/catch:
302
+ // the boundary owns the never-throws contract; duplicating it here would
303
+ // violate that ownership. The dispatch-time resolveSenderKey call
304
+ // (sendWorkflowRunDispatchToAllocation path) carries the same dependency
305
+ // note.
306
+ async function reresolveRunSenderKey(authenticatedSender) {
307
+ const resolveSenderKey = lookups.resolveSenderKey;
308
+ if (resolveSenderKey === undefined)
309
+ return null;
310
+ return resolveSenderKey(authenticatedSender);
311
+ }
312
+ // Replay a pending mail's lead frame (its run grants or a re-resolved sender
313
+ // key) and then the mail itself over `conn`. Awaits the resolve FIRST, then
314
+ // sends the lead frame and the mail back-to-back with NO await between them,
315
+ // so the co-delivered key always precedes the mail on the FIFO socket.
316
+ // Returns whether the mail was (re)sent, so the caller re-arms the retry timer
317
+ // only for an entry it actually redelivered.
318
+ async function replaySendPendingMail(conn, entry) {
319
+ const lead = await resolveReplayLeadFrame(entry);
320
+ // The resolve above may have awaited real I/O; during that gap a queued
321
+ // `mail.inbound.ack` can advance and run `resolvePendingMail` (delete +
322
+ // clearTimeout) on this entry. The window is opened by the timer-macrotask
323
+ // retry path, NOT by any bypass: `mail.inbound.ack` is a QUEUED frame
324
+ // (frameBypassesQueue returns false for it). It can interleave because the
325
+ // retry runs as an independent setTimeout macrotask (retryPendingMail), so
326
+ // the owning ws's message chain is free to advance the ack during the
327
+ // resolve await. On the reconnect/redeliver path the ack cannot interleave
328
+ // at all -- it queues behind the still-running reconnect handler on the
329
+ // same ws -- so here this guard is pure defense-in-depth. Re-confirm it is
330
+ // still the tracked entry before sending, or a post-ack redelivery would
331
+ // arm a retry timer on a detached entry.
332
+ if (pendingMail.get(entry.agentAddress)?.get(entry.messageId) !== entry) {
333
+ return false;
334
+ }
335
+ // The same gap can span a disconnect or a takeover that moves the address
336
+ // off `conn`. Sending on the stale conn would write to a dead socket and
337
+ // re-arm a retry that later drops a still-retained entry. Skip so the entry
338
+ // survives for the reconnect redelivery.
339
+ const ws = addressIndex.get(entry.agentAddress);
340
+ if (ws === undefined || connections.get(ws) !== conn)
341
+ return false;
342
+ if (lead !== undefined)
343
+ conn.send(lead);
344
+ conn.send(entry.frame);
345
+ return true;
346
+ }
347
+ function deletePendingMail(byId, agentAddress, messageId) {
348
+ byId.delete(messageId);
349
+ if (byId.size === 0) {
350
+ pendingMail.delete(agentAddress);
351
+ // The retention TTL guards a non-empty pending set; drop it once the set
352
+ // is empty so it never outlives the entries it was bounding.
353
+ const retention = pendingMailRetention.get(agentAddress);
354
+ if (retention !== undefined) {
355
+ clearTimeout(retention);
356
+ pendingMailRetention.delete(agentAddress);
357
+ }
358
+ }
359
+ }
360
+ async function retryPendingMail(agentAddress, messageId) {
361
+ const byId = pendingMail.get(agentAddress);
362
+ if (byId === undefined)
363
+ return;
364
+ const entry = byId.get(messageId);
117
365
  if (entry === undefined)
118
366
  return;
119
- clearTimeout(entry.timer);
120
- disconnectedAgents.delete(agentAddress);
121
- for (const frame of entry.queue) {
122
- conn.send(frame);
367
+ if (entry.attempts >= mailAckMaxRetries) {
368
+ // The sidecar never acked within the retry budget. The ack is withheld
369
+ // precisely because the sidecar's durable inbox write failed, so the
370
+ // mail was NOT delivered: surface it as undelivered so the host can relay
371
+ // it onto an external transport, then drop the pending entry so its timer
372
+ // does not leak.
373
+ if (entry.frame.type === "mail.inbound") {
374
+ events.emit("mail.outbound.undelivered", {
375
+ rawMessage: entry.frame.rawMessage,
376
+ recipients: [agentAddress],
377
+ });
378
+ }
379
+ deletePendingMail(byId, agentAddress, messageId);
380
+ logger.warn `Gave up redelivering mail ${messageId} to ${agentAddress} after ${String(entry.attempts)} un-acked attempt(s)`;
381
+ return;
123
382
  }
124
- if (entry.queue.length > 0) {
125
- logger.info `Flushed ${String(entry.queue.length)} queued message(s) to ${agentAddress}`;
383
+ // Redeliver over the address's CURRENT owner: a verified reconnect may have
384
+ // moved the address to a new connection since the original delivery.
385
+ const ws = addressIndex.get(agentAddress);
386
+ const conn = ws !== undefined ? connections.get(ws) : undefined;
387
+ const allocated = entry.allocatedTarget === undefined
388
+ ? undefined
389
+ : allocatedConnections.get(entry.allocatedTarget.allocationId);
390
+ const targetStillOwnsAddress = entry.allocatedTarget === undefined ||
391
+ (allocated !== undefined &&
392
+ allocated.identity.generation === entry.allocatedTarget.generation &&
393
+ allocated.ws === ws);
394
+ if (conn === undefined || !targetStillOwnsAddress) {
395
+ // No live connection to recover into. Connected-window redelivery only
396
+ // applies while the address is routable; a disconnected address is not
397
+ // retried here.
398
+ deletePendingMail(byId, agentAddress, messageId);
399
+ logger.warn `Dropping un-acked mail ${messageId} for ${agentAddress}: no live connection to redeliver over`;
400
+ return;
126
401
  }
402
+ if (!(await replaySendPendingMail(conn, entry)))
403
+ return;
404
+ entry.attempts += 1;
405
+ entry.cancelRetry = scheduleMailRetry(agentAddress, messageId);
127
406
  }
128
- function resetLivenessTimer(ws) {
129
- const existing = livenessTimers.get(ws);
407
+ function resolvePendingMail(agentAddress, messageId) {
408
+ const byId = pendingMail.get(agentAddress);
409
+ if (byId === undefined)
410
+ return;
411
+ const entry = byId.get(messageId);
412
+ if (entry === undefined)
413
+ return;
414
+ entry.cancelRetry();
415
+ deletePendingMail(byId, agentAddress, messageId);
416
+ }
417
+ // Hold an address's un-acked pending mail across a disconnect. The per-entry
418
+ // retry timers are cleared -- retrying over the dead socket is pointless --
419
+ // but the entries are KEPT so a verified reconnect can redeliver them. A
420
+ // retention TTL (the disconnect-queue horizon) bounds the hold so a sidecar
421
+ // that never reconnects does not leak; on expiry the still-un-acked entries
422
+ // are surfaced as `mail.outbound.undelivered` so the host can relay them,
423
+ // since a withheld ack means the sidecar's durable write never landed.
424
+ function retainPendingMailForAddress(agentAddress) {
425
+ const byId = pendingMail.get(agentAddress);
426
+ if (byId === undefined)
427
+ return;
428
+ for (const entry of byId.values())
429
+ entry.cancelRetry();
430
+ const existing = pendingMailRetention.get(agentAddress);
130
431
  if (existing !== undefined)
131
432
  clearTimeout(existing);
132
433
  const timer = setTimeout(() => {
434
+ pendingMailRetention.delete(agentAddress);
435
+ const expired = pendingMail.get(agentAddress);
436
+ pendingMail.delete(agentAddress);
437
+ if (expired !== undefined && expired.size > 0) {
438
+ for (const entry of expired.values()) {
439
+ if (entry.frame.type !== "mail.inbound")
440
+ continue;
441
+ events.emit("mail.outbound.undelivered", {
442
+ rawMessage: entry.frame.rawMessage,
443
+ recipients: [agentAddress],
444
+ });
445
+ }
446
+ logger.warn `Dropping ${String(expired.size)} un-acked message(s) for ${agentAddress}: pending-mail retention TTL expired`;
447
+ }
448
+ }, disconnectQueueTTLMs);
449
+ pendingMailRetention.set(agentAddress, timer);
450
+ }
451
+ // Redeliver an address's retained un-acked pending mail on a verified
452
+ // reconnect. Replays identical bytes (same messageId) over the new
453
+ // connection, so the sidecar's inbox dedups a message it already wrote
454
+ // (effectively-once) and processes one it had dropped (no loss). Re-arms the
455
+ // connected-window retry over the new connection with a fresh per-generation
456
+ // budget, so a redelivery that is itself dropped before its ack is retried.
457
+ async function redeliverPendingMail(agentAddress, conn) {
458
+ const retention = pendingMailRetention.get(agentAddress);
459
+ if (retention !== undefined) {
460
+ clearTimeout(retention);
461
+ pendingMailRetention.delete(agentAddress);
462
+ }
463
+ const byId = pendingMail.get(agentAddress);
464
+ if (byId === undefined)
465
+ return;
466
+ for (const entry of [...byId.values()]) {
467
+ if (entry.allocatedTarget !== undefined &&
468
+ (conn.identity.kind !== "allocated" ||
469
+ conn.identity.allocationId !== entry.allocatedTarget.allocationId ||
470
+ conn.identity.generation !== entry.allocatedTarget.generation)) {
471
+ // The Hub-owned dispatch row survives generation replacement and will
472
+ // be requeued by the allocation-ready callback. Do not leak or replay
473
+ // this generation-local retry entry onto a different worker.
474
+ entry.cancelRetry();
475
+ deletePendingMail(byId, agentAddress, entry.messageId);
476
+ continue;
477
+ }
478
+ if (!(await replaySendPendingMail(conn, entry)))
479
+ continue;
480
+ entry.attempts = 0;
481
+ entry.cancelRetry = scheduleMailRetry(agentAddress, entry.messageId);
482
+ }
483
+ if (byId.size > 0) {
484
+ logger.info `Redelivered ${String(byId.size)} un-acked message(s) to ${agentAddress} on reconnect`;
485
+ }
486
+ }
487
+ function resetLivenessTimer(ws) {
488
+ const existing = livenessTimers.get(ws);
489
+ if (existing !== undefined)
490
+ existing();
491
+ const cancel = scheduleTimeout(() => {
133
492
  livenessTimers.delete(ws);
134
493
  logger.warn `Sidecar ping timeout, closing connection`;
135
494
  ws.close();
136
495
  }, pingTimeoutMs);
137
- livenessTimers.set(ws, timer);
496
+ livenessTimers.set(ws, cancel);
138
497
  }
139
498
  function handlePing(ws) {
140
499
  resetLivenessTimer(ws);
@@ -197,11 +556,9 @@ export function createSidecarRouter(config) {
197
556
  // state. Such a frame has no ordering obligation against new inbound frames
198
557
  // (a response cannot resolve "too early" for a request that already went
199
558
  // out), and it is exactly what in-flight queued handlers block on, so it MUST
200
- // run out of band or the challenge round-trip deadlocks (challenge.response
201
- // -> agent.reconnected reaction -> sendSourcesUpdate awaits a later
202
- // session.ack). Every other frame establishes or reads routing, or carries an
203
- // inbound payload whose order matters, so it queues. The exhaustive switch +
204
- // assertNever makes adding a SidecarFrame variant without classifying it a
559
+ // run out of band. Every other frame establishes or reads routing, or carries
560
+ // an inbound payload whose order matters, so it queues. The exhaustive switch
561
+ // + assertNever makes adding a SidecarFrame variant without classifying it a
205
562
  // compile error, not a latent deadlock or a silent bypass hole.
206
563
  function frameBypassesQueue(frame) {
207
564
  switch (frame.type) {
@@ -213,13 +570,16 @@ export function createSidecarRouter(config) {
213
570
  case "agent.undeploy.ack":
214
571
  case "repo.pack.ack":
215
572
  case "repo.pack.reject":
573
+ case "workflow.probe.result":
574
+ case "workflow.probe.error":
216
575
  return true;
217
576
  case "register":
218
577
  case "reconnect":
219
- case "challenge.response":
220
578
  case "mail.outbound":
221
579
  case "agent.event":
222
580
  case "connector.state.changed":
581
+ case "mail.inbound.ack":
582
+ case "signal.correlation.register":
223
583
  case "repo.pack.push":
224
584
  case "repo.pack.done":
225
585
  return false;
@@ -229,50 +589,82 @@ export function createSidecarRouter(config) {
229
589
  }
230
590
  // Runs one frame's handler. Returns the handler's promise for async handlers
231
591
  // so the per-ws chain can await bounded completion; sync handlers return
232
- // void. Never awaits a promise that resolves on a LATER same-ws frame -- the
233
- // only such await (the challenge round-trip's session.ack) is reached via a
234
- // bypass frame, which does not queue.
592
+ // void. Never awaits a promise that resolves on a later same-ws frame.
235
593
  function dispatchFrame(ws, frame) {
594
+ const registeredIdentity = connections.get(ws)?.identity;
595
+ if (registeredIdentity?.kind === "probe" &&
596
+ frame.type !== "register" &&
597
+ frame.type !== "reconnect" &&
598
+ frame.type !== "ping" &&
599
+ frame.type !== "workflow.probe.result" &&
600
+ frame.type !== "workflow.probe.error") {
601
+ logger.warn `Rejected ${frame.type} from probe sidecar ${registeredIdentity.sidecarId}`;
602
+ handleClose(ws);
603
+ ws.close();
604
+ return;
605
+ }
236
606
  switch (frame.type) {
237
607
  case "register": {
238
608
  const agentAddresses = frame.agentAddresses;
239
- return authenticateHandshake(ws, frame, (identity) => handleRegister(ws, identity, agentAddresses));
609
+ const cachedSenderAddresses = frame.cachedSenderAddresses ?? [];
610
+ return authenticateHandshake(ws, frame, (identity) => handleRegister(ws, identity, agentAddresses, cachedSenderAddresses));
240
611
  }
241
612
  case "reconnect": {
242
613
  const agentAddresses = frame.agentAddresses;
243
- const deployRefs = frame.deployRefs ?? {};
244
- return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses, deployRefs));
614
+ const cachedSenderAddresses = frame.cachedSenderAddresses ?? [];
615
+ return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses, cachedSenderAddresses));
245
616
  }
246
- case "challenge.response":
247
- return handleChallengeResponse(ws, frame.responses);
248
617
  case "agent.deploy.ack":
249
- return handleDeployAck(frame.agentAddress, frame.publicKey);
618
+ return handleDeployAck(ws, frame);
250
619
  case "agent.error":
251
- rejectDeployPending(frame.agentAddress, frame.error);
252
- rejectUndeployPending(frame.agentAddress, frame.error);
620
+ rejectDeployPendingFromFrame(ws, frame.agentAddress, frame.error);
621
+ rejectUndeployPending(ws, frame.agentAddress, frame.error);
253
622
  return;
254
623
  case "agent.undeploy.ack":
255
- resolveUndeployPending(frame.agentAddress);
624
+ resolveUndeployPending(ws, frame.agentAddress);
256
625
  return;
257
626
  case "ping":
258
627
  handlePing(ws);
259
628
  return;
260
- case "mail.outbound":
261
- if (frame.delivered !== true) {
262
- handleMailOutbound(frame.rawMessage, frame.recipients);
629
+ case "mail.outbound": {
630
+ const conn = connections.get(ws);
631
+ if (conn === undefined)
632
+ return;
633
+ if (!connOwnsAddress(conn, frame.senderAddress)) {
634
+ logger.warn `Dropping mail.outbound from ${frame.senderAddress}: not registered to this sidecar`;
263
635
  return;
264
636
  }
265
- if (lookups.persistMail && frame.senderAddress) {
266
- return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
637
+ // The DoS backstop and the trust boundary for an untrusted sidecar's
638
+ // mail body: measure the true byte cost (a hostile sidecar can send
639
+ // multi-byte UTF-8, so `.length` would undercount) and drop an over-cap
640
+ // frame here before either delivery path allocates on it. The socket's
641
+ // maxPayloadLength has already closed the connection for a truly huge
642
+ // frame; this catches one between the mail cap and that ceiling.
643
+ const bodyBytes = Buffer.byteLength(frame.rawMessage, "utf8");
644
+ if (bodyBytes > MAX_MAIL_OUTBOUND_BODY_BYTES) {
645
+ logger.warn `Dropping mail.outbound from ${frame.senderAddress}: rawMessage of ${String(bodyBytes)} bytes exceeds the ${String(MAX_MAIL_OUTBOUND_BODY_BYTES)}-byte cap`;
646
+ return;
267
647
  }
268
- if (!frame.senderAddress) {
269
- logger.warn `Dropping delivered mail.outbound frame with no senderAddress`;
648
+ if (frame.delivered !== true) {
649
+ // frame.senderAddress is the sender this connection was just gated
650
+ // on by connOwnsAddress above -- a hub-verified value. Thread it so
651
+ // the relayed inbound frame is stamped with it, not the MIME From.
652
+ return handleMailOutbound(frame.rawMessage, frame.senderAddress, frame.recipients);
270
653
  }
271
- else {
272
- logger.warn `Dropping delivered mail.outbound frame: no persistMail lookup configured`;
654
+ if (lookups.persistMail) {
655
+ return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
273
656
  }
657
+ logger.warn `Dropping delivered mail.outbound frame: no persistMail lookup configured`;
274
658
  return;
275
- case "agent.event":
659
+ }
660
+ case "agent.event": {
661
+ const conn = connections.get(ws);
662
+ if (conn === undefined)
663
+ return;
664
+ if (!connOwnsAddress(conn, frame.agentAddress)) {
665
+ logger.warn `Dropping agent.event for ${frame.agentAddress}: not registered to this sidecar`;
666
+ return;
667
+ }
276
668
  events.emit("agent.event", {
277
669
  agentAddress: frame.agentAddress,
278
670
  sessionId: frame.sessionId,
@@ -280,6 +672,7 @@ export function createSidecarRouter(config) {
280
672
  });
281
673
  dispatchToSubscribers(frame.agentAddress, frame.event);
282
674
  return;
675
+ }
283
676
  case "connector.state.changed":
284
677
  // Gate the cache write on the sending sidecar actually owning
285
678
  // the named agent. A misbehaving sidecar that knows another
@@ -294,23 +687,66 @@ export function createSidecarRouter(config) {
294
687
  connectorState: frame.connectorState,
295
688
  });
296
689
  return;
690
+ case "mail.inbound.ack": {
691
+ // Terminal receipt for a connected-window `mail.inbound`: the sidecar
692
+ // has durably written the message to its inbox. Gate on ownership --
693
+ // like connector.state.changed and signal.correlation.register -- so a
694
+ // sidecar cannot clear another sidecar's pending mail. The messageId is
695
+ // a hub-minted id only the owning sidecar ever received on the frame,
696
+ // so the ownership check is defense-in-depth, not the sole guard.
697
+ const conn = connections.get(ws);
698
+ if (conn === undefined)
699
+ return;
700
+ if (!connOwnsAddress(conn, frame.agentAddress)) {
701
+ logger.warn `Dropping mail.inbound.ack for ${frame.agentAddress}: not registered to this sidecar`;
702
+ return;
703
+ }
704
+ resolvePendingMail(frame.agentAddress, frame.messageId);
705
+ events.emit("mail.inbound.acknowledged", {
706
+ agentAddress: frame.agentAddress,
707
+ messageId: frame.messageId,
708
+ ...(conn.identity.kind === "allocated"
709
+ ? {
710
+ allocated: {
711
+ allocationId: conn.identity.allocationId,
712
+ anchorRunId: conn.identity.anchorRunId,
713
+ generation: conn.identity.generation,
714
+ },
715
+ }
716
+ : {}),
717
+ });
718
+ return;
719
+ }
720
+ case "signal.correlation.register":
721
+ return handleSignalCorrelationRegister(ws, frame);
297
722
  case "session.ack":
298
- resolvePending(frame.requestId);
723
+ pendingRequests.resolve(frame.requestId);
299
724
  return;
300
725
  case "session.error":
301
- rejectPending(frame.requestId, frame.error);
726
+ pendingRequests.reject(frame.requestId, frame.error);
302
727
  return;
303
728
  case "repo.pack.ack":
304
- resolvePackPending(frame.transferId);
729
+ resolvePackPending(ws, frame);
305
730
  return;
306
731
  case "repo.pack.reject":
307
- rejectPackPending(frame.transferId, frame.reason);
732
+ rejectPackPending(ws, frame);
308
733
  return;
309
734
  case "repo.pack.push":
310
735
  handlePackPush(ws, frame);
311
736
  return;
312
737
  case "repo.pack.done":
313
738
  return handlePackDone(ws, frame);
739
+ case "workflow.probe.result":
740
+ resolveProbe(ws, frame.requestId, {
741
+ projection: frame.projection,
742
+ grants: frame.grants,
743
+ grantWalkSnapshot: frame.grantWalkSnapshot,
744
+ wireHash: frame.wireHash,
745
+ });
746
+ return;
747
+ case "workflow.probe.error":
748
+ rejectProbe(ws, frame.requestId, frame.error);
749
+ return;
314
750
  default:
315
751
  return assertNever(frame);
316
752
  }
@@ -343,466 +779,454 @@ export function createSidecarRouter(config) {
343
779
  if (identity.sidecarId !== frame.sidecarId) {
344
780
  logger.warn `Sidecar ${frame.type} claimed id ${frame.sidecarId} but token verifies as ${identity.sidecarId}; keying off the verified id`;
345
781
  }
782
+ if (!(await validateSidecarIdentity(identity, "registration"))) {
783
+ logger.warn `Rejected ${frame.type} from sidecar ${identity.sidecarId}: credential identity is no longer current`;
784
+ ws.close();
785
+ return;
786
+ }
346
787
  await run(identity);
347
788
  }
348
- async function handleRegister(ws, identity, agentAddresses) {
349
- const sidecarId = identity.sidecarId;
350
- // Key-existence gate. A register frame is token-authenticated but carries
351
- // no per-address ownership proof, so it may route an address ONLY if that
352
- // address has no stored key yet -- a genuine keyless first-deploy (the
353
- // token-bounded first-deploy trust model). An address that already has a
354
- // key must prove ownership through the challenged reconnect path; routing
355
- // it here on token auth alone is the register-frame sibling of the
356
- // reconnect hijack. The keyless-only set is computed up front, BEFORE the
357
- // ghost-cleanup and every routing mutation below, so a rejected address
358
- // touches nothing: no eviction of a live owner, hence no downgrade from
359
- // hijack to denial-of-service on the victim.
360
- const lookupKey = lookups.lookupPublicKey;
361
- const routableAddresses = [];
362
- for (const addr of agentAddresses) {
363
- if (lookupKey === undefined) {
364
- // Fail closed: without the ownership lookup a keyed address cannot be
365
- // told apart from a first-deploy, so route nothing and surface the
366
- // misconfiguration. Empty first-connect registers never reach here.
367
- logger.error `Cannot gate register routing for ${addr}: lookupPublicKey is not configured; refusing to route (challenged reconnect required)`;
368
- continue;
369
- }
370
- let existingKey;
371
- try {
372
- existingKey = await lookupKey(addr);
789
+ async function notifyAllocationWaiters(allocationId) {
790
+ const waiters = allocationWaiters.get(allocationId);
791
+ const current = allocatedConnections.get(allocationId);
792
+ if (waiters === undefined || current === undefined)
793
+ return;
794
+ const matchingWaiters = [...waiters].filter((waiter) => waiter.generation === current.identity.generation);
795
+ if (matchingWaiters.length === 0)
796
+ return;
797
+ const validation = Promise.resolve().then(() => validateSidecarIdentity(current.identity, "readiness"));
798
+ for (const waiter of matchingWaiters) {
799
+ waiter.validations.add(validation);
800
+ waiter.onValidation?.(validation);
801
+ }
802
+ let identityCurrent;
803
+ try {
804
+ identityCurrent = await validation;
805
+ }
806
+ catch (cause) {
807
+ // A failed revalidation leaves the waiters parked: a later register
808
+ // revalidates, and at expiry the wait reports the failure rather than a
809
+ // missed deadline. Registration itself was already gated, so this must
810
+ // not fail the connection that just registered.
811
+ const validationFailure = new SidecarIdentityValidationError(allocationId, current.identity.generation, cause);
812
+ for (const waiter of matchingWaiters) {
813
+ waiter.validationFailure = validationFailure;
373
814
  }
374
- catch (err) {
375
- // Fail closed on a lookup error (e.g. a transient DB failure): route
376
- // nothing for this address and surface the failure, rather than let
377
- // the rejection float out of this void-dispatched handler and take
378
- // down the hub.
379
- logger.error `Key lookup failed for ${addr} during register: ${err instanceof Error ? err.message : String(err)}; failing closed (challenged reconnect required)`;
380
- continue;
815
+ return;
816
+ }
817
+ finally {
818
+ for (const waiter of matchingWaiters) {
819
+ waiter.validations.delete(validation);
381
820
  }
382
- if (existingKey !== null) {
383
- logger.warn `Refusing to route ${addr} via register: address already has a stored key; ownership must be proven via challenged reconnect`;
821
+ }
822
+ // A clean validation supersedes earlier failures: expiry must report the
823
+ // current reading, not a stale transient.
824
+ for (const waiter of matchingWaiters) {
825
+ delete waiter.validationFailure;
826
+ }
827
+ if (!identityCurrent || allocatedConnections.get(allocationId) !== current)
828
+ return;
829
+ for (const waiter of matchingWaiters) {
830
+ if (!waiters.delete(waiter))
384
831
  continue;
385
- }
386
- routableAddresses.push(addr);
832
+ clearTimeout(waiter.timer);
833
+ waiter.resolve();
387
834
  }
388
- // Additive re-register: inherit every address this ws already owns and ADD
389
- // the frame's keyless first-deploys. Register never drops an owned route --
390
- // an address proved via challenged reconnect stays routed, and removal
391
- // happens via undeploy/disconnect, not register-omission.
392
- //
393
- // (Under the retired full-set model a register frame carried the sidecar's
394
- // complete live set, so an omitted address meant "removed". Now the frame
395
- // carries only keyless first-deploys the sidecar is adding -- keyed
396
- // addresses arrive via reconnect -- so omission is meaningless, and a
397
- // replace-on-register would wrongly drop an earlier first-deploy when a
398
- // later one is registered, as well as any reconnect-verified keyed route.)
399
- const existing = connections.get(ws);
400
- const inheritedAgent = new Set(existing?.agentAddresses);
401
- const inheritedWorkflow = new Set(existing?.workflowAddresses);
402
- // Clean up ghost entries from OTHER connections for the newly-claimed
403
- // addresses only. An inherited address is already owned by this ws
404
- // (prevWs === ws), so it needs no eviction and its in-flight deploy must
405
- // not be cancelled.
406
- for (const addr of routableAddresses) {
407
- const prevWs = addressIndex.get(addr);
408
- if (prevWs !== undefined && prevWs !== ws) {
409
- const prevConn = connections.get(prevWs);
410
- if (prevConn !== undefined) {
411
- prevConn.agentAddresses.delete(addr);
412
- }
413
- // The new owner is about to take over; the prior owner's
414
- // cached state must not survive into the new owner's window
415
- // before its bootstrap frame arrives.
416
- connectorStates.delete(addr);
417
- }
418
- // Cancel any in-flight deploy for this address since the
419
- // reconnecting sidecar is taking ownership.
420
- const staleDeployReq = pendingDeploys.get(addr);
421
- if (staleDeployReq !== undefined) {
422
- clearTimeout(staleDeployReq.timer);
423
- pendingDeploys.delete(addr);
424
- staleDeployReq.reject(`Sidecar ${sidecarId} reconnected and claimed address "${addr}"`);
835
+ if (waiters.size === 0 && allocationWaiters.get(allocationId) === waiters)
836
+ allocationWaiters.delete(allocationId);
837
+ }
838
+ async function handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses) {
839
+ if (allocationFences.get(identity.allocationId) !== identity.generation) {
840
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} generation ${String(identity.generation)} is not fenced as current`;
841
+ ws.close();
842
+ return;
843
+ }
844
+ if (identity.kind === "probe" && agentAddresses.length > 0) {
845
+ logger.warn `Rejected probe sidecar ${identity.sidecarId}: probe ${identity.allocationId} claimed workflow addresses`;
846
+ ws.close();
847
+ return;
848
+ }
849
+ if (agentAddresses.length > 0 &&
850
+ !(await validateSidecarIdentity(identity, "routing"))) {
851
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} is not ready to reclaim routes`;
852
+ ws.close();
853
+ return;
854
+ }
855
+ const existingOnSocket = connections.get(ws);
856
+ const newlyRoutedAddresses = new Set();
857
+ for (const address of agentAddresses) {
858
+ const alreadyOwned = existingOnSocket?.identity.kind === "allocated" &&
859
+ existingOnSocket.identity.allocationId === identity.allocationId &&
860
+ addressIndex.get(address) === ws &&
861
+ connOwnsAddress(existingOnSocket, address);
862
+ if (identity.kind !== "allocated" ||
863
+ (!alreadyOwned && address !== identity.workflowRunAddress)) {
864
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} claimed unrelated address ${address}`;
865
+ ws.close();
866
+ return;
425
867
  }
868
+ if (!alreadyOwned)
869
+ newlyRoutedAddresses.add(address);
870
+ }
871
+ if (allocationFences.get(identity.allocationId) !== identity.generation) {
872
+ ws.close();
873
+ return;
426
874
  }
427
- // New keyless first-deploys join the inherited session set; the workflow
428
- // set is inherited unchanged (register never adds to it -- a
429
- // workflow-derived address routes only through the challenged reconnect).
430
- for (const addr of routableAddresses) {
431
- inheritedAgent.add(addr);
432
- }
433
- const addrSet = inheritedAgent;
434
- const workflowSet = inheritedWorkflow;
435
- const conn = {
436
- sidecarId,
437
- agentAddresses: addrSet,
438
- workflowAddresses: workflowSet,
875
+ const current = allocatedConnections.get(identity.allocationId);
876
+ if (current !== undefined && current.ws !== ws) {
877
+ // A current-generation takeover is a reconnect, not a capacity loss.
878
+ // Remove the old socket from the allocation index before closing it so
879
+ // handleClose does not emit a false allocated-disconnect event.
880
+ allocatedConnections.delete(identity.allocationId);
881
+ handleClose(current.ws);
882
+ current.ws.close();
883
+ }
884
+ const conn = existingOnSocket ?? {
885
+ sidecarId: identity.sidecarId,
886
+ identity,
887
+ agentAddresses: new Set(),
888
+ workflowAddresses: new Set(),
439
889
  send(frame) {
440
890
  ws.send(JSON.stringify(frame));
441
891
  },
442
892
  };
443
- connections.set(ws, conn);
444
- // Only the newly-claimed addresses need a routing write + queue discard;
445
- // inherited addresses already point at this ws in addressIndex.
446
- for (const addr of routableAddresses) {
447
- addressIndex.set(addr, ws);
448
- // Discard any disconnect queue — register has no identity
449
- // verification, so flushing to an unverified connection is unsafe.
450
- // Use reconnect with challenge/response to preserve queued messages.
451
- const staleQueue = disconnectedAgents.get(addr);
452
- if (staleQueue !== undefined) {
453
- clearTimeout(staleQueue.timer);
454
- if (staleQueue.queue.length > 0) {
455
- logger.warn `Discarding ${String(staleQueue.queue.length)} queued message(s) for ${addr} on unverified register`;
456
- }
457
- disconnectedAgents.delete(addr);
458
- }
459
- }
460
- // Only keyless first-deploy addresses reach `addressIndex` here; an
461
- // address that already has a stored key was filtered out by the gate
462
- // above and must re-enter routing through the challenged reconnect path.
463
- // Because the gate runs before the ghost-cleanup, a rejected (keyed)
464
- // address never evicts its prior owner -- register cannot reclaim or
465
- // disrupt a victim's route on token auth alone.
466
- logger.info `Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`;
467
- }
468
- async function handleReconnect(ws, identity, agentAddresses, deployRefs = {}) {
469
- const sidecarId = identity.sidecarId;
470
- const lookupKey = lookups.lookupPublicKey;
471
- if (lookupKey === undefined) {
472
- logger.error `Received reconnect frame but no lookupPublicKey is configured`;
893
+ if (conn.identity.kind !== identity.kind ||
894
+ conn.identity.allocationId !== identity.allocationId ||
895
+ conn.identity.generation !== identity.generation) {
896
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: socket identity changed during registration`;
473
897
  ws.close();
474
898
  return;
475
899
  }
476
- // Cancel any existing pending challenge for this ws before doing
477
- // async work, so concurrent reconnect frames don't race.
478
- const existingChallenge = pendingChallenges.get(ws);
479
- if (existingChallenge !== undefined) {
480
- clearTimeout(existingChallenge.timer);
481
- pendingChallenges.delete(ws);
482
- }
483
- // Capture the addresses this live connection already owns. The
484
- // internal register below clears them, but ones the reconnect is not
485
- // re-challenging are still valid an agent provisioned during the
486
- // sidecar's restore window is routed into addressIndex by its deploy
487
- // and is not part of the disk-restored set this reconnect carries.
488
- // Without preserving them, the reconnect silently drops a
489
- // freshly-deployed agent from routing.
490
- const previouslyOwned = new Set(connections.get(ws)?.agentAddresses);
491
- // Register the sidecar connection immediately (with no addresses) so it
492
- // can receive frames while the ownership challenge is pending. Every
493
- // reconnect address -- session and workflow-derived alike -- enters
494
- // routing only through the verified path below, never unchallenged here.
495
- // Empty address list, so the key-existence gate has nothing to await.
496
- // The identity was already verified at the dispatch boundary, so the
497
- // internal register does not re-authenticate.
498
- await handleRegister(ws, identity, []);
499
- const conn = connections.get(ws);
500
- if (conn === undefined)
501
- return;
502
- // Re-add the still-owned addresses the register cleared but this
503
- // reconnect is not re-challenging. The challenged addresses below
504
- // re-enter addressIndex through the verified path instead.
505
- const claimedAddresses = new Set(agentAddresses);
506
- for (const addr of previouslyOwned) {
507
- if (claimedAddresses.has(addr))
508
- continue;
509
- conn.agentAddresses.add(addr);
510
- addressIndex.set(addr, ws);
511
- }
512
- // Look up stored public keys for all claimed addresses. Fail closed on a
513
- // lookup error (e.g. a transient DB failure): treat the address as
514
- // unverifiable so it fails its challenge and stays unrouted, rather than
515
- // letting the rejection float out of this void-dispatched handler as an
516
- // unhandled rejection that could take down the hub.
517
- const keyLookups = await Promise.all(agentAddresses.map(async (addr) => {
518
- try {
519
- return { address: addr, publicKeyHex: await lookupKey(addr) };
520
- }
521
- catch (err) {
522
- logger.error `Key lookup failed for ${addr} during reconnect: ${err instanceof Error ? err.message : String(err)}; failing closed`;
523
- return { address: addr, publicKeyHex: null };
524
- }
525
- }));
526
- // If the connection was closed or superseded while we were awaiting
527
- // key lookups, bail out.
528
- if (!connections.has(ws))
529
- return;
530
- const challenges = new Map();
531
- const challengeEntries = [];
532
- for (const { address, publicKeyHex } of keyLookups) {
533
- if (publicKeyHex === null) {
534
- conn.send({
535
- type: "challenge.failed",
536
- address,
537
- reason: "Unknown agent address",
538
- });
539
- continue;
540
- }
541
- let publicKey;
542
- try {
543
- publicKey = hexDecode(publicKeyHex);
900
+ connections.set(ws, conn);
901
+ for (const address of agentAddresses) {
902
+ conn.workflowAddresses.add(address);
903
+ addressIndex.set(address, ws);
904
+ }
905
+ allocatedConnections.set(identity.allocationId, { ws, identity });
906
+ for (const address of newlyRoutedAddresses) {
907
+ await redeliverPendingMail(address, conn);
908
+ }
909
+ // Reconcile a reconnecting deployment's credentials, closing the offline
910
+ // window: a credential revoked, deleted, or rotated while the sidecar was
911
+ // disconnected is applied to the child now. Fire-and-forget so
912
+ // registration is not blocked; the lookup no-ops for a run that persisted
913
+ // no credential refs.
914
+ const resyncCredentials = lookups.resyncCredentials;
915
+ if (resyncCredentials !== undefined) {
916
+ for (const address of newlyRoutedAddresses) {
917
+ if (!isRunAddress(address))
918
+ continue;
919
+ resyncCredentials(address);
544
920
  }
545
- catch {
546
- conn.send({
547
- type: "challenge.failed",
548
- address,
549
- reason: "Stored public key is corrupt",
550
- });
551
- logger.error `Corrupt stored public key for ${address}`;
552
- continue;
921
+ }
922
+ // Reconcile the sidecar's cached sender keys, closing the offline window: a
923
+ // user-principal key that rotated while the sidecar was disconnected is
924
+ // re-resolved and re-pushed, and a sender whose principal was DELETED while
925
+ // the sidecar was disconnected is evicted, so the recipient stops verifying
926
+ // either against a key the hub no longer vouches for. Only allocated
927
+ // sidecars host a sender cache worth reconciling. Resolve and push
928
+ // SEQUENTIALLY in one detached task: registration is never blocked, and a
929
+ // large cache cannot fan out into one concurrent DB query per reported
930
+ // sender on every reconnect.
931
+ const resolveSenderKeyStrict = lookups.resolveSenderKeyStrict;
932
+ if (identity.kind === "allocated" && resolveSenderKeyStrict !== undefined) {
933
+ const rotatableSenders = new Set(cachedSenderAddresses);
934
+ // Resolve-don't-trust applied to input SIZE: bound the reported set before
935
+ // acting on it. Run addresses count toward the cap by design -- the
936
+ // isRunAddress skip below is inside the loop, so the iteration, and thus
937
+ // the DB resolves, can never exceed the cap regardless of the run/non-run
938
+ // mix. Over the cap, reconcile the first MAX_RESYNC_SENDER_ADDRESSES and
939
+ // log the overflow so a misbehaving sidecar is detectable.
940
+ let sendersToResync = [...rotatableSenders];
941
+ if (sendersToResync.length > MAX_RESYNC_SENDER_ADDRESSES) {
942
+ logger.warn `Sidecar ${identity.sidecarId} reported ${String(sendersToResync.length)} cached sender addresses on allocation ${identity.allocationId} generation ${String(identity.generation)}, over the ${String(MAX_RESYNC_SENDER_ADDRESSES)} resync cap; reconciling the first ${String(MAX_RESYNC_SENDER_ADDRESSES)} and ignoring the rest`;
943
+ sendersToResync = sendersToResync.slice(0, MAX_RESYNC_SENDER_ADDRESSES);
553
944
  }
554
- const nonce = crypto.getRandomValues(new Uint8Array(32));
555
- challenges.set(address, { nonce, publicKey });
556
- challengeEntries.push({ address, nonce: hexEncode(nonce) });
945
+ void (async () => {
946
+ for (const address of sendersToResync) {
947
+ // The sidecar already reports only non-run senders, but do not trust
948
+ // the report: a run sender's key is the immutable
949
+ // workflow_run.public_key and is never refreshed or evicted, so skip
950
+ // it here too rather than couple correctness to the sidecar's filter.
951
+ if (isRunAddress(address))
952
+ continue;
953
+ // Tri-state, deleted-vs-fault distinguished by the STRICT resolver:
954
+ // - resolves to a key -> refresh the sidecar's cached key;
955
+ // - CONFIRMED null (no matching principal = a deleted sender) ->
956
+ // evict it;
957
+ // - THROWS (fault: ambiguous address, keyless-principal invariant
958
+ // break, DB error) -> keep the stale key, evict nothing.
959
+ // Never evicting on a fault is the load-bearing property: dropping a
960
+ // live key on a transient DB fault would be worse than doing nothing.
961
+ // Only the resolve is guarded here; conn.send stays outside so a
962
+ // socket-gone throw propagates to the outer catch and stops the loop.
963
+ let publicKey;
964
+ try {
965
+ publicKey = await resolveSenderKeyStrict(address);
966
+ }
967
+ catch (cause) {
968
+ const message = cause instanceof Error ? cause.message : String(cause);
969
+ logger.error `Keeping the stale cached key for ${address}: resolving it faulted (a fault, not a deleted sender): ${message}`;
970
+ continue;
971
+ }
972
+ if (publicKey !== null) {
973
+ conn.send({ type: "sender.key.refresh", address, publicKey });
974
+ }
975
+ else {
976
+ conn.send({ type: "sender.key.evict", address });
977
+ }
978
+ }
979
+ })().catch((cause) => {
980
+ // The per-address resolve is guarded above, so the only throw reaching
981
+ // here is conn.send (JSON.stringify + the socket write) once the sidecar
982
+ // is gone. That means the connection left, so stop -- the remaining
983
+ // sends would fail the same way.
984
+ const message = cause instanceof Error ? cause.message : String(cause);
985
+ logger.warn `Sender-key resync for sidecar ${identity.sidecarId} stopped: ${message}`;
986
+ });
557
987
  }
558
- if (challenges.size === 0)
559
- return;
560
- // If another reconnect completed while we were building the
561
- // challenge, it will have written its own entry. Don't overwrite.
562
- if (pendingChallenges.has(ws))
563
- return;
564
- const timer = setTimeout(() => {
565
- pendingChallenges.delete(ws);
566
- logger.warn `Challenge timed out for sidecar ${sidecarId}`;
567
- }, challengeTimeoutMs);
568
- pendingChallenges.set(ws, {
569
- sidecarId,
570
- challenges,
571
- deployRefs,
572
- timer,
573
- });
574
- conn.send({ type: "challenge", challenges: challengeEntries });
575
- }
576
- async function handleChallengeResponse(ws, responses) {
577
- const challenge = pendingChallenges.get(ws);
578
- if (challenge === undefined) {
579
- logger.warn `Received challenge.response with no pending challenge`;
580
- return;
988
+ logger.info `Provisioned sidecar ${identity.sidecarId} registered for allocation ${identity.allocationId} generation ${String(identity.generation)}`;
989
+ await notifyAllocationWaiters(identity.allocationId);
990
+ if (identity.kind === "allocated") {
991
+ events.emit("sidecar.allocated.connected", {
992
+ allocationId: identity.allocationId,
993
+ generation: identity.generation,
994
+ });
581
995
  }
582
- clearTimeout(challenge.timer);
583
- pendingChallenges.delete(ws);
584
- const conn = connections.get(ws);
585
- if (conn === undefined)
586
- return;
587
- const verified = [];
588
- const responded = new Set();
589
- for (const { address, signature } of responses) {
590
- responded.add(address);
591
- const entry = challenge.challenges.get(address);
592
- if (entry === undefined) {
593
- conn.send({
594
- type: "challenge.failed",
595
- address,
596
- reason: "Address was not challenged",
597
- });
598
- continue;
599
- }
600
- let valid = false;
601
- try {
602
- const nonceBytes = entry.nonce;
603
- const addressBytes = new TextEncoder().encode(address);
604
- const payload = new Uint8Array(nonceBytes.length + addressBytes.length);
605
- payload.set(nonceBytes);
606
- payload.set(addressBytes, nonceBytes.length);
607
- const sigBytes = hexDecode(signature);
608
- valid = await verifyEd25519(payload, sigBytes, entry.publicKey);
609
- }
610
- catch (err) {
611
- logger.warn `Challenge failed for ${address}: ${err instanceof Error ? err.message : String(err)}`;
612
- }
613
- if (valid) {
614
- verified.push(address);
615
- }
616
- else {
617
- conn.send({
618
- type: "challenge.failed",
619
- address,
620
- reason: "Signature verification failed",
996
+ }
997
+ async function handleRegister(ws, identity, agentAddresses, cachedSenderAddresses) {
998
+ await handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses);
999
+ }
1000
+ async function handleReconnect(ws, identity, agentAddresses, cachedSenderAddresses) {
1001
+ await handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses);
1002
+ }
1003
+ // Park a pre-ack sender's mail synchronously and return its entry. Registering
1004
+ // the entry BEFORE the caller awaits `resolveSenderKey` is the interlock that
1005
+ // guarantees a settle landing during the resolve has an entry to find: the
1006
+ // event loop is single-threaded, so no settle can interleave between this
1007
+ // synchronous registration and the caller's first await.
1008
+ function parkDeferredSenderMail(authenticatedSender, rawMessage, recipients) {
1009
+ let parked = deferredSenderMail.get(authenticatedSender);
1010
+ if (parked === undefined) {
1011
+ parked = new Set();
1012
+ deferredSenderMail.set(authenticatedSender, parked);
1013
+ }
1014
+ const entry = {
1015
+ authenticatedSender,
1016
+ rawMessage,
1017
+ recipients,
1018
+ timer: setTimeout(() => {
1019
+ // TTL backstop for the case where a settle never arrives (the sender's
1020
+ // deploy never acked and never failed loudly). Claim the entry and
1021
+ // surface it as undelivered so the mail is not held forever.
1022
+ if (!claimDeferredSenderEntry(entry))
1023
+ return;
1024
+ events.emit("mail.outbound.undelivered", {
1025
+ rawMessage: entry.rawMessage,
1026
+ recipients: entry.recipients,
621
1027
  });
622
- }
1028
+ logger.warn `Dropping mail from ${entry.authenticatedSender}: its sender key was not recorded before the deferred-mail TTL expired`;
1029
+ }, disconnectQueueTTLMs),
1030
+ };
1031
+ parked.add(entry);
1032
+ return entry;
1033
+ }
1034
+ // Remove one parked entry by identity, clearing its TTL timer. Returns whether
1035
+ // THIS call removed it. The inline-deliver path, a settle, and the TTL all
1036
+ // race to claim the same entry; only the claimer acts on it, so a claim that
1037
+ // finds nothing (already claimed) is a no-op. This is the idempotent
1038
+ // remove-by-key that keeps a settle and the inline non-null branch from both
1039
+ // delivering the same message.
1040
+ function claimDeferredSenderEntry(entry) {
1041
+ const parked = deferredSenderMail.get(entry.authenticatedSender);
1042
+ if (parked === undefined)
1043
+ return false;
1044
+ const claimed = parked.delete(entry);
1045
+ if (!claimed)
1046
+ return false;
1047
+ clearTimeout(entry.timer);
1048
+ if (parked.size === 0)
1049
+ deferredSenderMail.delete(entry.authenticatedSender);
1050
+ return true;
1051
+ }
1052
+ // Claim every entry parked for a sender, clearing their TTL timers. A later
1053
+ // settle or TTL for the same sender then finds nothing.
1054
+ function claimAllDeferredSenderMail(authenticatedSender) {
1055
+ const parked = deferredSenderMail.get(authenticatedSender);
1056
+ if (parked === undefined)
1057
+ return [];
1058
+ deferredSenderMail.delete(authenticatedSender);
1059
+ const entries = [...parked];
1060
+ for (const entry of entries)
1061
+ clearTimeout(entry.timer);
1062
+ return entries;
1063
+ }
1064
+ function drainDeferredSenderMail(authenticatedSender, reason) {
1065
+ const entries = claimAllDeferredSenderMail(authenticatedSender);
1066
+ if (entries.length === 0)
1067
+ return;
1068
+ for (const entry of entries) {
1069
+ events.emit("mail.outbound.undelivered", {
1070
+ rawMessage: entry.rawMessage,
1071
+ recipients: entry.recipients,
1072
+ });
623
1073
  }
624
- // Notify about challenged addresses that were omitted from the response.
625
- for (const address of challenge.challenges.keys()) {
626
- if (!responded.has(address)) {
627
- conn.send({
628
- type: "challenge.failed",
629
- address,
630
- reason: "No response provided for challenged address",
631
- });
632
- logger.warn `Challenge failed for ${address}: no response provided`;
633
- }
1074
+ logger.warn `Dropping ${String(entries.length)} deferred message(s) from ${authenticatedSender}: ${reason}`;
1075
+ }
1076
+ function noteSenderDeployStarted(address, attempt) {
1077
+ if (allocatedKeyRecordInFlight.has(address)) {
1078
+ throw new Error(`Sender deployment ${address} has an unresolved attempt`);
634
1079
  }
635
- // Add verified addresses to the routing table immediately so that
636
- // `agent.reconnected` subscribers can use sendRequest-based methods
637
- // (e.g. sendSourcesUpdate). Addresses that fail governance are
638
- // rolled back from the routing table afterward.
639
- for (const addr of verified) {
640
- // If a different ws still owns this address (live takeover via
641
- // verified reconnect), evict its cached connector state before
642
- // routing flips. The new owner's harness will bootstrap via
643
- // restore-fires-callback.
644
- const prevWs = addressIndex.get(addr);
645
- if (prevWs !== undefined && prevWs !== ws) {
646
- connectorStates.delete(addr);
647
- // Evict the reclaimed address from the superseded connection's owned
648
- // set. handleClose's cancelByAgent sweep iterates a connection's owned
649
- // union WITHOUT an ownership guard, so if the stale connection still
650
- // listed this address it would cancel THIS connection's in-flight pack
651
- // transfer for it when it finally closes. Delete from both sets: a
652
- // workflow-derived address lives on the workflow set, a launched agent
653
- // on the session set, and delete is a no-op for the absent one.
654
- const prevConn = connections.get(prevWs);
655
- if (prevConn !== undefined) {
656
- prevConn.workflowAddresses.delete(addr);
657
- prevConn.agentAddresses.delete(addr);
1080
+ allocatedKeyRecordInFlight.set(address, attempt);
1081
+ }
1082
+ function noteSenderDeploySettled(sender, outcome) {
1083
+ if (typeof sender !== "string") {
1084
+ for (const [address, attempt] of [...allocatedKeyRecordInFlight]) {
1085
+ if (attempt.allocationId !== sender.allocationId ||
1086
+ attempt.generation !== sender.generation ||
1087
+ ("leaseId" in sender && attempt.leaseId !== sender.leaseId)) {
1088
+ continue;
658
1089
  }
1090
+ allocatedKeyRecordInFlight.delete(address);
1091
+ settleSenderMail(address, outcome);
659
1092
  }
660
- // Track the address on the set that matches its lifecycle so
661
- // handleClose reclaims it correctly: a workflow-derived deployment
662
- // address goes on the workflow set (no disconnect queue -- its
663
- // in-flight state is reconstructed sidecar-locally on the next
664
- // reconnect), a launched agent on the session set (queued for
665
- // reconnect). The routing pointer is the same either way; only now
666
- // it is written behind a passed challenge.
667
- if (isWorkflowDerivedAddress(addr)) {
668
- conn.workflowAddresses.add(addr);
669
- }
670
- else {
671
- conn.agentAddresses.add(addr);
672
- }
673
- addressIndex.set(addr, ws);
674
- }
675
- const ready = [];
676
- const failed = [];
677
- for (const addr of verified) {
678
- // The `agent.reconnected` reaction is session lifecycle -- instance
679
- // status flip, event-collector restore -- owned by the agent_instance
680
- // concept. A workflow-derived deployment address has no agent_instance
681
- // row, so the reaction's `requireInstance` would throw and roll the
682
- // just-verified address back out of routing. It needs routing + queue
683
- // flush only, which the passed challenge has now made safe; skip the
684
- // session reaction for it.
685
- if (isWorkflowDerivedAddress(addr)) {
686
- ready.push(addr);
687
- continue;
688
- }
689
- if (events.listenerCount("agent.reconnected") === 0) {
690
- ready.push(addr);
691
- continue;
692
- }
693
- try {
694
- await events.emitAndAwait("agent.reconnected", { agentAddress: addr });
695
- ready.push(addr);
696
- }
697
- catch (err) {
698
- logger.error `Failed to handle reconnection for ${addr}: ${err instanceof Error ? err.message : String(err)}`;
699
- failed.push(addr);
700
- }
1093
+ return;
701
1094
  }
702
- // If a second reconnect arrived during the callback loop, our conn
703
- // is orphaned — handleRegister already rebuilt the connection and
704
- // cleared addressIndex. Bail out; the new reconnect flow will
705
- // re-verify these addresses from scratch.
706
- if (connections.get(ws) !== conn) {
707
- logger.warn `Challenge response processing aborted: connection superseded by new reconnect`;
708
- return;
709
- }
710
- // Roll back failed addresses from the routing table. Only the session set
711
- // is touched: a workflow-derived address can never be in `failed` -- it
712
- // early-`continue`s to `ready` above, before the reaction that populates
713
- // `failed` -- so it is never on the workflow set at this point.
714
- for (const addr of failed) {
715
- conn.agentAddresses.delete(addr);
716
- addressIndex.delete(addr);
717
- }
718
- // Flush queued messages only for ready addresses.
719
- for (const addr of ready) {
720
- flushDisconnectedQueue(addr, conn);
721
- }
722
- // Re-deploy agents whose deploy ref is stale or absent. Fire-and-forget
723
- // so reconnect completion is not blocked on pack transfer. The
724
- // wire layer owns the staleness comparison; the event fires only
725
- // when staleness is confirmed.
726
- const checkDeployRef = lookups.lookupDeployRef;
727
- if (checkDeployRef !== undefined) {
728
- for (const addr of ready) {
729
- // Workflow deployments are pinned-forever: a deployment keeps its
730
- // deploy-time definition until an explicit undeploy/redeploy, so the
731
- // deploy-ref freshness catch-up is deliberately NOT run for a
732
- // workflow-derived address. A definition edited on the hub while the
733
- // sidecar was disconnected does not reconcile on reconnect; it affects
734
- // only newly created deployments. The deployment's in-flight run state
735
- // is reconstructed sidecar-locally at restore, not re-fetched here. Do
736
- // NOT add a reconcile path for these addresses -- see the "Workflow
737
- // Definition Versioning: Pinned-Forever" note under "Reconnect
738
- // Sequencing" in docs/IMPLEMENTATION.md.
739
- if (isWorkflowDerivedAddress(addr))
740
- continue;
741
- void (async () => {
742
- try {
743
- const hubRef = await checkDeployRef(addr);
744
- if (hubRef === null)
745
- return;
746
- const sidecarRef = challenge.deployRefs[addr];
747
- if (sidecarRef === hubRef)
748
- return;
749
- logger.info `Re-deploying ${addr}: sidecar ref ${sidecarRef ?? "(none)"} != hub ref ${hubRef.slice(0, 8)}`;
750
- await events.emitAndAwait("deploy.ref.stale", {
751
- agentAddress: addr,
752
- });
753
- }
754
- catch (err) {
755
- logger.error `Failed to re-deploy ${addr} after reconnect: ${err instanceof Error ? err.message : String(err)}`;
756
- }
757
- })();
758
- }
1095
+ if (allocatedKeyRecordInFlight.has(sender))
1096
+ return;
1097
+ settleSenderMail(sender, outcome);
1098
+ }
1099
+ function settleSenderMail(address, outcome) {
1100
+ if ("failed" in outcome) {
1101
+ drainDeferredSenderMail(address, `sender deploy failed: ${outcome.failed}`);
1102
+ return;
759
1103
  }
760
- // Failed addresses: reset their queue TTL so messages survive until
761
- // the next reconnect attempt, and notify the sidecar.
762
- for (const addr of failed) {
763
- const entry = disconnectedAgents.get(addr);
764
- if (entry !== undefined) {
765
- clearTimeout(entry.timer);
766
- entry.timer = setTimeout(() => {
767
- const expired = disconnectedAgents.get(addr);
768
- disconnectedAgents.delete(addr);
769
- if (expired !== undefined) {
770
- surfaceDroppedFrames(addr, expired.queue, "disconnect queue TTL expired");
771
- }
772
- }, disconnectQueueTTLMs);
773
- }
774
- conn.send({
775
- type: "challenge.failed",
776
- address: addr,
777
- reason: "Reconnection rejected by governance",
1104
+ for (const entry of claimAllDeferredSenderMail(address)) {
1105
+ // Re-drive delivery as its OWN task, off the settle's stack, so delivery
1106
+ // work never runs on the deploy-ack handler's stack. Carry the confirmed
1107
+ // key: another attempt may start before this task runs, and must not
1108
+ // capture this mail or change the key that authenticates it.
1109
+ void Promise.resolve()
1110
+ .then(() => handleMailOutbound(entry.rawMessage, entry.authenticatedSender, entry.recipients, outcome.recorded))
1111
+ .catch((err) => {
1112
+ logger.error `Re-driving deferred mail from ${entry.authenticatedSender} failed: ${err instanceof Error ? err.message : String(err)}`;
778
1113
  });
779
1114
  }
780
- logger.info `Sidecar ${challenge.sidecarId} reconnected with ${String(ready.length)} verified agent(s)${failed.length > 0 ? `, ${String(failed.length)} rejected` : ""}`;
781
1115
  }
782
- function handleMailOutbound(rawMessage, recipients) {
1116
+ function senderIdentitiesFromKey(address, publicKey) {
1117
+ return publicKey !== null ? [{ address, publicKey }] : undefined;
1118
+ }
1119
+ // Resolve the co-delivered sender identities for a message, applying the
1120
+ // register-before-read interlock for a pre-ack run sender. Returns either
1121
+ // `deliver: true` with the resolved identities (undefined when there is no
1122
+ // resolvable key), or `deliver: false` when the message is parked and will be
1123
+ // driven later by a settle (`noteSenderDeploySettled`) or the TTL.
1124
+ async function resolveSenderIdentitiesOrPark(rawMessage, authenticatedSender, recipients) {
1125
+ const resolveSenderKey = lookups.resolveSenderKey;
1126
+ if (resolveSenderKey === undefined)
1127
+ return { deliver: true, senderIdentities: undefined };
1128
+ // The co-delivered key is consumed only by a run recipient caching it from the
1129
+ // run.grants frame. Purely external/federated mail never uses it and is never
1130
+ // locally verified, so resolve nothing and never park it.
1131
+ if (!recipients.some(isRunAddress))
1132
+ return { deliver: true, senderIdentities: undefined };
1133
+ // A stable-key (non-run) sender has no pre-ack window; resolve inline.
1134
+ if (!isRunAddress(authenticatedSender)) {
1135
+ const key = await resolveSenderKey(authenticatedSender);
1136
+ return {
1137
+ deliver: true,
1138
+ senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
1139
+ };
1140
+ }
1141
+ // Park a run sender ONLY while a key-record settle is guaranteed to arrive -- a
1142
+ // deploy is in flight. Without one, a null resolve is a transient fault or a
1143
+ // genuine absence on an already-settled run: no settle is coming, so parking
1144
+ // would strand the mail to the TTL. Deliver on the normal path instead.
1145
+ const settleGuaranteed = pendingDeploys.has(authenticatedSender) ||
1146
+ allocatedKeyRecordInFlight.has(authenticatedSender);
1147
+ if (!settleGuaranteed) {
1148
+ const key = await resolveSenderKey(authenticatedSender);
1149
+ return {
1150
+ deliver: true,
1151
+ senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
1152
+ };
1153
+ }
1154
+ // Register-before-read: park a waiter entry synchronously (NO await) so a
1155
+ // settle that lands while we resolve below has an entry to find, THEN
1156
+ // resolve. The single-threaded event loop cannot interleave a settle between
1157
+ // this registration and the await.
1158
+ const entry = parkDeferredSenderMail(authenticatedSender, rawMessage, recipients);
1159
+ const key = await resolveSenderKey(authenticatedSender);
1160
+ if (key === null) {
1161
+ // Not recorded yet. Leave the entry parked; a settle or the TTL drives it.
1162
+ return { deliver: false };
1163
+ }
1164
+ // The key was already recorded before we parked. Claim our entry and deliver
1165
+ // inline -- unless a concurrent settle already claimed it and is re-driving
1166
+ // this message, in which case claiming fails and we must NOT deliver again.
1167
+ if (!claimDeferredSenderEntry(entry)) {
1168
+ return { deliver: false };
1169
+ }
1170
+ return {
1171
+ deliver: true,
1172
+ senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
1173
+ };
1174
+ }
1175
+ async function handleMailOutbound(rawMessage, authenticatedSender, recipients, recordedSenderKey) {
1176
+ // A mail addressed to more than one workflow deployment would birth a
1177
+ // run per recipient from a single inbound mail. The stable runId
1178
+ // removed the Message-ID collision that originally forced this guard --
1179
+ // each recipient now derives its own per-deployment runId (its mail
1180
+ // address), so it is no longer a runId-collision guard. It stays a
1181
+ // deliberate one-workflow-recipient-per-mail restriction because the
1182
+ // fan-out is not verified end-to-end: per-recipient grants
1183
+ // materialization, consumed-tracking, and reply-addressing all assume a
1184
+ // single workflow recipient today. Lifting it means proving those three
1185
+ // hold per recipient, not just relaxing this check -- so fail loudly
1186
+ // rather than materialize a partial set. The guard only applies when a
1187
+ // materializer is wired -- absent one, no run is born from the mail, so
1188
+ // there is nothing to restrict.
1189
+ if (lookups.materializeMailTriggeredRunGrants !== undefined) {
1190
+ // A workflow recipient is one this hub owns: its address parses as a run
1191
+ // address. An external/federated address does not, and is not ours to
1192
+ // materialize a run for.
1193
+ const workflowRecipients = recipients.filter(isRunAddress);
1194
+ if (workflowRecipients.length > 1) {
1195
+ throw new Error(`mail addressed to multiple workflow-derived recipients (${workflowRecipients.join(", ")}); materializing a run for more than one workflow deployment from a single mail is unsupported`);
1196
+ }
1197
+ }
1198
+ // Resolve the sender's hub-held key ONCE for the whole message, ahead of the
1199
+ // recipient fan-out and any grant materialization, so every recipient in
1200
+ // this fan-out binds the same key snapshot. A run-address sender may be
1201
+ // pre-ack -- it minted its keypair locally and can send before the hub
1202
+ // records its public key. The register-before-read interlock holds such mail
1203
+ // until the key lands rather than delivering it keyless, which a strict
1204
+ // recipient drops as an unknown sender. A parked message returns here and is
1205
+ // re-driven later by a settle or the TTL.
1206
+ const resolution = recordedSenderKey === undefined
1207
+ ? await resolveSenderIdentitiesOrPark(rawMessage, authenticatedSender, recipients)
1208
+ : {
1209
+ deliver: true,
1210
+ senderIdentities: senderIdentitiesFromKey(authenticatedSender, recordedSenderKey),
1211
+ };
1212
+ if (!resolution.deliver)
1213
+ return;
1214
+ const senderIdentities = resolution.senderIdentities;
783
1215
  // Route to locally connected sidecars first, then try disconnect queues.
784
1216
  const unrouted = [];
785
1217
  for (const recipient of recipients) {
786
- const targetWs = addressIndex.get(recipient);
787
- if (targetWs !== undefined) {
788
- const conn = connections.get(targetWs);
789
- if (conn !== undefined) {
790
- conn.send({
791
- type: "mail.inbound",
792
- agentAddress: recipient,
793
- rawMessage,
794
- });
795
- continue;
796
- }
1218
+ // Each recipient is isolated: a materialization failure or a
1219
+ // fail-closed rejection for one must not drop the mail for its
1220
+ // co-recipients. The catch fails THIS recipient closed (its run never
1221
+ // starts under-authorized) and continues to the rest.
1222
+ try {
1223
+ const outcome = await deliverMailToRecipient(recipient, rawMessage, authenticatedSender, senderIdentities);
1224
+ if (outcome === "unrouted")
1225
+ unrouted.push(recipient);
1226
+ }
1227
+ catch (err) {
1228
+ logger.error `Failed to deliver mail to ${recipient}: ${err instanceof Error ? err.message : String(err)}`;
797
1229
  }
798
- const frame = {
799
- type: "mail.inbound",
800
- agentAddress: recipient,
801
- rawMessage,
802
- };
803
- if (enqueueForDisconnected(recipient, frame))
804
- continue;
805
- unrouted.push(recipient);
806
1230
  }
807
1231
  // Anything not routed locally is emitted as a notification. The
808
1232
  // host decides whether to relay onto an external transport, log,
@@ -814,6 +1238,95 @@ export function createSidecarRouter(config) {
814
1238
  });
815
1239
  }
816
1240
  }
1241
+ // Deliver an inbound mail to one recipient, materializing a
1242
+ // mail-triggered run's grants first when the recipient is a workflow
1243
+ // deployment. Returns:
1244
+ // - `routed`: the mail reached a live connection or disconnect queue.
1245
+ // - `unrouted`: the mail was locally undeliverable and should be
1246
+ // relayed externally by the host.
1247
+ // - `failed-closed`: the run's grants could not be materialized safely,
1248
+ // so the mail is deliberately DROPPED for this recipient (not relayed)
1249
+ // to keep its run from starting under-authorized.
1250
+ //
1251
+ // A workflow deployment is the only recipient whose inbound mail can first
1252
+ // fire its stable run. Its grants are reserved, and the `run.grants` frame is
1253
+ // sent BEFORE the mail. Same-address FIFO guarantees it lands ahead of the
1254
+ // mail that dispatches the run, so the run's `onRunStart` barrier resolves its
1255
+ // grants rather than failing closed. Reservation happens before routing so
1256
+ // concurrent first deliveries cannot send different snapshots; a routing
1257
+ // failure leaves a grants-only, still-unfired run.
1258
+ async function deliverMailToRecipient(recipient, rawMessage, authenticatedSender, senderIdentities) {
1259
+ if (lookups.materializeMailTriggeredRunGrants !== undefined &&
1260
+ isRunAddress(recipient)) {
1261
+ const runId = deriveWorkflowRunId(recipient);
1262
+ // This does NOT let mail mutate a run's authorization. First delivery
1263
+ // reserves and commits the run's grants (the mail IS the trigger);
1264
+ // every later delivery only RE-READS the current committed grants
1265
+ // (`loadCommittedRunGrants`) and re-asserts them ahead of the dispatch.
1266
+ // The committed rows already carry any standing-approval change (an
1267
+ // approve/reject-with-`always` resolution mutates them through its own
1268
+ // path), so this re-send is idempotent -- it re-establishes the run's
1269
+ // current floor on the sidecar, self-healing a `grants.json` a sidecar
1270
+ // may have lost, and never overwrites it with anything staler.
1271
+ const result = await lookups.materializeMailTriggeredRunGrants({
1272
+ agentAddress: recipient,
1273
+ runId,
1274
+ });
1275
+ if (result.outcome === "rejected") {
1276
+ // The run's grants could not be materialized with sufficient
1277
+ // authority or it is already terminal. Fail the mail closed for this
1278
+ // recipient: routing or external relay would bypass that decision.
1279
+ logger.error `Refusing mail-triggered run ${runId} for ${recipient}: grant materialization rejected (${result.code}): ${result.message}`;
1280
+ return "failed-closed";
1281
+ }
1282
+ if (result.outcome === "materialized") {
1283
+ // The sender's hub-held key was resolved ONCE in handleMailOutbound,
1284
+ // ahead of this fan-out, and threaded in as `senderIdentities`. Co-
1285
+ // deliver it on the run's grants barrier so a recipient that caches from
1286
+ // the `run.grants` frame binds the sender address to the key and can
1287
+ // verify the sender's mail locally. A null key is never carried (the
1288
+ // list is undefined then), so the "authorized-with-a-key implies key
1289
+ // cached" invariant holds; a recipient with no cached key resolves such
1290
+ // mail as `unknown`, which its admission policy rejects by default (a
1291
+ // workflow may relax `unknown` to admit).
1292
+ // Finish asynchronous preparation before sending the grants and mail
1293
+ // together, keeping another delivery's key out of the gap between them.
1294
+ const messageId = await deriveMessageId(base64Decode(rawMessage));
1295
+ // Send the run's grants ahead of the mail. A `false` here means the
1296
+ // deployment is unroutable. Do not route the mail that would dispatch
1297
+ // it; the grants-only reservation remains the canonical snapshot for a
1298
+ // later first-delivery attempt.
1299
+ if (!sendRunGrants(recipient, runId, result.stepGrants, senderIdentities)) {
1300
+ logger.error `Deployment ${recipient} is not routable for run ${runId}; retaining the unfired run's grant reservation for retry`;
1301
+ return "unrouted";
1302
+ }
1303
+ // Route through the messageId handshake `routeMail` -- NOT a
1304
+ // fire-and-forget send. This branch COMMITS a run, so a mail dropped in
1305
+ // the connected window (a socket that half-dies before the sidecar's
1306
+ // durable-write ack) would otherwise leave the run row "running"
1307
+ // forever with no body and no error. `routeMail` tracks the delivery
1308
+ // and redelivers identical bytes on reconnect, bringing the mail-relay
1309
+ // run-trigger to parity with the HTTP-trigger path. The messageId is the
1310
+ // mail's own id (derived over the same bytes the sidecar derives), so a
1311
+ // redelivery replays identically and the downstream RunStarted /
1312
+ // stable-runId dedup makes it effectively-once.
1313
+ const outcome = routeMail(recipient, rawMessage, authenticatedSender, messageId, {
1314
+ runId,
1315
+ stepGrants: result.stepGrants,
1316
+ ...(senderIdentities !== undefined ? { senderIdentities } : {}),
1317
+ })
1318
+ ? "routed"
1319
+ : "unrouted";
1320
+ return outcome;
1321
+ }
1322
+ // `skip`: the address named no deployed workflow deployment. Forward
1323
+ // the mail without grants -- the run, if any, is not ours to
1324
+ // authorize. No run is committed here, so no ack handshake is needed.
1325
+ }
1326
+ return routeMail(recipient, rawMessage, authenticatedSender)
1327
+ ? "routed"
1328
+ : "unrouted";
1329
+ }
817
1330
  async function handleMailPersist(persist, rawMessage, senderAddress, recipients) {
818
1331
  let results;
819
1332
  let raw;
@@ -835,15 +1348,60 @@ export function createSidecarRouter(config) {
835
1348
  raw,
836
1349
  createdAt: result.createdAt,
837
1350
  direction: result.direction,
838
- instanceId: result.instanceId,
1351
+ runId: result.runId,
839
1352
  address: result.address,
840
1353
  });
841
1354
  }
842
1355
  }
1356
+ async function handleSignalCorrelationRegister(ws, frame) {
1357
+ // Gate the co-write on the sending sidecar actually owning the named
1358
+ // deployment address, mirroring the connector.state.changed and pack
1359
+ // handlers. A workflow deployment routes on the keyless workflow set, so
1360
+ // ownership is the union check, not addressIndex identity alone. Without
1361
+ // it a misbehaving sidecar that knows another deployment's address could
1362
+ // register a spurious correlation against it.
1363
+ const conn = connections.get(ws);
1364
+ if (conn === undefined)
1365
+ return;
1366
+ if (!connOwnsAddress(conn, frame.agentAddress)) {
1367
+ logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: not registered to this sidecar`;
1368
+ return;
1369
+ }
1370
+ const register = lookups.registerSignalCorrelation;
1371
+ if (register === undefined) {
1372
+ logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: no registerSignalCorrelation lookup configured`;
1373
+ return;
1374
+ }
1375
+ try {
1376
+ await register({
1377
+ correlationId: frame.correlationId,
1378
+ runId: frame.runId,
1379
+ anchorRunId: frame.anchorRunId,
1380
+ agentAddress: frame.agentAddress,
1381
+ kind: frame.kind,
1382
+ approvalSnapshot: frame.snapshot,
1383
+ });
1384
+ // The co-write resolves only when a row exists -- freshly inserted or
1385
+ // already present (both stores are idempotent on the correlationId). Ack
1386
+ // so the sidecar's link stops retrying a register whose frame may have
1387
+ // been lost on an open socket. A thrown co-write (undeployed deployment,
1388
+ // id mismatch) means no row, so no ack: the sidecar keeps retrying and
1389
+ // the reconnect re-emit remains the ultimate backstop.
1390
+ conn.send({
1391
+ type: "signal.correlation.register.ack",
1392
+ agentAddress: frame.agentAddress,
1393
+ correlationId: frame.correlationId,
1394
+ });
1395
+ }
1396
+ catch (err) {
1397
+ logger.error `Failed to register signal correlation ${frame.correlationId} for ${frame.agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
1398
+ }
1399
+ }
843
1400
  function handleClose(ws) {
844
1401
  const conn = connections.get(ws);
845
1402
  if (conn === undefined)
846
1403
  return;
1404
+ let allocated;
847
1405
  for (const addr of conn.agentAddresses) {
848
1406
  // Only remove routing and pending state if this connection still
849
1407
  // owns the address. A reconnected sidecar may have already claimed it.
@@ -855,12 +1413,12 @@ export function createSidecarRouter(config) {
855
1413
  // reconnect re-bootstraps via the router's
856
1414
  // restore-fires-callback path.
857
1415
  connectorStates.delete(addr);
858
- const deployReq = pendingDeploys.get(addr);
859
- if (deployReq !== undefined && deployReq.ws === ws) {
860
- clearTimeout(deployReq.timer);
861
- pendingDeploys.delete(addr);
862
- deployReq.reject(`Sidecar ${conn.sidecarId} disconnected`);
863
- }
1416
+ // Retain this address's un-acked pending mail across the disconnect:
1417
+ // its in-flight retry timers target a dead socket (cleared), but the
1418
+ // entries are held so a verified reconnect redelivers them, closing the
1419
+ // connected-window drop rather than losing the mail. Bounded by a
1420
+ // retention TTL.
1421
+ retainPendingMailForAddress(addr);
864
1422
  // Create a queue entry so messages can accumulate while the
865
1423
  // sidecar is disconnected. Skip if the agent is being undeployed --
866
1424
  // there is no point queuing messages for an agent being torn down.
@@ -885,47 +1443,52 @@ export function createSidecarRouter(config) {
885
1443
  for (const addr of conn.workflowAddresses) {
886
1444
  if (addressIndex.get(addr) === ws) {
887
1445
  addressIndex.delete(addr);
1446
+ connectorStates.delete(addr);
1447
+ // Retain un-acked workflow trigger mail across the disconnect for the
1448
+ // same reason as the session loop above -- an authenticated reconnect
1449
+ // redelivers it. This is un-acked TRIGGER mail, distinct from the
1450
+ // deployment's in-flight run state (reconstructed sidecar-locally); the
1451
+ // "no disconnect queue" note above is about that run state, not this.
1452
+ retainPendingMailForAddress(addr);
1453
+ }
1454
+ }
1455
+ const current = allocatedConnections.get(conn.identity.allocationId);
1456
+ if (current?.ws === ws) {
1457
+ allocatedConnections.delete(conn.identity.allocationId);
1458
+ if (conn.identity.kind === "allocated") {
1459
+ allocated = {
1460
+ allocationId: conn.identity.allocationId,
1461
+ generation: conn.identity.generation,
1462
+ };
888
1463
  }
889
1464
  }
890
1465
  connections.delete(ws);
891
1466
  // Cancel the liveness timer for this connection.
892
- const livenessTimer = livenessTimers.get(ws);
893
- if (livenessTimer !== undefined) {
894
- clearTimeout(livenessTimer);
1467
+ const cancelLiveness = livenessTimers.get(ws);
1468
+ if (cancelLiveness !== undefined) {
1469
+ cancelLiveness();
895
1470
  livenessTimers.delete(ws);
896
1471
  }
897
1472
  // Drop the per-ws serialization chain; no more frames will queue on it.
898
1473
  messageChains.delete(ws);
899
- // Cancel any pending challenge for this connection.
900
- const challengeReq = pendingChallenges.get(ws);
901
- if (challengeReq !== undefined) {
902
- clearTimeout(challengeReq.timer);
903
- pendingChallenges.delete(ws);
904
- }
905
- // Reject any in-flight requests that were sent to this sidecar.
906
- for (const [requestId, req] of pending) {
907
- if (req.ws !== ws)
908
- continue;
909
- clearTimeout(req.timer);
910
- pending.delete(requestId);
911
- req.reject(`Sidecar ${conn.sidecarId} disconnected`);
912
- }
1474
+ // Reject any in-flight requests that were sent to this sidecar. Each
1475
+ // entry's reject closure runs its own per-site cleanup (the deploy and
1476
+ // undeploy closures roll routing back), exactly as a frame-error
1477
+ // rejection would.
1478
+ pendingRequests.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
1479
+ // Reject every deploy issued on this socket, including allocated
1480
+ // workflow deployments stored in `workflowAddresses` rather than
1481
+ // `agentAddresses`.
1482
+ pendingDeploys.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
913
1483
  // Reject any in-flight pack transfers for this sidecar.
914
- for (const [transferId, pack] of pendingPacks) {
915
- if (pack.ws !== ws)
916
- continue;
917
- clearTimeout(pack.timer);
918
- pendingPacks.delete(transferId);
919
- pack.reject(`Sidecar ${conn.sidecarId} disconnected`);
920
- }
1484
+ pendingPacks.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
921
1485
  // Reject any in-flight undeploys for this sidecar.
922
- for (const [addr, req] of pendingUndeploys) {
923
- if (req.ws !== ws)
924
- continue;
925
- clearTimeout(req.timer);
926
- pendingUndeploys.delete(addr);
927
- req.reject(`Sidecar ${conn.sidecarId} disconnected`);
928
- }
1486
+ pendingUndeploys.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
1487
+ // Reject any in-flight probes sent to this sidecar. A probe never enters
1488
+ // the address maps, so this ws-keyed sweep is its ONLY disconnect cleanup:
1489
+ // without it a probe whose sidecar drops mid-flight would hang until its
1490
+ // own timeout instead of failing fast on the disconnect.
1491
+ pendingProbes.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
929
1492
  // Cancel any in-flight inbound pack transfers from this sidecar
930
1493
  // across both receivers. The two receivers track their own in-
931
1494
  // flight transferIds, so a pending workflow-run transfer for an
@@ -944,6 +1507,7 @@ export function createSidecarRouter(config) {
944
1507
  }
945
1508
  events.emit("sidecar.disconnect", {
946
1509
  ownedAddresses: [...owned],
1510
+ ...(allocated !== undefined ? { allocated } : {}),
947
1511
  });
948
1512
  logger.info `Sidecar ${conn.sidecarId} disconnected`;
949
1513
  }
@@ -962,71 +1526,78 @@ export function createSidecarRouter(config) {
962
1526
  const requestId = nextRequestId();
963
1527
  const frame = buildFrame(requestId);
964
1528
  return new Promise((resolve, reject) => {
965
- const timer = setTimeout(() => {
966
- pending.delete(requestId);
967
- reject(new Error(`Request ${requestId} timed out after ${requestTimeoutMs}ms`));
968
- }, requestTimeoutMs);
969
- pending.set(requestId, {
970
- requestId,
971
- ws,
1529
+ pendingRequests.register(requestId, ws, {
1530
+ timeoutMs: requestTimeoutMs,
1531
+ timeoutMessage: `Request ${requestId} timed out after ${requestTimeoutMs}ms`,
972
1532
  resolve,
973
1533
  reject(error) {
974
1534
  reject(new Error(error));
975
1535
  },
976
- timer,
977
- });
1536
+ }, undefined);
978
1537
  conn.send(frame);
979
1538
  });
980
1539
  }
981
- function resolvePending(requestId) {
982
- const req = pending.get(requestId);
983
- if (req === undefined)
984
- return;
985
- clearTimeout(req.timer);
986
- pending.delete(requestId);
987
- req.resolve();
1540
+ function packResponseMatches(entry, ws, frame) {
1541
+ return (entry.ws === ws &&
1542
+ entry.meta.agentAddress === frame.agentAddress &&
1543
+ entry.meta.repoId.kind === frame.repoId.kind &&
1544
+ entry.meta.repoId.id === frame.repoId.id);
988
1545
  }
989
- function rejectPending(requestId, error) {
990
- const req = pending.get(requestId);
991
- if (req === undefined)
992
- return;
993
- clearTimeout(req.timer);
994
- pending.delete(requestId);
995
- req.reject(error);
996
- }
997
- function resolvePackPending(transferId) {
998
- const entry = pendingPacks.get(transferId);
1546
+ function resolvePackPending(ws, frame) {
1547
+ const entry = pendingPacks.get(frame.transferId);
999
1548
  if (entry === undefined)
1000
1549
  return;
1001
- clearTimeout(entry.timer);
1002
- pendingPacks.delete(transferId);
1003
- entry.resolve();
1550
+ if (!packResponseMatches(entry, ws, frame)) {
1551
+ logger.warn `Ignoring repo.pack.ack for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
1552
+ return;
1553
+ }
1554
+ pendingPacks.resolve(frame.transferId);
1004
1555
  }
1005
- function rejectPackPending(transferId, reason) {
1006
- const entry = pendingPacks.get(transferId);
1556
+ function rejectPackPending(ws, frame) {
1557
+ const entry = pendingPacks.get(frame.transferId);
1007
1558
  if (entry === undefined)
1008
1559
  return;
1009
- clearTimeout(entry.timer);
1010
- pendingPacks.delete(transferId);
1011
- entry.reject(reason);
1560
+ if (!packResponseMatches(entry, ws, frame)) {
1561
+ logger.warn `Ignoring repo.pack.reject for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
1562
+ return;
1563
+ }
1564
+ // Surface the receiver's specific cause when it carried one, so the awaiting
1565
+ // push sees "corrupt: <detail>" rather than only the coarse reason. The
1566
+ // "Pack rejected:" prefix is applied here rather than in the entry's
1567
+ // reject closure because a TIMEOUT rejection must not carry it.
1568
+ pendingPacks.reject(frame.transferId, `Pack rejected: ${frame.detail !== undefined
1569
+ ? `${frame.reason}: ${frame.detail}`
1570
+ : frame.reason}`);
1012
1571
  }
1013
- function resolveUndeployPending(agentAddress) {
1572
+ function resolveUndeployPending(ws, agentAddress) {
1014
1573
  const req = pendingUndeploys.get(agentAddress);
1015
1574
  if (req === undefined) {
1016
1575
  logger.warn `Received agent.undeploy.ack for "${agentAddress}" with no pending undeploy`;
1017
1576
  return;
1018
1577
  }
1019
- clearTimeout(req.timer);
1020
- pendingUndeploys.delete(agentAddress);
1021
- req.resolve();
1578
+ if (req.ws !== ws)
1579
+ return;
1580
+ pendingUndeploys.resolve(agentAddress);
1022
1581
  }
1023
- function rejectUndeployPending(agentAddress, error) {
1582
+ function rejectUndeployPending(ws, agentAddress, error) {
1024
1583
  const req = pendingUndeploys.get(agentAddress);
1025
1584
  if (req === undefined)
1026
1585
  return;
1027
- clearTimeout(req.timer);
1028
- pendingUndeploys.delete(agentAddress);
1029
- req.reject(error);
1586
+ if (req.ws !== ws)
1587
+ return;
1588
+ pendingUndeploys.reject(agentAddress, error);
1589
+ }
1590
+ function resolveProbe(ws, requestId, result) {
1591
+ const req = pendingProbes.get(requestId);
1592
+ if (req === undefined || req.ws !== ws)
1593
+ return;
1594
+ pendingProbes.resolve(requestId, result);
1595
+ }
1596
+ function rejectProbe(ws, requestId, error) {
1597
+ const req = pendingProbes.get(requestId);
1598
+ if (req === undefined || req.ws !== ws)
1599
+ return;
1600
+ pendingProbes.reject(requestId, error);
1030
1601
  }
1031
1602
  // Routing rule: pick the receiver dedicated to the repoId.kind the
1032
1603
  // frame carries. The receivers' in-flight state is independent, so a
@@ -1048,6 +1619,8 @@ export function createSidecarRouter(config) {
1048
1619
  function pickReceivePackLookup(repoId) {
1049
1620
  switch (repoId.kind) {
1050
1621
  case "agent-state":
1622
+ // The agent-state lookup ignores the `source` argument the workflow-run
1623
+ // lookup takes; the two are otherwise the same contract.
1051
1624
  return lookups.receiveAgentStatePack;
1052
1625
  case "workflow-run":
1053
1626
  return lookups.receiveWorkflowRunPack;
@@ -1063,6 +1636,19 @@ export function createSidecarRouter(config) {
1063
1636
  logger.warn `Received repo.pack.push for unrouted agent ${frame.agentAddress}`;
1064
1637
  return;
1065
1638
  }
1639
+ if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
1640
+ logger.warn `Rejected repo.pack.push outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
1641
+ conn.send({
1642
+ type: "repo.pack.reject",
1643
+ agentAddress: frame.agentAddress,
1644
+ repoId: frame.repoId,
1645
+ transferId: frame.transferId,
1646
+ reason: "path_violation",
1647
+ });
1648
+ return;
1649
+ }
1650
+ if (conn.identity.kind !== "allocated")
1651
+ return;
1066
1652
  const picked = pickPackReceiver(frame.repoId);
1067
1653
  if (picked === null) {
1068
1654
  logger.warn `Received repo.pack.push with unsupported repoId.kind ${frame.repoId.kind}`;
@@ -1094,6 +1680,20 @@ export function createSidecarRouter(config) {
1094
1680
  logger.warn `Received repo.pack.done for unrouted agent ${frame.agentAddress}`;
1095
1681
  return;
1096
1682
  }
1683
+ if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
1684
+ logger.warn `Rejected repo.pack.done outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
1685
+ conn.send({
1686
+ type: "repo.pack.reject",
1687
+ agentAddress: frame.agentAddress,
1688
+ repoId: frame.repoId,
1689
+ transferId: frame.transferId,
1690
+ reason: "path_violation",
1691
+ });
1692
+ return;
1693
+ }
1694
+ if (conn.identity.kind !== "allocated")
1695
+ return;
1696
+ const identity = conn.identity;
1097
1697
  const picked = pickPackReceiver(frame.repoId);
1098
1698
  if (picked === null) {
1099
1699
  logger.warn `Received repo.pack.done with unsupported repoId.kind ${frame.repoId.kind}`;
@@ -1127,7 +1727,13 @@ export function createSidecarRouter(config) {
1127
1727
  });
1128
1728
  return;
1129
1729
  }
1130
- const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha);
1730
+ const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha, {
1731
+ kind: "allocated",
1732
+ agentAddress: frame.agentAddress,
1733
+ allocationId: identity.allocationId,
1734
+ anchorRunId: identity.anchorRunId,
1735
+ generation: identity.generation,
1736
+ });
1131
1737
  // Connection may have closed during async verification.
1132
1738
  const currentConn = connections.get(ws);
1133
1739
  if (currentConn === undefined)
@@ -1155,53 +1761,220 @@ export function createSidecarRouter(config) {
1155
1761
  * window of a multi-step deploy, so `sendPack` can route the step's deploy
1156
1762
  * and asset packs before the deployment-level frame spawns the child.
1157
1763
  *
1158
- * The address is hub-minted and workflow-derived (no per-address key), so
1159
- * it enters the keyless `workflowAddresses` set -- never the challenged
1160
- * `agentAddresses` set -- and is torn down by `unbindStepRoute` once the
1764
+ * The address is Hub-minted and workflow-derived, so it enters the
1765
+ * `workflowAddresses` set rather than the legacy `agentAddresses` set and is
1766
+ * torn down by `unbindStepRoute` once the
1161
1767
  * step's packs land. `handleClose` reclaims it if the sidecar drops
1162
1768
  * mid-stage. Per-step addresses are not runtime-routed (mail, signals, and
1163
1769
  * drains use the deployment address), so the binding is transient: it is
1164
1770
  * never persisted into the reconnect set and never resurrected on
1165
1771
  * reconnect.
1166
1772
  */
1167
- function bindStepRoute(stepAddress) {
1168
- const ws = addressIndex.get(stepAddress) ?? findSidecarForNewAgent(stepAddress);
1169
- if (ws === undefined) {
1170
- throw new Error(`No sidecar available to stage workflow step "${stepAddress}"`);
1773
+ function fenceAllocation(allocationId, generation) {
1774
+ const existing = allocationFences.get(allocationId);
1775
+ if (existing !== undefined && generation < existing) {
1776
+ throw new Error(`Cannot move allocation ${allocationId} fence backward from ${String(existing)} to ${String(generation)}`);
1777
+ }
1778
+ allocationFences.set(allocationId, generation);
1779
+ // A durable generation advance resolves unfinished initialization as failed.
1780
+ // This also covers a cleanup transaction whose response was lost: the next
1781
+ // reconciliation rebuilds this fence before it can start a replacement.
1782
+ for (const attempt of [...allocatedKeyRecordInFlight.values()]) {
1783
+ if (attempt.allocationId === allocationId &&
1784
+ attempt.generation < generation) {
1785
+ noteSenderDeploySettled(attempt, {
1786
+ failed: `Allocation ${allocationId} advanced beyond the deployment attempt`,
1787
+ });
1788
+ }
1171
1789
  }
1172
- const conn = connections.get(ws);
1173
- if (conn === undefined) {
1174
- throw new Error(`No sidecar connected to stage workflow step "${stepAddress}"`);
1790
+ const current = allocatedConnections.get(allocationId);
1791
+ if (current !== undefined && current.identity.generation !== generation) {
1792
+ handleClose(current.ws);
1793
+ current.ws.close();
1794
+ }
1795
+ const waiters = allocationWaiters.get(allocationId);
1796
+ if (waiters === undefined)
1797
+ return;
1798
+ for (const waiter of [...waiters]) {
1799
+ if (waiter.generation === generation)
1800
+ continue;
1801
+ clearTimeout(waiter.timer);
1802
+ waiters.delete(waiter);
1803
+ waiter.reject(new Error(`Allocation ${allocationId} advanced to generation ${String(generation)}`));
1804
+ }
1805
+ if (waiters.size === 0)
1806
+ allocationWaiters.delete(allocationId);
1807
+ }
1808
+ function retireAllocation(target) {
1809
+ if (allocationFences.get(target.allocationId) !== target.generation)
1810
+ return;
1811
+ disconnectAllocation(target);
1812
+ allocationFences.delete(target.allocationId);
1813
+ // The fence is gone, so a lingering attempt can never settle normally.
1814
+ // Fail it here rather than leaving a marker that blocks the address.
1815
+ for (const attempt of [...allocatedKeyRecordInFlight.values()]) {
1816
+ if (attempt.allocationId === target.allocationId &&
1817
+ attempt.generation <= target.generation) {
1818
+ noteSenderDeploySettled(attempt, {
1819
+ failed: `Allocation ${target.allocationId} generation ${String(target.generation)} retired`,
1820
+ });
1821
+ }
1822
+ }
1823
+ const waiters = allocationWaiters.get(target.allocationId);
1824
+ if (waiters === undefined)
1825
+ return;
1826
+ allocationWaiters.delete(target.allocationId);
1827
+ for (const waiter of waiters) {
1828
+ clearTimeout(waiter.timer);
1829
+ waiter.reject(new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} retired`));
1830
+ }
1831
+ }
1832
+ async function getProvisionedConnection(target, use) {
1833
+ if (allocationFences.get(target.allocationId) !== target.generation) {
1834
+ throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
1835
+ }
1836
+ const current = allocatedConnections.get(target.allocationId);
1837
+ if (current === undefined ||
1838
+ current.identity.generation !== target.generation) {
1839
+ throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId} generation ${String(target.generation)}`);
1840
+ }
1841
+ let identityCurrent;
1842
+ try {
1843
+ identityCurrent = await validateSidecarIdentity(current.identity, use);
1844
+ }
1845
+ catch (cause) {
1846
+ throw new SidecarIdentityValidationError(target.allocationId, target.generation, cause);
1847
+ }
1848
+ if (!identityCurrent) {
1849
+ if (allocatedConnections.get(target.allocationId) === current) {
1850
+ handleClose(current.ws);
1851
+ current.ws.close();
1852
+ }
1853
+ throw new Error(`Allocated sidecar identity is no longer current for allocation ${target.allocationId}`);
1854
+ }
1855
+ if (allocatedConnections.get(target.allocationId) !== current) {
1856
+ throw new Error(`Allocated sidecar connection changed for allocation ${target.allocationId}`);
1857
+ }
1858
+ const conn = connections.get(current.ws);
1859
+ if (conn === undefined ||
1860
+ conn.identity.allocationId !== target.allocationId ||
1861
+ conn.identity.generation !== target.generation) {
1862
+ throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId}`);
1863
+ }
1864
+ return { ws: current.ws, conn };
1865
+ }
1866
+ async function getAllocatedConnection(target, use) {
1867
+ const current = await getProvisionedConnection(target, use);
1868
+ if (current.conn.identity.kind !== "allocated") {
1869
+ throw new Error(`Allocation ${target.allocationId} is connected as probe capacity`);
1870
+ }
1871
+ return {
1872
+ ws: current.ws,
1873
+ conn: { ...current.conn, identity: current.conn.identity },
1874
+ };
1875
+ }
1876
+ async function isAllocatedSidecarReady(target) {
1877
+ try {
1878
+ await getProvisionedConnection(target, "readiness");
1879
+ return true;
1880
+ }
1881
+ catch (error) {
1882
+ // A failed validation is unknown, not absent: the worker may be healthy
1883
+ // behind a failed lookup, so report it distinctly instead of answering
1884
+ // `false` and letting the caller release a live worker.
1885
+ if (error instanceof SidecarIdentityValidationError)
1886
+ throw error;
1887
+ return false;
1888
+ }
1889
+ }
1890
+ async function isAllocatedWorkflowActive(target) {
1891
+ try {
1892
+ const { conn } = await getAllocatedConnection(target, "readiness");
1893
+ if (conn.identity.kind !== "allocated")
1894
+ return false;
1895
+ return conn.workflowAddresses.has(conn.identity.workflowRunAddress);
1896
+ }
1897
+ catch (error) {
1898
+ if (error instanceof SidecarIdentityValidationError)
1899
+ throw error;
1900
+ return false;
1901
+ }
1902
+ }
1903
+ async function waitForAllocatedSidecar(target, timeoutMs, onValidation) {
1904
+ // An indeterminable worker waits out the unknown while time remains: only
1905
+ // confirmed absence may surface as a connection timeout. At expiry the
1906
+ // wait reports the validation failure rather than a missed deadline, so
1907
+ // the caller retries instead of releasing a worker that may be healthy.
1908
+ let validationFailure;
1909
+ try {
1910
+ if (await isAllocatedSidecarReady(target))
1911
+ return;
1912
+ }
1913
+ catch (error) {
1914
+ if (!(error instanceof SidecarIdentityValidationError))
1915
+ throw error;
1916
+ validationFailure = error;
1917
+ }
1918
+ if (allocationFences.get(target.allocationId) !== target.generation) {
1919
+ throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
1920
+ }
1921
+ if (timeoutMs <= 0) {
1922
+ if (validationFailure !== undefined)
1923
+ throw validationFailure;
1924
+ throw new Error(`Timed out waiting for allocated sidecar ${target.allocationId}`);
1925
+ }
1926
+ await new Promise((resolve, reject) => {
1927
+ const waiter = {
1928
+ generation: target.generation,
1929
+ validations: new Set(),
1930
+ ...(onValidation !== undefined ? { onValidation } : {}),
1931
+ resolve,
1932
+ reject,
1933
+ timer: setTimeout(() => {
1934
+ const current = allocationWaiters.get(target.allocationId);
1935
+ current?.delete(waiter);
1936
+ if (current?.size === 0) {
1937
+ allocationWaiters.delete(target.allocationId);
1938
+ }
1939
+ reject(waiter.validationFailure ??
1940
+ (waiter.validations.size > 0
1941
+ ? new SidecarIdentityValidationError(target.allocationId, target.generation)
1942
+ : new Error(`Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`)));
1943
+ }, timeoutMs),
1944
+ ...(validationFailure !== undefined ? { validationFailure } : {}),
1945
+ };
1946
+ let waiters = allocationWaiters.get(target.allocationId);
1947
+ if (waiters === undefined) {
1948
+ waiters = new Set();
1949
+ allocationWaiters.set(target.allocationId, waiters);
1950
+ }
1951
+ waiters.add(waiter);
1952
+ void notifyAllocationWaiters(target.allocationId);
1953
+ });
1954
+ }
1955
+ async function bindAllocatedStepRoute(target, stepAddress) {
1956
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
1957
+ const existing = addressIndex.get(stepAddress);
1958
+ if (existing !== undefined && existing !== ws) {
1959
+ throw new Error(`Workflow step ${stepAddress} is already routed to another sidecar`);
1175
1960
  }
1176
1961
  conn.workflowAddresses.add(stepAddress);
1177
1962
  addressIndex.set(stepAddress, ws);
1178
1963
  }
1179
- /**
1180
- * Remove a per-step route bound by `bindStepRoute` once the step's packs
1181
- * have landed. Idempotent: an address that was never bound (or already
1182
- * unbound, e.g. by a mid-stage `handleClose`) is a no-op.
1183
- */
1184
- function unbindStepRoute(stepAddress) {
1185
- const ws = addressIndex.get(stepAddress);
1186
- if (ws === undefined)
1964
+ function unbindAllocatedStepRoute(target, stepAddress) {
1965
+ const current = allocatedConnections.get(target.allocationId);
1966
+ if (current === undefined ||
1967
+ current.identity.generation !== target.generation) {
1187
1968
  return;
1188
- const conn = connections.get(ws);
1189
- if (conn !== undefined) {
1190
- conn.workflowAddresses.delete(stepAddress);
1191
1969
  }
1970
+ if (addressIndex.get(stepAddress) !== current.ws)
1971
+ return;
1972
+ connections.get(current.ws)?.workflowAddresses.delete(stepAddress);
1192
1973
  addressIndex.delete(stepAddress);
1193
1974
  }
1194
1975
  // Pack transfers may take longer than session requests due to data volume.
1195
1976
  const PACK_TIMEOUT_MS = requestTimeoutMs * 4;
1196
- function sendPack(agentAddress, pack, ref, commitSha, options) {
1197
- const ws = addressIndex.get(agentAddress);
1198
- if (ws === undefined) {
1199
- return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
1200
- }
1201
- const conn = connections.get(ws);
1202
- if (conn === undefined) {
1203
- return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
1204
- }
1977
+ function sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options) {
1205
1978
  const transferId = `pack-${++packCounter}`;
1206
1979
  // For the agent-state flow the destination agent and the source repo
1207
1980
  // are the same entity, so `repoId.id === agentAddress`. Asset packs
@@ -1215,19 +1988,14 @@ export function createSidecarRouter(config) {
1215
1988
  // Register pending entry before sending frames so that a synchronous
1216
1989
  // repo.pack.ack (e.g. in tests or loopback transports) resolves correctly.
1217
1990
  return new Promise((resolve, reject) => {
1218
- const timer = setTimeout(() => {
1219
- pendingPacks.delete(transferId);
1220
- reject(new Error(`Pack transfer ${transferId} timed out after ${PACK_TIMEOUT_MS}ms`));
1221
- }, PACK_TIMEOUT_MS);
1222
- pendingPacks.set(transferId, {
1223
- transferId,
1224
- ws,
1991
+ pendingPacks.register(transferId, ws, {
1992
+ timeoutMs: PACK_TIMEOUT_MS,
1993
+ timeoutMessage: `Pack transfer ${transferId} timed out after ${PACK_TIMEOUT_MS}ms`,
1225
1994
  resolve,
1226
1995
  reject(error) {
1227
- reject(new Error(`Pack rejected: ${error}`));
1996
+ reject(new Error(error));
1228
1997
  },
1229
- timer,
1230
- });
1998
+ }, { agentAddress, repoId });
1231
1999
  // Send chunks
1232
2000
  for (const chunk of chunkPack(pack)) {
1233
2001
  conn.send({
@@ -1251,128 +2019,291 @@ export function createSidecarRouter(config) {
1251
2019
  });
1252
2020
  });
1253
2021
  }
1254
- function routeMail(agentAddress, rawMessage) {
2022
+ async function sendPackToAllocation(target, agentAddress, pack, ref, commitSha, options) {
2023
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2024
+ if (addressIndex.get(agentAddress) !== ws) {
2025
+ throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
2026
+ }
2027
+ return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
2028
+ }
2029
+ async function sendWorkflowRunPackToAllocation(target, agentAddress, pack, ref, commitSha, signal) {
2030
+ signal?.throwIfAborted();
2031
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2032
+ signal?.throwIfAborted();
2033
+ if (agentAddress !== conn.identity.workflowRunAddress) {
2034
+ throw new Error(`Allocation ${target.allocationId} cannot restore unrelated address ${agentAddress}`);
2035
+ }
2036
+ if (conn.workflowAddresses.has(agentAddress)) {
2037
+ throw new Error(`Allocation ${target.allocationId} already hosts active workflow ${agentAddress}; refusing to overwrite its run history`);
2038
+ }
2039
+ return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, {
2040
+ repoId: {
2041
+ kind: "workflow-run",
2042
+ id: deriveWorkflowRunRepoId(agentAddress),
2043
+ },
2044
+ });
2045
+ }
2046
+ function routeMail(agentAddress, rawMessage, authenticatedSender, messageId, runGrants) {
2047
+ // `authenticatedSender` is hub-assigned by the caller from a hub-verified
2048
+ // value (the ownership-gated sender of a relayed mail, or the triggering
2049
+ // principal's address) -- never the message's own MIME `From`. It rides
2050
+ // the frame as the hub-verified sender of record, so a recipient can take
2051
+ // the sender from it rather than the forgeable `From`. The recipient's
2052
+ // signature check reads it as the sender of record -- resolving the
2053
+ // sender's key from its local cache to verify the signature -- and its
2054
+ // admission policy gates delivery on the verdict.
2055
+ //
2056
+ // Carry the hub-minted messageId on the frame so the sidecar's durable-
2057
+ // receipt ack (`mail.inbound.ack`) keys on the same id the hub tracks, and
2058
+ // a redelivery replays identical bytes for the downstream RunStarted dedup.
2059
+ // Optional: the workflow-trigger and session-conversation callers supply
2060
+ // it (they participate in the ack/retry handshake); a caller without a
2061
+ // hub-minted id omits it and the delivery is not tracked for redelivery.
2062
+ const frame = {
2063
+ type: "mail.inbound",
2064
+ agentAddress,
2065
+ rawMessage,
2066
+ authenticatedSender,
2067
+ ...(messageId !== undefined ? { messageId } : {}),
2068
+ };
1255
2069
  const ws = addressIndex.get(agentAddress);
1256
2070
  if (ws !== undefined) {
1257
2071
  const conn = connections.get(ws);
1258
2072
  if (conn !== undefined) {
1259
- conn.send({
1260
- type: "mail.inbound",
1261
- agentAddress,
1262
- rawMessage,
1263
- });
2073
+ conn.send(frame);
2074
+ // Track the delivery for redelivery until the sidecar acks its durable
2075
+ // inbox write. Only mail carrying a hub-minted messageId participates
2076
+ // in the ack handshake; relayed agent-to-agent mail omits it and is
2077
+ // delivered fire-and-forget as before. A mail that triggered a workflow
2078
+ // run carries the run's grants so redelivery can replay them ahead of
2079
+ // the mail.
2080
+ if (messageId !== undefined) {
2081
+ trackPendingMail(agentAddress, messageId, frame, runGrants);
2082
+ }
1264
2083
  return true;
1265
2084
  }
1266
2085
  }
1267
2086
  // If the agent recently disconnected, queue for delivery on reconnect.
1268
- const frame = { type: "mail.inbound", agentAddress, rawMessage };
1269
2087
  return enqueueForDisconnected(agentAddress, frame);
1270
2088
  }
1271
- async function handleDeployAck(agentAddress, publicKey) {
1272
- if (!pendingDeploys.has(agentAddress)) {
1273
- logger.warn `Received agent.deploy.ack for "${agentAddress}" with no pending deploy`;
2089
+ function sendRunGrants(agentAddress, runId, stepGrants, senderIdentities) {
2090
+ const frame = {
2091
+ type: "run.grants",
2092
+ agentAddress,
2093
+ runId,
2094
+ stepGrants,
2095
+ ...(senderIdentities !== undefined ? { senderIdentities } : {}),
2096
+ };
2097
+ const ws = addressIndex.get(agentAddress);
2098
+ if (ws !== undefined) {
2099
+ const conn = connections.get(ws);
2100
+ if (conn !== undefined) {
2101
+ conn.send(frame);
2102
+ return true;
2103
+ }
2104
+ }
2105
+ // Mirror routeMail: if the address has a live disconnect queue, ride it
2106
+ // so a run.grants issued in the window between deploy and the first
2107
+ // reconnect survives the same way the dispatching trigger mail does.
2108
+ // A queue exists only while the deployment address is still on
2109
+ // agentAddresses (pre-first-reconnect); after an authenticated reconnect it
2110
+ // moves to workflowAddresses, which handleClose leaves unqueued because
2111
+ // that generation's in-flight run state is reconstructed sidecar-locally.
2112
+ // Returning without enqueueing there is correct; enqueueing is what keeps
2113
+ // grants and mail from diverging in the pre-reconnect window.
2114
+ return enqueueForDisconnected(agentAddress, frame);
2115
+ }
2116
+ async function sendWorkflowRunDispatchToAllocation(target, agentAddress, runId, stepGrants, rawMessage, authenticatedSender, messageId, signal) {
2117
+ signal?.throwIfAborted();
2118
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2119
+ signal?.throwIfAborted();
2120
+ if (addressIndex.get(agentAddress) !== ws) {
2121
+ throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
2122
+ }
2123
+ // authenticatedSender is the sender persisted at enqueue on the dispatch
2124
+ // row (the triggering principal's hub-verified address); the caller reads
2125
+ // it from that row. It is never the message's MIME From.
2126
+ //
2127
+ // Resolve its key here, at dispatch (redelivery) time, from that persisted
2128
+ // address, to co-deliver on the run's grants barrier so the recipient
2129
+ // caches the sender's current hub-held key. A run sender's deployment key
2130
+ // is immutable once acked; a user sender's key rotating mid-flight would
2131
+ // leave the fixed signed bytes checked against the new key, which the
2132
+ // recipient logs as unverifiable. Null when unresolvable (no resolver
2133
+ // wired, or the sender has no durable key). The lookup contract (see
2134
+ // SidecarLookups.resolveSenderKey) is best-effort and never throws, so
2135
+ // resolving ahead of the run.grants send cannot block it.
2136
+ const authenticatedSenderPublicKey = lookups.resolveSenderKey !== undefined
2137
+ ? await lookups.resolveSenderKey(authenticatedSender)
2138
+ : null;
2139
+ signal?.throwIfAborted();
2140
+ // Co-deliver the resolved key on the run's grants barrier, omitting a null
2141
+ // key so it is never cached (see deliverMailToRecipient). The same list
2142
+ // rides the pending-mail entry so the reconnect replay carries it too.
2143
+ const senderIdentities = authenticatedSenderPublicKey !== null
2144
+ ? [
2145
+ {
2146
+ address: authenticatedSender,
2147
+ publicKey: authenticatedSenderPublicKey,
2148
+ },
2149
+ ]
2150
+ : undefined;
2151
+ const runGrants = {
2152
+ runId,
2153
+ stepGrants,
2154
+ ...(senderIdentities !== undefined ? { senderIdentities } : {}),
2155
+ };
2156
+ conn.send({
2157
+ type: "run.grants",
2158
+ agentAddress,
2159
+ runId,
2160
+ stepGrants,
2161
+ ...(senderIdentities !== undefined ? { senderIdentities } : {}),
2162
+ });
2163
+ const frame = {
2164
+ type: "mail.inbound",
2165
+ agentAddress,
2166
+ rawMessage,
2167
+ authenticatedSender,
2168
+ messageId,
2169
+ };
2170
+ conn.send(frame);
2171
+ trackPendingMail(agentAddress, messageId, frame, runGrants, target);
2172
+ }
2173
+ async function handleDeployAck(ws, frame) {
2174
+ const req = pendingDeploys.get(frame.agentAddress);
2175
+ if (req === undefined) {
2176
+ logger.warn `Received agent.deploy.ack for "${frame.agentAddress}" with no pending deploy`;
1274
2177
  return;
1275
2178
  }
2179
+ if (req.ws !== ws)
2180
+ return;
1276
2181
  if (events.listenerCount("agent.deploy.ack") > 0) {
1277
2182
  try {
2183
+ const identity = connections.get(ws)?.identity;
1278
2184
  await events.emitAndAwait("agent.deploy.ack", {
1279
- agentAddress,
1280
- publicKey,
2185
+ agentAddress: frame.agentAddress,
2186
+ publicKey: frame.publicKey,
2187
+ ...(identity?.kind === "allocated"
2188
+ ? {
2189
+ allocated: {
2190
+ allocationId: identity.allocationId,
2191
+ anchorRunId: identity.anchorRunId,
2192
+ generation: identity.generation,
2193
+ },
2194
+ }
2195
+ : {}),
1281
2196
  });
1282
2197
  }
1283
2198
  catch (err) {
1284
- rejectDeployPending(agentAddress, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
2199
+ pendingDeploys.reject(frame.agentAddress, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
1285
2200
  return;
1286
2201
  }
1287
2202
  }
1288
- resolveDeployPending(agentAddress);
1289
- }
1290
- function resolveDeployPending(agentAddress) {
1291
- const req = pendingDeploys.get(agentAddress);
1292
- if (req === undefined)
1293
- return;
1294
- clearTimeout(req.timer);
1295
- pendingDeploys.delete(agentAddress);
1296
- req.resolve();
2203
+ pendingDeploys.resolve(frame.agentAddress, frame.publicKey);
1297
2204
  }
1298
- function rejectDeployPending(agentAddress, error) {
2205
+ function rejectDeployPendingFromFrame(ws, agentAddress, error) {
1299
2206
  const req = pendingDeploys.get(agentAddress);
1300
- if (req === undefined)
2207
+ if (req === undefined || req.ws !== ws)
1301
2208
  return;
1302
- clearTimeout(req.timer);
1303
- pendingDeploys.delete(agentAddress);
1304
- req.reject(error);
2209
+ // Settle by key, not by the `req` object: a key lookup observes the
2210
+ // CURRENT entry, so a stale handle cannot settle a replaced round-trip.
2211
+ pendingDeploys.reject(agentAddress, error);
1305
2212
  }
1306
- async function sendAgentDeploy(agentAddress, harnessConfig, workflow) {
2213
+ function sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow) {
1307
2214
  if (hubPublicKeyHex === undefined) {
1308
- throw new Error("Hub signing key is required for agent deployment");
2215
+ throw deployFrameFailure("Hub signing key is required for agent deployment", false);
1309
2216
  }
1310
2217
  if (pendingDeploys.has(agentAddress)) {
1311
- throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
1312
- }
1313
- const ws = addressIndex.get(agentAddress) ?? findSidecarForNewAgent(agentAddress);
1314
- if (ws === undefined) {
1315
- throw new Error(`No sidecar available for agent "${agentAddress}"`);
1316
- }
1317
- const conn = connections.get(ws);
1318
- if (conn === undefined) {
1319
- throw new Error(`No sidecar connected for agent "${agentAddress}"`);
2218
+ throw deployFrameFailure(`Deploy already in progress for agent "${agentAddress}"`, false);
1320
2219
  }
1321
- conn.agentAddresses.add(agentAddress);
2220
+ const addressSet = conn.identity.kind === "allocated"
2221
+ ? conn.workflowAddresses
2222
+ : conn.agentAddresses;
2223
+ addressSet.add(agentAddress);
1322
2224
  addressIndex.set(agentAddress, ws);
1323
2225
  return new Promise((resolve, reject) => {
1324
- // The deploy-ack handler does not currently thread the
1325
- // sidecar-reported public key back through the pending-deploy
1326
- // resolver; it only fires `agent.deploy.ack` listeners and then
1327
- // resolves the pending deploy. Capture the key via a one-shot
1328
- // listener so the return value carries it without a wire-shape
1329
- // change to `agent.deploy.ack`.
1330
- let capturedPublicKey;
1331
- const detachListener = events.on("agent.deploy.ack", (payload) => {
1332
- if (payload.agentAddress === agentAddress) {
1333
- capturedPublicKey = payload.publicKey;
1334
- }
1335
- });
1336
- const timer = setTimeout(() => {
1337
- detachListener();
1338
- pendingDeploys.delete(agentAddress);
1339
- if (addressIndex.get(agentAddress) === ws) {
1340
- conn.agentAddresses.delete(agentAddress);
1341
- addressIndex.delete(agentAddress);
1342
- }
1343
- reject(new Error(`Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
1344
- }, requestTimeoutMs);
1345
- pendingDeploys.set(agentAddress, {
1346
- agentAddress,
1347
- ws,
1348
- resolve() {
1349
- detachListener();
1350
- if (capturedPublicKey === undefined) {
1351
- reject(new Error(`Deploy of "${agentAddress}" resolved without an agent.deploy.ack publicKey payload`));
1352
- return;
1353
- }
1354
- resolve({ publicKey: capturedPublicKey });
2226
+ // Timeout and frame-error rejections share this closure, so the routing
2227
+ // rollback and the `frameSent: true` tag live in one place.
2228
+ pendingDeploys.register(agentAddress, ws, {
2229
+ timeoutMs: requestTimeoutMs,
2230
+ timeoutMessage: `Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
2231
+ resolve(publicKey) {
2232
+ resolve({ publicKey });
1355
2233
  },
1356
2234
  reject(error) {
1357
- detachListener();
1358
2235
  if (addressIndex.get(agentAddress) === ws) {
1359
- conn.agentAddresses.delete(agentAddress);
2236
+ addressSet.delete(agentAddress);
1360
2237
  addressIndex.delete(agentAddress);
1361
2238
  }
1362
- reject(new Error(error));
2239
+ // A non-allocated deployment's key is recorded by the deploy-ack
2240
+ // projection, whose failure (reject/timeout/agent.error/disconnect)
2241
+ // is observed only here. Drain any pre-ack sender mail parked on
2242
+ // this address so it surfaces as undelivered rather than waiting out
2243
+ // the TTL. An allocated deployment's failure is drained by its
2244
+ // session-service owner instead, so skip it here to keep one owner
2245
+ // per case.
2246
+ if (conn.identity.kind !== "allocated") {
2247
+ drainDeferredSenderMail(agentAddress, `deploy failed: ${error}`);
2248
+ }
2249
+ reject(deployFrameFailure(error, true));
1363
2250
  },
1364
- timer,
1365
- });
1366
- conn.send({
1367
- type: "agent.deploy",
1368
- agentAddress,
1369
- agentId: harnessConfig.agentId,
1370
- config: harnessConfig,
1371
- hubPublicKey: hubPublicKeyHex,
1372
- ...(workflow !== undefined ? { workflow } : {}),
1373
- });
2251
+ }, undefined);
2252
+ try {
2253
+ conn.send({
2254
+ type: "agent.deploy",
2255
+ agentAddress,
2256
+ agentId: harnessConfig.agentId,
2257
+ config: harnessConfig,
2258
+ hubPublicKey: hubPublicKeyHex,
2259
+ ...(workflow !== undefined ? { workflow } : {}),
2260
+ });
2261
+ }
2262
+ catch (err) {
2263
+ // A synchronous send failure means the frame never reached the wire.
2264
+ // Drop the pending entry (and its armed timer) and reject as not-sent
2265
+ // so a caller may safely roll back what it staged. The drop bypasses
2266
+ // the entry's reject closure: this failure must report
2267
+ // `frameSent: false`, and the timer must not fire later and
2268
+ // double-reject.
2269
+ pendingDeploys.delete(agentAddress);
2270
+ if (addressIndex.get(agentAddress) === ws) {
2271
+ addressSet.delete(agentAddress);
2272
+ addressIndex.delete(agentAddress);
2273
+ }
2274
+ reject(deployFrameFailure(`Deploy of "${agentAddress}" failed to send: ${err instanceof Error ? err.message : String(err)}`, false));
2275
+ }
1374
2276
  });
1375
2277
  }
2278
+ async function sendAgentDeployToAllocation(target, agentAddress, harnessConfig, workflow, signal, beforeSend) {
2279
+ try {
2280
+ signal?.throwIfAborted();
2281
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2282
+ signal?.throwIfAborted();
2283
+ if (agentAddress !== conn.identity.workflowRunAddress) {
2284
+ throw new Error(`Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`);
2285
+ }
2286
+ const existing = addressIndex.get(agentAddress);
2287
+ if (existing !== undefined && existing !== ws) {
2288
+ throw new Error(`Deployment ${agentAddress} is already routed to another sidecar`);
2289
+ }
2290
+ if (hubPublicKeyHex === undefined)
2291
+ throw new Error("Hub signing key is required for agent deployment");
2292
+ if (pendingDeploys.has(agentAddress))
2293
+ throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
2294
+ await beforeSend?.();
2295
+ signal?.throwIfAborted();
2296
+ if (allocatedConnections.get(target.allocationId)?.ws !== ws ||
2297
+ allocationFences.get(target.allocationId) !== target.generation) {
2298
+ throw new Error(`Allocated sidecar connection changed for allocation ${target.allocationId}`);
2299
+ }
2300
+ // Return without awaiting: only pre-send failures belong to this catch.
2301
+ return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
2302
+ }
2303
+ catch (cause) {
2304
+ throw deployFrameFailure(cause instanceof Error ? cause.message : String(cause), false, cause);
2305
+ }
2306
+ }
1376
2307
  /**
1377
2308
  * Provision one step of a multi-step deploy on the sidecar WITHOUT
1378
2309
  * spawning: the sidecar initializes the step's agent-state repo and
@@ -1387,42 +2318,29 @@ export function createSidecarRouter(config) {
1387
2318
  * so the caller can safely deliver the deploy pack afterward. On failure
1388
2319
  * the caller owns tearing the route down via `unbindStepRoute`.
1389
2320
  */
1390
- function sendProvisionStep(agentAddress, harnessConfig) {
2321
+ function sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig) {
1391
2322
  if (hubPublicKeyHex === undefined) {
1392
2323
  throw new Error("Hub signing key is required for step provisioning");
1393
2324
  }
1394
2325
  if (pendingDeploys.has(agentAddress)) {
1395
2326
  throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
1396
2327
  }
1397
- const ws = addressIndex.get(agentAddress);
1398
- if (ws === undefined) {
1399
- throw new Error(`Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`);
1400
- }
1401
- const conn = connections.get(ws);
1402
- if (conn === undefined) {
1403
- throw new Error(`No sidecar connected for agent "${agentAddress}"`);
1404
- }
1405
2328
  const hubKey = hubPublicKeyHex;
1406
2329
  return new Promise((resolve, reject) => {
1407
- const timer = setTimeout(() => {
1408
- pendingDeploys.delete(agentAddress);
1409
- reject(new Error(`Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
1410
- }, requestTimeoutMs);
1411
2330
  // The sidecar's `agent.deploy.ack` resolves this through
1412
- // `resolveDeployPending` -> `req.resolve()`. The per-step address is
1413
- // workflow-derived and records no hub-side key, so the ack's public
1414
- // key is not needed and this resolves void.
1415
- pendingDeploys.set(agentAddress, {
1416
- agentAddress,
1417
- ws,
1418
- resolve() {
2331
+ // `pendingDeploys.resolve`. The per-step address is workflow-derived
2332
+ // and records no hub-side key, so the ack's public key is not needed
2333
+ // and this resolves void.
2334
+ pendingDeploys.register(agentAddress, ws, {
2335
+ timeoutMs: requestTimeoutMs,
2336
+ timeoutMessage: `Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
2337
+ resolve(_publicKey) {
1419
2338
  resolve();
1420
2339
  },
1421
2340
  reject(error) {
1422
2341
  reject(new Error(error));
1423
2342
  },
1424
- timer,
1425
- });
2343
+ }, undefined);
1426
2344
  conn.send({
1427
2345
  type: "agent.deploy",
1428
2346
  agentAddress,
@@ -1433,11 +2351,46 @@ export function createSidecarRouter(config) {
1433
2351
  });
1434
2352
  });
1435
2353
  }
1436
- function findSidecarForNewAgent(_agentAddress) {
1437
- const first = connections.entries().next();
1438
- if (first.done)
1439
- return undefined;
1440
- return first.value[0];
2354
+ async function sendProvisionStepToAllocation(target, agentAddress, harnessConfig) {
2355
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2356
+ if (addressIndex.get(agentAddress) !== ws) {
2357
+ throw new Error(`Step route ${agentAddress} is not bound to allocation ${target.allocationId}`);
2358
+ }
2359
+ return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
2360
+ }
2361
+ function sendProbeOnConnection(ws, conn, args) {
2362
+ const requestId = nextRequestId();
2363
+ return new Promise((resolve, reject) => {
2364
+ pendingProbes.register(requestId, ws, {
2365
+ timeoutMs: probeTimeoutMs,
2366
+ timeoutMessage: `Probe ${requestId} timed out after ${probeTimeoutMs}ms`,
2367
+ resolve,
2368
+ reject(error) {
2369
+ reject(new Error(error));
2370
+ },
2371
+ }, undefined);
2372
+ conn.send({
2373
+ type: "workflow.probe.request",
2374
+ requestId,
2375
+ source: args.source,
2376
+ closure: args.closure,
2377
+ entry: args.entry,
2378
+ ...(args.assets !== undefined ? { assets: args.assets } : {}),
2379
+ });
2380
+ });
2381
+ }
2382
+ async function sendProbeToAllocation(target, args) {
2383
+ const { ws, conn } = await getProvisionedConnection(target, "routing");
2384
+ return sendProbeOnConnection(ws, conn, args);
2385
+ }
2386
+ function disconnectAllocation(target) {
2387
+ const current = allocatedConnections.get(target.allocationId);
2388
+ if (current === undefined ||
2389
+ current.identity.generation !== target.generation) {
2390
+ return;
2391
+ }
2392
+ handleClose(current.ws);
2393
+ current.ws.close();
1441
2394
  }
1442
2395
  function sendAgentUndeploy(agentAddress, reason) {
1443
2396
  const ws = addressIndex.get(agentAddress);
@@ -1449,14 +2402,11 @@ export function createSidecarRouter(config) {
1449
2402
  return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
1450
2403
  }
1451
2404
  return new Promise((resolve, reject) => {
1452
- const timer = setTimeout(() => {
1453
- pendingUndeploys.delete(agentAddress);
1454
- removeAgentAddress(ws, agentAddress);
1455
- reject(new Error(`Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
1456
- }, requestTimeoutMs);
1457
- pendingUndeploys.set(agentAddress, {
1458
- agentAddress,
1459
- ws,
2405
+ // Timeout, ack, and error rejection share one closure so the routing
2406
+ // teardown runs exactly once no matter how the round-trip settles.
2407
+ pendingUndeploys.register(agentAddress, ws, {
2408
+ timeoutMs: requestTimeoutMs,
2409
+ timeoutMessage: `Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
1460
2410
  resolve() {
1461
2411
  removeAgentAddress(ws, agentAddress);
1462
2412
  resolve();
@@ -1465,8 +2415,7 @@ export function createSidecarRouter(config) {
1465
2415
  removeAgentAddress(ws, agentAddress);
1466
2416
  reject(new Error(error));
1467
2417
  },
1468
- timer,
1469
- });
2418
+ }, undefined);
1470
2419
  conn.send({
1471
2420
  type: "agent.undeploy",
1472
2421
  agentAddress,
@@ -1529,6 +2478,15 @@ export function createSidecarRouter(config) {
1529
2478
  defaultSource,
1530
2479
  }));
1531
2480
  }
2481
+ async function sendCredentialsUpdate(agentAddress, delivery, revoke) {
2482
+ await sendRequest(agentAddress, (requestId) => ({
2483
+ type: "credentials.update",
2484
+ requestId,
2485
+ agentAddress,
2486
+ delivery,
2487
+ ...(revoke !== undefined ? { revoke } : {}),
2488
+ }));
2489
+ }
1532
2490
  function sendSyncRequest(agentAddress) {
1533
2491
  const ws = addressIndex.get(agentAddress);
1534
2492
  if (ws === undefined) {
@@ -1563,6 +2521,15 @@ export function createSidecarRouter(config) {
1563
2521
  payload: opts.payload,
1564
2522
  });
1565
2523
  }
2524
+ async function sendSignalDeliverToAllocation(target, opts, signal) {
2525
+ signal?.throwIfAborted();
2526
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2527
+ signal?.throwIfAborted();
2528
+ if (addressIndex.get(opts.agentAddress) !== ws) {
2529
+ throw new Error(`Address ${opts.agentAddress} is not routed on allocation ${target.allocationId}`);
2530
+ }
2531
+ conn.send({ type: "signal.deliver", ...opts });
2532
+ }
1566
2533
  function sendDrain(opts) {
1567
2534
  const ws = addressIndex.get(opts.agentAddress);
1568
2535
  if (ws === undefined) {
@@ -1583,15 +2550,29 @@ export function createSidecarRouter(config) {
1583
2550
  handleMessage,
1584
2551
  handleClose,
1585
2552
  routeMail,
1586
- sendAgentDeploy,
2553
+ sendRunGrants,
2554
+ noteSenderDeployStarted,
2555
+ noteSenderDeploySettled,
2556
+ sendProbeToAllocation,
2557
+ disconnectAllocation,
1587
2558
  sendAgentUndeploy,
1588
2559
  sendSourcesUpdate,
1589
- sendPack,
1590
- bindStepRoute,
1591
- unbindStepRoute,
1592
- sendProvisionStep,
2560
+ sendCredentialsUpdate,
2561
+ sendPackToAllocation,
2562
+ sendWorkflowRunPackToAllocation,
2563
+ fenceAllocation,
2564
+ retireAllocation,
2565
+ waitForAllocatedSidecar,
2566
+ isAllocatedSidecarReady,
2567
+ isAllocatedWorkflowActive,
2568
+ sendAgentDeployToAllocation,
2569
+ bindAllocatedStepRoute,
2570
+ unbindAllocatedStepRoute,
2571
+ sendProvisionStepToAllocation,
2572
+ sendWorkflowRunDispatchToAllocation,
1593
2573
  sendSyncRequest,
1594
2574
  sendSignalDeliver,
2575
+ sendSignalDeliverToAllocation,
1595
2576
  sendDrain,
1596
2577
  subscribeAgent,
1597
2578
  dispatchAgentEvent: dispatchToSubscribers,