@intx/hub-sessions 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/agent-repo.d.ts +14 -2
  2. package/dist/agent-repo.js +17 -4
  3. package/dist/agent-state-kind.js +14 -63
  4. package/dist/asset-service.js +14 -10
  5. package/dist/credential-push.d.ts +48 -4
  6. package/dist/credential-push.js +138 -6
  7. package/dist/event-collector-registry.d.ts +2 -1
  8. package/dist/event-collector-registry.js +38 -9
  9. package/dist/event-collector.d.ts +11 -1
  10. package/dist/event-collector.js +36 -3
  11. package/dist/hub-session-lookups.d.ts +1 -1
  12. package/dist/hub-session-lookups.js +68 -72
  13. package/dist/hub-session-orchestrator.d.ts +2 -3
  14. package/dist/hub-session-orchestrator.js +13 -12
  15. package/dist/index.d.ts +7 -6
  16. package/dist/index.js +7 -6
  17. package/dist/reconciliation-scheduler.d.ts +14 -0
  18. package/dist/reconciliation-scheduler.js +55 -0
  19. package/dist/repo-store/index.d.ts +1 -0
  20. package/dist/repo-store/index.js +1 -0
  21. package/dist/repo-store/user-principal-gate.d.ts +26 -0
  22. package/dist/repo-store/user-principal-gate.js +78 -0
  23. package/dist/session-service.d.ts +66 -121
  24. package/dist/session-service.js +444 -411
  25. package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
  26. package/dist/sidecar-allocation/capability-policy.js +124 -0
  27. package/dist/sidecar-allocation/contracts.d.ts +29 -6
  28. package/dist/sidecar-allocation/contracts.js +7 -2
  29. package/dist/sidecar-allocation/index.d.ts +4 -3
  30. package/dist/sidecar-allocation/index.js +3 -2
  31. package/dist/sidecar-allocation/operation.d.ts +10 -0
  32. package/dist/sidecar-allocation/operation.js +54 -0
  33. package/dist/sidecar-allocation/plugin-registry.d.ts +16 -3
  34. package/dist/sidecar-allocation/plugin-registry.js +36 -12
  35. package/dist/sidecar-allocation/reconciler.d.ts +16 -4
  36. package/dist/sidecar-allocation/reconciler.js +486 -92
  37. package/dist/skill-kind.js +8 -62
  38. package/dist/substrate.d.ts +1 -1
  39. package/dist/substrate.js +1 -1
  40. package/dist/workflow-allocation-service.d.ts +21 -15
  41. package/dist/workflow-allocation-service.js +440 -125
  42. package/dist/workflow-dispatch-service.d.ts +4 -2
  43. package/dist/workflow-dispatch-service.js +89 -26
  44. package/dist/workflow-kind.d.ts +12 -0
  45. package/dist/workflow-kind.js +17 -60
  46. package/dist/workflow-probe-gate.d.ts +99 -27
  47. package/dist/workflow-probe-gate.js +196 -21
  48. package/dist/workflow-run-kind.d.ts +112 -19
  49. package/dist/workflow-run-kind.js +626 -210
  50. package/dist/workflow-run-restore.d.ts +1 -0
  51. package/dist/workflow-run-restore.js +5 -1
  52. package/dist/workflow-source-pins.d.ts +8 -0
  53. package/dist/workflow-source-pins.js +14 -0
  54. package/dist/ws/index.d.ts +1 -1
  55. package/dist/ws/index.js +1 -1
  56. package/dist/ws/pending-tracker.d.ts +93 -0
  57. package/dist/ws/pending-tracker.js +132 -0
  58. package/dist/ws/sidecar-events.d.ts +43 -29
  59. package/dist/ws/sidecar-events.js +0 -2
  60. package/dist/ws/sidecar-handler.d.ts +122 -85
  61. package/dist/ws/sidecar-handler.js +925 -878
  62. package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
  63. package/dist/ws/sidecar-handler.test-helpers.js +95 -0
  64. package/dist/ws/sidecar-token-authenticator.js +37 -23
  65. package/package.json +13 -13
  66. package/dist/sidecar-allocation/placement-policy.d.ts +0 -11
  67. package/dist/sidecar-allocation/placement-policy.js +0 -21
@@ -4,19 +4,37 @@
4
4
  // table of agentAddress → sidecar connection, and dispatches frames between
5
5
  // sidecars and the hub's internal systems.
6
6
  import { getLogger } from "@intx/log";
7
- import { verifyEd25519 } from "@intx/crypto";
8
7
  import { chunkPack, createPackReceiver } from "@intx/pack-transport";
9
- import { base64Decode, deriveMessageId, deriveWorkflowRunId, hexDecode, hexEncode, isRunAddress, } from "@intx/types";
8
+ import { base64Decode, deriveMessageId, deriveWorkflowRunId, isRunAddress, } from "@intx/types";
10
9
  import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy";
11
10
  import { type } from "arktype";
12
- import { SidecarFrame, } from "@intx/types/sidecar";
11
+ import { MAX_MAIL_OUTBOUND_BODY_BYTES, SidecarFrame, } from "@intx/types/sidecar";
13
12
  import { createSidecarEmitter, } from "./sidecar-events.js";
13
+ import { PendingTracker, } from "./pending-tracker.js";
14
14
  const logger = getLogger(["hub", "ws", "sidecar"]);
15
+ function deployFrameFailure(message, frameSent, cause) {
16
+ return Object.assign(new Error(message, { cause }), { frameSent });
17
+ }
18
+ export function isDeployFrameFailure(err) {
19
+ return (err instanceof Error &&
20
+ "frameSent" in err &&
21
+ typeof err.frameSent === "boolean");
22
+ }
15
23
  /**
16
- * Whether this connection owns `address` for routing/lifecycle purposes --
17
- * as a challenged session address OR a hub-minted workflow-substrate address.
18
- * The two sets are kept physically distinct (they differ on the challenge /
19
- * re-add path), but ownership readers -- pack-transfer authorization,
24
+ * Identity validation failed or remained pending at the connection deadline.
25
+ * Readiness is unknown: the worker may be healthy behind the lookup, so
26
+ * callers must retry rather than treat this as a missed connection deadline.
27
+ */
28
+ export class SidecarIdentityValidationError extends Error {
29
+ constructor(allocationId, generation, cause) {
30
+ super(`Cannot validate sidecar identity for allocation ${allocationId} generation ${String(generation)}`, { cause });
31
+ this.name = "SidecarIdentityValidationError";
32
+ }
33
+ }
34
+ /**
35
+ * Whether this connection owns `address` for routing/lifecycle purposes.
36
+ * The legacy and workflow sets remain physically distinct, but ownership
37
+ * readers -- pack-transfer authorization,
20
38
  * in-flight cancellation, disconnect teardown -- must see the union, or a
21
39
  * reconnected workflow deployment (which lives only in `workflowAddresses`)
22
40
  * is silently treated as unowned even though its mail routes.
@@ -31,12 +49,10 @@ function connOwnsAddress(conn, address) {
31
49
  * repository.
32
50
  */
33
51
  function connCanPushRepo(conn, agentAddress, repoId) {
34
- if (conn.identity.kind === "allocated" &&
35
- agentAddress !== conn.identity.workflowRunAddress) {
52
+ if (conn.identity.kind !== "allocated")
53
+ return false;
54
+ if (agentAddress !== conn.identity.workflowRunAddress) {
36
55
  return false;
37
- }
38
- if (repoId.kind === "agent-state") {
39
- return conn.identity.kind === "shared" && repoId.id === agentAddress;
40
56
  }
41
57
  return (repoId.kind === "workflow-run" &&
42
58
  repoId.id === deriveWorkflowRunRepoId(agentAddress));
@@ -46,7 +62,6 @@ function ownedAddresses(conn) {
46
62
  return new Set([...conn.agentAddresses, ...conn.workflowAddresses]);
47
63
  }
48
64
  const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
49
- const DEFAULT_CHALLENGE_TIMEOUT_MS = 30_000;
50
65
  // A probe fetches a workflow's dependency closure from a registry and
51
66
  // evaluates it on the sidecar, so it runs longer than a routine request; its
52
67
  // default timeout is correspondingly wider than DEFAULT_REQUEST_TIMEOUT_MS.
@@ -56,8 +71,25 @@ const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000;
56
71
  const DEFAULT_PING_TIMEOUT_MS = 60_000;
57
72
  const DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS = 10_000;
58
73
  const DEFAULT_MAIL_ACK_MAX_RETRIES = 5;
74
+ // The hub re-resolves and re-pushes a key for each rotatable sender a sidecar
75
+ // reports on (re)connect. A legitimate sidecar caches keys for tens, maybe low
76
+ // hundreds of distinct user senders, so this cap sits well above ten times that
77
+ // ceiling: it NEVER truncates a real report -- dropping a genuine sender would
78
+ // leave its key stale, the exact failure this refresh exists to prevent. It
79
+ // bounds only a hostile or buggy sidecar, since a compromised authenticated
80
+ // sidecar could otherwise report an unbounded set and drive that many sequential
81
+ // DB resolves on every reconnect. The cap lives in the handler, not on the
82
+ // arktype frame schema, on purpose: rejecting an over-cap frame at parse would
83
+ // fail the whole reconnect (a hard outage) rather than degrade gracefully to a
84
+ // bounded refresh.
85
+ export const MAX_RESYNC_SENDER_ADDRESSES = 2048;
59
86
  export function createSidecarRouter(config) {
60
- const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, challengeTimeoutMs = DEFAULT_CHALLENGE_TIMEOUT_MS, probeTimeoutMs = DEFAULT_PROBE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, validateSidecarIdentity = async () => true, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, lookups = {}, } = config;
87
+ const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, probeTimeoutMs = DEFAULT_PROBE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, validateSidecarIdentity, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, scheduleTimeout = (handler, ms) => {
88
+ const handle = setTimeout(handler, ms);
89
+ return () => {
90
+ clearTimeout(handle);
91
+ };
92
+ }, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, lookups = {}, } = config;
61
93
  // Receiver-dispatch surface. Wire-layer callsites emit events here;
62
94
  // host code subscribes via `router.events`.
63
95
  const events = createSidecarEmitter();
@@ -68,10 +100,20 @@ export function createSidecarRouter(config) {
68
100
  const allocationWaiters = new Map();
69
101
  // agentAddress → ws handle (routing table)
70
102
  const addressIndex = new Map();
71
- // requestId → pending promise
72
- const pending = new Map();
73
- const pendingDeploys = new Map();
74
- const pendingChallenges = new Map();
103
+ // requestId → pending promise (resolved by session.ack, rejected by
104
+ // session.error). `PendingTracker` owns the register/timeout/settle/sweep
105
+ // lifecycle shared by all five pending round-trips below; each entry's
106
+ // resolve/reject closures carry the per-round-trip cleanup.
107
+ const pendingRequests = new PendingTracker();
108
+ // agentAddress → pending deploy promise (matched by agent.deploy.ack/agent.error)
109
+ const pendingDeploys = new PendingTracker();
110
+ // Run addresses whose ALLOCATED deploy is mid-flight -- key-record has been
111
+ // started but not yet committed. pendingDeploys clears at the deploy ack, but an
112
+ // allocated run's key is recorded LATER by session-service's anchor-key update,
113
+ // so pendingDeploys alone under-covers the allocated pre-ack window. session-
114
+ // service brackets this marker across its deploy try/catch: set before the
115
+ // deploy emit, cleared by noteSenderDeploySettled on record or failure.
116
+ const allocatedKeyRecordInFlight = new Map();
75
117
  const disconnectedAgents = new Map();
76
118
  const pendingMail = new Map();
77
119
  // agentAddress → retention TTL timer for un-acked pending mail held across a
@@ -81,6 +123,7 @@ export function createSidecarRouter(config) {
81
123
  // does not leak entries. Cleared when the address reconnects (redelivery) or
82
124
  // its last pending entry is acked.
83
125
  const pendingMailRetention = new Map();
126
+ const deferredSenderMail = new Map();
84
127
  // agentAddress → set of subscriber callbacks for agent events
85
128
  const agentSubscribers = new Map();
86
129
  // agentAddress → cached connector-thread state, populated by
@@ -100,10 +143,17 @@ export function createSidecarRouter(config) {
100
143
  // a single in-flight promise per ws (replaced each queued frame), cleared on
101
144
  // close.
102
145
  const messageChains = new Map();
103
- const pendingPacks = new Map();
146
+ const pendingPacks = new PendingTracker();
104
147
  let packCounter = 0;
105
- const pendingUndeploys = new Map();
106
- const pendingProbes = new Map();
148
+ // agentAddress pending undeploy (resolved by agent.undeploy.ack)
149
+ const pendingUndeploys = new PendingTracker();
150
+ // requestId → pending workflow probe (resolved by workflow.probe.result,
151
+ // rejected by workflow.probe.error). Result-carrying, unlike the other
152
+ // trackers (which resolve void): a probe returns the sidecar's inert
153
+ // projection + grant set + wire hash. Keyed on requestId alone -- the
154
+ // probe runs in the sidecar's pre-deploy state and enters no address map,
155
+ // so `handleClose`'s ws-keyed sweep is its ONLY disconnect cleanup.
156
+ const pendingProbes = new PendingTracker();
107
157
  // Receives agent-state packs pushed from sidecars. The wire frames
108
158
  // (`repo.pack.push` / `repo.pack.done`) are shared with the
109
159
  // workflow-run flow; dispatch on `repoId.kind` picks which receiver
@@ -147,18 +197,16 @@ export function createSidecarRouter(config) {
147
197
  entry.queue.push(frame);
148
198
  return true;
149
199
  }
150
- function flushDisconnectedQueue(agentAddress, conn) {
151
- const entry = disconnectedAgents.get(agentAddress);
152
- if (entry === undefined)
153
- return;
154
- clearTimeout(entry.timer);
155
- disconnectedAgents.delete(agentAddress);
156
- for (const frame of entry.queue) {
157
- conn.send(frame);
158
- }
159
- if (entry.queue.length > 0) {
160
- logger.info `Flushed ${String(entry.queue.length)} queued message(s) to ${agentAddress}`;
161
- }
200
+ // Arm a redelivery-retry timer for a tracked pending mail. Wraps the async
201
+ // `retryPendingMail` so a rejection -- a socket write that throws once the
202
+ // sidecar is gone -- is logged rather than floating out of the timer as an
203
+ // unhandled rejection.
204
+ function scheduleMailRetry(agentAddress, messageId) {
205
+ return scheduleTimeout(() => {
206
+ void retryPendingMail(agentAddress, messageId).catch((err) => {
207
+ logger.warn `Redelivery retry for mail ${messageId} to ${agentAddress} failed: ${err instanceof Error ? err.message : String(err)}`;
208
+ });
209
+ }, mailAckRetryIntervalMs);
162
210
  }
163
211
  // Track a connected-window `mail.inbound` for redelivery until the sidecar
164
212
  // acks its durable inbox write. Replaces any prior entry for the same
@@ -172,30 +220,129 @@ export function createSidecarRouter(config) {
172
220
  }
173
221
  const existing = byId.get(messageId);
174
222
  if (existing !== undefined)
175
- clearTimeout(existing.timer);
223
+ existing.cancelRetry();
176
224
  byId.set(messageId, {
177
225
  agentAddress,
178
226
  messageId,
179
227
  frame,
180
228
  attempts: 0,
181
- timer: setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs),
229
+ cancelRetry: scheduleMailRetry(agentAddress, messageId),
182
230
  ...(runGrants !== undefined ? { runGrants } : {}),
183
231
  ...(allocatedTarget !== undefined ? { allocatedTarget } : {}),
184
232
  });
185
233
  }
186
- // Replay a mail-triggered run's grants ahead of a redelivery of its trigger
187
- // mail. Same-connection FIFO lands the `run.grants` frame before the mail, so
188
- // the redelivered run resolves its onRunStart barrier instead of failing
189
- // closed on missing grants.
190
- function replayRunGrantsAhead(conn, entry) {
191
- if (entry.runGrants === undefined)
192
- return;
193
- conn.send({
234
+ // Resolve the frame that must precede a redelivery of a trigger mail on the
235
+ // FIFO socket, re-resolving a keyless run sender's key so it still
236
+ // co-delivers. Returns:
237
+ // - a `run.grants` frame when the entry carries run grants (the redelivered
238
+ // run resolves its onRunStart barrier instead of failing closed on
239
+ // missing grants);
240
+ // - a bare `sender.key.refresh` frame when the entry carries NO run grants
241
+ // but its run sender's key was never co-delivered, so the recipient still
242
+ // caches the key ahead of the mail;
243
+ // - `undefined` when nothing must precede the mail.
244
+ //
245
+ // The re-resolve is KIND-GATED to run-address senders only. A run's
246
+ // deployment key is immutable once acked, so the re-resolved key equals the
247
+ // signing-time key -- safe. A user (non-run) sender's key may have rotated
248
+ // since it signed, so re-resolving would check the fixed signed bytes against
249
+ // a newer key and turn a valid message into a false `invalid`; such a sender
250
+ // lacking a captured key stays keyless (an honest `unknown`). An entry that
251
+ // captured `senderIdentities` at track time replays that snapshot as-is: it
252
+ // holds the signing-time key and is never re-resolved.
253
+ //
254
+ // Awaits any key resolve so the caller sends the returned frame and the mail
255
+ // back-to-back with no await between them, keeping the co-delivered key ahead
256
+ // of the mail on the FIFO socket.
257
+ async function resolveReplayLeadFrame(entry) {
258
+ const authenticatedSender = entry.frame.type === "mail.inbound"
259
+ ? entry.frame.authenticatedSender
260
+ : undefined;
261
+ const senderIsRun = authenticatedSender !== undefined && isRunAddress(authenticatedSender);
262
+ if (entry.runGrants === undefined) {
263
+ if (authenticatedSender === undefined || !senderIsRun)
264
+ return undefined;
265
+ const key = await reresolveRunSenderKey(authenticatedSender);
266
+ if (key === null)
267
+ return undefined;
268
+ return {
269
+ type: "sender.key.refresh",
270
+ address: authenticatedSender,
271
+ publicKey: key,
272
+ };
273
+ }
274
+ let senderIdentities = entry.runGrants.senderIdentities;
275
+ if (senderIdentities === undefined &&
276
+ authenticatedSender !== undefined &&
277
+ senderIsRun) {
278
+ const key = await reresolveRunSenderKey(authenticatedSender);
279
+ senderIdentities = senderIdentitiesFromKey(authenticatedSender, key);
280
+ }
281
+ return {
194
282
  type: "run.grants",
195
283
  agentAddress: entry.agentAddress,
196
284
  runId: entry.runGrants.runId,
197
285
  stepGrants: entry.runGrants.stepGrants,
198
- });
286
+ ...(senderIdentities !== undefined ? { senderIdentities } : {}),
287
+ };
288
+ }
289
+ // Best-effort re-resolve of a run sender's hub-held key at replay time. Only
290
+ // called for a run-address sender, whose deployment key is immutable once
291
+ // acked, so the current key equals the signing-time key. Returns null when no
292
+ // resolver is wired or the sender has no durable key.
293
+ //
294
+ // This relies on `lookups.resolveSenderKey` being the BEST-EFFORT,
295
+ // NEVER-THROWS resolver (the contract at sidecar-events.ts:274-279, wired to
296
+ // resolveFrameSenderKey, which swallows faults to null). That contract is
297
+ // load-bearing here: `redeliverPendingMail` clears the retention TTL up-front
298
+ // and re-arms each entry's per-entry timer only on a successful send, so a
299
+ // resolver that THREW would abort the redeliver loop and strand the
300
+ // not-yet-processed entries with no timer and no TTL until a process restart.
301
+ // A strict/throwing resolver must NOT be wired here. Do not add a try/catch:
302
+ // the boundary owns the never-throws contract; duplicating it here would
303
+ // violate that ownership. The dispatch-time resolveSenderKey call
304
+ // (sendWorkflowRunDispatchToAllocation path) carries the same dependency
305
+ // note.
306
+ async function reresolveRunSenderKey(authenticatedSender) {
307
+ const resolveSenderKey = lookups.resolveSenderKey;
308
+ if (resolveSenderKey === undefined)
309
+ return null;
310
+ return resolveSenderKey(authenticatedSender);
311
+ }
312
+ // Replay a pending mail's lead frame (its run grants or a re-resolved sender
313
+ // key) and then the mail itself over `conn`. Awaits the resolve FIRST, then
314
+ // sends the lead frame and the mail back-to-back with NO await between them,
315
+ // so the co-delivered key always precedes the mail on the FIFO socket.
316
+ // Returns whether the mail was (re)sent, so the caller re-arms the retry timer
317
+ // only for an entry it actually redelivered.
318
+ async function replaySendPendingMail(conn, entry) {
319
+ const lead = await resolveReplayLeadFrame(entry);
320
+ // The resolve above may have awaited real I/O; during that gap a queued
321
+ // `mail.inbound.ack` can advance and run `resolvePendingMail` (delete +
322
+ // clearTimeout) on this entry. The window is opened by the timer-macrotask
323
+ // retry path, NOT by any bypass: `mail.inbound.ack` is a QUEUED frame
324
+ // (frameBypassesQueue returns false for it). It can interleave because the
325
+ // retry runs as an independent setTimeout macrotask (retryPendingMail), so
326
+ // the owning ws's message chain is free to advance the ack during the
327
+ // resolve await. On the reconnect/redeliver path the ack cannot interleave
328
+ // at all -- it queues behind the still-running reconnect handler on the
329
+ // same ws -- so here this guard is pure defense-in-depth. Re-confirm it is
330
+ // still the tracked entry before sending, or a post-ack redelivery would
331
+ // arm a retry timer on a detached entry.
332
+ if (pendingMail.get(entry.agentAddress)?.get(entry.messageId) !== entry) {
333
+ return false;
334
+ }
335
+ // The same gap can span a disconnect or a takeover that moves the address
336
+ // off `conn`. Sending on the stale conn would write to a dead socket and
337
+ // re-arm a retry that later drops a still-retained entry. Skip so the entry
338
+ // survives for the reconnect redelivery.
339
+ const ws = addressIndex.get(entry.agentAddress);
340
+ if (ws === undefined || connections.get(ws) !== conn)
341
+ return false;
342
+ if (lead !== undefined)
343
+ conn.send(lead);
344
+ conn.send(entry.frame);
345
+ return true;
199
346
  }
200
347
  function deletePendingMail(byId, agentAddress, messageId) {
201
348
  byId.delete(messageId);
@@ -210,7 +357,7 @@ export function createSidecarRouter(config) {
210
357
  }
211
358
  }
212
359
  }
213
- function retryPendingMail(agentAddress, messageId) {
360
+ async function retryPendingMail(agentAddress, messageId) {
214
361
  const byId = pendingMail.get(agentAddress);
215
362
  if (byId === undefined)
216
363
  return;
@@ -252,10 +399,10 @@ export function createSidecarRouter(config) {
252
399
  logger.warn `Dropping un-acked mail ${messageId} for ${agentAddress}: no live connection to redeliver over`;
253
400
  return;
254
401
  }
402
+ if (!(await replaySendPendingMail(conn, entry)))
403
+ return;
255
404
  entry.attempts += 1;
256
- replayRunGrantsAhead(conn, entry);
257
- conn.send(entry.frame);
258
- entry.timer = setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs);
405
+ entry.cancelRetry = scheduleMailRetry(agentAddress, messageId);
259
406
  }
260
407
  function resolvePendingMail(agentAddress, messageId) {
261
408
  const byId = pendingMail.get(agentAddress);
@@ -264,7 +411,7 @@ export function createSidecarRouter(config) {
264
411
  const entry = byId.get(messageId);
265
412
  if (entry === undefined)
266
413
  return;
267
- clearTimeout(entry.timer);
414
+ entry.cancelRetry();
268
415
  deletePendingMail(byId, agentAddress, messageId);
269
416
  }
270
417
  // Hold an address's un-acked pending mail across a disconnect. The per-entry
@@ -279,7 +426,7 @@ export function createSidecarRouter(config) {
279
426
  if (byId === undefined)
280
427
  return;
281
428
  for (const entry of byId.values())
282
- clearTimeout(entry.timer);
429
+ entry.cancelRetry();
283
430
  const existing = pendingMailRetention.get(agentAddress);
284
431
  if (existing !== undefined)
285
432
  clearTimeout(existing);
@@ -307,7 +454,7 @@ export function createSidecarRouter(config) {
307
454
  // (effectively-once) and processes one it had dropped (no loss). Re-arms the
308
455
  // connected-window retry over the new connection with a fresh per-generation
309
456
  // budget, so a redelivery that is itself dropped before its ack is retried.
310
- function redeliverPendingMail(agentAddress, conn) {
457
+ async function redeliverPendingMail(agentAddress, conn) {
311
458
  const retention = pendingMailRetention.get(agentAddress);
312
459
  if (retention !== undefined) {
313
460
  clearTimeout(retention);
@@ -324,14 +471,14 @@ export function createSidecarRouter(config) {
324
471
  // The Hub-owned dispatch row survives generation replacement and will
325
472
  // be requeued by the allocation-ready callback. Do not leak or replay
326
473
  // this generation-local retry entry onto a different worker.
327
- clearTimeout(entry.timer);
474
+ entry.cancelRetry();
328
475
  deletePendingMail(byId, agentAddress, entry.messageId);
329
476
  continue;
330
477
  }
331
- replayRunGrantsAhead(conn, entry);
332
- conn.send(entry.frame);
478
+ if (!(await replaySendPendingMail(conn, entry)))
479
+ continue;
333
480
  entry.attempts = 0;
334
- entry.timer = setTimeout(() => retryPendingMail(agentAddress, entry.messageId), mailAckRetryIntervalMs);
481
+ entry.cancelRetry = scheduleMailRetry(agentAddress, entry.messageId);
335
482
  }
336
483
  if (byId.size > 0) {
337
484
  logger.info `Redelivered ${String(byId.size)} un-acked message(s) to ${agentAddress} on reconnect`;
@@ -340,13 +487,13 @@ export function createSidecarRouter(config) {
340
487
  function resetLivenessTimer(ws) {
341
488
  const existing = livenessTimers.get(ws);
342
489
  if (existing !== undefined)
343
- clearTimeout(existing);
344
- const timer = setTimeout(() => {
490
+ existing();
491
+ const cancel = scheduleTimeout(() => {
345
492
  livenessTimers.delete(ws);
346
493
  logger.warn `Sidecar ping timeout, closing connection`;
347
494
  ws.close();
348
495
  }, pingTimeoutMs);
349
- livenessTimers.set(ws, timer);
496
+ livenessTimers.set(ws, cancel);
350
497
  }
351
498
  function handlePing(ws) {
352
499
  resetLivenessTimer(ws);
@@ -409,11 +556,9 @@ export function createSidecarRouter(config) {
409
556
  // state. Such a frame has no ordering obligation against new inbound frames
410
557
  // (a response cannot resolve "too early" for a request that already went
411
558
  // out), and it is exactly what in-flight queued handlers block on, so it MUST
412
- // run out of band or the challenge round-trip deadlocks (challenge.response
413
- // -> agent.reconnected reaction -> sendSourcesUpdate awaits a later
414
- // session.ack). Every other frame establishes or reads routing, or carries an
415
- // inbound payload whose order matters, so it queues. The exhaustive switch +
416
- // assertNever makes adding a SidecarFrame variant without classifying it a
559
+ // run out of band. Every other frame establishes or reads routing, or carries
560
+ // an inbound payload whose order matters, so it queues. The exhaustive switch
561
+ // + assertNever makes adding a SidecarFrame variant without classifying it a
417
562
  // compile error, not a latent deadlock or a silent bypass hole.
418
563
  function frameBypassesQueue(frame) {
419
564
  switch (frame.type) {
@@ -430,7 +575,6 @@ export function createSidecarRouter(config) {
430
575
  return true;
431
576
  case "register":
432
577
  case "reconnect":
433
- case "challenge.response":
434
578
  case "mail.outbound":
435
579
  case "agent.event":
436
580
  case "connector.state.changed":
@@ -445,22 +589,31 @@ export function createSidecarRouter(config) {
445
589
  }
446
590
  // Runs one frame's handler. Returns the handler's promise for async handlers
447
591
  // so the per-ws chain can await bounded completion; sync handlers return
448
- // void. Never awaits a promise that resolves on a LATER same-ws frame -- the
449
- // only such await (the challenge round-trip's session.ack) is reached via a
450
- // bypass frame, which does not queue.
592
+ // void. Never awaits a promise that resolves on a later same-ws frame.
451
593
  function dispatchFrame(ws, frame) {
594
+ const registeredIdentity = connections.get(ws)?.identity;
595
+ if (registeredIdentity?.kind === "probe" &&
596
+ frame.type !== "register" &&
597
+ frame.type !== "reconnect" &&
598
+ frame.type !== "ping" &&
599
+ frame.type !== "workflow.probe.result" &&
600
+ frame.type !== "workflow.probe.error") {
601
+ logger.warn `Rejected ${frame.type} from probe sidecar ${registeredIdentity.sidecarId}`;
602
+ handleClose(ws);
603
+ ws.close();
604
+ return;
605
+ }
452
606
  switch (frame.type) {
453
607
  case "register": {
454
608
  const agentAddresses = frame.agentAddresses;
455
- return authenticateHandshake(ws, frame, (identity) => handleRegister(ws, identity, agentAddresses));
609
+ const cachedSenderAddresses = frame.cachedSenderAddresses ?? [];
610
+ return authenticateHandshake(ws, frame, (identity) => handleRegister(ws, identity, agentAddresses, cachedSenderAddresses));
456
611
  }
457
612
  case "reconnect": {
458
613
  const agentAddresses = frame.agentAddresses;
459
- const deployRefs = frame.deployRefs ?? {};
460
- return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses, deployRefs));
614
+ const cachedSenderAddresses = frame.cachedSenderAddresses ?? [];
615
+ return authenticateHandshake(ws, frame, (identity) => handleReconnect(ws, identity, agentAddresses, cachedSenderAddresses));
461
616
  }
462
- case "challenge.response":
463
- return handleChallengeResponse(ws, frame.responses);
464
617
  case "agent.deploy.ack":
465
618
  return handleDeployAck(ws, frame);
466
619
  case "agent.error":
@@ -473,21 +626,45 @@ export function createSidecarRouter(config) {
473
626
  case "ping":
474
627
  handlePing(ws);
475
628
  return;
476
- case "mail.outbound":
477
- if (frame.delivered !== true) {
478
- return handleMailOutbound(frame.rawMessage, frame.recipients);
629
+ case "mail.outbound": {
630
+ const conn = connections.get(ws);
631
+ if (conn === undefined)
632
+ return;
633
+ if (!connOwnsAddress(conn, frame.senderAddress)) {
634
+ logger.warn `Dropping mail.outbound from ${frame.senderAddress}: not registered to this sidecar`;
635
+ return;
479
636
  }
480
- if (lookups.persistMail && frame.senderAddress) {
481
- return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
637
+ // The DoS backstop and the trust boundary for an untrusted sidecar's
638
+ // mail body: measure the true byte cost (a hostile sidecar can send
639
+ // multi-byte UTF-8, so `.length` would undercount) and drop an over-cap
640
+ // frame here before either delivery path allocates on it. The socket's
641
+ // maxPayloadLength has already closed the connection for a truly huge
642
+ // frame; this catches one between the mail cap and that ceiling.
643
+ const bodyBytes = Buffer.byteLength(frame.rawMessage, "utf8");
644
+ if (bodyBytes > MAX_MAIL_OUTBOUND_BODY_BYTES) {
645
+ logger.warn `Dropping mail.outbound from ${frame.senderAddress}: rawMessage of ${String(bodyBytes)} bytes exceeds the ${String(MAX_MAIL_OUTBOUND_BODY_BYTES)}-byte cap`;
646
+ return;
482
647
  }
483
- if (!frame.senderAddress) {
484
- logger.warn `Dropping delivered mail.outbound frame with no senderAddress`;
648
+ if (frame.delivered !== true) {
649
+ // frame.senderAddress is the sender this connection was just gated
650
+ // on by connOwnsAddress above -- a hub-verified value. Thread it so
651
+ // the relayed inbound frame is stamped with it, not the MIME From.
652
+ return handleMailOutbound(frame.rawMessage, frame.senderAddress, frame.recipients);
485
653
  }
486
- else {
487
- logger.warn `Dropping delivered mail.outbound frame: no persistMail lookup configured`;
654
+ if (lookups.persistMail) {
655
+ return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
488
656
  }
657
+ logger.warn `Dropping delivered mail.outbound frame: no persistMail lookup configured`;
489
658
  return;
490
- case "agent.event":
659
+ }
660
+ case "agent.event": {
661
+ const conn = connections.get(ws);
662
+ if (conn === undefined)
663
+ return;
664
+ if (!connOwnsAddress(conn, frame.agentAddress)) {
665
+ logger.warn `Dropping agent.event for ${frame.agentAddress}: not registered to this sidecar`;
666
+ return;
667
+ }
491
668
  events.emit("agent.event", {
492
669
  agentAddress: frame.agentAddress,
493
670
  sessionId: frame.sessionId,
@@ -495,6 +672,7 @@ export function createSidecarRouter(config) {
495
672
  });
496
673
  dispatchToSubscribers(frame.agentAddress, frame.event);
497
674
  return;
675
+ }
498
676
  case "connector.state.changed":
499
677
  // Gate the cache write on the sending sidecar actually owning
500
678
  // the named agent. A misbehaving sidecar that knows another
@@ -542,10 +720,10 @@ export function createSidecarRouter(config) {
542
720
  case "signal.correlation.register":
543
721
  return handleSignalCorrelationRegister(ws, frame);
544
722
  case "session.ack":
545
- resolvePending(frame.requestId);
723
+ pendingRequests.resolve(frame.requestId);
546
724
  return;
547
725
  case "session.error":
548
- rejectPending(frame.requestId, frame.error);
726
+ pendingRequests.reject(frame.requestId, frame.error);
549
727
  return;
550
728
  case "repo.pack.ack":
551
729
  resolvePackPending(ws, frame);
@@ -559,7 +737,7 @@ export function createSidecarRouter(config) {
559
737
  case "repo.pack.done":
560
738
  return handlePackDone(ws, frame);
561
739
  case "workflow.probe.result":
562
- resolveProbe(frame.requestId, {
740
+ resolveProbe(ws, frame.requestId, {
563
741
  projection: frame.projection,
564
742
  grants: frame.grants,
565
743
  grantWalkSnapshot: frame.grantWalkSnapshot,
@@ -567,7 +745,7 @@ export function createSidecarRouter(config) {
567
745
  });
568
746
  return;
569
747
  case "workflow.probe.error":
570
- rejectProbe(frame.requestId, frame.error);
748
+ rejectProbe(ws, frame.requestId, frame.error);
571
749
  return;
572
750
  default:
573
751
  return assertNever(frame);
@@ -613,24 +791,61 @@ export function createSidecarRouter(config) {
613
791
  const current = allocatedConnections.get(allocationId);
614
792
  if (waiters === undefined || current === undefined)
615
793
  return;
616
- if (!(await validateSidecarIdentity(current.identity, "readiness")))
794
+ const matchingWaiters = [...waiters].filter((waiter) => waiter.generation === current.identity.generation);
795
+ if (matchingWaiters.length === 0)
617
796
  return;
618
- for (const waiter of [...waiters]) {
619
- if (waiter.generation !== current.identity.generation)
797
+ const validation = Promise.resolve().then(() => validateSidecarIdentity(current.identity, "readiness"));
798
+ for (const waiter of matchingWaiters) {
799
+ waiter.validations.add(validation);
800
+ waiter.onValidation?.(validation);
801
+ }
802
+ let identityCurrent;
803
+ try {
804
+ identityCurrent = await validation;
805
+ }
806
+ catch (cause) {
807
+ // A failed revalidation leaves the waiters parked: a later register
808
+ // revalidates, and at expiry the wait reports the failure rather than a
809
+ // missed deadline. Registration itself was already gated, so this must
810
+ // not fail the connection that just registered.
811
+ const validationFailure = new SidecarIdentityValidationError(allocationId, current.identity.generation, cause);
812
+ for (const waiter of matchingWaiters) {
813
+ waiter.validationFailure = validationFailure;
814
+ }
815
+ return;
816
+ }
817
+ finally {
818
+ for (const waiter of matchingWaiters) {
819
+ waiter.validations.delete(validation);
820
+ }
821
+ }
822
+ // A clean validation supersedes earlier failures: expiry must report the
823
+ // current reading, not a stale transient.
824
+ for (const waiter of matchingWaiters) {
825
+ delete waiter.validationFailure;
826
+ }
827
+ if (!identityCurrent || allocatedConnections.get(allocationId) !== current)
828
+ return;
829
+ for (const waiter of matchingWaiters) {
830
+ if (!waiters.delete(waiter))
620
831
  continue;
621
832
  clearTimeout(waiter.timer);
622
- waiters.delete(waiter);
623
833
  waiter.resolve();
624
834
  }
625
- if (waiters.size === 0)
835
+ if (waiters.size === 0 && allocationWaiters.get(allocationId) === waiters)
626
836
  allocationWaiters.delete(allocationId);
627
837
  }
628
- async function handleAllocatedRegister(ws, identity, agentAddresses) {
838
+ async function handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses) {
629
839
  if (allocationFences.get(identity.allocationId) !== identity.generation) {
630
840
  logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} generation ${String(identity.generation)} is not fenced as current`;
631
841
  ws.close();
632
842
  return;
633
843
  }
844
+ if (identity.kind === "probe" && agentAddresses.length > 0) {
845
+ logger.warn `Rejected probe sidecar ${identity.sidecarId}: probe ${identity.allocationId} claimed workflow addresses`;
846
+ ws.close();
847
+ return;
848
+ }
634
849
  if (agentAddresses.length > 0 &&
635
850
  !(await validateSidecarIdentity(identity, "routing"))) {
636
851
  logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} is not ready to reclaim routes`;
@@ -644,7 +859,8 @@ export function createSidecarRouter(config) {
644
859
  existingOnSocket.identity.allocationId === identity.allocationId &&
645
860
  addressIndex.get(address) === ws &&
646
861
  connOwnsAddress(existingOnSocket, address);
647
- if (!alreadyOwned && address !== identity.workflowRunAddress) {
862
+ if (identity.kind !== "allocated" ||
863
+ (!alreadyOwned && address !== identity.workflowRunAddress)) {
648
864
  logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} claimed unrelated address ${address}`;
649
865
  ws.close();
650
866
  return;
@@ -674,7 +890,7 @@ export function createSidecarRouter(config) {
674
890
  ws.send(JSON.stringify(frame));
675
891
  },
676
892
  };
677
- if (conn.identity.kind !== "allocated" ||
893
+ if (conn.identity.kind !== identity.kind ||
678
894
  conn.identity.allocationId !== identity.allocationId ||
679
895
  conn.identity.generation !== identity.generation) {
680
896
  logger.warn `Rejected allocated sidecar ${identity.sidecarId}: socket identity changed during registration`;
@@ -688,472 +904,275 @@ export function createSidecarRouter(config) {
688
904
  }
689
905
  allocatedConnections.set(identity.allocationId, { ws, identity });
690
906
  for (const address of newlyRoutedAddresses) {
691
- redeliverPendingMail(address, conn);
692
- }
693
- logger.info `Allocated sidecar ${identity.sidecarId} registered for allocation ${identity.allocationId} generation ${String(identity.generation)}`;
694
- await notifyAllocationWaiters(identity.allocationId);
695
- events.emit("sidecar.allocated.connected", {
696
- allocationId: identity.allocationId,
697
- generation: identity.generation,
698
- });
699
- }
700
- async function handleRegister(ws, identity, agentAddresses) {
701
- if (identity.kind === "allocated") {
702
- await handleAllocatedRegister(ws, identity, agentAddresses);
703
- return;
704
- }
705
- const sidecarId = identity.sidecarId;
706
- // Key-existence gate. A register frame is token-authenticated but carries
707
- // no per-address ownership proof, so it may route an address ONLY if that
708
- // address has no stored key yet -- a genuine keyless first-deploy (the
709
- // token-bounded first-deploy trust model). An address that already has a
710
- // key must prove ownership through the challenged reconnect path; routing
711
- // it here on token auth alone is the register-frame sibling of the
712
- // reconnect hijack. The keyless-only set is computed up front, BEFORE the
713
- // ghost-cleanup and every routing mutation below, so a rejected address
714
- // touches nothing: no eviction of a live owner, hence no downgrade from
715
- // hijack to denial-of-service on the victim.
716
- const lookupKey = lookups.lookupPublicKey;
717
- const routableAddresses = [];
718
- for (const addr of agentAddresses) {
719
- if (lookupKey === undefined) {
720
- // Fail closed: without the ownership lookup a keyed address cannot be
721
- // told apart from a first-deploy, so route nothing and surface the
722
- // misconfiguration. Empty first-connect registers never reach here.
723
- logger.error `Cannot gate register routing for ${addr}: lookupPublicKey is not configured; refusing to route (challenged reconnect required)`;
724
- continue;
725
- }
726
- let existingKey;
727
- try {
728
- existingKey = await lookupKey(addr);
729
- }
730
- catch (err) {
731
- // Fail closed on a lookup error (e.g. a transient DB failure): route
732
- // nothing for this address and surface the failure, rather than let
733
- // the rejection float out of this void-dispatched handler and take
734
- // down the hub.
735
- logger.error `Key lookup failed for ${addr} during register: ${err instanceof Error ? err.message : String(err)}; failing closed (challenged reconnect required)`;
736
- continue;
737
- }
738
- if (existingKey !== null) {
739
- logger.warn `Refusing to route ${addr} via register: address already has a stored key; ownership must be proven via challenged reconnect`;
740
- continue;
907
+ await redeliverPendingMail(address, conn);
908
+ }
909
+ // Reconcile a reconnecting deployment's credentials, closing the offline
910
+ // window: a credential revoked, deleted, or rotated while the sidecar was
911
+ // disconnected is applied to the child now. Fire-and-forget so
912
+ // registration is not blocked; the lookup no-ops for a run that persisted
913
+ // no credential refs.
914
+ const resyncCredentials = lookups.resyncCredentials;
915
+ if (resyncCredentials !== undefined) {
916
+ for (const address of newlyRoutedAddresses) {
917
+ if (!isRunAddress(address))
918
+ continue;
919
+ resyncCredentials(address);
741
920
  }
742
- routableAddresses.push(addr);
743
921
  }
744
- // Additive re-register: inherit every address this ws already owns and ADD
745
- // the frame's keyless first-deploys. Register never drops an owned route --
746
- // an address proved via challenged reconnect stays routed, and removal
747
- // happens via undeploy/disconnect, not register-omission.
748
- //
749
- // (Under the retired full-set model a register frame carried the sidecar's
750
- // complete live set, so an omitted address meant "removed". Now the frame
751
- // carries only keyless first-deploys the sidecar is adding -- keyed
752
- // addresses arrive via reconnect -- so omission is meaningless, and a
753
- // replace-on-register would wrongly drop an earlier first-deploy when a
754
- // later one is registered, as well as any reconnect-verified keyed route.)
755
- const existing = connections.get(ws);
756
- const inheritedAgent = new Set(existing?.agentAddresses);
757
- const inheritedWorkflow = new Set(existing?.workflowAddresses);
758
- // Clean up ghost entries from OTHER connections for the newly-claimed
759
- // addresses only. An inherited address is already owned by this ws
760
- // (prevWs === ws), so it needs no eviction and its in-flight deploy must
761
- // not be cancelled.
762
- for (const addr of routableAddresses) {
763
- const prevWs = addressIndex.get(addr);
764
- if (prevWs !== undefined && prevWs !== ws) {
765
- const prevConn = connections.get(prevWs);
766
- if (prevConn !== undefined) {
767
- prevConn.agentAddresses.delete(addr);
768
- }
769
- // The new owner is about to take over; the prior owner's
770
- // cached state must not survive into the new owner's window
771
- // before its bootstrap frame arrives.
772
- connectorStates.delete(addr);
922
+ // Reconcile the sidecar's cached sender keys, closing the offline window: a
923
+ // user-principal key that rotated while the sidecar was disconnected is
924
+ // re-resolved and re-pushed, and a sender whose principal was DELETED while
925
+ // the sidecar was disconnected is evicted, so the recipient stops verifying
926
+ // either against a key the hub no longer vouches for. Only allocated
927
+ // sidecars host a sender cache worth reconciling. Resolve and push
928
+ // SEQUENTIALLY in one detached task: registration is never blocked, and a
929
+ // large cache cannot fan out into one concurrent DB query per reported
930
+ // sender on every reconnect.
931
+ const resolveSenderKeyStrict = lookups.resolveSenderKeyStrict;
932
+ if (identity.kind === "allocated" && resolveSenderKeyStrict !== undefined) {
933
+ const rotatableSenders = new Set(cachedSenderAddresses);
934
+ // Resolve-don't-trust applied to input SIZE: bound the reported set before
935
+ // acting on it. Run addresses count toward the cap by design -- the
936
+ // isRunAddress skip below is inside the loop, so the iteration, and thus
937
+ // the DB resolves, can never exceed the cap regardless of the run/non-run
938
+ // mix. Over the cap, reconcile the first MAX_RESYNC_SENDER_ADDRESSES and
939
+ // log the overflow so a misbehaving sidecar is detectable.
940
+ let sendersToResync = [...rotatableSenders];
941
+ if (sendersToResync.length > MAX_RESYNC_SENDER_ADDRESSES) {
942
+ logger.warn `Sidecar ${identity.sidecarId} reported ${String(sendersToResync.length)} cached sender addresses on allocation ${identity.allocationId} generation ${String(identity.generation)}, over the ${String(MAX_RESYNC_SENDER_ADDRESSES)} resync cap; reconciling the first ${String(MAX_RESYNC_SENDER_ADDRESSES)} and ignoring the rest`;
943
+ sendersToResync = sendersToResync.slice(0, MAX_RESYNC_SENDER_ADDRESSES);
773
944
  }
774
- // Cancel any in-flight deploy for this address since the
775
- // reconnecting sidecar is taking ownership.
776
- const staleDeployReq = pendingDeploys.get(addr);
777
- if (staleDeployReq !== undefined) {
778
- clearTimeout(staleDeployReq.timer);
779
- pendingDeploys.delete(addr);
780
- staleDeployReq.reject(`Sidecar ${sidecarId} reconnected and claimed address "${addr}"`);
781
- }
782
- }
783
- // New keyless first-deploys join the inherited session set; the workflow
784
- // set is inherited unchanged (register never adds to it -- a
785
- // workflow-derived address routes only through the challenged reconnect).
786
- for (const addr of routableAddresses) {
787
- inheritedAgent.add(addr);
788
- }
789
- const addrSet = inheritedAgent;
790
- const workflowSet = inheritedWorkflow;
791
- const conn = {
792
- sidecarId,
793
- identity,
794
- agentAddresses: addrSet,
795
- workflowAddresses: workflowSet,
796
- send(frame) {
797
- ws.send(JSON.stringify(frame));
798
- },
799
- };
800
- connections.set(ws, conn);
801
- // Only the newly-claimed addresses need a routing write + queue discard;
802
- // inherited addresses already point at this ws in addressIndex.
803
- for (const addr of routableAddresses) {
804
- addressIndex.set(addr, ws);
805
- // Discard any disconnect queue — register has no identity
806
- // verification, so flushing to an unverified connection is unsafe.
807
- // Use reconnect with challenge/response to preserve queued messages.
808
- const staleQueue = disconnectedAgents.get(addr);
809
- if (staleQueue !== undefined) {
810
- clearTimeout(staleQueue.timer);
811
- if (staleQueue.queue.length > 0) {
812
- logger.warn `Discarding ${String(staleQueue.queue.length)} queued message(s) for ${addr} on unverified register`;
945
+ void (async () => {
946
+ for (const address of sendersToResync) {
947
+ // The sidecar already reports only non-run senders, but do not trust
948
+ // the report: a run sender's key is the immutable
949
+ // workflow_run.public_key and is never refreshed or evicted, so skip
950
+ // it here too rather than couple correctness to the sidecar's filter.
951
+ if (isRunAddress(address))
952
+ continue;
953
+ // Tri-state, deleted-vs-fault distinguished by the STRICT resolver:
954
+ // - resolves to a key -> refresh the sidecar's cached key;
955
+ // - CONFIRMED null (no matching principal = a deleted sender) ->
956
+ // evict it;
957
+ // - THROWS (fault: ambiguous address, keyless-principal invariant
958
+ // break, DB error) -> keep the stale key, evict nothing.
959
+ // Never evicting on a fault is the load-bearing property: dropping a
960
+ // live key on a transient DB fault would be worse than doing nothing.
961
+ // Only the resolve is guarded here; conn.send stays outside so a
962
+ // socket-gone throw propagates to the outer catch and stops the loop.
963
+ let publicKey;
964
+ try {
965
+ publicKey = await resolveSenderKeyStrict(address);
966
+ }
967
+ catch (cause) {
968
+ const message = cause instanceof Error ? cause.message : String(cause);
969
+ logger.error `Keeping the stale cached key for ${address}: resolving it faulted (a fault, not a deleted sender): ${message}`;
970
+ continue;
971
+ }
972
+ if (publicKey !== null) {
973
+ conn.send({ type: "sender.key.refresh", address, publicKey });
974
+ }
975
+ else {
976
+ conn.send({ type: "sender.key.evict", address });
977
+ }
813
978
  }
814
- disconnectedAgents.delete(addr);
815
- }
979
+ })().catch((cause) => {
980
+ // The per-address resolve is guarded above, so the only throw reaching
981
+ // here is conn.send (JSON.stringify + the socket write) once the sidecar
982
+ // is gone. That means the connection left, so stop -- the remaining
983
+ // sends would fail the same way.
984
+ const message = cause instanceof Error ? cause.message : String(cause);
985
+ logger.warn `Sender-key resync for sidecar ${identity.sidecarId} stopped: ${message}`;
986
+ });
816
987
  }
817
- // Only keyless first-deploy addresses reach `addressIndex` here; an
818
- // address that already has a stored key was filtered out by the gate
819
- // above and must re-enter routing through the challenged reconnect path.
820
- // Because the gate runs before the ghost-cleanup, a rejected (keyed)
821
- // address never evicts its prior owner -- register cannot reclaim or
822
- // disrupt a victim's route on token auth alone.
823
- logger.info `Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`;
824
- }
825
- async function handleReconnect(ws, identity, agentAddresses, deployRefs = {}) {
988
+ logger.info `Provisioned sidecar ${identity.sidecarId} registered for allocation ${identity.allocationId} generation ${String(identity.generation)}`;
989
+ await notifyAllocationWaiters(identity.allocationId);
826
990
  if (identity.kind === "allocated") {
827
- await handleAllocatedRegister(ws, identity, agentAddresses);
828
- return;
991
+ events.emit("sidecar.allocated.connected", {
992
+ allocationId: identity.allocationId,
993
+ generation: identity.generation,
994
+ });
829
995
  }
830
- const sidecarId = identity.sidecarId;
831
- const lookupKey = lookups.lookupPublicKey;
832
- if (lookupKey === undefined) {
833
- logger.error `Received reconnect frame but no lookupPublicKey is configured`;
834
- ws.close();
835
- return;
996
+ }
997
+ async function handleRegister(ws, identity, agentAddresses, cachedSenderAddresses) {
998
+ await handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses);
999
+ }
1000
+ async function handleReconnect(ws, identity, agentAddresses, cachedSenderAddresses) {
1001
+ await handleAllocatedRegister(ws, identity, agentAddresses, cachedSenderAddresses);
1002
+ }
1003
+ // Park a pre-ack sender's mail synchronously and return its entry. Registering
1004
+ // the entry BEFORE the caller awaits `resolveSenderKey` is the interlock that
1005
+ // guarantees a settle landing during the resolve has an entry to find: the
1006
+ // event loop is single-threaded, so no settle can interleave between this
1007
+ // synchronous registration and the caller's first await.
1008
+ function parkDeferredSenderMail(authenticatedSender, rawMessage, recipients) {
1009
+ let parked = deferredSenderMail.get(authenticatedSender);
1010
+ if (parked === undefined) {
1011
+ parked = new Set();
1012
+ deferredSenderMail.set(authenticatedSender, parked);
836
1013
  }
837
- // Cancel any existing pending challenge for this ws before doing
838
- // async work, so concurrent reconnect frames don't race.
839
- const existingChallenge = pendingChallenges.get(ws);
840
- if (existingChallenge !== undefined) {
841
- clearTimeout(existingChallenge.timer);
842
- pendingChallenges.delete(ws);
843
- }
844
- // Capture the addresses this live connection already owns. The
845
- // internal register below clears them, but ones the reconnect is not
846
- // re-challenging are still valid — an agent provisioned during the
847
- // sidecar's restore window is routed into addressIndex by its deploy
848
- // and is not part of the disk-restored set this reconnect carries.
849
- // Without preserving them, the reconnect silently drops a
850
- // freshly-deployed agent from routing.
851
- const previouslyOwned = new Set(connections.get(ws)?.agentAddresses);
852
- // Register the sidecar connection immediately (with no addresses) so it
853
- // can receive frames while the ownership challenge is pending. Every
854
- // reconnect address -- session and workflow-derived alike -- enters
855
- // routing only through the verified path below, never unchallenged here.
856
- // Empty address list, so the key-existence gate has nothing to await.
857
- // The identity was already verified at the dispatch boundary, so the
858
- // internal register does not re-authenticate.
859
- await handleRegister(ws, identity, []);
860
- const conn = connections.get(ws);
861
- if (conn === undefined)
862
- return;
863
- // Re-add the still-owned addresses the register cleared but this
864
- // reconnect is not re-challenging. The challenged addresses below
865
- // re-enter addressIndex through the verified path instead.
866
- const claimedAddresses = new Set(agentAddresses);
867
- for (const addr of previouslyOwned) {
868
- if (claimedAddresses.has(addr))
869
- continue;
870
- conn.agentAddresses.add(addr);
871
- addressIndex.set(addr, ws);
872
- }
873
- // Look up stored public keys for all claimed addresses. Fail closed on a
874
- // lookup error (e.g. a transient DB failure): treat the address as
875
- // unverifiable so it fails its challenge and stays unrouted, rather than
876
- // letting the rejection float out of this void-dispatched handler as an
877
- // unhandled rejection that could take down the hub.
878
- const keyLookups = await Promise.all(agentAddresses.map(async (addr) => {
879
- try {
880
- return { address: addr, publicKeyHex: await lookupKey(addr) };
881
- }
882
- catch (err) {
883
- logger.error `Key lookup failed for ${addr} during reconnect: ${err instanceof Error ? err.message : String(err)}; failing closed`;
884
- return { address: addr, publicKeyHex: null };
885
- }
886
- }));
887
- // If the connection was closed or superseded while we were awaiting
888
- // key lookups, bail out.
889
- if (!connections.has(ws))
890
- return;
891
- const challenges = new Map();
892
- const challengeEntries = [];
893
- for (const { address, publicKeyHex } of keyLookups) {
894
- if (publicKeyHex === null) {
895
- conn.send({
896
- type: "challenge.failed",
897
- address,
898
- reason: "Unknown run address",
899
- });
900
- continue;
901
- }
902
- let publicKey;
903
- try {
904
- publicKey = hexDecode(publicKeyHex);
905
- }
906
- catch {
907
- conn.send({
908
- type: "challenge.failed",
909
- address,
910
- reason: "Stored public key is corrupt",
1014
+ const entry = {
1015
+ authenticatedSender,
1016
+ rawMessage,
1017
+ recipients,
1018
+ timer: setTimeout(() => {
1019
+ // TTL backstop for the case where a settle never arrives (the sender's
1020
+ // deploy never acked and never failed loudly). Claim the entry and
1021
+ // surface it as undelivered so the mail is not held forever.
1022
+ if (!claimDeferredSenderEntry(entry))
1023
+ return;
1024
+ events.emit("mail.outbound.undelivered", {
1025
+ rawMessage: entry.rawMessage,
1026
+ recipients: entry.recipients,
911
1027
  });
912
- logger.error `Corrupt stored public key for ${address}`;
913
- continue;
914
- }
915
- const nonce = crypto.getRandomValues(new Uint8Array(32));
916
- challenges.set(address, { nonce, publicKey });
917
- challengeEntries.push({ address, nonce: hexEncode(nonce) });
918
- }
919
- if (challenges.size === 0)
920
- return;
921
- // If another reconnect completed while we were building the
922
- // challenge, it will have written its own entry. Don't overwrite.
923
- if (pendingChallenges.has(ws))
924
- return;
925
- const timer = setTimeout(() => {
926
- pendingChallenges.delete(ws);
927
- logger.warn `Challenge timed out for sidecar ${sidecarId}`;
928
- }, challengeTimeoutMs);
929
- pendingChallenges.set(ws, {
930
- sidecarId,
931
- challenges,
932
- deployRefs,
933
- timer,
934
- });
935
- conn.send({ type: "challenge", challenges: challengeEntries });
1028
+ logger.warn `Dropping mail from ${entry.authenticatedSender}: its sender key was not recorded before the deferred-mail TTL expired`;
1029
+ }, disconnectQueueTTLMs),
1030
+ };
1031
+ parked.add(entry);
1032
+ return entry;
1033
+ }
1034
+ // Remove one parked entry by identity, clearing its TTL timer. Returns whether
1035
+ // THIS call removed it. The inline-deliver path, a settle, and the TTL all
1036
+ // race to claim the same entry; only the claimer acts on it, so a claim that
1037
+ // finds nothing (already claimed) is a no-op. This is the idempotent
1038
+ // remove-by-key that keeps a settle and the inline non-null branch from both
1039
+ // delivering the same message.
1040
+ function claimDeferredSenderEntry(entry) {
1041
+ const parked = deferredSenderMail.get(entry.authenticatedSender);
1042
+ if (parked === undefined)
1043
+ return false;
1044
+ const claimed = parked.delete(entry);
1045
+ if (!claimed)
1046
+ return false;
1047
+ clearTimeout(entry.timer);
1048
+ if (parked.size === 0)
1049
+ deferredSenderMail.delete(entry.authenticatedSender);
1050
+ return true;
936
1051
  }
937
- async function handleChallengeResponse(ws, responses) {
938
- const challenge = pendingChallenges.get(ws);
939
- if (challenge === undefined) {
940
- logger.warn `Received challenge.response with no pending challenge`;
941
- return;
942
- }
943
- clearTimeout(challenge.timer);
944
- pendingChallenges.delete(ws);
945
- const conn = connections.get(ws);
946
- if (conn === undefined)
1052
+ // Claim every entry parked for a sender, clearing their TTL timers. A later
1053
+ // settle or TTL for the same sender then finds nothing.
1054
+ function claimAllDeferredSenderMail(authenticatedSender) {
1055
+ const parked = deferredSenderMail.get(authenticatedSender);
1056
+ if (parked === undefined)
1057
+ return [];
1058
+ deferredSenderMail.delete(authenticatedSender);
1059
+ const entries = [...parked];
1060
+ for (const entry of entries)
1061
+ clearTimeout(entry.timer);
1062
+ return entries;
1063
+ }
1064
+ function drainDeferredSenderMail(authenticatedSender, reason) {
1065
+ const entries = claimAllDeferredSenderMail(authenticatedSender);
1066
+ if (entries.length === 0)
947
1067
  return;
948
- const verified = [];
949
- const responded = new Set();
950
- for (const { address, signature } of responses) {
951
- responded.add(address);
952
- const entry = challenge.challenges.get(address);
953
- if (entry === undefined) {
954
- conn.send({
955
- type: "challenge.failed",
956
- address,
957
- reason: "Address was not challenged",
958
- });
959
- continue;
960
- }
961
- let valid = false;
962
- try {
963
- const nonceBytes = entry.nonce;
964
- const addressBytes = new TextEncoder().encode(address);
965
- const payload = new Uint8Array(nonceBytes.length + addressBytes.length);
966
- payload.set(nonceBytes);
967
- payload.set(addressBytes, nonceBytes.length);
968
- const sigBytes = hexDecode(signature);
969
- valid = await verifyEd25519(payload, sigBytes, entry.publicKey);
970
- }
971
- catch (err) {
972
- logger.warn `Challenge failed for ${address}: ${err instanceof Error ? err.message : String(err)}`;
973
- }
974
- if (valid) {
975
- verified.push(address);
976
- }
977
- else {
978
- conn.send({
979
- type: "challenge.failed",
980
- address,
981
- reason: "Signature verification failed",
982
- });
983
- }
1068
+ for (const entry of entries) {
1069
+ events.emit("mail.outbound.undelivered", {
1070
+ rawMessage: entry.rawMessage,
1071
+ recipients: entry.recipients,
1072
+ });
984
1073
  }
985
- // Notify about challenged addresses that were omitted from the response.
986
- for (const address of challenge.challenges.keys()) {
987
- if (!responded.has(address)) {
988
- conn.send({
989
- type: "challenge.failed",
990
- address,
991
- reason: "No response provided for challenged address",
992
- });
993
- logger.warn `Challenge failed for ${address}: no response provided`;
994
- }
1074
+ logger.warn `Dropping ${String(entries.length)} deferred message(s) from ${authenticatedSender}: ${reason}`;
1075
+ }
1076
+ function noteSenderDeployStarted(address, attempt) {
1077
+ if (allocatedKeyRecordInFlight.has(address)) {
1078
+ throw new Error(`Sender deployment ${address} has an unresolved attempt`);
995
1079
  }
996
- // Add verified addresses to the routing table immediately so that
997
- // `agent.reconnected` subscribers can use sendRequest-based methods
998
- // (e.g. sendSourcesUpdate). Addresses that fail governance are
999
- // rolled back from the routing table afterward.
1000
- for (const addr of verified) {
1001
- // If a different ws still owns this address (live takeover via
1002
- // verified reconnect), evict its cached connector state before
1003
- // routing flips. The new owner's harness will bootstrap via
1004
- // restore-fires-callback.
1005
- const prevWs = addressIndex.get(addr);
1006
- if (prevWs !== undefined && prevWs !== ws) {
1007
- connectorStates.delete(addr);
1008
- // Evict the reclaimed address from the superseded connection's owned
1009
- // set. handleClose's cancelByAgent sweep iterates a connection's owned
1010
- // union WITHOUT an ownership guard, so if the stale connection still
1011
- // listed this address it would cancel THIS connection's in-flight pack
1012
- // transfer for it when it finally closes. Delete from both sets: a
1013
- // workflow-derived address lives on the workflow set, a launched agent
1014
- // on the session set, and delete is a no-op for the absent one.
1015
- const prevConn = connections.get(prevWs);
1016
- if (prevConn !== undefined) {
1017
- prevConn.workflowAddresses.delete(addr);
1018
- prevConn.agentAddresses.delete(addr);
1080
+ allocatedKeyRecordInFlight.set(address, attempt);
1081
+ }
1082
+ function noteSenderDeploySettled(sender, outcome) {
1083
+ if (typeof sender !== "string") {
1084
+ for (const [address, attempt] of [...allocatedKeyRecordInFlight]) {
1085
+ if (attempt.allocationId !== sender.allocationId ||
1086
+ attempt.generation !== sender.generation ||
1087
+ ("leaseId" in sender && attempt.leaseId !== sender.leaseId)) {
1088
+ continue;
1019
1089
  }
1090
+ allocatedKeyRecordInFlight.delete(address);
1091
+ settleSenderMail(address, outcome);
1020
1092
  }
1021
- // A run address goes on the workflow set (no disconnect queue -- its
1022
- // in-flight state is reconstructed sidecar-locally on the next reconnect),
1023
- // so handleClose reclaims it correctly. The routing pointer is the same
1024
- // either way; only now it is written behind a passed challenge.
1025
- //
1026
- // The `else` (session set, queued for reconnect) is the retired
1027
- // launched-agent path: launched agents no longer exist (the folded-launch
1028
- // route was removed), so no current producer reaches it. It is left in
1029
- // place for the reconnect-subsystem teardown to remove as its own
1030
- // reviewable change, not folded into this collapse.
1031
- if (isRunAddress(addr)) {
1032
- conn.workflowAddresses.add(addr);
1033
- }
1034
- else {
1035
- conn.agentAddresses.add(addr);
1036
- }
1037
- addressIndex.set(addr, ws);
1038
- }
1039
- const ready = [];
1040
- const failed = [];
1041
- for (const addr of verified) {
1042
- // A workflow run needs routing + queue flush only, which the passed
1043
- // challenge has now made safe: its in-flight state is reconstructed
1044
- // sidecar-locally, so it does not enrol in the `agent.reconnected` session
1045
- // reaction (event-collector restore). Skip the reaction for it.
1046
- //
1047
- // The reaction below was the retired launched-agent path (restore a
1048
- // launched agent's collector). Launched agents no longer exist, so no
1049
- // current producer reaches it; the `agent.reconnected` listener is gone,
1050
- // so `listenerCount` is 0 and the emit is inert. Left for the
1051
- // reconnect-subsystem teardown to remove.
1052
- if (isRunAddress(addr)) {
1053
- ready.push(addr);
1054
- continue;
1055
- }
1056
- if (events.listenerCount("agent.reconnected") === 0) {
1057
- ready.push(addr);
1058
- continue;
1059
- }
1060
- try {
1061
- await events.emitAndAwait("agent.reconnected", { agentAddress: addr });
1062
- ready.push(addr);
1063
- }
1064
- catch (err) {
1065
- logger.error `Failed to handle reconnection for ${addr}: ${err instanceof Error ? err.message : String(err)}`;
1066
- failed.push(addr);
1067
- }
1093
+ return;
1068
1094
  }
1069
- // If a second reconnect arrived during the callback loop, our conn
1070
- // is orphaned — handleRegister already rebuilt the connection and
1071
- // cleared addressIndex. Bail out; the new reconnect flow will
1072
- // re-verify these addresses from scratch.
1073
- if (connections.get(ws) !== conn) {
1074
- logger.warn `Challenge response processing aborted: connection superseded by new reconnect`;
1075
- return;
1076
- }
1077
- // Roll back failed addresses from the routing table. Only the session set
1078
- // is touched: a workflow-derived address can never be in `failed` -- it
1079
- // early-`continue`s to `ready` above, before the reaction that populates
1080
- // `failed` -- so it is never on the workflow set at this point.
1081
- for (const addr of failed) {
1082
- conn.agentAddresses.delete(addr);
1083
- addressIndex.delete(addr);
1084
- }
1085
- // Flush queued messages and redeliver retained un-acked mail, only for
1086
- // ready addresses. The disconnect queue carries mail that arrived WHILE
1087
- // disconnected; pending-mail redelivery carries mail that was delivered
1088
- // over the prior live connection but never acked (the connected-window
1089
- // drop). Both replay over the verified new connection.
1090
- for (const addr of ready) {
1091
- flushDisconnectedQueue(addr, conn);
1092
- redeliverPendingMail(addr, conn);
1093
- }
1094
- // Re-deploy agents whose deploy ref is stale or absent. Fire-and-forget
1095
- // so reconnect completion is not blocked on pack transfer. The
1096
- // wire layer owns the staleness comparison; the event fires only
1097
- // when staleness is confirmed.
1098
- const checkDeployRef = lookups.lookupDeployRef;
1099
- if (checkDeployRef !== undefined) {
1100
- for (const addr of ready) {
1101
- // Workflow deployments are pinned-forever: a deployment keeps its
1102
- // deploy-time definition until an explicit undeploy/redeploy, so the
1103
- // deploy-ref freshness catch-up is deliberately NOT run for a run
1104
- // address. A definition edited on the hub while the sidecar was
1105
- // disconnected does not reconcile on reconnect; it affects only newly
1106
- // created deployments. The deployment's in-flight run state is
1107
- // reconstructed sidecar-locally at restore, not re-fetched here. Do NOT
1108
- // add a reconcile path for these addresses -- see the "Workflow
1109
- // Definition Versioning: Pinned-Forever" note under "Reconnect
1110
- // Sequencing" in docs/IMPLEMENTATION.md. (Every routable address is a
1111
- // run address now; the fall-through was the retired launched-agent
1112
- // catch-up path.)
1113
- if (isRunAddress(addr))
1114
- continue;
1115
- void (async () => {
1116
- try {
1117
- const hubRef = await checkDeployRef(addr);
1118
- if (hubRef === null)
1119
- return;
1120
- const sidecarRef = challenge.deployRefs[addr];
1121
- if (sidecarRef === hubRef)
1122
- return;
1123
- logger.info `Re-deploying ${addr}: sidecar ref ${sidecarRef ?? "(none)"} != hub ref ${hubRef.slice(0, 8)}`;
1124
- await events.emitAndAwait("deploy.ref.stale", {
1125
- agentAddress: addr,
1126
- });
1127
- }
1128
- catch (err) {
1129
- logger.error `Failed to re-deploy ${addr} after reconnect: ${err instanceof Error ? err.message : String(err)}`;
1130
- }
1131
- })();
1132
- }
1095
+ if (allocatedKeyRecordInFlight.has(sender))
1096
+ return;
1097
+ settleSenderMail(sender, outcome);
1098
+ }
1099
+ function settleSenderMail(address, outcome) {
1100
+ if ("failed" in outcome) {
1101
+ drainDeferredSenderMail(address, `sender deploy failed: ${outcome.failed}`);
1102
+ return;
1133
1103
  }
1134
- // Failed addresses: reset their queue TTL so messages survive until
1135
- // the next reconnect attempt, and notify the sidecar.
1136
- for (const addr of failed) {
1137
- const entry = disconnectedAgents.get(addr);
1138
- if (entry !== undefined) {
1139
- clearTimeout(entry.timer);
1140
- entry.timer = setTimeout(() => {
1141
- const expired = disconnectedAgents.get(addr);
1142
- disconnectedAgents.delete(addr);
1143
- if (expired !== undefined) {
1144
- surfaceDroppedFrames(addr, expired.queue, "disconnect queue TTL expired");
1145
- }
1146
- }, disconnectQueueTTLMs);
1147
- }
1148
- conn.send({
1149
- type: "challenge.failed",
1150
- address: addr,
1151
- reason: "Reconnection rejected by governance",
1104
+ for (const entry of claimAllDeferredSenderMail(address)) {
1105
+ // Re-drive delivery as its OWN task, off the settle's stack, so delivery
1106
+ // work never runs on the deploy-ack handler's stack. Carry the confirmed
1107
+ // key: another attempt may start before this task runs, and must not
1108
+ // capture this mail or change the key that authenticates it.
1109
+ void Promise.resolve()
1110
+ .then(() => handleMailOutbound(entry.rawMessage, entry.authenticatedSender, entry.recipients, outcome.recorded))
1111
+ .catch((err) => {
1112
+ logger.error `Re-driving deferred mail from ${entry.authenticatedSender} failed: ${err instanceof Error ? err.message : String(err)}`;
1152
1113
  });
1153
1114
  }
1154
- logger.info `Sidecar ${challenge.sidecarId} reconnected with ${String(ready.length)} verified agent(s)${failed.length > 0 ? `, ${String(failed.length)} rejected` : ""}`;
1155
1115
  }
1156
- async function handleMailOutbound(rawMessage, recipients) {
1116
+ function senderIdentitiesFromKey(address, publicKey) {
1117
+ return publicKey !== null ? [{ address, publicKey }] : undefined;
1118
+ }
1119
+ // Resolve the co-delivered sender identities for a message, applying the
1120
+ // register-before-read interlock for a pre-ack run sender. Returns either
1121
+ // `deliver: true` with the resolved identities (undefined when there is no
1122
+ // resolvable key), or `deliver: false` when the message is parked and will be
1123
+ // driven later by a settle (`noteSenderDeploySettled`) or the TTL.
1124
+ async function resolveSenderIdentitiesOrPark(rawMessage, authenticatedSender, recipients) {
1125
+ const resolveSenderKey = lookups.resolveSenderKey;
1126
+ if (resolveSenderKey === undefined)
1127
+ return { deliver: true, senderIdentities: undefined };
1128
+ // The co-delivered key is consumed only by a run recipient caching it from the
1129
+ // run.grants frame. Purely external/federated mail never uses it and is never
1130
+ // locally verified, so resolve nothing and never park it.
1131
+ if (!recipients.some(isRunAddress))
1132
+ return { deliver: true, senderIdentities: undefined };
1133
+ // A stable-key (non-run) sender has no pre-ack window; resolve inline.
1134
+ if (!isRunAddress(authenticatedSender)) {
1135
+ const key = await resolveSenderKey(authenticatedSender);
1136
+ return {
1137
+ deliver: true,
1138
+ senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
1139
+ };
1140
+ }
1141
+ // Park a run sender ONLY while a key-record settle is guaranteed to arrive -- a
1142
+ // deploy is in flight. Without one, a null resolve is a transient fault or a
1143
+ // genuine absence on an already-settled run: no settle is coming, so parking
1144
+ // would strand the mail to the TTL. Deliver on the normal path instead.
1145
+ const settleGuaranteed = pendingDeploys.has(authenticatedSender) ||
1146
+ allocatedKeyRecordInFlight.has(authenticatedSender);
1147
+ if (!settleGuaranteed) {
1148
+ const key = await resolveSenderKey(authenticatedSender);
1149
+ return {
1150
+ deliver: true,
1151
+ senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
1152
+ };
1153
+ }
1154
+ // Register-before-read: park a waiter entry synchronously (NO await) so a
1155
+ // settle that lands while we resolve below has an entry to find, THEN
1156
+ // resolve. The single-threaded event loop cannot interleave a settle between
1157
+ // this registration and the await.
1158
+ const entry = parkDeferredSenderMail(authenticatedSender, rawMessage, recipients);
1159
+ const key = await resolveSenderKey(authenticatedSender);
1160
+ if (key === null) {
1161
+ // Not recorded yet. Leave the entry parked; a settle or the TTL drives it.
1162
+ return { deliver: false };
1163
+ }
1164
+ // The key was already recorded before we parked. Claim our entry and deliver
1165
+ // inline -- unless a concurrent settle already claimed it and is re-driving
1166
+ // this message, in which case claiming fails and we must NOT deliver again.
1167
+ if (!claimDeferredSenderEntry(entry)) {
1168
+ return { deliver: false };
1169
+ }
1170
+ return {
1171
+ deliver: true,
1172
+ senderIdentities: senderIdentitiesFromKey(authenticatedSender, key),
1173
+ };
1174
+ }
1175
+ async function handleMailOutbound(rawMessage, authenticatedSender, recipients, recordedSenderKey) {
1157
1176
  // A mail addressed to more than one workflow deployment would birth a
1158
1177
  // run per recipient from a single inbound mail. The stable runId
1159
1178
  // removed the Message-ID collision that originally forced this guard --
@@ -1176,6 +1195,23 @@ export function createSidecarRouter(config) {
1176
1195
  throw new Error(`mail addressed to multiple workflow-derived recipients (${workflowRecipients.join(", ")}); materializing a run for more than one workflow deployment from a single mail is unsupported`);
1177
1196
  }
1178
1197
  }
1198
+ // Resolve the sender's hub-held key ONCE for the whole message, ahead of the
1199
+ // recipient fan-out and any grant materialization, so every recipient in
1200
+ // this fan-out binds the same key snapshot. A run-address sender may be
1201
+ // pre-ack -- it minted its keypair locally and can send before the hub
1202
+ // records its public key. The register-before-read interlock holds such mail
1203
+ // until the key lands rather than delivering it keyless, which a strict
1204
+ // recipient drops as an unknown sender. A parked message returns here and is
1205
+ // re-driven later by a settle or the TTL.
1206
+ const resolution = recordedSenderKey === undefined
1207
+ ? await resolveSenderIdentitiesOrPark(rawMessage, authenticatedSender, recipients)
1208
+ : {
1209
+ deliver: true,
1210
+ senderIdentities: senderIdentitiesFromKey(authenticatedSender, recordedSenderKey),
1211
+ };
1212
+ if (!resolution.deliver)
1213
+ return;
1214
+ const senderIdentities = resolution.senderIdentities;
1179
1215
  // Route to locally connected sidecars first, then try disconnect queues.
1180
1216
  const unrouted = [];
1181
1217
  for (const recipient of recipients) {
@@ -1184,7 +1220,7 @@ export function createSidecarRouter(config) {
1184
1220
  // co-recipients. The catch fails THIS recipient closed (its run never
1185
1221
  // starts under-authorized) and continues to the rest.
1186
1222
  try {
1187
- const outcome = await deliverMailToRecipient(recipient, rawMessage);
1223
+ const outcome = await deliverMailToRecipient(recipient, rawMessage, authenticatedSender, senderIdentities);
1188
1224
  if (outcome === "unrouted")
1189
1225
  unrouted.push(recipient);
1190
1226
  }
@@ -1219,10 +1255,19 @@ export function createSidecarRouter(config) {
1219
1255
  // grants rather than failing closed. Reservation happens before routing so
1220
1256
  // concurrent first deliveries cannot send different snapshots; a routing
1221
1257
  // failure leaves a grants-only, still-unfired run.
1222
- async function deliverMailToRecipient(recipient, rawMessage) {
1258
+ async function deliverMailToRecipient(recipient, rawMessage, authenticatedSender, senderIdentities) {
1223
1259
  if (lookups.materializeMailTriggeredRunGrants !== undefined &&
1224
1260
  isRunAddress(recipient)) {
1225
1261
  const runId = deriveWorkflowRunId(recipient);
1262
+ // This does NOT let mail mutate a run's authorization. First delivery
1263
+ // reserves and commits the run's grants (the mail IS the trigger);
1264
+ // every later delivery only RE-READS the current committed grants
1265
+ // (`loadCommittedRunGrants`) and re-asserts them ahead of the dispatch.
1266
+ // The committed rows already carry any standing-approval change (an
1267
+ // approve/reject-with-`always` resolution mutates them through its own
1268
+ // path), so this re-send is idempotent -- it re-establishes the run's
1269
+ // current floor on the sidecar, self-healing a `grants.json` a sidecar
1270
+ // may have lost, and never overwrites it with anything staler.
1226
1271
  const result = await lookups.materializeMailTriggeredRunGrants({
1227
1272
  agentAddress: recipient,
1228
1273
  runId,
@@ -1235,11 +1280,23 @@ export function createSidecarRouter(config) {
1235
1280
  return "failed-closed";
1236
1281
  }
1237
1282
  if (result.outcome === "materialized") {
1283
+ // The sender's hub-held key was resolved ONCE in handleMailOutbound,
1284
+ // ahead of this fan-out, and threaded in as `senderIdentities`. Co-
1285
+ // deliver it on the run's grants barrier so a recipient that caches from
1286
+ // the `run.grants` frame binds the sender address to the key and can
1287
+ // verify the sender's mail locally. A null key is never carried (the
1288
+ // list is undefined then), so the "authorized-with-a-key implies key
1289
+ // cached" invariant holds; a recipient with no cached key resolves such
1290
+ // mail as `unknown`, which its admission policy rejects by default (a
1291
+ // workflow may relax `unknown` to admit).
1292
+ // Finish asynchronous preparation before sending the grants and mail
1293
+ // together, keeping another delivery's key out of the gap between them.
1294
+ const messageId = await deriveMessageId(base64Decode(rawMessage));
1238
1295
  // Send the run's grants ahead of the mail. A `false` here means the
1239
1296
  // deployment is unroutable. Do not route the mail that would dispatch
1240
1297
  // it; the grants-only reservation remains the canonical snapshot for a
1241
1298
  // later first-delivery attempt.
1242
- if (!sendRunGrants(recipient, runId, result.stepGrants)) {
1299
+ if (!sendRunGrants(recipient, runId, result.stepGrants, senderIdentities)) {
1243
1300
  logger.error `Deployment ${recipient} is not routable for run ${runId}; retaining the unfired run's grant reservation for retry`;
1244
1301
  return "unrouted";
1245
1302
  }
@@ -1253,8 +1310,11 @@ export function createSidecarRouter(config) {
1253
1310
  // mail's own id (derived over the same bytes the sidecar derives), so a
1254
1311
  // redelivery replays identically and the downstream RunStarted /
1255
1312
  // stable-runId dedup makes it effectively-once.
1256
- const messageId = await deriveMessageId(base64Decode(rawMessage));
1257
- const outcome = routeMail(recipient, rawMessage, messageId, { runId, stepGrants: result.stepGrants })
1313
+ const outcome = routeMail(recipient, rawMessage, authenticatedSender, messageId, {
1314
+ runId,
1315
+ stepGrants: result.stepGrants,
1316
+ ...(senderIdentities !== undefined ? { senderIdentities } : {}),
1317
+ })
1258
1318
  ? "routed"
1259
1319
  : "unrouted";
1260
1320
  return outcome;
@@ -1263,7 +1323,9 @@ export function createSidecarRouter(config) {
1263
1323
  // the mail without grants -- the run, if any, is not ours to
1264
1324
  // authorize. No run is committed here, so no ack handshake is needed.
1265
1325
  }
1266
- return routeMail(recipient, rawMessage) ? "routed" : "unrouted";
1326
+ return routeMail(recipient, rawMessage, authenticatedSender)
1327
+ ? "routed"
1328
+ : "unrouted";
1267
1329
  }
1268
1330
  async function handleMailPersist(persist, rawMessage, senderAddress, recipients) {
1269
1331
  let results;
@@ -1381,18 +1443,19 @@ export function createSidecarRouter(config) {
1381
1443
  for (const addr of conn.workflowAddresses) {
1382
1444
  if (addressIndex.get(addr) === ws) {
1383
1445
  addressIndex.delete(addr);
1446
+ connectorStates.delete(addr);
1384
1447
  // Retain un-acked workflow trigger mail across the disconnect for the
1385
- // same reason as the session loop above -- a challenged reconnect
1448
+ // same reason as the session loop above -- an authenticated reconnect
1386
1449
  // redelivers it. This is un-acked TRIGGER mail, distinct from the
1387
1450
  // deployment's in-flight run state (reconstructed sidecar-locally); the
1388
1451
  // "no disconnect queue" note above is about that run state, not this.
1389
1452
  retainPendingMailForAddress(addr);
1390
1453
  }
1391
1454
  }
1392
- if (conn.identity.kind === "allocated") {
1393
- const current = allocatedConnections.get(conn.identity.allocationId);
1394
- if (current?.ws === ws) {
1395
- allocatedConnections.delete(conn.identity.allocationId);
1455
+ const current = allocatedConnections.get(conn.identity.allocationId);
1456
+ if (current?.ws === ws) {
1457
+ allocatedConnections.delete(conn.identity.allocationId);
1458
+ if (conn.identity.kind === "allocated") {
1396
1459
  allocated = {
1397
1460
  allocationId: conn.identity.allocationId,
1398
1461
  generation: conn.identity.generation,
@@ -1401,64 +1464,31 @@ export function createSidecarRouter(config) {
1401
1464
  }
1402
1465
  connections.delete(ws);
1403
1466
  // Cancel the liveness timer for this connection.
1404
- const livenessTimer = livenessTimers.get(ws);
1405
- if (livenessTimer !== undefined) {
1406
- clearTimeout(livenessTimer);
1467
+ const cancelLiveness = livenessTimers.get(ws);
1468
+ if (cancelLiveness !== undefined) {
1469
+ cancelLiveness();
1407
1470
  livenessTimers.delete(ws);
1408
1471
  }
1409
1472
  // Drop the per-ws serialization chain; no more frames will queue on it.
1410
1473
  messageChains.delete(ws);
1411
- // Cancel any pending challenge for this connection.
1412
- const challengeReq = pendingChallenges.get(ws);
1413
- if (challengeReq !== undefined) {
1414
- clearTimeout(challengeReq.timer);
1415
- pendingChallenges.delete(ws);
1416
- }
1417
- // Reject any in-flight requests that were sent to this sidecar.
1418
- for (const [requestId, req] of pending) {
1419
- if (req.ws !== ws)
1420
- continue;
1421
- clearTimeout(req.timer);
1422
- pending.delete(requestId);
1423
- req.reject(`Sidecar ${conn.sidecarId} disconnected`);
1424
- }
1474
+ // Reject any in-flight requests that were sent to this sidecar. Each
1475
+ // entry's reject closure runs its own per-site cleanup (the deploy and
1476
+ // undeploy closures roll routing back), exactly as a frame-error
1477
+ // rejection would.
1478
+ pendingRequests.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
1425
1479
  // Reject every deploy issued on this socket, including allocated
1426
1480
  // workflow deployments stored in `workflowAddresses` rather than
1427
1481
  // `agentAddresses`.
1428
- for (const [agentAddress, req] of pendingDeploys) {
1429
- if (req.ws !== ws)
1430
- continue;
1431
- clearTimeout(req.timer);
1432
- pendingDeploys.delete(agentAddress);
1433
- req.reject(`Sidecar ${conn.sidecarId} disconnected`);
1434
- }
1482
+ pendingDeploys.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
1435
1483
  // Reject any in-flight pack transfers for this sidecar.
1436
- for (const [transferId, pack] of pendingPacks) {
1437
- if (pack.ws !== ws)
1438
- continue;
1439
- clearTimeout(pack.timer);
1440
- pendingPacks.delete(transferId);
1441
- pack.reject(`Sidecar ${conn.sidecarId} disconnected`);
1442
- }
1484
+ pendingPacks.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
1443
1485
  // Reject any in-flight undeploys for this sidecar.
1444
- for (const [addr, req] of pendingUndeploys) {
1445
- if (req.ws !== ws)
1446
- continue;
1447
- clearTimeout(req.timer);
1448
- pendingUndeploys.delete(addr);
1449
- req.reject(`Sidecar ${conn.sidecarId} disconnected`);
1450
- }
1486
+ pendingUndeploys.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
1451
1487
  // Reject any in-flight probes sent to this sidecar. A probe never enters
1452
1488
  // the address maps, so this ws-keyed sweep is its ONLY disconnect cleanup:
1453
1489
  // without it a probe whose sidecar drops mid-flight would hang until its
1454
1490
  // own timeout instead of failing fast on the disconnect.
1455
- for (const [requestId, probe] of pendingProbes) {
1456
- if (probe.ws !== ws)
1457
- continue;
1458
- clearTimeout(probe.timer);
1459
- pendingProbes.delete(requestId);
1460
- probe.reject(`Sidecar ${conn.sidecarId} disconnected`);
1461
- }
1491
+ pendingProbes.rejectAllForWs(ws, `Sidecar ${conn.sidecarId} disconnected`);
1462
1492
  // Cancel any in-flight inbound pack transfers from this sidecar
1463
1493
  // across both receivers. The two receivers track their own in-
1464
1494
  // flight transferIds, so a pending workflow-run transfer for an
@@ -1496,43 +1526,22 @@ export function createSidecarRouter(config) {
1496
1526
  const requestId = nextRequestId();
1497
1527
  const frame = buildFrame(requestId);
1498
1528
  return new Promise((resolve, reject) => {
1499
- const timer = setTimeout(() => {
1500
- pending.delete(requestId);
1501
- reject(new Error(`Request ${requestId} timed out after ${requestTimeoutMs}ms`));
1502
- }, requestTimeoutMs);
1503
- pending.set(requestId, {
1504
- requestId,
1505
- ws,
1529
+ pendingRequests.register(requestId, ws, {
1530
+ timeoutMs: requestTimeoutMs,
1531
+ timeoutMessage: `Request ${requestId} timed out after ${requestTimeoutMs}ms`,
1506
1532
  resolve,
1507
1533
  reject(error) {
1508
1534
  reject(new Error(error));
1509
1535
  },
1510
- timer,
1511
- });
1536
+ }, undefined);
1512
1537
  conn.send(frame);
1513
1538
  });
1514
1539
  }
1515
- function resolvePending(requestId) {
1516
- const req = pending.get(requestId);
1517
- if (req === undefined)
1518
- return;
1519
- clearTimeout(req.timer);
1520
- pending.delete(requestId);
1521
- req.resolve();
1522
- }
1523
- function rejectPending(requestId, error) {
1524
- const req = pending.get(requestId);
1525
- if (req === undefined)
1526
- return;
1527
- clearTimeout(req.timer);
1528
- pending.delete(requestId);
1529
- req.reject(error);
1530
- }
1531
1540
  function packResponseMatches(entry, ws, frame) {
1532
1541
  return (entry.ws === ws &&
1533
- entry.agentAddress === frame.agentAddress &&
1534
- entry.repoId.kind === frame.repoId.kind &&
1535
- entry.repoId.id === frame.repoId.id);
1542
+ entry.meta.agentAddress === frame.agentAddress &&
1543
+ entry.meta.repoId.kind === frame.repoId.kind &&
1544
+ entry.meta.repoId.id === frame.repoId.id);
1536
1545
  }
1537
1546
  function resolvePackPending(ws, frame) {
1538
1547
  const entry = pendingPacks.get(frame.transferId);
@@ -1542,9 +1551,7 @@ export function createSidecarRouter(config) {
1542
1551
  logger.warn `Ignoring repo.pack.ack for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
1543
1552
  return;
1544
1553
  }
1545
- clearTimeout(entry.timer);
1546
- pendingPacks.delete(frame.transferId);
1547
- entry.resolve();
1554
+ pendingPacks.resolve(frame.transferId);
1548
1555
  }
1549
1556
  function rejectPackPending(ws, frame) {
1550
1557
  const entry = pendingPacks.get(frame.transferId);
@@ -1554,13 +1561,13 @@ export function createSidecarRouter(config) {
1554
1561
  logger.warn `Ignoring repo.pack.reject for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
1555
1562
  return;
1556
1563
  }
1557
- clearTimeout(entry.timer);
1558
- pendingPacks.delete(frame.transferId);
1559
1564
  // Surface the receiver's specific cause when it carried one, so the awaiting
1560
- // push sees "corrupt: <detail>" rather than only the coarse reason.
1561
- entry.reject(frame.detail !== undefined
1565
+ // push sees "corrupt: <detail>" rather than only the coarse reason. The
1566
+ // "Pack rejected:" prefix is applied here rather than in the entry's
1567
+ // reject closure because a TIMEOUT rejection must not carry it.
1568
+ pendingPacks.reject(frame.transferId, `Pack rejected: ${frame.detail !== undefined
1562
1569
  ? `${frame.reason}: ${frame.detail}`
1563
- : frame.reason);
1570
+ : frame.reason}`);
1564
1571
  }
1565
1572
  function resolveUndeployPending(ws, agentAddress) {
1566
1573
  const req = pendingUndeploys.get(agentAddress);
@@ -1570,9 +1577,7 @@ export function createSidecarRouter(config) {
1570
1577
  }
1571
1578
  if (req.ws !== ws)
1572
1579
  return;
1573
- clearTimeout(req.timer);
1574
- pendingUndeploys.delete(agentAddress);
1575
- req.resolve();
1580
+ pendingUndeploys.resolve(agentAddress);
1576
1581
  }
1577
1582
  function rejectUndeployPending(ws, agentAddress, error) {
1578
1583
  const req = pendingUndeploys.get(agentAddress);
@@ -1580,25 +1585,19 @@ export function createSidecarRouter(config) {
1580
1585
  return;
1581
1586
  if (req.ws !== ws)
1582
1587
  return;
1583
- clearTimeout(req.timer);
1584
- pendingUndeploys.delete(agentAddress);
1585
- req.reject(error);
1588
+ pendingUndeploys.reject(agentAddress, error);
1586
1589
  }
1587
- function resolveProbe(requestId, result) {
1590
+ function resolveProbe(ws, requestId, result) {
1588
1591
  const req = pendingProbes.get(requestId);
1589
- if (req === undefined)
1592
+ if (req === undefined || req.ws !== ws)
1590
1593
  return;
1591
- clearTimeout(req.timer);
1592
- pendingProbes.delete(requestId);
1593
- req.resolve(result);
1594
+ pendingProbes.resolve(requestId, result);
1594
1595
  }
1595
- function rejectProbe(requestId, error) {
1596
+ function rejectProbe(ws, requestId, error) {
1596
1597
  const req = pendingProbes.get(requestId);
1597
- if (req === undefined)
1598
+ if (req === undefined || req.ws !== ws)
1598
1599
  return;
1599
- clearTimeout(req.timer);
1600
- pendingProbes.delete(requestId);
1601
- req.reject(error);
1600
+ pendingProbes.reject(requestId, error);
1602
1601
  }
1603
1602
  // Routing rule: pick the receiver dedicated to the repoId.kind the
1604
1603
  // frame carries. The receivers' in-flight state is independent, so a
@@ -1648,6 +1647,8 @@ export function createSidecarRouter(config) {
1648
1647
  });
1649
1648
  return;
1650
1649
  }
1650
+ if (conn.identity.kind !== "allocated")
1651
+ return;
1651
1652
  const picked = pickPackReceiver(frame.repoId);
1652
1653
  if (picked === null) {
1653
1654
  logger.warn `Received repo.pack.push with unsupported repoId.kind ${frame.repoId.kind}`;
@@ -1690,6 +1691,9 @@ export function createSidecarRouter(config) {
1690
1691
  });
1691
1692
  return;
1692
1693
  }
1694
+ if (conn.identity.kind !== "allocated")
1695
+ return;
1696
+ const identity = conn.identity;
1693
1697
  const picked = pickPackReceiver(frame.repoId);
1694
1698
  if (picked === null) {
1695
1699
  logger.warn `Received repo.pack.done with unsupported repoId.kind ${frame.repoId.kind}`;
@@ -1723,15 +1727,13 @@ export function createSidecarRouter(config) {
1723
1727
  });
1724
1728
  return;
1725
1729
  }
1726
- const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha, conn.identity.kind === "allocated"
1727
- ? {
1728
- kind: "allocated",
1729
- agentAddress: frame.agentAddress,
1730
- allocationId: conn.identity.allocationId,
1731
- anchorRunId: conn.identity.anchorRunId,
1732
- generation: conn.identity.generation,
1733
- }
1734
- : { kind: "shared", agentAddress: frame.agentAddress });
1730
+ const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha, {
1731
+ kind: "allocated",
1732
+ agentAddress: frame.agentAddress,
1733
+ allocationId: identity.allocationId,
1734
+ anchorRunId: identity.anchorRunId,
1735
+ generation: identity.generation,
1736
+ });
1735
1737
  // Connection may have closed during async verification.
1736
1738
  const currentConn = connections.get(ws);
1737
1739
  if (currentConn === undefined)
@@ -1759,9 +1761,9 @@ export function createSidecarRouter(config) {
1759
1761
  * window of a multi-step deploy, so `sendPack` can route the step's deploy
1760
1762
  * and asset packs before the deployment-level frame spawns the child.
1761
1763
  *
1762
- * The address is hub-minted and workflow-derived (no per-address key), so
1763
- * it enters the keyless `workflowAddresses` set -- never the challenged
1764
- * `agentAddresses` set -- and is torn down by `unbindStepRoute` once the
1764
+ * The address is Hub-minted and workflow-derived, so it enters the
1765
+ * `workflowAddresses` set rather than the legacy `agentAddresses` set and is
1766
+ * torn down by `unbindStepRoute` once the
1765
1767
  * step's packs land. `handleClose` reclaims it if the sidecar drops
1766
1768
  * mid-stage. Per-step addresses are not runtime-routed (mail, signals, and
1767
1769
  * drains use the deployment address), so the binding is transient: it is
@@ -1774,6 +1776,17 @@ export function createSidecarRouter(config) {
1774
1776
  throw new Error(`Cannot move allocation ${allocationId} fence backward from ${String(existing)} to ${String(generation)}`);
1775
1777
  }
1776
1778
  allocationFences.set(allocationId, generation);
1779
+ // A durable generation advance resolves unfinished initialization as failed.
1780
+ // This also covers a cleanup transaction whose response was lost: the next
1781
+ // reconciliation rebuilds this fence before it can start a replacement.
1782
+ for (const attempt of [...allocatedKeyRecordInFlight.values()]) {
1783
+ if (attempt.allocationId === allocationId &&
1784
+ attempt.generation < generation) {
1785
+ noteSenderDeploySettled(attempt, {
1786
+ failed: `Allocation ${allocationId} advanced beyond the deployment attempt`,
1787
+ });
1788
+ }
1789
+ }
1777
1790
  const current = allocatedConnections.get(allocationId);
1778
1791
  if (current !== undefined && current.identity.generation !== generation) {
1779
1792
  handleClose(current.ws);
@@ -1792,7 +1805,31 @@ export function createSidecarRouter(config) {
1792
1805
  if (waiters.size === 0)
1793
1806
  allocationWaiters.delete(allocationId);
1794
1807
  }
1795
- async function getAllocatedConnection(target, use) {
1808
+ function retireAllocation(target) {
1809
+ if (allocationFences.get(target.allocationId) !== target.generation)
1810
+ return;
1811
+ disconnectAllocation(target);
1812
+ allocationFences.delete(target.allocationId);
1813
+ // The fence is gone, so a lingering attempt can never settle normally.
1814
+ // Fail it here rather than leaving a marker that blocks the address.
1815
+ for (const attempt of [...allocatedKeyRecordInFlight.values()]) {
1816
+ if (attempt.allocationId === target.allocationId &&
1817
+ attempt.generation <= target.generation) {
1818
+ noteSenderDeploySettled(attempt, {
1819
+ failed: `Allocation ${target.allocationId} generation ${String(target.generation)} retired`,
1820
+ });
1821
+ }
1822
+ }
1823
+ const waiters = allocationWaiters.get(target.allocationId);
1824
+ if (waiters === undefined)
1825
+ return;
1826
+ allocationWaiters.delete(target.allocationId);
1827
+ for (const waiter of waiters) {
1828
+ clearTimeout(waiter.timer);
1829
+ waiter.reject(new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} retired`));
1830
+ }
1831
+ }
1832
+ async function getProvisionedConnection(target, use) {
1796
1833
  if (allocationFences.get(target.allocationId) !== target.generation) {
1797
1834
  throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
1798
1835
  }
@@ -1801,7 +1838,14 @@ export function createSidecarRouter(config) {
1801
1838
  current.identity.generation !== target.generation) {
1802
1839
  throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId} generation ${String(target.generation)}`);
1803
1840
  }
1804
- if (!(await validateSidecarIdentity(current.identity, use))) {
1841
+ let identityCurrent;
1842
+ try {
1843
+ identityCurrent = await validateSidecarIdentity(current.identity, use);
1844
+ }
1845
+ catch (cause) {
1846
+ throw new SidecarIdentityValidationError(target.allocationId, target.generation, cause);
1847
+ }
1848
+ if (!identityCurrent) {
1805
1849
  if (allocatedConnections.get(target.allocationId) === current) {
1806
1850
  handleClose(current.ws);
1807
1851
  current.ws.close();
@@ -1813,19 +1857,33 @@ export function createSidecarRouter(config) {
1813
1857
  }
1814
1858
  const conn = connections.get(current.ws);
1815
1859
  if (conn === undefined ||
1816
- conn.identity.kind !== "allocated" ||
1817
1860
  conn.identity.allocationId !== target.allocationId ||
1818
1861
  conn.identity.generation !== target.generation) {
1819
1862
  throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId}`);
1820
1863
  }
1821
1864
  return { ws: current.ws, conn };
1822
1865
  }
1866
+ async function getAllocatedConnection(target, use) {
1867
+ const current = await getProvisionedConnection(target, use);
1868
+ if (current.conn.identity.kind !== "allocated") {
1869
+ throw new Error(`Allocation ${target.allocationId} is connected as probe capacity`);
1870
+ }
1871
+ return {
1872
+ ws: current.ws,
1873
+ conn: { ...current.conn, identity: current.conn.identity },
1874
+ };
1875
+ }
1823
1876
  async function isAllocatedSidecarReady(target) {
1824
1877
  try {
1825
- await getAllocatedConnection(target, "readiness");
1878
+ await getProvisionedConnection(target, "readiness");
1826
1879
  return true;
1827
1880
  }
1828
- catch {
1881
+ catch (error) {
1882
+ // A failed validation is unknown, not absent: the worker may be healthy
1883
+ // behind a failed lookup, so report it distinctly instead of answering
1884
+ // `false` and letting the caller release a live worker.
1885
+ if (error instanceof SidecarIdentityValidationError)
1886
+ throw error;
1829
1887
  return false;
1830
1888
  }
1831
1889
  }
@@ -1836,22 +1894,40 @@ export function createSidecarRouter(config) {
1836
1894
  return false;
1837
1895
  return conn.workflowAddresses.has(conn.identity.workflowRunAddress);
1838
1896
  }
1839
- catch {
1897
+ catch (error) {
1898
+ if (error instanceof SidecarIdentityValidationError)
1899
+ throw error;
1840
1900
  return false;
1841
1901
  }
1842
1902
  }
1843
- async function waitForAllocatedSidecar(target, timeoutMs) {
1844
- if (await isAllocatedSidecarReady(target))
1845
- return;
1903
+ async function waitForAllocatedSidecar(target, timeoutMs, onValidation) {
1904
+ // An indeterminable worker waits out the unknown while time remains: only
1905
+ // confirmed absence may surface as a connection timeout. At expiry the
1906
+ // wait reports the validation failure rather than a missed deadline, so
1907
+ // the caller retries instead of releasing a worker that may be healthy.
1908
+ let validationFailure;
1909
+ try {
1910
+ if (await isAllocatedSidecarReady(target))
1911
+ return;
1912
+ }
1913
+ catch (error) {
1914
+ if (!(error instanceof SidecarIdentityValidationError))
1915
+ throw error;
1916
+ validationFailure = error;
1917
+ }
1846
1918
  if (allocationFences.get(target.allocationId) !== target.generation) {
1847
1919
  throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
1848
1920
  }
1849
1921
  if (timeoutMs <= 0) {
1922
+ if (validationFailure !== undefined)
1923
+ throw validationFailure;
1850
1924
  throw new Error(`Timed out waiting for allocated sidecar ${target.allocationId}`);
1851
1925
  }
1852
1926
  await new Promise((resolve, reject) => {
1853
1927
  const waiter = {
1854
1928
  generation: target.generation,
1929
+ validations: new Set(),
1930
+ ...(onValidation !== undefined ? { onValidation } : {}),
1855
1931
  resolve,
1856
1932
  reject,
1857
1933
  timer: setTimeout(() => {
@@ -1860,8 +1936,12 @@ export function createSidecarRouter(config) {
1860
1936
  if (current?.size === 0) {
1861
1937
  allocationWaiters.delete(target.allocationId);
1862
1938
  }
1863
- reject(new Error(`Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`));
1939
+ reject(waiter.validationFailure ??
1940
+ (waiter.validations.size > 0
1941
+ ? new SidecarIdentityValidationError(target.allocationId, target.generation)
1942
+ : new Error(`Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`)));
1864
1943
  }, timeoutMs),
1944
+ ...(validationFailure !== undefined ? { validationFailure } : {}),
1865
1945
  };
1866
1946
  let waiters = allocationWaiters.get(target.allocationId);
1867
1947
  if (waiters === undefined) {
@@ -1872,21 +1952,6 @@ export function createSidecarRouter(config) {
1872
1952
  void notifyAllocationWaiters(target.allocationId);
1873
1953
  });
1874
1954
  }
1875
- function bindStepRoute(stepAddress) {
1876
- const ws = addressIndex.get(stepAddress) ?? findSidecarForNewAgent(stepAddress);
1877
- if (ws === undefined) {
1878
- throw new Error(`No sidecar available to stage workflow step "${stepAddress}"`);
1879
- }
1880
- const conn = connections.get(ws);
1881
- if (conn === undefined) {
1882
- throw new Error(`No sidecar connected to stage workflow step "${stepAddress}"`);
1883
- }
1884
- if (conn.identity.kind !== "shared") {
1885
- throw new Error(`Allocated sidecar ${conn.sidecarId} cannot receive ordinary workflow step ${stepAddress}`);
1886
- }
1887
- conn.workflowAddresses.add(stepAddress);
1888
- addressIndex.set(stepAddress, ws);
1889
- }
1890
1955
  async function bindAllocatedStepRoute(target, stepAddress) {
1891
1956
  const { ws, conn } = await getAllocatedConnection(target, "routing");
1892
1957
  const existing = addressIndex.get(stepAddress);
@@ -1896,21 +1961,6 @@ export function createSidecarRouter(config) {
1896
1961
  conn.workflowAddresses.add(stepAddress);
1897
1962
  addressIndex.set(stepAddress, ws);
1898
1963
  }
1899
- /**
1900
- * Remove a per-step route bound by `bindStepRoute` once the step's packs
1901
- * have landed. Idempotent: an address that was never bound (or already
1902
- * unbound, e.g. by a mid-stage `handleClose`) is a no-op.
1903
- */
1904
- function unbindStepRoute(stepAddress) {
1905
- const ws = addressIndex.get(stepAddress);
1906
- if (ws === undefined)
1907
- return;
1908
- const conn = connections.get(ws);
1909
- if (conn !== undefined) {
1910
- conn.workflowAddresses.delete(stepAddress);
1911
- }
1912
- addressIndex.delete(stepAddress);
1913
- }
1914
1964
  function unbindAllocatedStepRoute(target, stepAddress) {
1915
1965
  const current = allocatedConnections.get(target.allocationId);
1916
1966
  if (current === undefined ||
@@ -1924,20 +1974,6 @@ export function createSidecarRouter(config) {
1924
1974
  }
1925
1975
  // Pack transfers may take longer than session requests due to data volume.
1926
1976
  const PACK_TIMEOUT_MS = requestTimeoutMs * 4;
1927
- function sendPack(agentAddress, pack, ref, commitSha, options) {
1928
- const ws = addressIndex.get(agentAddress);
1929
- if (ws === undefined) {
1930
- return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
1931
- }
1932
- const conn = connections.get(ws);
1933
- if (conn === undefined) {
1934
- return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
1935
- }
1936
- if (conn.identity.kind !== "shared") {
1937
- return Promise.reject(new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound pack routing for "${agentAddress}"`));
1938
- }
1939
- return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
1940
- }
1941
1977
  function sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options) {
1942
1978
  const transferId = `pack-${++packCounter}`;
1943
1979
  // For the agent-state flow the destination agent and the source repo
@@ -1952,21 +1988,14 @@ export function createSidecarRouter(config) {
1952
1988
  // Register pending entry before sending frames so that a synchronous
1953
1989
  // repo.pack.ack (e.g. in tests or loopback transports) resolves correctly.
1954
1990
  return new Promise((resolve, reject) => {
1955
- const timer = setTimeout(() => {
1956
- pendingPacks.delete(transferId);
1957
- reject(new Error(`Pack transfer ${transferId} timed out after ${PACK_TIMEOUT_MS}ms`));
1958
- }, PACK_TIMEOUT_MS);
1959
- pendingPacks.set(transferId, {
1960
- transferId,
1961
- ws,
1962
- agentAddress,
1963
- repoId,
1991
+ pendingPacks.register(transferId, ws, {
1992
+ timeoutMs: PACK_TIMEOUT_MS,
1993
+ timeoutMessage: `Pack transfer ${transferId} timed out after ${PACK_TIMEOUT_MS}ms`,
1964
1994
  resolve,
1965
1995
  reject(error) {
1966
- reject(new Error(`Pack rejected: ${error}`));
1996
+ reject(new Error(error));
1967
1997
  },
1968
- timer,
1969
- });
1998
+ }, { agentAddress, repoId });
1970
1999
  // Send chunks
1971
2000
  for (const chunk of chunkPack(pack)) {
1972
2001
  conn.send({
@@ -1997,11 +2026,10 @@ export function createSidecarRouter(config) {
1997
2026
  }
1998
2027
  return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
1999
2028
  }
2000
- async function sendWorkflowRunPackToAllocation(target, agentAddress, pack, ref, commitSha) {
2029
+ async function sendWorkflowRunPackToAllocation(target, agentAddress, pack, ref, commitSha, signal) {
2030
+ signal?.throwIfAborted();
2001
2031
  const { ws, conn } = await getAllocatedConnection(target, "routing");
2002
- if (conn.identity.kind !== "allocated") {
2003
- throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
2004
- }
2032
+ signal?.throwIfAborted();
2005
2033
  if (agentAddress !== conn.identity.workflowRunAddress) {
2006
2034
  throw new Error(`Allocation ${target.allocationId} cannot restore unrelated address ${agentAddress}`);
2007
2035
  }
@@ -2015,7 +2043,16 @@ export function createSidecarRouter(config) {
2015
2043
  },
2016
2044
  });
2017
2045
  }
2018
- function routeMail(agentAddress, rawMessage, messageId, runGrants) {
2046
+ function routeMail(agentAddress, rawMessage, authenticatedSender, messageId, runGrants) {
2047
+ // `authenticatedSender` is hub-assigned by the caller from a hub-verified
2048
+ // value (the ownership-gated sender of a relayed mail, or the triggering
2049
+ // principal's address) -- never the message's own MIME `From`. It rides
2050
+ // the frame as the hub-verified sender of record, so a recipient can take
2051
+ // the sender from it rather than the forgeable `From`. The recipient's
2052
+ // signature check reads it as the sender of record -- resolving the
2053
+ // sender's key from its local cache to verify the signature -- and its
2054
+ // admission policy gates delivery on the verdict.
2055
+ //
2019
2056
  // Carry the hub-minted messageId on the frame so the sidecar's durable-
2020
2057
  // receipt ack (`mail.inbound.ack`) keys on the same id the hub tracks, and
2021
2058
  // a redelivery replays identical bytes for the downstream RunStarted dedup.
@@ -2026,6 +2063,7 @@ export function createSidecarRouter(config) {
2026
2063
  type: "mail.inbound",
2027
2064
  agentAddress,
2028
2065
  rawMessage,
2066
+ authenticatedSender,
2029
2067
  ...(messageId !== undefined ? { messageId } : {}),
2030
2068
  };
2031
2069
  const ws = addressIndex.get(agentAddress);
@@ -2048,12 +2086,13 @@ export function createSidecarRouter(config) {
2048
2086
  // If the agent recently disconnected, queue for delivery on reconnect.
2049
2087
  return enqueueForDisconnected(agentAddress, frame);
2050
2088
  }
2051
- function sendRunGrants(agentAddress, runId, stepGrants) {
2089
+ function sendRunGrants(agentAddress, runId, stepGrants, senderIdentities) {
2052
2090
  const frame = {
2053
2091
  type: "run.grants",
2054
2092
  agentAddress,
2055
2093
  runId,
2056
2094
  stepGrants,
2095
+ ...(senderIdentities !== undefined ? { senderIdentities } : {}),
2057
2096
  };
2058
2097
  const ws = addressIndex.get(agentAddress);
2059
2098
  if (ws !== undefined) {
@@ -2067,29 +2106,65 @@ export function createSidecarRouter(config) {
2067
2106
  // so a run.grants issued in the window between deploy and the first
2068
2107
  // reconnect survives the same way the dispatching trigger mail does.
2069
2108
  // A queue exists only while the deployment address is still on
2070
- // agentAddresses (pre-first-reconnect); after a challenged reconnect it
2109
+ // agentAddresses (pre-first-reconnect); after an authenticated reconnect it
2071
2110
  // moves to workflowAddresses, which handleClose leaves unqueued because
2072
2111
  // that generation's in-flight run state is reconstructed sidecar-locally.
2073
2112
  // Returning without enqueueing there is correct; enqueueing is what keeps
2074
2113
  // grants and mail from diverging in the pre-reconnect window.
2075
2114
  return enqueueForDisconnected(agentAddress, frame);
2076
2115
  }
2077
- async function sendWorkflowRunDispatchToAllocation(target, agentAddress, runId, stepGrants, rawMessage, messageId) {
2116
+ async function sendWorkflowRunDispatchToAllocation(target, agentAddress, runId, stepGrants, rawMessage, authenticatedSender, messageId, signal) {
2117
+ signal?.throwIfAborted();
2078
2118
  const { ws, conn } = await getAllocatedConnection(target, "routing");
2119
+ signal?.throwIfAborted();
2079
2120
  if (addressIndex.get(agentAddress) !== ws) {
2080
2121
  throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
2081
2122
  }
2082
- const runGrants = { runId, stepGrants };
2123
+ // authenticatedSender is the sender persisted at enqueue on the dispatch
2124
+ // row (the triggering principal's hub-verified address); the caller reads
2125
+ // it from that row. It is never the message's MIME From.
2126
+ //
2127
+ // Resolve its key here, at dispatch (redelivery) time, from that persisted
2128
+ // address, to co-deliver on the run's grants barrier so the recipient
2129
+ // caches the sender's current hub-held key. A run sender's deployment key
2130
+ // is immutable once acked; a user sender's key rotating mid-flight would
2131
+ // leave the fixed signed bytes checked against the new key, which the
2132
+ // recipient logs as unverifiable. Null when unresolvable (no resolver
2133
+ // wired, or the sender has no durable key). The lookup contract (see
2134
+ // SidecarLookups.resolveSenderKey) is best-effort and never throws, so
2135
+ // resolving ahead of the run.grants send cannot block it.
2136
+ const authenticatedSenderPublicKey = lookups.resolveSenderKey !== undefined
2137
+ ? await lookups.resolveSenderKey(authenticatedSender)
2138
+ : null;
2139
+ signal?.throwIfAborted();
2140
+ // Co-deliver the resolved key on the run's grants barrier, omitting a null
2141
+ // key so it is never cached (see deliverMailToRecipient). The same list
2142
+ // rides the pending-mail entry so the reconnect replay carries it too.
2143
+ const senderIdentities = authenticatedSenderPublicKey !== null
2144
+ ? [
2145
+ {
2146
+ address: authenticatedSender,
2147
+ publicKey: authenticatedSenderPublicKey,
2148
+ },
2149
+ ]
2150
+ : undefined;
2151
+ const runGrants = {
2152
+ runId,
2153
+ stepGrants,
2154
+ ...(senderIdentities !== undefined ? { senderIdentities } : {}),
2155
+ };
2083
2156
  conn.send({
2084
2157
  type: "run.grants",
2085
2158
  agentAddress,
2086
2159
  runId,
2087
2160
  stepGrants,
2161
+ ...(senderIdentities !== undefined ? { senderIdentities } : {}),
2088
2162
  });
2089
2163
  const frame = {
2090
2164
  type: "mail.inbound",
2091
2165
  agentAddress,
2092
2166
  rawMessage,
2167
+ authenticatedSender,
2093
2168
  messageId,
2094
2169
  };
2095
2170
  conn.send(frame);
@@ -2121,55 +2196,26 @@ export function createSidecarRouter(config) {
2121
2196
  });
2122
2197
  }
2123
2198
  catch (err) {
2124
- rejectDeployPending(req, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
2199
+ pendingDeploys.reject(frame.agentAddress, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
2125
2200
  return;
2126
2201
  }
2127
2202
  }
2128
- resolveDeployPending(req, frame.publicKey);
2129
- }
2130
- function resolveDeployPending(req, publicKey) {
2131
- if (pendingDeploys.get(req.agentAddress) !== req)
2132
- return;
2133
- clearTimeout(req.timer);
2134
- pendingDeploys.delete(req.agentAddress);
2135
- req.resolve(publicKey);
2136
- }
2137
- function rejectDeployPending(req, error) {
2138
- if (pendingDeploys.get(req.agentAddress) !== req)
2139
- return;
2140
- clearTimeout(req.timer);
2141
- pendingDeploys.delete(req.agentAddress);
2142
- req.reject(error);
2203
+ pendingDeploys.resolve(frame.agentAddress, frame.publicKey);
2143
2204
  }
2144
2205
  function rejectDeployPendingFromFrame(ws, agentAddress, error) {
2145
2206
  const req = pendingDeploys.get(agentAddress);
2146
2207
  if (req === undefined || req.ws !== ws)
2147
2208
  return;
2148
- rejectDeployPending(req, error);
2149
- }
2150
- async function sendAgentDeploy(agentAddress, harnessConfig, workflow) {
2151
- if (hubPublicKeyHex === undefined) {
2152
- throw new Error("Hub signing key is required for agent deployment");
2153
- }
2154
- const ws = addressIndex.get(agentAddress) ?? findSidecarForNewAgent(agentAddress);
2155
- if (ws === undefined) {
2156
- throw new Error(`No sidecar available for agent "${agentAddress}"`);
2157
- }
2158
- const conn = connections.get(ws);
2159
- if (conn === undefined) {
2160
- throw new Error(`No sidecar connected for agent "${agentAddress}"`);
2161
- }
2162
- if (conn.identity.kind !== "shared") {
2163
- throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound deploy routing`);
2164
- }
2165
- return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
2209
+ // Settle by key, not by the `req` object: a key lookup observes the
2210
+ // CURRENT entry, so a stale handle cannot settle a replaced round-trip.
2211
+ pendingDeploys.reject(agentAddress, error);
2166
2212
  }
2167
2213
  function sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow) {
2168
2214
  if (hubPublicKeyHex === undefined) {
2169
- throw new Error("Hub signing key is required for agent deployment");
2215
+ throw deployFrameFailure("Hub signing key is required for agent deployment", false);
2170
2216
  }
2171
2217
  if (pendingDeploys.has(agentAddress)) {
2172
- throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
2218
+ throw deployFrameFailure(`Deploy already in progress for agent "${agentAddress}"`, false);
2173
2219
  }
2174
2220
  const addressSet = conn.identity.kind === "allocated"
2175
2221
  ? conn.workflowAddresses
@@ -2177,17 +2223,11 @@ export function createSidecarRouter(config) {
2177
2223
  addressSet.add(agentAddress);
2178
2224
  addressIndex.set(agentAddress, ws);
2179
2225
  return new Promise((resolve, reject) => {
2180
- const timer = setTimeout(() => {
2181
- pendingDeploys.delete(agentAddress);
2182
- if (addressIndex.get(agentAddress) === ws) {
2183
- addressSet.delete(agentAddress);
2184
- addressIndex.delete(agentAddress);
2185
- }
2186
- reject(new Error(`Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
2187
- }, requestTimeoutMs);
2188
- pendingDeploys.set(agentAddress, {
2189
- agentAddress,
2190
- ws,
2226
+ // Timeout and frame-error rejections share this closure, so the routing
2227
+ // rollback and the `frameSent: true` tag live in one place.
2228
+ pendingDeploys.register(agentAddress, ws, {
2229
+ timeoutMs: requestTimeoutMs,
2230
+ timeoutMessage: `Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
2191
2231
  resolve(publicKey) {
2192
2232
  resolve({ publicKey });
2193
2233
  },
@@ -2196,33 +2236,73 @@ export function createSidecarRouter(config) {
2196
2236
  addressSet.delete(agentAddress);
2197
2237
  addressIndex.delete(agentAddress);
2198
2238
  }
2199
- reject(new Error(error));
2239
+ // A non-allocated deployment's key is recorded by the deploy-ack
2240
+ // projection, whose failure (reject/timeout/agent.error/disconnect)
2241
+ // is observed only here. Drain any pre-ack sender mail parked on
2242
+ // this address so it surfaces as undelivered rather than waiting out
2243
+ // the TTL. An allocated deployment's failure is drained by its
2244
+ // session-service owner instead, so skip it here to keep one owner
2245
+ // per case.
2246
+ if (conn.identity.kind !== "allocated") {
2247
+ drainDeferredSenderMail(agentAddress, `deploy failed: ${error}`);
2248
+ }
2249
+ reject(deployFrameFailure(error, true));
2200
2250
  },
2201
- timer,
2202
- });
2203
- conn.send({
2204
- type: "agent.deploy",
2205
- agentAddress,
2206
- agentId: harnessConfig.agentId,
2207
- config: harnessConfig,
2208
- hubPublicKey: hubPublicKeyHex,
2209
- ...(workflow !== undefined ? { workflow } : {}),
2210
- });
2251
+ }, undefined);
2252
+ try {
2253
+ conn.send({
2254
+ type: "agent.deploy",
2255
+ agentAddress,
2256
+ agentId: harnessConfig.agentId,
2257
+ config: harnessConfig,
2258
+ hubPublicKey: hubPublicKeyHex,
2259
+ ...(workflow !== undefined ? { workflow } : {}),
2260
+ });
2261
+ }
2262
+ catch (err) {
2263
+ // A synchronous send failure means the frame never reached the wire.
2264
+ // Drop the pending entry (and its armed timer) and reject as not-sent
2265
+ // so a caller may safely roll back what it staged. The drop bypasses
2266
+ // the entry's reject closure: this failure must report
2267
+ // `frameSent: false`, and the timer must not fire later and
2268
+ // double-reject.
2269
+ pendingDeploys.delete(agentAddress);
2270
+ if (addressIndex.get(agentAddress) === ws) {
2271
+ addressSet.delete(agentAddress);
2272
+ addressIndex.delete(agentAddress);
2273
+ }
2274
+ reject(deployFrameFailure(`Deploy of "${agentAddress}" failed to send: ${err instanceof Error ? err.message : String(err)}`, false));
2275
+ }
2211
2276
  });
2212
2277
  }
2213
- async function sendAgentDeployToAllocation(target, agentAddress, harnessConfig, workflow) {
2214
- const { ws, conn } = await getAllocatedConnection(target, "routing");
2215
- if (conn.identity.kind !== "allocated") {
2216
- throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
2217
- }
2218
- if (agentAddress !== conn.identity.workflowRunAddress) {
2219
- throw new Error(`Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`);
2278
+ async function sendAgentDeployToAllocation(target, agentAddress, harnessConfig, workflow, signal, beforeSend) {
2279
+ try {
2280
+ signal?.throwIfAborted();
2281
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2282
+ signal?.throwIfAborted();
2283
+ if (agentAddress !== conn.identity.workflowRunAddress) {
2284
+ throw new Error(`Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`);
2285
+ }
2286
+ const existing = addressIndex.get(agentAddress);
2287
+ if (existing !== undefined && existing !== ws) {
2288
+ throw new Error(`Deployment ${agentAddress} is already routed to another sidecar`);
2289
+ }
2290
+ if (hubPublicKeyHex === undefined)
2291
+ throw new Error("Hub signing key is required for agent deployment");
2292
+ if (pendingDeploys.has(agentAddress))
2293
+ throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
2294
+ await beforeSend?.();
2295
+ signal?.throwIfAborted();
2296
+ if (allocatedConnections.get(target.allocationId)?.ws !== ws ||
2297
+ allocationFences.get(target.allocationId) !== target.generation) {
2298
+ throw new Error(`Allocated sidecar connection changed for allocation ${target.allocationId}`);
2299
+ }
2300
+ // Return without awaiting: only pre-send failures belong to this catch.
2301
+ return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
2220
2302
  }
2221
- const existing = addressIndex.get(agentAddress);
2222
- if (existing !== undefined && existing !== ws) {
2223
- throw new Error(`Deployment ${agentAddress} is already routed to another sidecar`);
2303
+ catch (cause) {
2304
+ throw deployFrameFailure(cause instanceof Error ? cause.message : String(cause), false, cause);
2224
2305
  }
2225
- return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
2226
2306
  }
2227
2307
  /**
2228
2308
  * Provision one step of a multi-step deploy on the sidecar WITHOUT
@@ -2238,20 +2318,6 @@ export function createSidecarRouter(config) {
2238
2318
  * so the caller can safely deliver the deploy pack afterward. On failure
2239
2319
  * the caller owns tearing the route down via `unbindStepRoute`.
2240
2320
  */
2241
- function sendProvisionStep(agentAddress, harnessConfig) {
2242
- const ws = addressIndex.get(agentAddress);
2243
- if (ws === undefined) {
2244
- throw new Error(`Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`);
2245
- }
2246
- const conn = connections.get(ws);
2247
- if (conn === undefined) {
2248
- throw new Error(`No sidecar connected for agent "${agentAddress}"`);
2249
- }
2250
- if (conn.identity.kind !== "shared") {
2251
- throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound step provisioning`);
2252
- }
2253
- return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
2254
- }
2255
2321
  function sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig) {
2256
2322
  if (hubPublicKeyHex === undefined) {
2257
2323
  throw new Error("Hub signing key is required for step provisioning");
@@ -2261,25 +2327,20 @@ export function createSidecarRouter(config) {
2261
2327
  }
2262
2328
  const hubKey = hubPublicKeyHex;
2263
2329
  return new Promise((resolve, reject) => {
2264
- const timer = setTimeout(() => {
2265
- pendingDeploys.delete(agentAddress);
2266
- reject(new Error(`Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
2267
- }, requestTimeoutMs);
2268
2330
  // The sidecar's `agent.deploy.ack` resolves this through
2269
- // `resolveDeployPending` -> `req.resolve()`. The per-step address is
2270
- // workflow-derived and records no hub-side key, so the ack's public
2271
- // key is not needed and this resolves void.
2272
- pendingDeploys.set(agentAddress, {
2273
- agentAddress,
2274
- ws,
2331
+ // `pendingDeploys.resolve`. The per-step address is workflow-derived
2332
+ // and records no hub-side key, so the ack's public key is not needed
2333
+ // and this resolves void.
2334
+ pendingDeploys.register(agentAddress, ws, {
2335
+ timeoutMs: requestTimeoutMs,
2336
+ timeoutMessage: `Step provision of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
2275
2337
  resolve(_publicKey) {
2276
2338
  resolve();
2277
2339
  },
2278
2340
  reject(error) {
2279
2341
  reject(new Error(error));
2280
2342
  },
2281
- timer,
2282
- });
2343
+ }, undefined);
2283
2344
  conn.send({
2284
2345
  type: "agent.deploy",
2285
2346
  agentAddress,
@@ -2297,42 +2358,17 @@ export function createSidecarRouter(config) {
2297
2358
  }
2298
2359
  return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
2299
2360
  }
2300
- function findSidecarForNewAgent(_agentAddress) {
2301
- for (const [ws, conn] of connections) {
2302
- if (conn.identity.kind === "shared")
2303
- return ws;
2304
- }
2305
- return undefined;
2306
- }
2307
- function sendProbe(args) {
2308
- // Select any connected sidecar, exactly as the pre-deploy window does.
2309
- // A probe runs in the sidecar's pre-deploy state, so it targets a
2310
- // connection, not an address -- `findSidecarForNewAgent` ignores its
2311
- // argument and returns the first connection. Throw immediately on an empty
2312
- // registry rather than register a probe that could only time out.
2313
- const ws = findSidecarForNewAgent("");
2314
- if (ws === undefined) {
2315
- return Promise.reject(new Error("No sidecar available to probe workflow"));
2316
- }
2317
- const conn = connections.get(ws);
2318
- if (conn === undefined) {
2319
- return Promise.reject(new Error("No sidecar connected to probe workflow"));
2320
- }
2361
+ function sendProbeOnConnection(ws, conn, args) {
2321
2362
  const requestId = nextRequestId();
2322
2363
  return new Promise((resolve, reject) => {
2323
- const timer = setTimeout(() => {
2324
- pendingProbes.delete(requestId);
2325
- reject(new Error(`Probe ${requestId} timed out after ${probeTimeoutMs}ms`));
2326
- }, probeTimeoutMs);
2327
- pendingProbes.set(requestId, {
2328
- requestId,
2329
- ws,
2364
+ pendingProbes.register(requestId, ws, {
2365
+ timeoutMs: probeTimeoutMs,
2366
+ timeoutMessage: `Probe ${requestId} timed out after ${probeTimeoutMs}ms`,
2330
2367
  resolve,
2331
2368
  reject(error) {
2332
2369
  reject(new Error(error));
2333
2370
  },
2334
- timer,
2335
- });
2371
+ }, undefined);
2336
2372
  conn.send({
2337
2373
  type: "workflow.probe.request",
2338
2374
  requestId,
@@ -2343,6 +2379,19 @@ export function createSidecarRouter(config) {
2343
2379
  });
2344
2380
  });
2345
2381
  }
2382
+ async function sendProbeToAllocation(target, args) {
2383
+ const { ws, conn } = await getProvisionedConnection(target, "routing");
2384
+ return sendProbeOnConnection(ws, conn, args);
2385
+ }
2386
+ function disconnectAllocation(target) {
2387
+ const current = allocatedConnections.get(target.allocationId);
2388
+ if (current === undefined ||
2389
+ current.identity.generation !== target.generation) {
2390
+ return;
2391
+ }
2392
+ handleClose(current.ws);
2393
+ current.ws.close();
2394
+ }
2346
2395
  function sendAgentUndeploy(agentAddress, reason) {
2347
2396
  const ws = addressIndex.get(agentAddress);
2348
2397
  if (ws === undefined) {
@@ -2353,14 +2402,11 @@ export function createSidecarRouter(config) {
2353
2402
  return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
2354
2403
  }
2355
2404
  return new Promise((resolve, reject) => {
2356
- const timer = setTimeout(() => {
2357
- pendingUndeploys.delete(agentAddress);
2358
- removeAgentAddress(ws, agentAddress);
2359
- reject(new Error(`Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
2360
- }, requestTimeoutMs);
2361
- pendingUndeploys.set(agentAddress, {
2362
- agentAddress,
2363
- ws,
2405
+ // Timeout, ack, and error rejection share one closure so the routing
2406
+ // teardown runs exactly once no matter how the round-trip settles.
2407
+ pendingUndeploys.register(agentAddress, ws, {
2408
+ timeoutMs: requestTimeoutMs,
2409
+ timeoutMessage: `Undeploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`,
2364
2410
  resolve() {
2365
2411
  removeAgentAddress(ws, agentAddress);
2366
2412
  resolve();
@@ -2369,8 +2415,7 @@ export function createSidecarRouter(config) {
2369
2415
  removeAgentAddress(ws, agentAddress);
2370
2416
  reject(new Error(error));
2371
2417
  },
2372
- timer,
2373
- });
2418
+ }, undefined);
2374
2419
  conn.send({
2375
2420
  type: "agent.undeploy",
2376
2421
  agentAddress,
@@ -2433,12 +2478,13 @@ export function createSidecarRouter(config) {
2433
2478
  defaultSource,
2434
2479
  }));
2435
2480
  }
2436
- async function sendCredentialsUpdate(agentAddress, delivery) {
2481
+ async function sendCredentialsUpdate(agentAddress, delivery, revoke) {
2437
2482
  await sendRequest(agentAddress, (requestId) => ({
2438
2483
  type: "credentials.update",
2439
2484
  requestId,
2440
2485
  agentAddress,
2441
2486
  delivery,
2487
+ ...(revoke !== undefined ? { revoke } : {}),
2442
2488
  }));
2443
2489
  }
2444
2490
  function sendSyncRequest(agentAddress) {
@@ -2475,8 +2521,10 @@ export function createSidecarRouter(config) {
2475
2521
  payload: opts.payload,
2476
2522
  });
2477
2523
  }
2478
- async function sendSignalDeliverToAllocation(target, opts) {
2524
+ async function sendSignalDeliverToAllocation(target, opts, signal) {
2525
+ signal?.throwIfAborted();
2479
2526
  const { ws, conn } = await getAllocatedConnection(target, "routing");
2527
+ signal?.throwIfAborted();
2480
2528
  if (addressIndex.get(opts.agentAddress) !== ws) {
2481
2529
  throw new Error(`Address ${opts.agentAddress} is not routed on allocation ${target.allocationId}`);
2482
2530
  }
@@ -2503,24 +2551,23 @@ export function createSidecarRouter(config) {
2503
2551
  handleClose,
2504
2552
  routeMail,
2505
2553
  sendRunGrants,
2506
- sendAgentDeploy,
2507
- sendProbe,
2554
+ noteSenderDeployStarted,
2555
+ noteSenderDeploySettled,
2556
+ sendProbeToAllocation,
2557
+ disconnectAllocation,
2508
2558
  sendAgentUndeploy,
2509
2559
  sendSourcesUpdate,
2510
2560
  sendCredentialsUpdate,
2511
- sendPack,
2512
2561
  sendPackToAllocation,
2513
2562
  sendWorkflowRunPackToAllocation,
2514
2563
  fenceAllocation,
2564
+ retireAllocation,
2515
2565
  waitForAllocatedSidecar,
2516
2566
  isAllocatedSidecarReady,
2517
2567
  isAllocatedWorkflowActive,
2518
2568
  sendAgentDeployToAllocation,
2519
- bindStepRoute,
2520
2569
  bindAllocatedStepRoute,
2521
- unbindStepRoute,
2522
2570
  unbindAllocatedStepRoute,
2523
- sendProvisionStep,
2524
2571
  sendProvisionStepToAllocation,
2525
2572
  sendWorkflowRunDispatchToAllocation,
2526
2573
  sendSyncRequest,