@intx/hub-sessions 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +3 -5
  2. package/dist/agent-repo.d.ts +9 -5
  3. package/dist/agent-repo.js +2 -2
  4. package/dist/agent-state-kind.js +4 -0
  5. package/dist/asset-service.d.ts +1 -20
  6. package/dist/asset-service.js +9 -91
  7. package/dist/committed-source-tree.d.ts +10 -0
  8. package/dist/committed-source-tree.js +35 -0
  9. package/dist/credential-push.d.ts +7 -6
  10. package/dist/credential-push.js +42 -18
  11. package/dist/event-collector-registry.d.ts +1 -1
  12. package/dist/event-collector-registry.js +4 -4
  13. package/dist/event-collector.d.ts +1 -1
  14. package/dist/event-collector.js +10 -2
  15. package/dist/hub-session-lookups.d.ts +125 -7
  16. package/dist/hub-session-lookups.js +539 -80
  17. package/dist/hub-session-orchestrator.js +14 -49
  18. package/dist/index.d.ts +17 -8
  19. package/dist/index.js +14 -6
  20. package/dist/repo-store/index.d.ts +1 -1
  21. package/dist/repo-store/store.d.ts +1 -1
  22. package/dist/repo-store/store.js +138 -1
  23. package/dist/repo-store/subscribe-kind.d.ts +6 -3
  24. package/dist/repo-store/subscribe-kind.js +42 -77
  25. package/dist/repo-store/types.d.ts +94 -6
  26. package/dist/session-service.d.ts +277 -96
  27. package/dist/session-service.js +741 -547
  28. package/dist/sidecar-allocation/contracts.d.ts +78 -0
  29. package/dist/sidecar-allocation/contracts.js +21 -0
  30. package/dist/sidecar-allocation/index.d.ts +4 -0
  31. package/dist/sidecar-allocation/index.js +3 -0
  32. package/dist/sidecar-allocation/placement-policy.d.ts +11 -0
  33. package/dist/sidecar-allocation/placement-policy.js +21 -0
  34. package/dist/sidecar-allocation/plugin-registry.d.ts +11 -0
  35. package/dist/sidecar-allocation/plugin-registry.js +37 -0
  36. package/dist/sidecar-allocation/reconciler.d.ts +42 -0
  37. package/dist/sidecar-allocation/reconciler.js +431 -0
  38. package/dist/skill-kind.js +4 -0
  39. package/dist/substrate.d.ts +3 -3
  40. package/dist/substrate.js +1 -1
  41. package/dist/workflow-allocation-service.d.ts +58 -0
  42. package/dist/workflow-allocation-service.js +239 -0
  43. package/dist/workflow-closure-resolution.d.ts +106 -0
  44. package/dist/workflow-closure-resolution.js +123 -0
  45. package/dist/workflow-definition-ensure.d.ts +24 -0
  46. package/dist/workflow-definition-ensure.js +75 -0
  47. package/dist/workflow-dispatch-service.d.ts +40 -0
  48. package/dist/workflow-dispatch-service.js +146 -0
  49. package/dist/workflow-dispatch-settlement.d.ts +29 -0
  50. package/dist/workflow-dispatch-settlement.js +140 -0
  51. package/dist/workflow-kind.d.ts +17 -1
  52. package/dist/workflow-kind.js +127 -80
  53. package/dist/workflow-probe-gate.d.ts +214 -0
  54. package/dist/workflow-probe-gate.js +207 -0
  55. package/dist/workflow-run-kind.d.ts +128 -14
  56. package/dist/workflow-run-kind.js +353 -83
  57. package/dist/workflow-run-reader.d.ts +1 -1
  58. package/dist/workflow-run-reader.js +3 -7
  59. package/dist/workflow-run-restore.d.ts +15 -0
  60. package/dist/workflow-run-restore.js +26 -0
  61. package/dist/workflow-source-closure.d.ts +35 -0
  62. package/dist/workflow-source-closure.js +342 -0
  63. package/dist/ws/index.d.ts +3 -3
  64. package/dist/ws/index.js +1 -1
  65. package/dist/ws/sidecar-events.d.ts +100 -12
  66. package/dist/ws/sidecar-events.js +2 -0
  67. package/dist/ws/sidecar-handler.d.ts +128 -7
  68. package/dist/ws/sidecar-handler.js +1069 -135
  69. package/dist/ws/sidecar-token-authenticator.d.ts +3 -1
  70. package/dist/ws/sidecar-token-authenticator.js +64 -7
  71. package/package.json +14 -13
  72. package/dist/available-skills-stanza.d.ts +0 -21
  73. package/dist/available-skills-stanza.js +0 -32
@@ -6,8 +6,8 @@
6
6
  import { getLogger } from "@intx/log";
7
7
  import { verifyEd25519 } from "@intx/crypto";
8
8
  import { chunkPack, createPackReceiver } from "@intx/pack-transport";
9
- import { hexDecode, hexEncode } from "@intx/types";
10
- import { isWorkflowDerivedAddress } from "@intx/workflow-deploy";
9
+ import { base64Decode, deriveMessageId, deriveWorkflowRunId, hexDecode, hexEncode, isRunAddress, } from "@intx/types";
10
+ import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy";
11
11
  import { type } from "arktype";
12
12
  import { SidecarFrame, } from "@intx/types/sidecar";
13
13
  import { createSidecarEmitter, } from "./sidecar-events.js";
@@ -24,22 +24,48 @@ const logger = getLogger(["hub", "ws", "sidecar"]);
24
24
  function connOwnsAddress(conn, address) {
25
25
  return (conn.agentAddresses.has(address) || conn.workflowAddresses.has(address));
26
26
  }
27
+ /**
28
+ * Bind pack writes to the repository implied by the authenticated address.
29
+ * An allocated credential is narrower still: it may only write its one
30
+ * deployment's workflow-run repository and never a standalone agent-state
31
+ * repository.
32
+ */
33
+ function connCanPushRepo(conn, agentAddress, repoId) {
34
+ if (conn.identity.kind === "allocated" &&
35
+ agentAddress !== conn.identity.workflowRunAddress) {
36
+ return false;
37
+ }
38
+ if (repoId.kind === "agent-state") {
39
+ return conn.identity.kind === "shared" && repoId.id === agentAddress;
40
+ }
41
+ return (repoId.kind === "workflow-run" &&
42
+ repoId.id === deriveWorkflowRunRepoId(agentAddress));
43
+ }
27
44
  /** The deduped set of every address this connection owns (session + workflow). */
28
45
  function ownedAddresses(conn) {
29
46
  return new Set([...conn.agentAddresses, ...conn.workflowAddresses]);
30
47
  }
31
48
  const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
32
49
  const DEFAULT_CHALLENGE_TIMEOUT_MS = 30_000;
50
+ // A probe fetches a workflow's dependency closure from a registry and
51
+ // evaluates it on the sidecar, so it runs longer than a routine request; its
52
+ // default timeout is correspondingly wider than DEFAULT_REQUEST_TIMEOUT_MS.
53
+ export const DEFAULT_PROBE_TIMEOUT_MS = 60_000;
33
54
  const DEFAULT_DISCONNECT_QUEUE_MAX_SIZE = 100;
34
55
  const DEFAULT_DISCONNECT_QUEUE_TTL_MS = 5 * 60 * 1000;
35
56
  const DEFAULT_PING_TIMEOUT_MS = 60_000;
57
+ const DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS = 10_000;
58
+ const DEFAULT_MAIL_ACK_MAX_RETRIES = 5;
36
59
  export function createSidecarRouter(config) {
37
- const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, challengeTimeoutMs = DEFAULT_CHALLENGE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, lookups = {}, } = config;
60
+ const { requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, challengeTimeoutMs = DEFAULT_CHALLENGE_TIMEOUT_MS, probeTimeoutMs = DEFAULT_PROBE_TIMEOUT_MS, hubPublicKey: hubPublicKeyHex, authenticateSidecar, validateSidecarIdentity = async () => true, disconnectQueueMaxSize = DEFAULT_DISCONNECT_QUEUE_MAX_SIZE, disconnectQueueTTLMs = DEFAULT_DISCONNECT_QUEUE_TTL_MS, pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS, mailAckRetryIntervalMs = DEFAULT_MAIL_ACK_RETRY_INTERVAL_MS, mailAckMaxRetries = DEFAULT_MAIL_ACK_MAX_RETRIES, lookups = {}, } = config;
38
61
  // Receiver-dispatch surface. Wire-layer callsites emit events here;
39
62
  // host code subscribes via `router.events`.
40
63
  const events = createSidecarEmitter();
41
64
  // ws handle → registered connection
42
65
  const connections = new Map();
66
+ const allocatedConnections = new Map();
67
+ const allocationFences = new Map();
68
+ const allocationWaiters = new Map();
43
69
  // agentAddress → ws handle (routing table)
44
70
  const addressIndex = new Map();
45
71
  // requestId → pending promise
@@ -47,6 +73,14 @@ export function createSidecarRouter(config) {
47
73
  const pendingDeploys = new Map();
48
74
  const pendingChallenges = new Map();
49
75
  const disconnectedAgents = new Map();
76
+ const pendingMail = new Map();
77
+ // agentAddress → retention TTL timer for un-acked pending mail held across a
78
+ // disconnect. On close the per-entry retry timers are cleared (the socket is
79
+ // gone) but the entries are RETAINED so a verified reconnect can redeliver
80
+ // them; this timer bounds that retention so a sidecar that never reconnects
81
+ // does not leak entries. Cleared when the address reconnects (redelivery) or
82
+ // its last pending entry is acked.
83
+ const pendingMailRetention = new Map();
50
84
  // agentAddress → set of subscriber callbacks for agent events
51
85
  const agentSubscribers = new Map();
52
86
  // agentAddress → cached connector-thread state, populated by
@@ -69,6 +103,7 @@ export function createSidecarRouter(config) {
69
103
  const pendingPacks = new Map();
70
104
  let packCounter = 0;
71
105
  const pendingUndeploys = new Map();
106
+ const pendingProbes = new Map();
72
107
  // Receives agent-state packs pushed from sidecars. The wire frames
73
108
  // (`repo.pack.push` / `repo.pack.done`) are shared with the
74
109
  // workflow-run flow; dispatch on `repoId.kind` picks which receiver
@@ -125,6 +160,183 @@ export function createSidecarRouter(config) {
125
160
  logger.info `Flushed ${String(entry.queue.length)} queued message(s) to ${agentAddress}`;
126
161
  }
127
162
  }
163
+ // Track a connected-window `mail.inbound` for redelivery until the sidecar
164
+ // acks its durable inbox write. Replaces any prior entry for the same
165
+ // (agentAddress, messageId) -- clearing its timer first so no timer leaks --
166
+ // which keeps a re-sent delivery from arming a second concurrent retry loop.
167
+ function trackPendingMail(agentAddress, messageId, frame, runGrants, allocatedTarget) {
168
+ let byId = pendingMail.get(agentAddress);
169
+ if (byId === undefined) {
170
+ byId = new Map();
171
+ pendingMail.set(agentAddress, byId);
172
+ }
173
+ const existing = byId.get(messageId);
174
+ if (existing !== undefined)
175
+ clearTimeout(existing.timer);
176
+ byId.set(messageId, {
177
+ agentAddress,
178
+ messageId,
179
+ frame,
180
+ attempts: 0,
181
+ timer: setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs),
182
+ ...(runGrants !== undefined ? { runGrants } : {}),
183
+ ...(allocatedTarget !== undefined ? { allocatedTarget } : {}),
184
+ });
185
+ }
186
+ // Replay a mail-triggered run's grants ahead of a redelivery of its trigger
187
+ // mail. Same-connection FIFO lands the `run.grants` frame before the mail, so
188
+ // the redelivered run resolves its onRunStart barrier instead of failing
189
+ // closed on missing grants.
190
+ function replayRunGrantsAhead(conn, entry) {
191
+ if (entry.runGrants === undefined)
192
+ return;
193
+ conn.send({
194
+ type: "run.grants",
195
+ agentAddress: entry.agentAddress,
196
+ runId: entry.runGrants.runId,
197
+ stepGrants: entry.runGrants.stepGrants,
198
+ });
199
+ }
200
+ function deletePendingMail(byId, agentAddress, messageId) {
201
+ byId.delete(messageId);
202
+ if (byId.size === 0) {
203
+ pendingMail.delete(agentAddress);
204
+ // The retention TTL guards a non-empty pending set; drop it once the set
205
+ // is empty so it never outlives the entries it was bounding.
206
+ const retention = pendingMailRetention.get(agentAddress);
207
+ if (retention !== undefined) {
208
+ clearTimeout(retention);
209
+ pendingMailRetention.delete(agentAddress);
210
+ }
211
+ }
212
+ }
213
+ function retryPendingMail(agentAddress, messageId) {
214
+ const byId = pendingMail.get(agentAddress);
215
+ if (byId === undefined)
216
+ return;
217
+ const entry = byId.get(messageId);
218
+ if (entry === undefined)
219
+ return;
220
+ if (entry.attempts >= mailAckMaxRetries) {
221
+ // The sidecar never acked within the retry budget. The ack is withheld
222
+ // precisely because the sidecar's durable inbox write failed, so the
223
+ // mail was NOT delivered: surface it as undelivered so the host can relay
224
+ // it onto an external transport, then drop the pending entry so its timer
225
+ // does not leak.
226
+ if (entry.frame.type === "mail.inbound") {
227
+ events.emit("mail.outbound.undelivered", {
228
+ rawMessage: entry.frame.rawMessage,
229
+ recipients: [agentAddress],
230
+ });
231
+ }
232
+ deletePendingMail(byId, agentAddress, messageId);
233
+ logger.warn `Gave up redelivering mail ${messageId} to ${agentAddress} after ${String(entry.attempts)} un-acked attempt(s)`;
234
+ return;
235
+ }
236
+ // Redeliver over the address's CURRENT owner: a verified reconnect may have
237
+ // moved the address to a new connection since the original delivery.
238
+ const ws = addressIndex.get(agentAddress);
239
+ const conn = ws !== undefined ? connections.get(ws) : undefined;
240
+ const allocated = entry.allocatedTarget === undefined
241
+ ? undefined
242
+ : allocatedConnections.get(entry.allocatedTarget.allocationId);
243
+ const targetStillOwnsAddress = entry.allocatedTarget === undefined ||
244
+ (allocated !== undefined &&
245
+ allocated.identity.generation === entry.allocatedTarget.generation &&
246
+ allocated.ws === ws);
247
+ if (conn === undefined || !targetStillOwnsAddress) {
248
+ // No live connection to recover into. Connected-window redelivery only
249
+ // applies while the address is routable; a disconnected address is not
250
+ // retried here.
251
+ deletePendingMail(byId, agentAddress, messageId);
252
+ logger.warn `Dropping un-acked mail ${messageId} for ${agentAddress}: no live connection to redeliver over`;
253
+ return;
254
+ }
255
+ entry.attempts += 1;
256
+ replayRunGrantsAhead(conn, entry);
257
+ conn.send(entry.frame);
258
+ entry.timer = setTimeout(() => retryPendingMail(agentAddress, messageId), mailAckRetryIntervalMs);
259
+ }
260
+ function resolvePendingMail(agentAddress, messageId) {
261
+ const byId = pendingMail.get(agentAddress);
262
+ if (byId === undefined)
263
+ return;
264
+ const entry = byId.get(messageId);
265
+ if (entry === undefined)
266
+ return;
267
+ clearTimeout(entry.timer);
268
+ deletePendingMail(byId, agentAddress, messageId);
269
+ }
270
+ // Hold an address's un-acked pending mail across a disconnect. The per-entry
271
+ // retry timers are cleared -- retrying over the dead socket is pointless --
272
+ // but the entries are KEPT so a verified reconnect can redeliver them. A
273
+ // retention TTL (the disconnect-queue horizon) bounds the hold so a sidecar
274
+ // that never reconnects does not leak; on expiry the still-un-acked entries
275
+ // are surfaced as `mail.outbound.undelivered` so the host can relay them,
276
+ // since a withheld ack means the sidecar's durable write never landed.
277
+ function retainPendingMailForAddress(agentAddress) {
278
+ const byId = pendingMail.get(agentAddress);
279
+ if (byId === undefined)
280
+ return;
281
+ for (const entry of byId.values())
282
+ clearTimeout(entry.timer);
283
+ const existing = pendingMailRetention.get(agentAddress);
284
+ if (existing !== undefined)
285
+ clearTimeout(existing);
286
+ const timer = setTimeout(() => {
287
+ pendingMailRetention.delete(agentAddress);
288
+ const expired = pendingMail.get(agentAddress);
289
+ pendingMail.delete(agentAddress);
290
+ if (expired !== undefined && expired.size > 0) {
291
+ for (const entry of expired.values()) {
292
+ if (entry.frame.type !== "mail.inbound")
293
+ continue;
294
+ events.emit("mail.outbound.undelivered", {
295
+ rawMessage: entry.frame.rawMessage,
296
+ recipients: [agentAddress],
297
+ });
298
+ }
299
+ logger.warn `Dropping ${String(expired.size)} un-acked message(s) for ${agentAddress}: pending-mail retention TTL expired`;
300
+ }
301
+ }, disconnectQueueTTLMs);
302
+ pendingMailRetention.set(agentAddress, timer);
303
+ }
304
+ // Redeliver an address's retained un-acked pending mail on a verified
305
+ // reconnect. Replays identical bytes (same messageId) over the new
306
+ // connection, so the sidecar's inbox dedups a message it already wrote
307
+ // (effectively-once) and processes one it had dropped (no loss). Re-arms the
308
+ // connected-window retry over the new connection with a fresh per-generation
309
+ // budget, so a redelivery that is itself dropped before its ack is retried.
310
+ function redeliverPendingMail(agentAddress, conn) {
311
+ const retention = pendingMailRetention.get(agentAddress);
312
+ if (retention !== undefined) {
313
+ clearTimeout(retention);
314
+ pendingMailRetention.delete(agentAddress);
315
+ }
316
+ const byId = pendingMail.get(agentAddress);
317
+ if (byId === undefined)
318
+ return;
319
+ for (const entry of [...byId.values()]) {
320
+ if (entry.allocatedTarget !== undefined &&
321
+ (conn.identity.kind !== "allocated" ||
322
+ conn.identity.allocationId !== entry.allocatedTarget.allocationId ||
323
+ conn.identity.generation !== entry.allocatedTarget.generation)) {
324
+ // The Hub-owned dispatch row survives generation replacement and will
325
+ // be requeued by the allocation-ready callback. Do not leak or replay
326
+ // this generation-local retry entry onto a different worker.
327
+ clearTimeout(entry.timer);
328
+ deletePendingMail(byId, agentAddress, entry.messageId);
329
+ continue;
330
+ }
331
+ replayRunGrantsAhead(conn, entry);
332
+ conn.send(entry.frame);
333
+ entry.attempts = 0;
334
+ entry.timer = setTimeout(() => retryPendingMail(agentAddress, entry.messageId), mailAckRetryIntervalMs);
335
+ }
336
+ if (byId.size > 0) {
337
+ logger.info `Redelivered ${String(byId.size)} un-acked message(s) to ${agentAddress} on reconnect`;
338
+ }
339
+ }
128
340
  function resetLivenessTimer(ws) {
129
341
  const existing = livenessTimers.get(ws);
130
342
  if (existing !== undefined)
@@ -213,6 +425,8 @@ export function createSidecarRouter(config) {
213
425
  case "agent.undeploy.ack":
214
426
  case "repo.pack.ack":
215
427
  case "repo.pack.reject":
428
+ case "workflow.probe.result":
429
+ case "workflow.probe.error":
216
430
  return true;
217
431
  case "register":
218
432
  case "reconnect":
@@ -220,6 +434,8 @@ export function createSidecarRouter(config) {
220
434
  case "mail.outbound":
221
435
  case "agent.event":
222
436
  case "connector.state.changed":
437
+ case "mail.inbound.ack":
438
+ case "signal.correlation.register":
223
439
  case "repo.pack.push":
224
440
  case "repo.pack.done":
225
441
  return false;
@@ -246,21 +462,20 @@ export function createSidecarRouter(config) {
246
462
  case "challenge.response":
247
463
  return handleChallengeResponse(ws, frame.responses);
248
464
  case "agent.deploy.ack":
249
- return handleDeployAck(frame.agentAddress, frame.publicKey);
465
+ return handleDeployAck(ws, frame);
250
466
  case "agent.error":
251
- rejectDeployPending(frame.agentAddress, frame.error);
252
- rejectUndeployPending(frame.agentAddress, frame.error);
467
+ rejectDeployPendingFromFrame(ws, frame.agentAddress, frame.error);
468
+ rejectUndeployPending(ws, frame.agentAddress, frame.error);
253
469
  return;
254
470
  case "agent.undeploy.ack":
255
- resolveUndeployPending(frame.agentAddress);
471
+ resolveUndeployPending(ws, frame.agentAddress);
256
472
  return;
257
473
  case "ping":
258
474
  handlePing(ws);
259
475
  return;
260
476
  case "mail.outbound":
261
477
  if (frame.delivered !== true) {
262
- handleMailOutbound(frame.rawMessage, frame.recipients);
263
- return;
478
+ return handleMailOutbound(frame.rawMessage, frame.recipients);
264
479
  }
265
480
  if (lookups.persistMail && frame.senderAddress) {
266
481
  return handleMailPersist(lookups.persistMail, frame.rawMessage, frame.senderAddress, frame.recipients);
@@ -294,6 +509,38 @@ export function createSidecarRouter(config) {
294
509
  connectorState: frame.connectorState,
295
510
  });
296
511
  return;
512
+ case "mail.inbound.ack": {
513
+ // Terminal receipt for a connected-window `mail.inbound`: the sidecar
514
+ // has durably written the message to its inbox. Gate on ownership --
515
+ // like connector.state.changed and signal.correlation.register -- so a
516
+ // sidecar cannot clear another sidecar's pending mail. The messageId is
517
+ // a hub-minted id only the owning sidecar ever received on the frame,
518
+ // so the ownership check is defense-in-depth, not the sole guard.
519
+ const conn = connections.get(ws);
520
+ if (conn === undefined)
521
+ return;
522
+ if (!connOwnsAddress(conn, frame.agentAddress)) {
523
+ logger.warn `Dropping mail.inbound.ack for ${frame.agentAddress}: not registered to this sidecar`;
524
+ return;
525
+ }
526
+ resolvePendingMail(frame.agentAddress, frame.messageId);
527
+ events.emit("mail.inbound.acknowledged", {
528
+ agentAddress: frame.agentAddress,
529
+ messageId: frame.messageId,
530
+ ...(conn.identity.kind === "allocated"
531
+ ? {
532
+ allocated: {
533
+ allocationId: conn.identity.allocationId,
534
+ anchorRunId: conn.identity.anchorRunId,
535
+ generation: conn.identity.generation,
536
+ },
537
+ }
538
+ : {}),
539
+ });
540
+ return;
541
+ }
542
+ case "signal.correlation.register":
543
+ return handleSignalCorrelationRegister(ws, frame);
297
544
  case "session.ack":
298
545
  resolvePending(frame.requestId);
299
546
  return;
@@ -301,16 +548,27 @@ export function createSidecarRouter(config) {
301
548
  rejectPending(frame.requestId, frame.error);
302
549
  return;
303
550
  case "repo.pack.ack":
304
- resolvePackPending(frame.transferId);
551
+ resolvePackPending(ws, frame);
305
552
  return;
306
553
  case "repo.pack.reject":
307
- rejectPackPending(frame.transferId, frame.reason);
554
+ rejectPackPending(ws, frame);
308
555
  return;
309
556
  case "repo.pack.push":
310
557
  handlePackPush(ws, frame);
311
558
  return;
312
559
  case "repo.pack.done":
313
560
  return handlePackDone(ws, frame);
561
+ case "workflow.probe.result":
562
+ resolveProbe(frame.requestId, {
563
+ projection: frame.projection,
564
+ grants: frame.grants,
565
+ grantWalkSnapshot: frame.grantWalkSnapshot,
566
+ wireHash: frame.wireHash,
567
+ });
568
+ return;
569
+ case "workflow.probe.error":
570
+ rejectProbe(frame.requestId, frame.error);
571
+ return;
314
572
  default:
315
573
  return assertNever(frame);
316
574
  }
@@ -343,9 +601,107 @@ export function createSidecarRouter(config) {
343
601
  if (identity.sidecarId !== frame.sidecarId) {
344
602
  logger.warn `Sidecar ${frame.type} claimed id ${frame.sidecarId} but token verifies as ${identity.sidecarId}; keying off the verified id`;
345
603
  }
604
+ if (!(await validateSidecarIdentity(identity, "registration"))) {
605
+ logger.warn `Rejected ${frame.type} from sidecar ${identity.sidecarId}: credential identity is no longer current`;
606
+ ws.close();
607
+ return;
608
+ }
346
609
  await run(identity);
347
610
  }
611
+ async function notifyAllocationWaiters(allocationId) {
612
+ const waiters = allocationWaiters.get(allocationId);
613
+ const current = allocatedConnections.get(allocationId);
614
+ if (waiters === undefined || current === undefined)
615
+ return;
616
+ if (!(await validateSidecarIdentity(current.identity, "readiness")))
617
+ return;
618
+ for (const waiter of [...waiters]) {
619
+ if (waiter.generation !== current.identity.generation)
620
+ continue;
621
+ clearTimeout(waiter.timer);
622
+ waiters.delete(waiter);
623
+ waiter.resolve();
624
+ }
625
+ if (waiters.size === 0)
626
+ allocationWaiters.delete(allocationId);
627
+ }
628
+ async function handleAllocatedRegister(ws, identity, agentAddresses) {
629
+ if (allocationFences.get(identity.allocationId) !== identity.generation) {
630
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} generation ${String(identity.generation)} is not fenced as current`;
631
+ ws.close();
632
+ return;
633
+ }
634
+ if (agentAddresses.length > 0 &&
635
+ !(await validateSidecarIdentity(identity, "routing"))) {
636
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} is not ready to reclaim routes`;
637
+ ws.close();
638
+ return;
639
+ }
640
+ const existingOnSocket = connections.get(ws);
641
+ const newlyRoutedAddresses = new Set();
642
+ for (const address of agentAddresses) {
643
+ const alreadyOwned = existingOnSocket?.identity.kind === "allocated" &&
644
+ existingOnSocket.identity.allocationId === identity.allocationId &&
645
+ addressIndex.get(address) === ws &&
646
+ connOwnsAddress(existingOnSocket, address);
647
+ if (!alreadyOwned && address !== identity.workflowRunAddress) {
648
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: allocation ${identity.allocationId} claimed unrelated address ${address}`;
649
+ ws.close();
650
+ return;
651
+ }
652
+ if (!alreadyOwned)
653
+ newlyRoutedAddresses.add(address);
654
+ }
655
+ if (allocationFences.get(identity.allocationId) !== identity.generation) {
656
+ ws.close();
657
+ return;
658
+ }
659
+ const current = allocatedConnections.get(identity.allocationId);
660
+ if (current !== undefined && current.ws !== ws) {
661
+ // A current-generation takeover is a reconnect, not a capacity loss.
662
+ // Remove the old socket from the allocation index before closing it so
663
+ // handleClose does not emit a false allocated-disconnect event.
664
+ allocatedConnections.delete(identity.allocationId);
665
+ handleClose(current.ws);
666
+ current.ws.close();
667
+ }
668
+ const conn = existingOnSocket ?? {
669
+ sidecarId: identity.sidecarId,
670
+ identity,
671
+ agentAddresses: new Set(),
672
+ workflowAddresses: new Set(),
673
+ send(frame) {
674
+ ws.send(JSON.stringify(frame));
675
+ },
676
+ };
677
+ if (conn.identity.kind !== "allocated" ||
678
+ conn.identity.allocationId !== identity.allocationId ||
679
+ conn.identity.generation !== identity.generation) {
680
+ logger.warn `Rejected allocated sidecar ${identity.sidecarId}: socket identity changed during registration`;
681
+ ws.close();
682
+ return;
683
+ }
684
+ connections.set(ws, conn);
685
+ for (const address of agentAddresses) {
686
+ conn.workflowAddresses.add(address);
687
+ addressIndex.set(address, ws);
688
+ }
689
+ allocatedConnections.set(identity.allocationId, { ws, identity });
690
+ for (const address of newlyRoutedAddresses) {
691
+ redeliverPendingMail(address, conn);
692
+ }
693
+ logger.info `Allocated sidecar ${identity.sidecarId} registered for allocation ${identity.allocationId} generation ${String(identity.generation)}`;
694
+ await notifyAllocationWaiters(identity.allocationId);
695
+ events.emit("sidecar.allocated.connected", {
696
+ allocationId: identity.allocationId,
697
+ generation: identity.generation,
698
+ });
699
+ }
348
700
  async function handleRegister(ws, identity, agentAddresses) {
701
+ if (identity.kind === "allocated") {
702
+ await handleAllocatedRegister(ws, identity, agentAddresses);
703
+ return;
704
+ }
349
705
  const sidecarId = identity.sidecarId;
350
706
  // Key-existence gate. A register frame is token-authenticated but carries
351
707
  // no per-address ownership proof, so it may route an address ONLY if that
@@ -434,6 +790,7 @@ export function createSidecarRouter(config) {
434
790
  const workflowSet = inheritedWorkflow;
435
791
  const conn = {
436
792
  sidecarId,
793
+ identity,
437
794
  agentAddresses: addrSet,
438
795
  workflowAddresses: workflowSet,
439
796
  send(frame) {
@@ -466,6 +823,10 @@ export function createSidecarRouter(config) {
466
823
  logger.info `Sidecar ${sidecarId} registered; routed ${String(addrSet.size)} of ${String(agentAddresses.length)} address(es) (keyless first-deploy only)`;
467
824
  }
468
825
  async function handleReconnect(ws, identity, agentAddresses, deployRefs = {}) {
826
+ if (identity.kind === "allocated") {
827
+ await handleAllocatedRegister(ws, identity, agentAddresses);
828
+ return;
829
+ }
469
830
  const sidecarId = identity.sidecarId;
470
831
  const lookupKey = lookups.lookupPublicKey;
471
832
  if (lookupKey === undefined) {
@@ -534,7 +895,7 @@ export function createSidecarRouter(config) {
534
895
  conn.send({
535
896
  type: "challenge.failed",
536
897
  address,
537
- reason: "Unknown agent address",
898
+ reason: "Unknown run address",
538
899
  });
539
900
  continue;
540
901
  }
@@ -657,14 +1018,17 @@ export function createSidecarRouter(config) {
657
1018
  prevConn.agentAddresses.delete(addr);
658
1019
  }
659
1020
  }
660
- // Track the address on the set that matches its lifecycle so
661
- // handleClose reclaims it correctly: a workflow-derived deployment
662
- // address goes on the workflow set (no disconnect queue -- its
663
- // in-flight state is reconstructed sidecar-locally on the next
664
- // reconnect), a launched agent on the session set (queued for
665
- // reconnect). The routing pointer is the same either way; only now
666
- // it is written behind a passed challenge.
667
- if (isWorkflowDerivedAddress(addr)) {
1021
+ // A run address goes on the workflow set (no disconnect queue -- its
1022
+ // in-flight state is reconstructed sidecar-locally on the next reconnect),
1023
+ // so handleClose reclaims it correctly. The routing pointer is the same
1024
+ // either way; only now it is written behind a passed challenge.
1025
+ //
1026
+ // The `else` (session set, queued for reconnect) is the retired
1027
+ // launched-agent path: launched agents no longer exist (the folded-launch
1028
+ // route was removed), so no current producer reaches it. It is left in
1029
+ // place for the reconnect-subsystem teardown to remove as its own
1030
+ // reviewable change, not folded into this collapse.
1031
+ if (isRunAddress(addr)) {
668
1032
  conn.workflowAddresses.add(addr);
669
1033
  }
670
1034
  else {
@@ -675,14 +1039,17 @@ export function createSidecarRouter(config) {
675
1039
  const ready = [];
676
1040
  const failed = [];
677
1041
  for (const addr of verified) {
678
- // The `agent.reconnected` reaction is session lifecycle -- instance
679
- // status flip, event-collector restore -- owned by the agent_instance
680
- // concept. A workflow-derived deployment address has no agent_instance
681
- // row, so the reaction's `requireInstance` would throw and roll the
682
- // just-verified address back out of routing. It needs routing + queue
683
- // flush only, which the passed challenge has now made safe; skip the
684
- // session reaction for it.
685
- if (isWorkflowDerivedAddress(addr)) {
1042
+ // A workflow run needs routing + queue flush only, which the passed
1043
+ // challenge has now made safe: its in-flight state is reconstructed
1044
+ // sidecar-locally, so it does not enrol in the `agent.reconnected` session
1045
+ // reaction (event-collector restore). Skip the reaction for it.
1046
+ //
1047
+ // The reaction below was the retired launched-agent path (restore a
1048
+ // launched agent's collector). Launched agents no longer exist, so no
1049
+ // current producer reaches it; the `agent.reconnected` listener is gone,
1050
+ // so `listenerCount` is 0 and the emit is inert. Left for the
1051
+ // reconnect-subsystem teardown to remove.
1052
+ if (isRunAddress(addr)) {
686
1053
  ready.push(addr);
687
1054
  continue;
688
1055
  }
@@ -715,9 +1082,14 @@ export function createSidecarRouter(config) {
715
1082
  conn.agentAddresses.delete(addr);
716
1083
  addressIndex.delete(addr);
717
1084
  }
718
- // Flush queued messages only for ready addresses.
1085
+ // Flush queued messages and redeliver retained un-acked mail, only for
1086
+ // ready addresses. The disconnect queue carries mail that arrived WHILE
1087
+ // disconnected; pending-mail redelivery carries mail that was delivered
1088
+ // over the prior live connection but never acked (the connected-window
1089
+ // drop). Both replay over the verified new connection.
719
1090
  for (const addr of ready) {
720
1091
  flushDisconnectedQueue(addr, conn);
1092
+ redeliverPendingMail(addr, conn);
721
1093
  }
722
1094
  // Re-deploy agents whose deploy ref is stale or absent. Fire-and-forget
723
1095
  // so reconnect completion is not blocked on pack transfer. The
@@ -728,15 +1100,17 @@ export function createSidecarRouter(config) {
728
1100
  for (const addr of ready) {
729
1101
  // Workflow deployments are pinned-forever: a deployment keeps its
730
1102
  // deploy-time definition until an explicit undeploy/redeploy, so the
731
- // deploy-ref freshness catch-up is deliberately NOT run for a
732
- // workflow-derived address. A definition edited on the hub while the
733
- // sidecar was disconnected does not reconcile on reconnect; it affects
734
- // only newly created deployments. The deployment's in-flight run state
735
- // is reconstructed sidecar-locally at restore, not re-fetched here. Do
736
- // NOT add a reconcile path for these addresses -- see the "Workflow
1103
+ // deploy-ref freshness catch-up is deliberately NOT run for a run
1104
+ // address. A definition edited on the hub while the sidecar was
1105
+ // disconnected does not reconcile on reconnect; it affects only newly
1106
+ // created deployments. The deployment's in-flight run state is
1107
+ // reconstructed sidecar-locally at restore, not re-fetched here. Do NOT
1108
+ // add a reconcile path for these addresses -- see the "Workflow
737
1109
  // Definition Versioning: Pinned-Forever" note under "Reconnect
738
- // Sequencing" in docs/IMPLEMENTATION.md.
739
- if (isWorkflowDerivedAddress(addr))
1110
+ // Sequencing" in docs/IMPLEMENTATION.md. (Every routable address is a
1111
+ // run address now; the fall-through was the retired launched-agent
1112
+ // catch-up path.)
1113
+ if (isRunAddress(addr))
740
1114
  continue;
741
1115
  void (async () => {
742
1116
  try {
@@ -779,30 +1153,44 @@ export function createSidecarRouter(config) {
779
1153
  }
780
1154
  logger.info `Sidecar ${challenge.sidecarId} reconnected with ${String(ready.length)} verified agent(s)${failed.length > 0 ? `, ${String(failed.length)} rejected` : ""}`;
781
1155
  }
782
- function handleMailOutbound(rawMessage, recipients) {
1156
+ async function handleMailOutbound(rawMessage, recipients) {
1157
+ // A mail addressed to more than one workflow deployment would birth a
1158
+ // run per recipient from a single inbound mail. The stable runId
1159
+ // removed the Message-ID collision that originally forced this guard --
1160
+ // each recipient now derives its own per-deployment runId (its mail
1161
+ // address), so it is no longer a runId-collision guard. It stays a
1162
+ // deliberate one-workflow-recipient-per-mail restriction because the
1163
+ // fan-out is not verified end-to-end: per-recipient grants
1164
+ // materialization, consumed-tracking, and reply-addressing all assume a
1165
+ // single workflow recipient today. Lifting it means proving those three
1166
+ // hold per recipient, not just relaxing this check -- so fail loudly
1167
+ // rather than materialize a partial set. The guard only applies when a
1168
+ // materializer is wired -- absent one, no run is born from the mail, so
1169
+ // there is nothing to restrict.
1170
+ if (lookups.materializeMailTriggeredRunGrants !== undefined) {
1171
+ // A workflow recipient is one this hub owns: its address parses as a run
1172
+ // address. An external/federated address does not, and is not ours to
1173
+ // materialize a run for.
1174
+ const workflowRecipients = recipients.filter(isRunAddress);
1175
+ if (workflowRecipients.length > 1) {
1176
+ throw new Error(`mail addressed to multiple workflow-derived recipients (${workflowRecipients.join(", ")}); materializing a run for more than one workflow deployment from a single mail is unsupported`);
1177
+ }
1178
+ }
783
1179
  // Route to locally connected sidecars first, then try disconnect queues.
784
1180
  const unrouted = [];
785
1181
  for (const recipient of recipients) {
786
- const targetWs = addressIndex.get(recipient);
787
- if (targetWs !== undefined) {
788
- const conn = connections.get(targetWs);
789
- if (conn !== undefined) {
790
- conn.send({
791
- type: "mail.inbound",
792
- agentAddress: recipient,
793
- rawMessage,
794
- });
795
- continue;
796
- }
1182
+ // Each recipient is isolated: a materialization failure or a
1183
+ // fail-closed rejection for one must not drop the mail for its
1184
+ // co-recipients. The catch fails THIS recipient closed (its run never
1185
+ // starts under-authorized) and continues to the rest.
1186
+ try {
1187
+ const outcome = await deliverMailToRecipient(recipient, rawMessage);
1188
+ if (outcome === "unrouted")
1189
+ unrouted.push(recipient);
1190
+ }
1191
+ catch (err) {
1192
+ logger.error `Failed to deliver mail to ${recipient}: ${err instanceof Error ? err.message : String(err)}`;
797
1193
  }
798
- const frame = {
799
- type: "mail.inbound",
800
- agentAddress: recipient,
801
- rawMessage,
802
- };
803
- if (enqueueForDisconnected(recipient, frame))
804
- continue;
805
- unrouted.push(recipient);
806
1194
  }
807
1195
  // Anything not routed locally is emitted as a notification. The
808
1196
  // host decides whether to relay onto an external transport, log,
@@ -814,6 +1202,69 @@ export function createSidecarRouter(config) {
814
1202
  });
815
1203
  }
816
1204
  }
1205
+ // Deliver an inbound mail to one recipient, materializing a
1206
+ // mail-triggered run's grants first when the recipient is a workflow
1207
+ // deployment. Returns:
1208
+ // - `routed`: the mail reached a live connection or disconnect queue.
1209
+ // - `unrouted`: the mail was locally undeliverable and should be
1210
+ // relayed externally by the host.
1211
+ // - `failed-closed`: the run's grants could not be materialized safely,
1212
+ // so the mail is deliberately DROPPED for this recipient (not relayed)
1213
+ // to keep its run from starting under-authorized.
1214
+ //
1215
+ // A workflow deployment is the only recipient whose inbound mail can first
1216
+ // fire its stable run. Its grants are reserved, and the `run.grants` frame is
1217
+ // sent BEFORE the mail. Same-address FIFO guarantees it lands ahead of the
1218
+ // mail that dispatches the run, so the run's `onRunStart` barrier resolves its
1219
+ // grants rather than failing closed. Reservation happens before routing so
1220
+ // concurrent first deliveries cannot send different snapshots; a routing
1221
+ // failure leaves a grants-only, still-unfired run.
1222
+ async function deliverMailToRecipient(recipient, rawMessage) {
1223
+ if (lookups.materializeMailTriggeredRunGrants !== undefined &&
1224
+ isRunAddress(recipient)) {
1225
+ const runId = deriveWorkflowRunId(recipient);
1226
+ const result = await lookups.materializeMailTriggeredRunGrants({
1227
+ agentAddress: recipient,
1228
+ runId,
1229
+ });
1230
+ if (result.outcome === "rejected") {
1231
+ // The run's grants could not be materialized with sufficient
1232
+ // authority or it is already terminal. Fail the mail closed for this
1233
+ // recipient: routing or external relay would bypass that decision.
1234
+ logger.error `Refusing mail-triggered run ${runId} for ${recipient}: grant materialization rejected (${result.code}): ${result.message}`;
1235
+ return "failed-closed";
1236
+ }
1237
+ if (result.outcome === "materialized") {
1238
+ // Send the run's grants ahead of the mail. A `false` here means the
1239
+ // deployment is unroutable. Do not route the mail that would dispatch
1240
+ // it; the grants-only reservation remains the canonical snapshot for a
1241
+ // later first-delivery attempt.
1242
+ if (!sendRunGrants(recipient, runId, result.stepGrants)) {
1243
+ logger.error `Deployment ${recipient} is not routable for run ${runId}; retaining the unfired run's grant reservation for retry`;
1244
+ return "unrouted";
1245
+ }
1246
+ // Route through the messageId handshake `routeMail` -- NOT a
1247
+ // fire-and-forget send. This branch COMMITS a run, so a mail dropped in
1248
+ // the connected window (a socket that half-dies before the sidecar's
1249
+ // durable-write ack) would otherwise leave the run row "running"
1250
+ // forever with no body and no error. `routeMail` tracks the delivery
1251
+ // and redelivers identical bytes on reconnect, bringing the mail-relay
1252
+ // run-trigger to parity with the HTTP-trigger path. The messageId is the
1253
+ // mail's own id (derived over the same bytes the sidecar derives), so a
1254
+ // redelivery replays identically and the downstream RunStarted /
1255
+ // stable-runId dedup makes it effectively-once.
1256
+ const messageId = await deriveMessageId(base64Decode(rawMessage));
1257
+ const outcome = routeMail(recipient, rawMessage, messageId, { runId, stepGrants: result.stepGrants })
1258
+ ? "routed"
1259
+ : "unrouted";
1260
+ return outcome;
1261
+ }
1262
+ // `skip`: the address named no deployed workflow deployment. Forward
1263
+ // the mail without grants -- the run, if any, is not ours to
1264
+ // authorize. No run is committed here, so no ack handshake is needed.
1265
+ }
1266
+ return routeMail(recipient, rawMessage) ? "routed" : "unrouted";
1267
+ }
817
1268
  async function handleMailPersist(persist, rawMessage, senderAddress, recipients) {
818
1269
  let results;
819
1270
  let raw;
@@ -835,15 +1286,60 @@ export function createSidecarRouter(config) {
835
1286
  raw,
836
1287
  createdAt: result.createdAt,
837
1288
  direction: result.direction,
838
- instanceId: result.instanceId,
1289
+ runId: result.runId,
839
1290
  address: result.address,
840
1291
  });
841
1292
  }
842
1293
  }
1294
+ async function handleSignalCorrelationRegister(ws, frame) {
1295
+ // Gate the co-write on the sending sidecar actually owning the named
1296
+ // deployment address, mirroring the connector.state.changed and pack
1297
+ // handlers. A workflow deployment routes on the keyless workflow set, so
1298
+ // ownership is the union check, not addressIndex identity alone. Without
1299
+ // it a misbehaving sidecar that knows another deployment's address could
1300
+ // register a spurious correlation against it.
1301
+ const conn = connections.get(ws);
1302
+ if (conn === undefined)
1303
+ return;
1304
+ if (!connOwnsAddress(conn, frame.agentAddress)) {
1305
+ logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: not registered to this sidecar`;
1306
+ return;
1307
+ }
1308
+ const register = lookups.registerSignalCorrelation;
1309
+ if (register === undefined) {
1310
+ logger.warn `Dropping signal.correlation.register for ${frame.agentAddress}: no registerSignalCorrelation lookup configured`;
1311
+ return;
1312
+ }
1313
+ try {
1314
+ await register({
1315
+ correlationId: frame.correlationId,
1316
+ runId: frame.runId,
1317
+ anchorRunId: frame.anchorRunId,
1318
+ agentAddress: frame.agentAddress,
1319
+ kind: frame.kind,
1320
+ approvalSnapshot: frame.snapshot,
1321
+ });
1322
+ // The co-write resolves only when a row exists -- freshly inserted or
1323
+ // already present (both stores are idempotent on the correlationId). Ack
1324
+ // so the sidecar's link stops retrying a register whose frame may have
1325
+ // been lost on an open socket. A thrown co-write (undeployed deployment,
1326
+ // id mismatch) means no row, so no ack: the sidecar keeps retrying and
1327
+ // the reconnect re-emit remains the ultimate backstop.
1328
+ conn.send({
1329
+ type: "signal.correlation.register.ack",
1330
+ agentAddress: frame.agentAddress,
1331
+ correlationId: frame.correlationId,
1332
+ });
1333
+ }
1334
+ catch (err) {
1335
+ logger.error `Failed to register signal correlation ${frame.correlationId} for ${frame.agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
1336
+ }
1337
+ }
843
1338
  function handleClose(ws) {
844
1339
  const conn = connections.get(ws);
845
1340
  if (conn === undefined)
846
1341
  return;
1342
+ let allocated;
847
1343
  for (const addr of conn.agentAddresses) {
848
1344
  // Only remove routing and pending state if this connection still
849
1345
  // owns the address. A reconnected sidecar may have already claimed it.
@@ -855,12 +1351,12 @@ export function createSidecarRouter(config) {
855
1351
  // reconnect re-bootstraps via the router's
856
1352
  // restore-fires-callback path.
857
1353
  connectorStates.delete(addr);
858
- const deployReq = pendingDeploys.get(addr);
859
- if (deployReq !== undefined && deployReq.ws === ws) {
860
- clearTimeout(deployReq.timer);
861
- pendingDeploys.delete(addr);
862
- deployReq.reject(`Sidecar ${conn.sidecarId} disconnected`);
863
- }
1354
+ // Retain this address's un-acked pending mail across the disconnect:
1355
+ // its in-flight retry timers target a dead socket (cleared), but the
1356
+ // entries are held so a verified reconnect redelivers them, closing the
1357
+ // connected-window drop rather than losing the mail. Bounded by a
1358
+ // retention TTL.
1359
+ retainPendingMailForAddress(addr);
864
1360
  // Create a queue entry so messages can accumulate while the
865
1361
  // sidecar is disconnected. Skip if the agent is being undeployed --
866
1362
  // there is no point queuing messages for an agent being torn down.
@@ -885,6 +1381,22 @@ export function createSidecarRouter(config) {
885
1381
  for (const addr of conn.workflowAddresses) {
886
1382
  if (addressIndex.get(addr) === ws) {
887
1383
  addressIndex.delete(addr);
1384
+ // Retain un-acked workflow trigger mail across the disconnect for the
1385
+ // same reason as the session loop above -- a challenged reconnect
1386
+ // redelivers it. This is un-acked TRIGGER mail, distinct from the
1387
+ // deployment's in-flight run state (reconstructed sidecar-locally); the
1388
+ // "no disconnect queue" note above is about that run state, not this.
1389
+ retainPendingMailForAddress(addr);
1390
+ }
1391
+ }
1392
+ if (conn.identity.kind === "allocated") {
1393
+ const current = allocatedConnections.get(conn.identity.allocationId);
1394
+ if (current?.ws === ws) {
1395
+ allocatedConnections.delete(conn.identity.allocationId);
1396
+ allocated = {
1397
+ allocationId: conn.identity.allocationId,
1398
+ generation: conn.identity.generation,
1399
+ };
888
1400
  }
889
1401
  }
890
1402
  connections.delete(ws);
@@ -910,6 +1422,16 @@ export function createSidecarRouter(config) {
910
1422
  pending.delete(requestId);
911
1423
  req.reject(`Sidecar ${conn.sidecarId} disconnected`);
912
1424
  }
1425
+ // Reject every deploy issued on this socket, including allocated
1426
+ // workflow deployments stored in `workflowAddresses` rather than
1427
+ // `agentAddresses`.
1428
+ for (const [agentAddress, req] of pendingDeploys) {
1429
+ if (req.ws !== ws)
1430
+ continue;
1431
+ clearTimeout(req.timer);
1432
+ pendingDeploys.delete(agentAddress);
1433
+ req.reject(`Sidecar ${conn.sidecarId} disconnected`);
1434
+ }
913
1435
  // Reject any in-flight pack transfers for this sidecar.
914
1436
  for (const [transferId, pack] of pendingPacks) {
915
1437
  if (pack.ws !== ws)
@@ -926,6 +1448,17 @@ export function createSidecarRouter(config) {
926
1448
  pendingUndeploys.delete(addr);
927
1449
  req.reject(`Sidecar ${conn.sidecarId} disconnected`);
928
1450
  }
1451
+ // Reject any in-flight probes sent to this sidecar. A probe never enters
1452
+ // the address maps, so this ws-keyed sweep is its ONLY disconnect cleanup:
1453
+ // without it a probe whose sidecar drops mid-flight would hang until its
1454
+ // own timeout instead of failing fast on the disconnect.
1455
+ for (const [requestId, probe] of pendingProbes) {
1456
+ if (probe.ws !== ws)
1457
+ continue;
1458
+ clearTimeout(probe.timer);
1459
+ pendingProbes.delete(requestId);
1460
+ probe.reject(`Sidecar ${conn.sidecarId} disconnected`);
1461
+ }
929
1462
  // Cancel any in-flight inbound pack transfers from this sidecar
930
1463
  // across both receivers. The two receivers track their own in-
931
1464
  // flight transferIds, so a pending workflow-run transfer for an
@@ -944,6 +1477,7 @@ export function createSidecarRouter(config) {
944
1477
  }
945
1478
  events.emit("sidecar.disconnect", {
946
1479
  ownedAddresses: [...owned],
1480
+ ...(allocated !== undefined ? { allocated } : {}),
947
1481
  });
948
1482
  logger.info `Sidecar ${conn.sidecarId} disconnected`;
949
1483
  }
@@ -994,40 +1528,78 @@ export function createSidecarRouter(config) {
994
1528
  pending.delete(requestId);
995
1529
  req.reject(error);
996
1530
  }
997
- function resolvePackPending(transferId) {
998
- const entry = pendingPacks.get(transferId);
1531
+ function packResponseMatches(entry, ws, frame) {
1532
+ return (entry.ws === ws &&
1533
+ entry.agentAddress === frame.agentAddress &&
1534
+ entry.repoId.kind === frame.repoId.kind &&
1535
+ entry.repoId.id === frame.repoId.id);
1536
+ }
1537
+ function resolvePackPending(ws, frame) {
1538
+ const entry = pendingPacks.get(frame.transferId);
999
1539
  if (entry === undefined)
1000
1540
  return;
1541
+ if (!packResponseMatches(entry, ws, frame)) {
1542
+ logger.warn `Ignoring repo.pack.ack for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
1543
+ return;
1544
+ }
1001
1545
  clearTimeout(entry.timer);
1002
- pendingPacks.delete(transferId);
1546
+ pendingPacks.delete(frame.transferId);
1003
1547
  entry.resolve();
1004
1548
  }
1005
- function rejectPackPending(transferId, reason) {
1006
- const entry = pendingPacks.get(transferId);
1549
+ function rejectPackPending(ws, frame) {
1550
+ const entry = pendingPacks.get(frame.transferId);
1007
1551
  if (entry === undefined)
1008
1552
  return;
1553
+ if (!packResponseMatches(entry, ws, frame)) {
1554
+ logger.warn `Ignoring repo.pack.reject for transfer ${frame.transferId} from a connection that does not own the pending transfer`;
1555
+ return;
1556
+ }
1009
1557
  clearTimeout(entry.timer);
1010
- pendingPacks.delete(transferId);
1011
- entry.reject(reason);
1558
+ pendingPacks.delete(frame.transferId);
1559
+ // Surface the receiver's specific cause when it carried one, so the awaiting
1560
+ // push sees "corrupt: <detail>" rather than only the coarse reason.
1561
+ entry.reject(frame.detail !== undefined
1562
+ ? `${frame.reason}: ${frame.detail}`
1563
+ : frame.reason);
1012
1564
  }
1013
- function resolveUndeployPending(agentAddress) {
1565
+ function resolveUndeployPending(ws, agentAddress) {
1014
1566
  const req = pendingUndeploys.get(agentAddress);
1015
1567
  if (req === undefined) {
1016
1568
  logger.warn `Received agent.undeploy.ack for "${agentAddress}" with no pending undeploy`;
1017
1569
  return;
1018
1570
  }
1571
+ if (req.ws !== ws)
1572
+ return;
1019
1573
  clearTimeout(req.timer);
1020
1574
  pendingUndeploys.delete(agentAddress);
1021
1575
  req.resolve();
1022
1576
  }
1023
- function rejectUndeployPending(agentAddress, error) {
1577
+ function rejectUndeployPending(ws, agentAddress, error) {
1024
1578
  const req = pendingUndeploys.get(agentAddress);
1025
1579
  if (req === undefined)
1026
1580
  return;
1581
+ if (req.ws !== ws)
1582
+ return;
1027
1583
  clearTimeout(req.timer);
1028
1584
  pendingUndeploys.delete(agentAddress);
1029
1585
  req.reject(error);
1030
1586
  }
1587
+ function resolveProbe(requestId, result) {
1588
+ const req = pendingProbes.get(requestId);
1589
+ if (req === undefined)
1590
+ return;
1591
+ clearTimeout(req.timer);
1592
+ pendingProbes.delete(requestId);
1593
+ req.resolve(result);
1594
+ }
1595
+ function rejectProbe(requestId, error) {
1596
+ const req = pendingProbes.get(requestId);
1597
+ if (req === undefined)
1598
+ return;
1599
+ clearTimeout(req.timer);
1600
+ pendingProbes.delete(requestId);
1601
+ req.reject(error);
1602
+ }
1031
1603
  // Routing rule: pick the receiver dedicated to the repoId.kind the
1032
1604
  // frame carries. The receivers' in-flight state is independent, so a
1033
1605
  // workflow-run transferId can never collide with or evict an
@@ -1048,6 +1620,8 @@ export function createSidecarRouter(config) {
1048
1620
  function pickReceivePackLookup(repoId) {
1049
1621
  switch (repoId.kind) {
1050
1622
  case "agent-state":
1623
+ // The agent-state lookup ignores the `source` argument the workflow-run
1624
+ // lookup takes; the two are otherwise the same contract.
1051
1625
  return lookups.receiveAgentStatePack;
1052
1626
  case "workflow-run":
1053
1627
  return lookups.receiveWorkflowRunPack;
@@ -1063,6 +1637,17 @@ export function createSidecarRouter(config) {
1063
1637
  logger.warn `Received repo.pack.push for unrouted agent ${frame.agentAddress}`;
1064
1638
  return;
1065
1639
  }
1640
+ if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
1641
+ logger.warn `Rejected repo.pack.push outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
1642
+ conn.send({
1643
+ type: "repo.pack.reject",
1644
+ agentAddress: frame.agentAddress,
1645
+ repoId: frame.repoId,
1646
+ transferId: frame.transferId,
1647
+ reason: "path_violation",
1648
+ });
1649
+ return;
1650
+ }
1066
1651
  const picked = pickPackReceiver(frame.repoId);
1067
1652
  if (picked === null) {
1068
1653
  logger.warn `Received repo.pack.push with unsupported repoId.kind ${frame.repoId.kind}`;
@@ -1094,6 +1679,17 @@ export function createSidecarRouter(config) {
1094
1679
  logger.warn `Received repo.pack.done for unrouted agent ${frame.agentAddress}`;
1095
1680
  return;
1096
1681
  }
1682
+ if (!connCanPushRepo(conn, frame.agentAddress, frame.repoId)) {
1683
+ logger.warn `Rejected repo.pack.done outside sidecar ${conn.sidecarId}'s authenticated repository scope`;
1684
+ conn.send({
1685
+ type: "repo.pack.reject",
1686
+ agentAddress: frame.agentAddress,
1687
+ repoId: frame.repoId,
1688
+ transferId: frame.transferId,
1689
+ reason: "path_violation",
1690
+ });
1691
+ return;
1692
+ }
1097
1693
  const picked = pickPackReceiver(frame.repoId);
1098
1694
  if (picked === null) {
1099
1695
  logger.warn `Received repo.pack.done with unsupported repoId.kind ${frame.repoId.kind}`;
@@ -1127,7 +1723,15 @@ export function createSidecarRouter(config) {
1127
1723
  });
1128
1724
  return;
1129
1725
  }
1130
- const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha);
1726
+ const verdict = await receivePackLookup(frame.repoId, result.pack, result.ref, result.commitSha, conn.identity.kind === "allocated"
1727
+ ? {
1728
+ kind: "allocated",
1729
+ agentAddress: frame.agentAddress,
1730
+ allocationId: conn.identity.allocationId,
1731
+ anchorRunId: conn.identity.anchorRunId,
1732
+ generation: conn.identity.generation,
1733
+ }
1734
+ : { kind: "shared", agentAddress: frame.agentAddress });
1131
1735
  // Connection may have closed during async verification.
1132
1736
  const currentConn = connections.get(ws);
1133
1737
  if (currentConn === undefined)
@@ -1164,6 +1768,110 @@ export function createSidecarRouter(config) {
1164
1768
  * never persisted into the reconnect set and never resurrected on
1165
1769
  * reconnect.
1166
1770
  */
1771
+ function fenceAllocation(allocationId, generation) {
1772
+ const existing = allocationFences.get(allocationId);
1773
+ if (existing !== undefined && generation < existing) {
1774
+ throw new Error(`Cannot move allocation ${allocationId} fence backward from ${String(existing)} to ${String(generation)}`);
1775
+ }
1776
+ allocationFences.set(allocationId, generation);
1777
+ const current = allocatedConnections.get(allocationId);
1778
+ if (current !== undefined && current.identity.generation !== generation) {
1779
+ handleClose(current.ws);
1780
+ current.ws.close();
1781
+ }
1782
+ const waiters = allocationWaiters.get(allocationId);
1783
+ if (waiters === undefined)
1784
+ return;
1785
+ for (const waiter of [...waiters]) {
1786
+ if (waiter.generation === generation)
1787
+ continue;
1788
+ clearTimeout(waiter.timer);
1789
+ waiters.delete(waiter);
1790
+ waiter.reject(new Error(`Allocation ${allocationId} advanced to generation ${String(generation)}`));
1791
+ }
1792
+ if (waiters.size === 0)
1793
+ allocationWaiters.delete(allocationId);
1794
+ }
1795
+ async function getAllocatedConnection(target, use) {
1796
+ if (allocationFences.get(target.allocationId) !== target.generation) {
1797
+ throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
1798
+ }
1799
+ const current = allocatedConnections.get(target.allocationId);
1800
+ if (current === undefined ||
1801
+ current.identity.generation !== target.generation) {
1802
+ throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId} generation ${String(target.generation)}`);
1803
+ }
1804
+ if (!(await validateSidecarIdentity(current.identity, use))) {
1805
+ if (allocatedConnections.get(target.allocationId) === current) {
1806
+ handleClose(current.ws);
1807
+ current.ws.close();
1808
+ }
1809
+ throw new Error(`Allocated sidecar identity is no longer current for allocation ${target.allocationId}`);
1810
+ }
1811
+ if (allocatedConnections.get(target.allocationId) !== current) {
1812
+ throw new Error(`Allocated sidecar connection changed for allocation ${target.allocationId}`);
1813
+ }
1814
+ const conn = connections.get(current.ws);
1815
+ if (conn === undefined ||
1816
+ conn.identity.kind !== "allocated" ||
1817
+ conn.identity.allocationId !== target.allocationId ||
1818
+ conn.identity.generation !== target.generation) {
1819
+ throw new Error(`Allocated sidecar is not connected for allocation ${target.allocationId}`);
1820
+ }
1821
+ return { ws: current.ws, conn };
1822
+ }
1823
+ async function isAllocatedSidecarReady(target) {
1824
+ try {
1825
+ await getAllocatedConnection(target, "readiness");
1826
+ return true;
1827
+ }
1828
+ catch {
1829
+ return false;
1830
+ }
1831
+ }
1832
+ async function isAllocatedWorkflowActive(target) {
1833
+ try {
1834
+ const { conn } = await getAllocatedConnection(target, "readiness");
1835
+ if (conn.identity.kind !== "allocated")
1836
+ return false;
1837
+ return conn.workflowAddresses.has(conn.identity.workflowRunAddress);
1838
+ }
1839
+ catch {
1840
+ return false;
1841
+ }
1842
+ }
1843
+ async function waitForAllocatedSidecar(target, timeoutMs) {
1844
+ if (await isAllocatedSidecarReady(target))
1845
+ return;
1846
+ if (allocationFences.get(target.allocationId) !== target.generation) {
1847
+ throw new Error(`Allocation ${target.allocationId} generation ${String(target.generation)} is not current`);
1848
+ }
1849
+ if (timeoutMs <= 0) {
1850
+ throw new Error(`Timed out waiting for allocated sidecar ${target.allocationId}`);
1851
+ }
1852
+ await new Promise((resolve, reject) => {
1853
+ const waiter = {
1854
+ generation: target.generation,
1855
+ resolve,
1856
+ reject,
1857
+ timer: setTimeout(() => {
1858
+ const current = allocationWaiters.get(target.allocationId);
1859
+ current?.delete(waiter);
1860
+ if (current?.size === 0) {
1861
+ allocationWaiters.delete(target.allocationId);
1862
+ }
1863
+ reject(new Error(`Timed out waiting for allocated sidecar ${target.allocationId} generation ${String(target.generation)}`));
1864
+ }, timeoutMs),
1865
+ };
1866
+ let waiters = allocationWaiters.get(target.allocationId);
1867
+ if (waiters === undefined) {
1868
+ waiters = new Set();
1869
+ allocationWaiters.set(target.allocationId, waiters);
1870
+ }
1871
+ waiters.add(waiter);
1872
+ void notifyAllocationWaiters(target.allocationId);
1873
+ });
1874
+ }
1167
1875
  function bindStepRoute(stepAddress) {
1168
1876
  const ws = addressIndex.get(stepAddress) ?? findSidecarForNewAgent(stepAddress);
1169
1877
  if (ws === undefined) {
@@ -1173,6 +1881,18 @@ export function createSidecarRouter(config) {
1173
1881
  if (conn === undefined) {
1174
1882
  throw new Error(`No sidecar connected to stage workflow step "${stepAddress}"`);
1175
1883
  }
1884
+ if (conn.identity.kind !== "shared") {
1885
+ throw new Error(`Allocated sidecar ${conn.sidecarId} cannot receive ordinary workflow step ${stepAddress}`);
1886
+ }
1887
+ conn.workflowAddresses.add(stepAddress);
1888
+ addressIndex.set(stepAddress, ws);
1889
+ }
1890
+ async function bindAllocatedStepRoute(target, stepAddress) {
1891
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
1892
+ const existing = addressIndex.get(stepAddress);
1893
+ if (existing !== undefined && existing !== ws) {
1894
+ throw new Error(`Workflow step ${stepAddress} is already routed to another sidecar`);
1895
+ }
1176
1896
  conn.workflowAddresses.add(stepAddress);
1177
1897
  addressIndex.set(stepAddress, ws);
1178
1898
  }
@@ -1191,6 +1911,17 @@ export function createSidecarRouter(config) {
1191
1911
  }
1192
1912
  addressIndex.delete(stepAddress);
1193
1913
  }
1914
+ function unbindAllocatedStepRoute(target, stepAddress) {
1915
+ const current = allocatedConnections.get(target.allocationId);
1916
+ if (current === undefined ||
1917
+ current.identity.generation !== target.generation) {
1918
+ return;
1919
+ }
1920
+ if (addressIndex.get(stepAddress) !== current.ws)
1921
+ return;
1922
+ connections.get(current.ws)?.workflowAddresses.delete(stepAddress);
1923
+ addressIndex.delete(stepAddress);
1924
+ }
1194
1925
  // Pack transfers may take longer than session requests due to data volume.
1195
1926
  const PACK_TIMEOUT_MS = requestTimeoutMs * 4;
1196
1927
  function sendPack(agentAddress, pack, ref, commitSha, options) {
@@ -1202,6 +1933,12 @@ export function createSidecarRouter(config) {
1202
1933
  if (conn === undefined) {
1203
1934
  return Promise.reject(new Error(`No sidecar connected for agent "${agentAddress}"`));
1204
1935
  }
1936
+ if (conn.identity.kind !== "shared") {
1937
+ return Promise.reject(new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound pack routing for "${agentAddress}"`));
1938
+ }
1939
+ return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
1940
+ }
1941
+ function sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options) {
1205
1942
  const transferId = `pack-${++packCounter}`;
1206
1943
  // For the agent-state flow the destination agent and the source repo
1207
1944
  // are the same entity, so `repoId.id === agentAddress`. Asset packs
@@ -1222,6 +1959,8 @@ export function createSidecarRouter(config) {
1222
1959
  pendingPacks.set(transferId, {
1223
1960
  transferId,
1224
1961
  ws,
1962
+ agentAddress,
1963
+ repoId,
1225
1964
  resolve,
1226
1965
  reject(error) {
1227
1966
  reject(new Error(`Pack rejected: ${error}`));
@@ -1251,65 +1990,167 @@ export function createSidecarRouter(config) {
1251
1990
  });
1252
1991
  });
1253
1992
  }
1254
- function routeMail(agentAddress, rawMessage) {
1993
+ async function sendPackToAllocation(target, agentAddress, pack, ref, commitSha, options) {
1994
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
1995
+ if (addressIndex.get(agentAddress) !== ws) {
1996
+ throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
1997
+ }
1998
+ return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, options);
1999
+ }
2000
+ async function sendWorkflowRunPackToAllocation(target, agentAddress, pack, ref, commitSha) {
2001
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2002
+ if (conn.identity.kind !== "allocated") {
2003
+ throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
2004
+ }
2005
+ if (agentAddress !== conn.identity.workflowRunAddress) {
2006
+ throw new Error(`Allocation ${target.allocationId} cannot restore unrelated address ${agentAddress}`);
2007
+ }
2008
+ if (conn.workflowAddresses.has(agentAddress)) {
2009
+ throw new Error(`Allocation ${target.allocationId} already hosts active workflow ${agentAddress}; refusing to overwrite its run history`);
2010
+ }
2011
+ return sendPackOnConnection(ws, conn, agentAddress, pack, ref, commitSha, {
2012
+ repoId: {
2013
+ kind: "workflow-run",
2014
+ id: deriveWorkflowRunRepoId(agentAddress),
2015
+ },
2016
+ });
2017
+ }
2018
+ function routeMail(agentAddress, rawMessage, messageId, runGrants) {
2019
+ // Carry the hub-minted messageId on the frame so the sidecar's durable-
2020
+ // receipt ack (`mail.inbound.ack`) keys on the same id the hub tracks, and
2021
+ // a redelivery replays identical bytes for the downstream RunStarted dedup.
2022
+ // Optional: the workflow-trigger and session-conversation callers supply
2023
+ // it (they participate in the ack/retry handshake); a caller without a
2024
+ // hub-minted id omits it and the delivery is not tracked for redelivery.
2025
+ const frame = {
2026
+ type: "mail.inbound",
2027
+ agentAddress,
2028
+ rawMessage,
2029
+ ...(messageId !== undefined ? { messageId } : {}),
2030
+ };
1255
2031
  const ws = addressIndex.get(agentAddress);
1256
2032
  if (ws !== undefined) {
1257
2033
  const conn = connections.get(ws);
1258
2034
  if (conn !== undefined) {
1259
- conn.send({
1260
- type: "mail.inbound",
1261
- agentAddress,
1262
- rawMessage,
1263
- });
2035
+ conn.send(frame);
2036
+ // Track the delivery for redelivery until the sidecar acks its durable
2037
+ // inbox write. Only mail carrying a hub-minted messageId participates
2038
+ // in the ack handshake; relayed agent-to-agent mail omits it and is
2039
+ // delivered fire-and-forget as before. A mail that triggered a workflow
2040
+ // run carries the run's grants so redelivery can replay them ahead of
2041
+ // the mail.
2042
+ if (messageId !== undefined) {
2043
+ trackPendingMail(agentAddress, messageId, frame, runGrants);
2044
+ }
1264
2045
  return true;
1265
2046
  }
1266
2047
  }
1267
2048
  // If the agent recently disconnected, queue for delivery on reconnect.
1268
- const frame = { type: "mail.inbound", agentAddress, rawMessage };
1269
2049
  return enqueueForDisconnected(agentAddress, frame);
1270
2050
  }
1271
- async function handleDeployAck(agentAddress, publicKey) {
1272
- if (!pendingDeploys.has(agentAddress)) {
1273
- logger.warn `Received agent.deploy.ack for "${agentAddress}" with no pending deploy`;
2051
+ function sendRunGrants(agentAddress, runId, stepGrants) {
2052
+ const frame = {
2053
+ type: "run.grants",
2054
+ agentAddress,
2055
+ runId,
2056
+ stepGrants,
2057
+ };
2058
+ const ws = addressIndex.get(agentAddress);
2059
+ if (ws !== undefined) {
2060
+ const conn = connections.get(ws);
2061
+ if (conn !== undefined) {
2062
+ conn.send(frame);
2063
+ return true;
2064
+ }
2065
+ }
2066
+ // Mirror routeMail: if the address has a live disconnect queue, ride it
2067
+ // so a run.grants issued in the window between deploy and the first
2068
+ // reconnect survives the same way the dispatching trigger mail does.
2069
+ // A queue exists only while the deployment address is still on
2070
+ // agentAddresses (pre-first-reconnect); after a challenged reconnect it
2071
+ // moves to workflowAddresses, which handleClose leaves unqueued because
2072
+ // that generation's in-flight run state is reconstructed sidecar-locally.
2073
+ // Returning without enqueueing there is correct; enqueueing is what keeps
2074
+ // grants and mail from diverging in the pre-reconnect window.
2075
+ return enqueueForDisconnected(agentAddress, frame);
2076
+ }
2077
+ async function sendWorkflowRunDispatchToAllocation(target, agentAddress, runId, stepGrants, rawMessage, messageId) {
2078
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2079
+ if (addressIndex.get(agentAddress) !== ws) {
2080
+ throw new Error(`Address ${agentAddress} is not routed on allocation ${target.allocationId}`);
2081
+ }
2082
+ const runGrants = { runId, stepGrants };
2083
+ conn.send({
2084
+ type: "run.grants",
2085
+ agentAddress,
2086
+ runId,
2087
+ stepGrants,
2088
+ });
2089
+ const frame = {
2090
+ type: "mail.inbound",
2091
+ agentAddress,
2092
+ rawMessage,
2093
+ messageId,
2094
+ };
2095
+ conn.send(frame);
2096
+ trackPendingMail(agentAddress, messageId, frame, runGrants, target);
2097
+ }
2098
+ async function handleDeployAck(ws, frame) {
2099
+ const req = pendingDeploys.get(frame.agentAddress);
2100
+ if (req === undefined) {
2101
+ logger.warn `Received agent.deploy.ack for "${frame.agentAddress}" with no pending deploy`;
1274
2102
  return;
1275
2103
  }
2104
+ if (req.ws !== ws)
2105
+ return;
1276
2106
  if (events.listenerCount("agent.deploy.ack") > 0) {
1277
2107
  try {
2108
+ const identity = connections.get(ws)?.identity;
1278
2109
  await events.emitAndAwait("agent.deploy.ack", {
1279
- agentAddress,
1280
- publicKey,
2110
+ agentAddress: frame.agentAddress,
2111
+ publicKey: frame.publicKey,
2112
+ ...(identity?.kind === "allocated"
2113
+ ? {
2114
+ allocated: {
2115
+ allocationId: identity.allocationId,
2116
+ anchorRunId: identity.anchorRunId,
2117
+ generation: identity.generation,
2118
+ },
2119
+ }
2120
+ : {}),
1281
2121
  });
1282
2122
  }
1283
2123
  catch (err) {
1284
- rejectDeployPending(agentAddress, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
2124
+ rejectDeployPending(req, `Failed to store public key: ${err instanceof Error ? err.message : String(err)}`);
1285
2125
  return;
1286
2126
  }
1287
2127
  }
1288
- resolveDeployPending(agentAddress);
2128
+ resolveDeployPending(req, frame.publicKey);
1289
2129
  }
1290
- function resolveDeployPending(agentAddress) {
1291
- const req = pendingDeploys.get(agentAddress);
1292
- if (req === undefined)
2130
+ function resolveDeployPending(req, publicKey) {
2131
+ if (pendingDeploys.get(req.agentAddress) !== req)
1293
2132
  return;
1294
2133
  clearTimeout(req.timer);
1295
- pendingDeploys.delete(agentAddress);
1296
- req.resolve();
2134
+ pendingDeploys.delete(req.agentAddress);
2135
+ req.resolve(publicKey);
1297
2136
  }
1298
- function rejectDeployPending(agentAddress, error) {
1299
- const req = pendingDeploys.get(agentAddress);
1300
- if (req === undefined)
2137
+ function rejectDeployPending(req, error) {
2138
+ if (pendingDeploys.get(req.agentAddress) !== req)
1301
2139
  return;
1302
2140
  clearTimeout(req.timer);
1303
- pendingDeploys.delete(agentAddress);
2141
+ pendingDeploys.delete(req.agentAddress);
1304
2142
  req.reject(error);
1305
2143
  }
2144
+ function rejectDeployPendingFromFrame(ws, agentAddress, error) {
2145
+ const req = pendingDeploys.get(agentAddress);
2146
+ if (req === undefined || req.ws !== ws)
2147
+ return;
2148
+ rejectDeployPending(req, error);
2149
+ }
1306
2150
  async function sendAgentDeploy(agentAddress, harnessConfig, workflow) {
1307
2151
  if (hubPublicKeyHex === undefined) {
1308
2152
  throw new Error("Hub signing key is required for agent deployment");
1309
2153
  }
1310
- if (pendingDeploys.has(agentAddress)) {
1311
- throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
1312
- }
1313
2154
  const ws = addressIndex.get(agentAddress) ?? findSidecarForNewAgent(agentAddress);
1314
2155
  if (ws === undefined) {
1315
2156
  throw new Error(`No sidecar available for agent "${agentAddress}"`);
@@ -1318,26 +2159,28 @@ export function createSidecarRouter(config) {
1318
2159
  if (conn === undefined) {
1319
2160
  throw new Error(`No sidecar connected for agent "${agentAddress}"`);
1320
2161
  }
1321
- conn.agentAddresses.add(agentAddress);
2162
+ if (conn.identity.kind !== "shared") {
2163
+ throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound deploy routing`);
2164
+ }
2165
+ return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
2166
+ }
2167
+ function sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow) {
2168
+ if (hubPublicKeyHex === undefined) {
2169
+ throw new Error("Hub signing key is required for agent deployment");
2170
+ }
2171
+ if (pendingDeploys.has(agentAddress)) {
2172
+ throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
2173
+ }
2174
+ const addressSet = conn.identity.kind === "allocated"
2175
+ ? conn.workflowAddresses
2176
+ : conn.agentAddresses;
2177
+ addressSet.add(agentAddress);
1322
2178
  addressIndex.set(agentAddress, ws);
1323
2179
  return new Promise((resolve, reject) => {
1324
- // The deploy-ack handler does not currently thread the
1325
- // sidecar-reported public key back through the pending-deploy
1326
- // resolver; it only fires `agent.deploy.ack` listeners and then
1327
- // resolves the pending deploy. Capture the key via a one-shot
1328
- // listener so the return value carries it without a wire-shape
1329
- // change to `agent.deploy.ack`.
1330
- let capturedPublicKey;
1331
- const detachListener = events.on("agent.deploy.ack", (payload) => {
1332
- if (payload.agentAddress === agentAddress) {
1333
- capturedPublicKey = payload.publicKey;
1334
- }
1335
- });
1336
2180
  const timer = setTimeout(() => {
1337
- detachListener();
1338
2181
  pendingDeploys.delete(agentAddress);
1339
2182
  if (addressIndex.get(agentAddress) === ws) {
1340
- conn.agentAddresses.delete(agentAddress);
2183
+ addressSet.delete(agentAddress);
1341
2184
  addressIndex.delete(agentAddress);
1342
2185
  }
1343
2186
  reject(new Error(`Deploy of "${agentAddress}" timed out after ${requestTimeoutMs}ms`));
@@ -1345,18 +2188,12 @@ export function createSidecarRouter(config) {
1345
2188
  pendingDeploys.set(agentAddress, {
1346
2189
  agentAddress,
1347
2190
  ws,
1348
- resolve() {
1349
- detachListener();
1350
- if (capturedPublicKey === undefined) {
1351
- reject(new Error(`Deploy of "${agentAddress}" resolved without an agent.deploy.ack publicKey payload`));
1352
- return;
1353
- }
1354
- resolve({ publicKey: capturedPublicKey });
2191
+ resolve(publicKey) {
2192
+ resolve({ publicKey });
1355
2193
  },
1356
2194
  reject(error) {
1357
- detachListener();
1358
2195
  if (addressIndex.get(agentAddress) === ws) {
1359
- conn.agentAddresses.delete(agentAddress);
2196
+ addressSet.delete(agentAddress);
1360
2197
  addressIndex.delete(agentAddress);
1361
2198
  }
1362
2199
  reject(new Error(error));
@@ -1373,6 +2210,20 @@ export function createSidecarRouter(config) {
1373
2210
  });
1374
2211
  });
1375
2212
  }
2213
+ async function sendAgentDeployToAllocation(target, agentAddress, harnessConfig, workflow) {
2214
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2215
+ if (conn.identity.kind !== "allocated") {
2216
+ throw new Error(`Allocation ${target.allocationId} resolved to a shared sidecar`);
2217
+ }
2218
+ if (agentAddress !== conn.identity.workflowRunAddress) {
2219
+ throw new Error(`Allocation ${target.allocationId} cannot deploy unrelated address ${agentAddress}`);
2220
+ }
2221
+ const existing = addressIndex.get(agentAddress);
2222
+ if (existing !== undefined && existing !== ws) {
2223
+ throw new Error(`Deployment ${agentAddress} is already routed to another sidecar`);
2224
+ }
2225
+ return sendAgentDeployOnConnection(ws, conn, agentAddress, harnessConfig, workflow);
2226
+ }
1376
2227
  /**
1377
2228
  * Provision one step of a multi-step deploy on the sidecar WITHOUT
1378
2229
  * spawning: the sidecar initializes the step's agent-state repo and
@@ -1388,12 +2239,6 @@ export function createSidecarRouter(config) {
1388
2239
  * the caller owns tearing the route down via `unbindStepRoute`.
1389
2240
  */
1390
2241
  function sendProvisionStep(agentAddress, harnessConfig) {
1391
- if (hubPublicKeyHex === undefined) {
1392
- throw new Error("Hub signing key is required for step provisioning");
1393
- }
1394
- if (pendingDeploys.has(agentAddress)) {
1395
- throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
1396
- }
1397
2242
  const ws = addressIndex.get(agentAddress);
1398
2243
  if (ws === undefined) {
1399
2244
  throw new Error(`Step route for "${agentAddress}" is not bound; call bindStepRoute before provisioning`);
@@ -1402,6 +2247,18 @@ export function createSidecarRouter(config) {
1402
2247
  if (conn === undefined) {
1403
2248
  throw new Error(`No sidecar connected for agent "${agentAddress}"`);
1404
2249
  }
2250
+ if (conn.identity.kind !== "shared") {
2251
+ throw new Error(`Allocated sidecar ${conn.sidecarId} requires allocation-bound step provisioning`);
2252
+ }
2253
+ return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
2254
+ }
2255
+ function sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig) {
2256
+ if (hubPublicKeyHex === undefined) {
2257
+ throw new Error("Hub signing key is required for step provisioning");
2258
+ }
2259
+ if (pendingDeploys.has(agentAddress)) {
2260
+ throw new Error(`Deploy already in progress for agent "${agentAddress}"`);
2261
+ }
1405
2262
  const hubKey = hubPublicKeyHex;
1406
2263
  return new Promise((resolve, reject) => {
1407
2264
  const timer = setTimeout(() => {
@@ -1415,7 +2272,7 @@ export function createSidecarRouter(config) {
1415
2272
  pendingDeploys.set(agentAddress, {
1416
2273
  agentAddress,
1417
2274
  ws,
1418
- resolve() {
2275
+ resolve(_publicKey) {
1419
2276
  resolve();
1420
2277
  },
1421
2278
  reject(error) {
@@ -1433,11 +2290,58 @@ export function createSidecarRouter(config) {
1433
2290
  });
1434
2291
  });
1435
2292
  }
2293
+ async function sendProvisionStepToAllocation(target, agentAddress, harnessConfig) {
2294
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2295
+ if (addressIndex.get(agentAddress) !== ws) {
2296
+ throw new Error(`Step route ${agentAddress} is not bound to allocation ${target.allocationId}`);
2297
+ }
2298
+ return sendProvisionStepOnConnection(ws, conn, agentAddress, harnessConfig);
2299
+ }
1436
2300
  function findSidecarForNewAgent(_agentAddress) {
1437
- const first = connections.entries().next();
1438
- if (first.done)
1439
- return undefined;
1440
- return first.value[0];
2301
+ for (const [ws, conn] of connections) {
2302
+ if (conn.identity.kind === "shared")
2303
+ return ws;
2304
+ }
2305
+ return undefined;
2306
+ }
2307
+ function sendProbe(args) {
2308
+ // Select any connected sidecar, exactly as the pre-deploy window does.
2309
+ // A probe runs in the sidecar's pre-deploy state, so it targets a
2310
+ // connection, not an address -- `findSidecarForNewAgent` ignores its
2311
+ // argument and returns the first connection. Throw immediately on an empty
2312
+ // registry rather than register a probe that could only time out.
2313
+ const ws = findSidecarForNewAgent("");
2314
+ if (ws === undefined) {
2315
+ return Promise.reject(new Error("No sidecar available to probe workflow"));
2316
+ }
2317
+ const conn = connections.get(ws);
2318
+ if (conn === undefined) {
2319
+ return Promise.reject(new Error("No sidecar connected to probe workflow"));
2320
+ }
2321
+ const requestId = nextRequestId();
2322
+ return new Promise((resolve, reject) => {
2323
+ const timer = setTimeout(() => {
2324
+ pendingProbes.delete(requestId);
2325
+ reject(new Error(`Probe ${requestId} timed out after ${probeTimeoutMs}ms`));
2326
+ }, probeTimeoutMs);
2327
+ pendingProbes.set(requestId, {
2328
+ requestId,
2329
+ ws,
2330
+ resolve,
2331
+ reject(error) {
2332
+ reject(new Error(error));
2333
+ },
2334
+ timer,
2335
+ });
2336
+ conn.send({
2337
+ type: "workflow.probe.request",
2338
+ requestId,
2339
+ source: args.source,
2340
+ closure: args.closure,
2341
+ entry: args.entry,
2342
+ ...(args.assets !== undefined ? { assets: args.assets } : {}),
2343
+ });
2344
+ });
1441
2345
  }
1442
2346
  function sendAgentUndeploy(agentAddress, reason) {
1443
2347
  const ws = addressIndex.get(agentAddress);
@@ -1529,6 +2433,14 @@ export function createSidecarRouter(config) {
1529
2433
  defaultSource,
1530
2434
  }));
1531
2435
  }
2436
+ async function sendCredentialsUpdate(agentAddress, delivery) {
2437
+ await sendRequest(agentAddress, (requestId) => ({
2438
+ type: "credentials.update",
2439
+ requestId,
2440
+ agentAddress,
2441
+ delivery,
2442
+ }));
2443
+ }
1532
2444
  function sendSyncRequest(agentAddress) {
1533
2445
  const ws = addressIndex.get(agentAddress);
1534
2446
  if (ws === undefined) {
@@ -1563,6 +2475,13 @@ export function createSidecarRouter(config) {
1563
2475
  payload: opts.payload,
1564
2476
  });
1565
2477
  }
2478
+ async function sendSignalDeliverToAllocation(target, opts) {
2479
+ const { ws, conn } = await getAllocatedConnection(target, "routing");
2480
+ if (addressIndex.get(opts.agentAddress) !== ws) {
2481
+ throw new Error(`Address ${opts.agentAddress} is not routed on allocation ${target.allocationId}`);
2482
+ }
2483
+ conn.send({ type: "signal.deliver", ...opts });
2484
+ }
1566
2485
  function sendDrain(opts) {
1567
2486
  const ws = addressIndex.get(opts.agentAddress);
1568
2487
  if (ws === undefined) {
@@ -1583,15 +2502,30 @@ export function createSidecarRouter(config) {
1583
2502
  handleMessage,
1584
2503
  handleClose,
1585
2504
  routeMail,
2505
+ sendRunGrants,
1586
2506
  sendAgentDeploy,
2507
+ sendProbe,
1587
2508
  sendAgentUndeploy,
1588
2509
  sendSourcesUpdate,
2510
+ sendCredentialsUpdate,
1589
2511
  sendPack,
2512
+ sendPackToAllocation,
2513
+ sendWorkflowRunPackToAllocation,
2514
+ fenceAllocation,
2515
+ waitForAllocatedSidecar,
2516
+ isAllocatedSidecarReady,
2517
+ isAllocatedWorkflowActive,
2518
+ sendAgentDeployToAllocation,
1590
2519
  bindStepRoute,
2520
+ bindAllocatedStepRoute,
1591
2521
  unbindStepRoute,
2522
+ unbindAllocatedStepRoute,
1592
2523
  sendProvisionStep,
2524
+ sendProvisionStepToAllocation,
2525
+ sendWorkflowRunDispatchToAllocation,
1593
2526
  sendSyncRequest,
1594
2527
  sendSignalDeliver,
2528
+ sendSignalDeliverToAllocation,
1595
2529
  sendDrain,
1596
2530
  subscribeAgent,
1597
2531
  dispatchAgentEvent: dispatchToSubscribers,