@bastani/atomic 0.9.39 → 0.9.40-alpha.2

package/dist/builtin/intercom/broker/broker.ts

@@ -3,9 +3,18 @@
 import "./bounded-stderr-install.js";
 import net from "net";
 import { chmodSync, writeFileSync, unlinkSync, mkdirSync, readFileSync, statSync } from "fs";
+import { dirname } from "path";
 import { randomUUID } from "crypto";
 import { createMessageReader, type JsonWireValue } from "./framing.js";
-import { writeMessageIfOpen, writeMessageWithOutcome } from "./socket-writes.js";
+import {
+  discardHeldWrites,
+  flushHeldWrites,
+  type HeldWrite,
+  holdWrites,
+  releaseHeldWrites,
+  writeMessageIfOpen,
+  writeMessageWithOutcome,
+} from "./socket-writes.js";
 import { isSocketAnswering } from "./socket-liveness.js";
 import { acquireSocketReplacementLock } from "./socket-replacement-lock.js";
 import {
@@ -39,9 +48,24 @@ import {
 	handleBrokerSend,
 	type BrokerConnectedSession,
 	type PendingStageRoute,
+	type RemoteSendRequest,
+	type RemoteSendRouter,
+	type RestoringRecipientHold,
+	type SendClientMessage,
 	senderGroupIdentity,
 } from "./send-handler.js";
+import {
+  type BrokerStateOpener,
+  LOCAL_RETRY_LOOKUP_BOUND_MS,
+  MultiNodeRouter,
+  readSharedStorageSetting,
+  resolveMultiNodeStartup,
+  SHARED_STORAGE_UNAVAILABLE_REASON,
+} from "./multinode-router.js";
+import { OUTBOX_BACKUP_POLL_INTERVAL_MS } from "./multinode-state-store.js";
 import { SupervisorChannelCache } from "./supervisor-channel.js";
+import { SessionIdReclaim } from "./session-id-reclaim.js";
+import { type HeldSend, RestoringRecipients } from "./restoring-recipients.js";
 import { hasGroup, normalizeGroup, normalizeGroups } from "../group.js";
 import { handleBrokerPresence } from "./presence-handler.js";
 import {
@@ -52,6 +76,7 @@ import {
 	setSessionGroups,
 } from "./group-membership.js";
 import { PendingQuestionIndex } from "./pending-question-index.js";
+import { type ReceiptSender, RecipientReceipts, UNCERTAIN_RECEIPT_REASON } from "./recipient-receipts.js";
 import { matchStagePathSegments } from "../workflow-stage-path-matching.js";
 import { DELIVERED_MESSAGE_TTL_MS } from "../retry-policy.js";
 import { isAgentRecipient } from "../recipient-purpose.js";
@@ -69,6 +94,7 @@ const PENDING_STAGE_MESSAGE_TIMEOUT_MS = 10_000;
 const LISTEN_ATTEMPTS = 5;
 const LISTEN_RETRY_DELAY_MS = 100;
 const SOCKET_OWNERSHIP_CHECK_MS = 1_000;
+const MULTI_NODE_CLOSE_BOUND_MS = 2_000;
 
 function exitBecauseLiveBrokerOwnsSocket(): never {
   console.error(`Intercom broker not started: a live broker already answers on ${SOCKET_PATH}`);
@@ -79,6 +105,8 @@ interface PendingStageRouteRegistration {
   readonly sessionId: string;
   readonly group: string;
   readonly capability: string;
+  /** Registered by a restarted owner for an interrupted run; yields to the process that resumes it (#3760). */
+  readonly dormant?: true;
 }
 
 interface WorkflowRosterRegistration {
@@ -173,6 +201,22 @@ function isSupervisorRegistration(value: unknown): value is SupervisorRegistrati
     && typeof registration.supervisorSessionId === "string";
 }
 
+const RESTORING_RECIPIENT_ALIASES_MAX = 4;
+const RESTORING_RECIPIENT_GONE_REASON_MAX = 1_000;
+const SESSION_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+function isRestoringRecipientAliasList(value: unknown): value is string[] {
+  return Array.isArray(value) && value.length <= RESTORING_RECIPIENT_ALIASES_MAX && value.every((alias) => typeof alias === "string");
+}
+
+/**
+ * A restoring subagent's extra targets are ids only: its Atomic session id and its pre-restart Intercom id (#3760).
+ * Its name comes from the supervisor capability, so no reservation can claim another child's name.
+ */
+function isRestoringRecipientAlias(alias: string): boolean {
+  return SESSION_ID_PATTERN.test(alias.trim());
+}
+
 function invocationOwnsGroup(invocationGroup: string, candidateGroup: string): boolean {
 	const owner = normalizeGroup(invocationGroup);
 	const candidate = normalizeGroup(candidateGroup);
@@ -269,13 +313,24 @@ class IntercomBroker {
 	getBrokerDeliveredMessagesPath(),
   );
   private supervisorChannel = new SupervisorChannelCache();
+  private sessionIdReclaim = new SessionIdReclaim();
   private pendingQuestions = new PendingQuestionIndex();
+  private receipts = new RecipientReceipts<ConnectedSession>();
+  private restoringRecipients = new RestoringRecipients((held, reason) => this.failHeldSend(held, reason));
   private pendingStageRoutes = new Map<string, PendingStageRouteRegistration>();
   private pendingStageAcknowledgments = new Map<string, PendingStageAcknowledgment>();
 	private pendingStageNotificationAcknowledgments = new Map<string, PendingStageNotificationAcknowledgment>();
   private liveWorkflowStageRoutes = new Map<string, LiveWorkflowStageRouteRegistration>();
   private workflowRosters = new Map<string, WorkflowRosterRegistration>();
   private liveWorkflowStageRouteActivations = new Map<string, LiveWorkflowStageRouteActivation>();
+  /** Multi-node S5: set only when the agent directory's settings select a top-level shared `storage`. */
+  private readonly sharedStorageSetting = readSharedStorageSetting(dirname(INTERCOM_DIR));
+  private readonly routeRemote: RemoteSendRouter | undefined =
+    this.sharedStorageSetting === undefined ? undefined : (request) => this.routeRemoteSend(request);
+  private multiNode: MultiNodeRouter | undefined;
+  private multiNodeStatus: "starting" | "on" | "off" | "unavailable" = "off";
+  private multiNodeFirstAttempt: Promise<void> = Promise.resolve();
+  private shuttingDown = false;
 
   constructor() {
     mkdirSync(INTERCOM_DIR, { recursive: true, mode: 0o700 });
@@ -285,6 +340,7 @@ class IntercomBroker {
 
   start(): void {
     this.listen(1);
+    if (this.sharedStorageSetting !== undefined) this.startMultiNode(this.sharedStorageSetting);
     process.on("SIGTERM", () => this.shutdown());
     process.on("SIGINT", () => this.shutdown());
   }
@@ -957,7 +1013,9 @@ class IntercomBroker {
 			pending.signature,
 			targetIdentity,
 		);
-		if (deliveredMatch !== "miss") {
+		// #3761: a recipient that confirms saves ignores a copy it already saved, so a reserved message is sent again.
+		const confirmsSaves = target.acknowledgesReceipt === true;
+		if (deliveredMatch !== "miss" && !(deliveredMatch === "uncertain" && confirmsSaves)) {
 			if (deliveredMatch === "match") {
 				if (pending.message.expectsReply === true) {
 					const acceptedQuestionTarget = this.deliveredMessages.lookupQuestionTarget(
@@ -1041,7 +1099,7 @@ class IntercomBroker {
 			});
 			return;
 		}
-		if (reservation !== "recorded") {
+		if (reservation !== "recorded" && !(reservation === "uncertain" && confirmsSaves)) {
 			writeMessageIfOpen(pending.senderSocket, {
 				type: "delivery_failed",
 				messageId: pending.messageId,
@@ -1058,49 +1116,81 @@ class IntercomBroker {
 		}
 		if (pending.message.expectsReply === true && pending.resolveReplyTarget === true && pending.attemptId !== undefined)
 			writeMessageIfOpen(pending.senderSocket, { type: "question_target", messageId: pending.messageId, attemptId: pending.attemptId, sessionId: target.info.id });
-		if (!writeMessageIfOpen(target.socket, { type: "message", from: pending.sender, message: pending.message })) {
-			this.deliveredMessages.forget(pending.messageId, pending.signature);
-			writeMessageIfOpen(pending.senderSocket, {
-				type: "delivery_failed",
-				messageId: pending.messageId,
-				...(pending.attemptId ? { attemptId: pending.attemptId } : {}),
-				reason: "Session not found",
-			});
-			return;
-		}
-		let accepted: ReturnType<DeliveredMessageCache["accept"]>;
-		try {
-			accepted = this.deliveredMessages.accept(pending.messageId, pending.signature, targetIdentity);
-		} catch {
-			writeMessageIfOpen(pending.senderSocket, {
-				type: "delivery_failed",
-				messageId: pending.messageId,
-				...(pending.attemptId ? { attemptId: pending.attemptId } : {}),
-				reason: "Intercom could not durably confirm the forwarded operation; refusing a possibly unsafe retry",
-			});
-			return;
+		const frame: BrokerMessage = { type: "message", from: pending.sender, message: pending.message };
+		const signature = pending.signature;
+		const finish = (recipient: ConnectedSession, sender: ReceiptSender | undefined): void => {
+			const notify = (message: BrokerMessage): void => {
+				if (sender !== undefined) writeMessageIfOpen(sender.socket, message);
+			};
+			const senderAttempt = sender?.attemptId === undefined ? {} : { attemptId: sender.attemptId };
+			let accepted: ReturnType<DeliveredMessageCache["accept"]>;
+			try {
+				accepted = this.deliveredMessages.accept(pending.messageId, signature, targetIdentity);
+			} catch {
+				notify({
+					type: "delivery_failed",
+					messageId: pending.messageId,
+					...senderAttempt,
+					reason: "Intercom could not durably confirm the forwarded operation; refusing a possibly unsafe retry",
+				});
+				return;
+			}
+			if (accepted !== "accepted") {
+				notify({
+					type: "delivery_failed",
+					messageId: pending.messageId,
+					...senderAttempt,
+					reason: "Intercom could not durably confirm the forwarded operation; refusing a possibly unsafe retry",
+					...(accepted === "conflict" ? { reasonCode: "message_id_conflict" as const } : {}),
+				});
+				return;
+			}
+			if (pending.message.expectsReply === true && sender !== undefined) {
+				this.pendingQuestions.record(from.info.id, recipient.info.id, pending.messageId);
+				// #3761: a recipient that reconnected before confirming answers under its new session id.
+				if (recipient.info.id !== target.info.id && pending.resolveReplyTarget === true && sender.attemptId !== undefined)
+					notify({ type: "question_target", messageId: pending.messageId, attemptId: sender.attemptId, sessionId: recipient.info.id });
+			}
+			if (pending.message.replyTo !== undefined && this.pendingQuestions.clearReply(from.info.id, recipient.info.id, pending.message.replyTo)) {
+				this.deliveredMessages.settleQuestion(pending.message.replyTo);
+			}
+			notify({ type: "delivered", messageId: pending.messageId, ...senderAttempt });
+		};
+		const sender: ReceiptSender = {
+			socket: pending.senderSocket,
+			...(pending.attemptId ? { attemptId: pending.attemptId } : {}),
+		};
+		if (confirmsSaves) {
+			this.receipts.await(
+				{
+					messageId: pending.messageId,
+					targetIdentity,
+					frame,
+					complete: finish,
+					expire: (waiting) => {
+						writeMessageIfOpen(waiting.socket, {
+							type: "delivery_failed",
+							messageId: pending.messageId,
+							...(waiting.attemptId === undefined ? {} : { attemptId: waiting.attemptId }),
+							reason: UNCERTAIN_RECEIPT_REASON,
+						});
+					},
+				},
+				sender,
+			);
 		}
-		if (accepted !== "accepted") {
+		if (!writeMessageIfOpen(target.socket, frame)) {
+			if (confirmsSaves) this.receipts.drop(pending.messageId);
+			if (reservation === "recorded") this.deliveredMessages.forget(pending.messageId, pending.signature);
 			writeMessageIfOpen(pending.senderSocket, {
 				type: "delivery_failed",
 				messageId: pending.messageId,
 				...(pending.attemptId ? { attemptId: pending.attemptId } : {}),
-				reason: "Intercom could not durably confirm the forwarded operation; refusing a possibly unsafe retry",
-				...(accepted === "conflict" ? { reasonCode: "message_id_conflict" as const } : {}),
+				reason: "Session not found",
 			});
 			return;
 		}
-		if (pending.message.expectsReply === true) {
-			this.pendingQuestions.record(from.info.id, target.info.id, pending.messageId);
-		}
-		if (pending.message.replyTo !== undefined && this.pendingQuestions.clearReply(from.info.id, target.info.id, pending.message.replyTo)) {
-			this.deliveredMessages.settleQuestion(pending.message.replyTo);
-		}
-		writeMessageIfOpen(pending.senderSocket, {
-			type: "delivered",
-			messageId: pending.messageId,
-			...(pending.attemptId ? { attemptId: pending.attemptId } : {}),
-		});
+		if (!confirmsSaves) finish(target, sender);
 	}
 
 
@@ -1405,6 +1495,12 @@ class IntercomBroker {
 			&& (typeof clientMessage.supervisorOwnerToken !== "string" || !clientMessage.supervisorOwnerToken)) {
 			throw new Error("Invalid supervisor owner token");
 		}
+		if (clientMessage.preferredSessionId !== undefined && typeof clientMessage.preferredSessionId !== "string") {
+			throw new Error("Invalid preferred session id");
+		}
+		if (clientMessage.acknowledgesReceipt !== undefined && clientMessage.acknowledgesReceipt !== true) {
+			throw new Error("Invalid receipt acknowledgement capability");
+		}
 
         if (currentId) {
           throw new Error("Received duplicate register message");
@@ -1424,7 +1520,12 @@ class IntercomBroker {
           supervisorId = claimedSupervisorId;
         }
 
-        const id = randomUUID();
+        // #3760: a restored or reconnected session keeps the id peers know when it is free; never a live one.
+        const id = this.sessionIdReclaim.grant(
+          clientMessage.preferredSessionId,
+          clientMessage.returnAddress,
+          (sessionId) => this.sessions.has(sessionId),
+        ) ?? randomUUID();
         setId(id);
         const initialGroups = normalizeGroups(clientMessage.session.groups, clientMessage.session.group);
         const legacyGroup = normalizeGroup(clientMessage.session.group ?? initialGroups.values().next().value);
@@ -1448,6 +1549,7 @@ class IntercomBroker {
           ...(typeof clientMessage.supervisorOwnerToken === "string"
             ? { supervisorOwnerToken: clientMessage.supervisorOwnerToken }
             : {}),
+          ...(clientMessage.acknowledgesReceipt === true ? { acknowledgesReceipt: true as const } : {}),
         });
 
         if (this.shutdownTimer) {
@@ -1455,10 +1557,63 @@ class IntercomBroker {
           this.shutdownTimer = null;
         }
 
-        writeMessageIfOpen(socket, supervisorId
-          ? { type: "registered", sessionId: id, supervisorSessionId: supervisorId }
-          : { type: "registered", sessionId: id });
-        if (isAgentRecipient(info)) this.broadcastToMemberships({ type: "session_joined", session: info }, sessionGroups(info), id);
+        const rejoined = this.sessions.get(id);
+        const consumedCapability =
+          supervisorId && isSupervisorRegistration(clientMessage.supervisor) ? clientMessage.supervisor.capability : undefined;
+        const completeRegistration = (held: readonly HeldWrite[] = []): void => {
+          // #3761: echoing the capability tells the client this broker understands `received`; an old broker never does.
+          writeMessageIfOpen(socket, {
+            type: "registered",
+            sessionId: id,
+            ...(supervisorId ? { supervisorSessionId: supervisorId } : {}),
+            ...(clientMessage.acknowledgesReceipt === true ? { acknowledgesReceipt: true as const } : {}),
+          });
+          if (isAgentRecipient(info)) this.broadcastToMemberships({ type: "session_joined", session: info }, sessionGroups(info), id);
+          // #3761: messages written to this recipient's previous connection that it never confirmed saving go out again.
+          if (rejoined?.acknowledgesReceipt === true) {
+            const heldFrames = new Set(held.map((write) => write.message));
+            for (const frame of this.receipts.framesFor(deliveryTargetIdentity(rejoined))) {
+              if (!heldFrames.has(frame)) writeMessageIfOpen(socket, frame);
+            }
+          }
+          // Multi-node S5: frames written while registration waited for the presence lease follow `registered`.
+          flushHeldWrites(socket, held);
+          // #3760: a restored subagent is back under the capability its owner reserved it with; the sends held
+          // for it while it was being restored go out now, once each and in the order they arrived.
+          if (consumedCapability !== undefined) {
+            const registered = info.name?.trim();
+            for (const send of this.restoringRecipients.consume(
+              consumedCapability,
+              registered ? { childName: registered, groups: sessionGroups(info) } : undefined,
+            ))
+              this.dispatchSend(send.socket, send.clientMessage, send.senderId);
+          }
+        };
+        // Multi-node S5: brokers on other hosts find this session through its presence lease, so registration
+        // completes once the lease is written (or bounded attempts found it still held elsewhere). The session is
+        // routable meanwhile, so its frames are held until `registered` is written.
+        const presence =
+          rejoined === undefined
+            ? undefined
+            : this.multiNode?.sessionRegistered(
+                rejoined,
+                consumedCapability === undefined
+                  ? undefined
+                  : {
+                      capability: consumedCapability,
+                      ...(info.name?.trim() ? { childName: info.name.trim() } : {}),
+                      groups: sessionGroups(info),
+                    },
+              );
+        if (presence === undefined) completeRegistration();
+        else {
+          holdWrites(socket);
+          void presence.then(() => {
+            const held = releaseHeldWrites(socket);
+            if (this.sessions.get(id) === rejoined) completeRegistration(held);
+            else discardHeldWrites(held);
+          });
+        }
         break;
       }
 
@@ -1479,19 +1634,27 @@ class IntercomBroker {
 
         const requester = currentId ? this.sessions.get(currentId) : undefined;
 		if (requester === undefined) throw new Error("Session not found");
-		const sessions =
-			typeof clientMessage.group === "string"
-				? this.canInspectSelectedGroup(requester, clientMessage.group)
-					? sessionsInGroup(this.sessions, clientMessage.group)
+		const requestId = clientMessage.requestId;
+		const selectedGroup = clientMessage.group;
+		const visible = (sessions: ReadonlyMap<string, Pick<ConnectedSession, "info">>): SessionInfo[] =>
+			typeof selectedGroup === "string"
+				? this.canInspectSelectedGroup(requester, selectedGroup)
+					? sessionsInGroup(sessions, selectedGroup)
 					: []
-				: sessionsVisibleTo(this.sessions, requester.info);
-        writeMessageIfOpen(socket, {
-			type: "sessions",
-			requestId: clientMessage.requestId,
-			sessions,
-			workflowStages: this.workflowStagesVisibleTo(requester, clientMessage.group),
-			workflowFutureStages: this.workflowFutureStagesVisibleTo(requester, clientMessage.group),
-		});
+				: sessionsVisibleTo(sessions, requester.info);
+		const respond = (sessions: SessionInfo[]): void => {
+			writeMessageIfOpen(socket, {
+				type: "sessions",
+				requestId,
+				sessions,
+				workflowStages: this.workflowStagesVisibleTo(requester, selectedGroup),
+				workflowFutureStages: this.workflowFutureStagesVisibleTo(requester, selectedGroup),
+			});
+		};
+		const multiNode = this.multiNode;
+		if (multiNode === undefined) respond(visible(this.sessions));
+		// Multi-node S5: a session connected through another node is a live peer too, under the same visibility.
+		else void multiNode.remoteSessions().then((remote) => respond([...visible(this.sessions), ...visible(remote)]));
         break;
       }
 
@@ -1531,6 +1694,78 @@ class IntercomBroker {
         break;
       }
 
+      case "reserve_restoring_recipient": {
+        const owner = this.sessions.get(currentId);
+        if (
+          owner === undefined || typeof clientMessage.requestId !== "string" ||
+          typeof clientMessage.capability !== "string" || !clientMessage.capability ||
+          (clientMessage.group !== undefined && typeof clientMessage.group !== "string") ||
+          (clientMessage.dormant !== undefined && typeof clientMessage.dormant !== "boolean") ||
+          (clientMessage.aliases !== undefined && !isRestoringRecipientAliasList(clientMessage.aliases))
+        ) {
+          throw new Error("Invalid reserve_restoring_recipient message");
+        }
+        // Only the supervisor the capability was issued to may hold sends for that child (#3760).
+        const childName = this.supervisorChannel.supervisedChildName(clientMessage.capability, currentId);
+        const aliases = (clientMessage.aliases ?? []) as string[];
+        if (childName === undefined || !aliases.every(isRestoringRecipientAlias)) {
+          writeMessageIfOpen(socket, {
+            type: "restoring_recipient_ack",
+            requestId: clientMessage.requestId,
+            error: childName === undefined
+              ? "Only the supervisor authorized for this subagent can reserve its Intercom targets"
+              : "A restoring subagent can be reserved only under its name and session ids",
+          });
+          break;
+        }
+        this.restoringRecipients.reserve(
+          clientMessage.capability,
+          currentId,
+          childName,
+          normalizeGroup(typeof clientMessage.group === "string" ? clientMessage.group : owner.registrationGroup),
+          aliases,
+          { dormant: clientMessage.dormant === true },
+        );
+        this.multiNode?.restoringReserved(
+          clientMessage.capability,
+          currentId,
+          childName,
+          normalizeGroup(typeof clientMessage.group === "string" ? clientMessage.group : owner.registrationGroup),
+          aliases,
+          clientMessage.dormant === true,
+        );
+        writeMessageIfOpen(socket, { type: "restoring_recipient_ack", requestId: clientMessage.requestId });
+        break;
+      }
+
+      case "release_restoring_recipient": {
+        if (
+          typeof clientMessage.requestId !== "string" || typeof clientMessage.capability !== "string" ||
+          (clientMessage.gone !== undefined && typeof clientMessage.gone !== "string") ||
+          (clientMessage.refusal !== undefined && typeof clientMessage.refusal !== "string")
+        ) {
+          throw new Error("Invalid release_restoring_recipient message");
+        }
+        const released = this.restoringRecipients.release(
+          clientMessage.capability,
+          currentId,
+          typeof clientMessage.gone === "string" ? clientMessage.gone.slice(0, RESTORING_RECIPIENT_GONE_REASON_MAX) : undefined,
+          typeof clientMessage.refusal === "string" && clientMessage.refusal.trim()
+            ? clientMessage.refusal.slice(0, RESTORING_RECIPIENT_GONE_REASON_MAX)
+            : undefined,
+        );
+        if (released !== undefined) {
+          this.multiNode?.restoringReleased(clientMessage.capability, currentId, released.gone, released.refusal);
+        }
+        writeMessageIfOpen(socket, { type: "restoring_recipient_ack", requestId: clientMessage.requestId });
+        for (const send of released?.held ?? []) {
+          if (released?.gone !== undefined) this.failHeldSend(send, released.gone);
+          else if (released?.refusal !== undefined) this.failHeldSend(send, released.refusal);
+          else this.dispatchSend(send.socket, send.clientMessage, send.senderId);
+        }
+        break;
+      }
+
       case "register_pending_stage_route": {
         if (
           typeof clientMessage.runId !== "string" ||
@@ -1544,7 +1779,21 @@ class IntercomBroker {
         const owner = this.sessions.get(currentId);
         const existing = this.pendingStageRoutes.get(clientMessage.runId);
         const ownerGroup = normalizeGroup(owner?.registrationGroup);
-        const activeExisting = existing !== undefined && this.sessions.has(existing.sessionId) ? existing : undefined;
+        const dormant = clientMessage.dormant === true;
+        const liveExisting = existing !== undefined && this.sessions.has(existing.sessionId) ? existing : undefined;
+        // #3760: a restarted owner queues for its interrupted run under a dormant route. The process that
+        // resumes the run holds the run's lease and a capability of its own, so it takes the route over and
+        // the dormant owner is told to stop answering for it; a dormant route never takes over another.
+        const yieldedDormant =
+          liveExisting?.dormant === true &&
+          !dormant &&
+          liveExisting.sessionId !== currentId &&
+          liveExisting.capability !== clientMessage.capability &&
+          normalizeGroup(clientMessage.group) === ownerGroup &&
+          liveExisting.group === ownerGroup
+            ? liveExisting
+            : undefined;
+        const activeExisting = yieldedDormant === undefined ? liveExisting : undefined;
         if (
           owner === undefined ||
           normalizeGroup(clientMessage.group) !== ownerGroup ||
@@ -1611,7 +1860,13 @@ class IntercomBroker {
           sessionId: currentId,
           group: ownerGroup,
           capability: clientMessage.capability,
+          ...(dormant ? { dormant: true as const } : {}),
         });
+        if (yieldedDormant !== undefined) {
+          const revoked = this.sessions.get(yieldedDormant.sessionId);
+          if (revoked !== undefined)
+            writeMessageIfOpen(revoked.socket, { type: "pending_stage_route_revoked", runId: clientMessage.runId });
+        }
         if (clientMessage.stages !== undefined || possibleStages !== undefined) {
           this.workflowRosters.set(clientMessage.runId, {
             ownerSessionId: currentId,
@@ -1728,22 +1983,17 @@ class IntercomBroker {
 
 		case "send":
 		case "supervisor_send": {
-			handleBrokerSend(
-				socket,
-				clientMessage,
-				currentId,
-				this.sessions,
-				this.deliveredMessages,
-				writeMessageIfOpen,
-				this.supervisorChannel,
-				this.pendingQuestions,
-				this.routePendingStage,
-				this.resolveLiveWorkflowStage,
-				this.canControlLiveWorkflowStage,
-				this.resolveLegacyWorkflowStageTarget,
-				writeMessageWithOutcome,
-				this.isNonAgentWorkflowTarget,
-			);
+			this.dispatchSend(socket, clientMessage, currentId);
+			break;
+		}
+
+		case "received": {
+			if (typeof clientMessage.messageId !== "string") throw new Error("Invalid received message");
+			const recipient = this.sessions.get(currentId);
+			if (recipient?.acknowledgesReceipt === true) {
+				this.multiNode?.received(recipient, clientMessage.messageId);
+				this.receipts.settle(clientMessage.messageId, deliveryTargetIdentity(recipient), recipient);
+			}
 			break;
 		}
 
@@ -1771,6 +2021,8 @@ class IntercomBroker {
           (target, message) => writeMessageIfOpen(target, message),
           this.pendingQuestions,
         );
+        const updated = this.sessions.get(currentId);
+        if (updated !== undefined) this.multiNode?.sessionUpdated(updated);
         break;
       }
 
@@ -1779,10 +2031,191 @@ class IntercomBroker {
     }
   }
 
+  /** Route one send or ask frame, as received or again after a restoring subagent's hold ends (#3760). */
+  private dispatchSend(socket: net.Socket, clientMessage: SendClientMessage, senderId: string | null): void {
+    handleBrokerSend(
+      socket,
+      clientMessage,
+      senderId,
+      this.sessions,
+      this.deliveredMessages,
+      writeMessageIfOpen,
+      this.supervisorChannel,
+      this.pendingQuestions,
+      this.routePendingStage,
+      this.resolveLiveWorkflowStage,
+      this.canControlLiveWorkflowStage,
+      this.resolveLegacyWorkflowStageTarget,
+      writeMessageWithOutcome,
+      this.isNonAgentWorkflowTarget,
+      this.holdForRestoringRecipient,
+      this.receipts,
+      this.routeRemote,
+    );
+  }
+
+  /** Multi-node S5: a send no local session resolved goes to the shared state once it is open. */
+  private routeRemoteSend(request: RemoteSendRequest): void {
+    if (this.multiNode !== undefined) {
+      this.multiNode.routeSend(request);
+      return;
+    }
+    // Off, or a send to a session on this broker: local delivery never depends on the shared database (spec §8).
+    if (this.multiNodeStatus === "off" || (this.multiNodeStatus === "unavailable" && request.localTarget !== undefined)) {
+      request.notFound();
+      return;
+    }
+    if (this.multiNodeStatus === "starting") {
+      if (request.localTarget === undefined) {
+        void this.multiNodeFirstAttempt.then(() => this.routeRemoteSend(request));
+        return;
+      }
+      // A send to a session on this broker waits for the first opening at most as long as the router's own retry
+      // lookup would, then is delivered locally; whichever comes second does nothing, so it is answered once.
+      let answered = false;
+      const bound = setTimeout(() => {
+        answered = true;
+        request.notFound();
+      }, LOCAL_RETRY_LOOKUP_BOUND_MS);
+      void this.multiNodeFirstAttempt.then(() => {
+        if (answered) return;
+        answered = true;
+        clearTimeout(bound);
+        this.routeRemoteSend(request);
+      });
+      return;
+    }
+    writeMessageIfOpen(request.socket, {
+      type: "delivery_failed",
+      messageId: request.message.id,
+      ...(request.attemptId === undefined ? {} : { attemptId: request.attemptId }),
+      reason: SHARED_STORAGE_UNAVAILABLE_REASON,
+      ...(request.requirePendingReply ? { reasonCode: "session_not_found" as const } : {}),
+    });
+  }
+
+  /**
+   * The shared state is loaded lazily, so a single-node broker never loads the host storage layer. A database that
+   * cannot be opened is retried with exponential backoff; meanwhile local delivery continues.
+   */
+  private startMultiNode(setting: Record<string, unknown>): void {
+    this.multiNodeStatus = "starting";
+    let settleFirstAttempt: () => void = () => {};
+    this.multiNodeFirstAttempt = new Promise<void>((resolve) => {
+      settleFirstAttempt = resolve;
+    });
+    const settle = (status: "on" | "off" | "unavailable"): void => {
+      this.multiNodeStatus = status;
+      settleFirstAttempt();
+    };
+    const report = (error: unknown): void => {
+      console.error(`Intercom multi-node routing is unavailable: ${error instanceof Error ? error.message : String(error)}`);
+    };
+    void (async () => {
+      const startup = await resolveMultiNodeStartup(setting, dirname(INTERCOM_DIR), async () => {
+        const loaded = await import("./open-broker-state.js");
+        type Source = Parameters<typeof loaded.openIntercomBrokerState>[0];
+        const opener: BrokerStateOpener<Source> = {
+          resolveBrokerStateSource: (value, agentDir) =>
+            loaded.resolveBrokerStateSource(value as Parameters<typeof loaded.resolveBrokerStateSource>[0], agentDir),
+          openIntercomBrokerState: (source) => loaded.openIntercomBrokerState(source),
+        };
+        return opener;
+      });
+      if (startup.status === "off") {
+        // A standalone binary has no host storage module, and a selection may not resolve to a database: both
+        // leave this broker single-node, exactly as without a shared `storage`.
+        if (startup.error !== undefined) console.error(`Intercom multi-node routing is off: ${startup.reason}`);
+        settle("off");
+        return;
+      }
+      const { opener, source } = startup;
+      for (let attempt = 0; !this.shuttingDown; attempt++) {
+        try {
+          const opened = await opener.openIntercomBrokerState(source);
+          try {
+            this.multiNode = await MultiNodeRouter.start({
+              ...opened,
+              host: {
+                sessions: this.sessions,
+                write: writeMessageIfOpen,
+                writeConfirmed: writeMessageWithOutcome,
+                supervisorCache: this.supervisorChannel,
+                pendingQuestions: this.pendingQuestions,
+                deliveredMessages: this.deliveredMessages,
+                dispatch: (socket, clientMessage, senderId) => this.dispatchSend(socket, clientMessage, senderId),
+              },
+            });
+          } catch (error) {
+            await opened.bus.close().catch(() => {});
+            await opened.state.close().catch(() => {});
+            throw error;
+          }
+          // Reservations acknowledged before routing started (or while an earlier opening failed) were kept only
+          // here; mirror the ones still in force so other brokers hold sends for them too. Releases and owner
+          // departures from now on reach the router behind these reserves.
+          for (const reservation of this.restoringRecipients.active()) {
+            this.multiNode.restoringReserved(
+              reservation.capability,
+              reservation.ownerSessionId,
+              reservation.childName,
+              reservation.group,
+              reservation.aliases,
+              reservation.dormant,
+            );
+            if (reservation.gone !== undefined) {
+              this.multiNode.restoringReleased(reservation.capability, reservation.ownerSessionId, reservation.gone);
+            }
+          }
+          if (this.shuttingDown) await this.multiNode.close().catch(() => {});
+          settle("on");
+          return;
+        } catch (error) {
+          report(error);
+          settle("unavailable");
+          await new Promise<void>((resolve) => {
+            setTimeout(resolve, Math.min(1_000 * 2 ** attempt, OUTBOX_BACKUP_POLL_INTERVAL_MS)).unref();
+          });
+        }
+      }
+    })();
+  }
+
+  /**
+   * #3760: a send to a subagent its owner is restoring is held, by whichever sender, instead of failing with
+   * `Session not found`; a send to one its owner reported gone gets the gone refusal.
+   */
+  private readonly holdForRestoringRecipient: RestoringRecipientHold = ({ socket, from, clientMessage, target, messageId, attemptId }) => {
+    const held: HeldSend = { socket, senderId: from.info.id, messageId, clientMessage, ...(attemptId === undefined ? {} : { attemptId }) };
+    const admission = this.restoringRecipients.admit(
+      target,
+      { id: from.info.id, info: from.info },
+      held,
+      clientMessage.awaitRestoringRecipient === true,
+    );
+    if (admission.kind === "none") return false;
+    if (admission.kind === "refused") this.failHeldSend(held, admission.reason);
+    else writeMessageIfOpen(socket, { type: "send_held", messageId, ...(attemptId === undefined ? {} : { attemptId }), timeoutMs: admission.timeoutMs });
+    return true;
+  };
+
+  private failHeldSend(held: HeldSend, reason: string): void {
+    writeMessageIfOpen(held.socket, {
+      type: "delivery_failed",
+      messageId: held.messageId,
+      ...(held.attemptId === undefined ? {} : { attemptId: held.attemptId }),
+      reason,
+    });
+  }
+
   private disconnectSession(sessionId: string): void {
     const departed = this.sessions.get(sessionId);
     if (!departed) return;
 
+    this.restoringRecipients.dropSender(sessionId);
+    // #3760: a departed owner restores nothing more; what it held goes through ordinary routing once it is gone.
+    const unreserved = this.restoringRecipients.dropOwner(sessionId);
+
     this.pendingQuestions.pruneSender(sessionId);
     for (const question of this.pendingQuestions.takeForTarget(sessionId)) {
       const asker = this.sessions.get(question.senderSessionId);
@@ -1832,7 +2265,10 @@ class IntercomBroker {
 		if (activation.sessionId === sessionId) this.liveWorkflowStageRouteActivations.delete(requestId);
 	}
 	this.sessions.delete(sessionId);
+	this.multiNode?.sessionLeft(departed);
+	this.sessionIdReclaim.retire(sessionId, departed.registrationReturnAddress);
 	if (isAgentRecipient(departed.info)) this.broadcastToMemberships({ type: "session_left", sessionId }, sessionGroups(departed.info), sessionId);
+	for (const send of unreserved) this.dispatchSend(send.socket, send.clientMessage, send.senderId);
 	}
   /** Deliver a broadcast once to each session represented in any given group. */
   private broadcastToMemberships(msg: BrokerMessage, groups: ReadonlySet<string>, exclude?: string): void {
@@ -1863,6 +2299,8 @@ class IntercomBroker {
   }
 
   private shutdown(): void {
+    if (this.shuttingDown) return;
+    this.shuttingDown = true;
     console.log("Broker shutting down");
 
     for (const session of this.sessions.values()) {
@@ -1873,9 +2311,22 @@ class IntercomBroker {
     this.pendingStageAcknowledgments.clear();
     this.liveWorkflowStageRoutes.clear();
     this.liveWorkflowStageRouteActivations.clear();
+    this.restoringRecipients.clear();
+    this.receipts.clear();
 	for (const pending of this.pendingStageNotificationAcknowledgments.values()) clearTimeout(pending.timeout);
 	this.pendingStageNotificationAcknowledgments.clear();
 	this.deliveredMessages.close();
+    const multiNode = this.multiNode;
+    if (multiNode === undefined) {
+      this.exitAfterShutdown();
+      return;
+    }
+    // Give this broker's leases up so other brokers see its sessions offline at once, within a bound.
+    const bound = new Promise<void>((resolve) => setTimeout(resolve, MULTI_NODE_CLOSE_BOUND_MS).unref());
+    void Promise.race([multiNode.close().catch(() => {}), bound]).finally(() => this.exitAfterShutdown());
+  }
+
+  private exitAfterShutdown(): never {
     const ownsRuntime = this.readPidFile() === process.pid;
     this.unlinkRuntimeFilesIfOwned();
     // Node unlinks a Unix socket path on server.close() even after a successor