@bastani/atomic 0.9.39 → 0.9.40-alpha.1

package/dist/builtin/intercom/broker/open-broker-state.ts

@@ -0,0 +1,160 @@
+import { homedir } from "node:os";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+import {
+	loadStorageAdapter,
+	openNodeSqliteDatabase,
+	resolveAdapterOptions,
+	type SqliteDatabase,
+	type StorageOpenContext,
+	type StorageSettingsSelection,
+	StorageUnavailableError,
+	selectStorage,
+	validateStoragePrefix,
+} from "@bastani/atomic/storage";
+import { type DatabaseWakeUpBusOptions, openDatabaseWakeUpBus } from "./database-wake-up-bus.js";
+import type { IntercomBrokerStateStore } from "./multinode-state-store.js";
+import { openSqlHeldSendStore } from "./sql-held-send-store.js";
+import { openSqlKnownSessionStore } from "./sql-known-session-store.js";
+import { openSqlBrokerLeaseStore } from "./sql-lease-store.js";
+import { openSqlOutboxStore } from "./sql-outbox-store.js";
+import { openSqlPendingAskStore } from "./sql-pending-ask-store.js";
+import { openSqlRestoringReservationStore } from "./sql-restoring-reservation-store.js";
+import { openSqlSentMessageAuthority } from "./sql-sent-message-authority.js";
+import type { WakeUpBus } from "./wake-up-bus.js";
+
+/**
+ * S5 (specs/2026-10-19-intercom-multi-node.md §4, §14): opens the shared Intercom broker state on the top-level
+ * `storage` selection of the agent directory's settings. The broker loads this module lazily, only when that
+ * selection is set, so single-node brokers never load the host storage layer.
+ */
+
+/** Where the shared Intercom state lives: a database opener and the selection's table prefix. */
+export interface SharedDatabaseSource {
+	/** Opens the connection the stores and the wake-up bus share. */
+	readonly openDatabase: () => Promise<SqliteDatabase>;
+	readonly prefix?: string;
+}
+
+export interface OpenedBrokerState {
+	readonly state: IntercomBrokerStateStore;
+	readonly bus: WakeUpBus;
+}
+
+/** Why multi-node routing is off: there is no `SqliteDatabase` to share, so Intercom behaves exactly as today. */
+export type BrokerStateOffReason = "no-storage" | "memory" | "domain-only-adapter";
+
+export type BrokerStateSource =
+	| { readonly status: "off"; readonly reason: BrokerStateOffReason }
+	| { readonly status: "shared"; readonly source: SharedDatabaseSource };
+
+function resolveSettingsPath(path: string, agentDir: string): string {
+	if (path === "~") return homedir();
+	if (path.startsWith("~/") || (process.platform === "win32" && path.startsWith("~\\"))) {
+		return join(homedir(), path.slice(2));
+	}
+	return isAbsolute(path) ? path : resolve(dirname(agentDir), path);
+}
+
+/**
+ * Decide from a top-level `storage` value where the shared state lives. Invalid configuration throws the storage
+ * layer's own `StorageConfigError` or `StorageAdapterLoadError`; it never selects another store.
+ */
+export async function resolveBrokerStateSource(
+	value: StorageSettingsSelection | undefined,
+	agentDir: string,
+): Promise<BrokerStateSource> {
+	if (value === undefined) return { status: "off", reason: "no-storage" };
+	const selection = selectStorage(value, "settings", "storage", join(dirname(agentDir), "storage.sqlite"));
+	if (selection.kind === "memory") return { status: "off", reason: "memory" };
+	if (selection.kind === "sqlite") {
+		if (selection.path === "" || selection.path === ":memory:") return { status: "off", reason: "memory" };
+		const path = resolveSettingsPath(selection.path, agentDir);
+		validateStoragePrefix(selection.prefix);
+		return {
+			status: "shared",
+			source: { openDatabase: () => openNodeSqliteDatabase(path), prefix: selection.prefix },
+		};
+	}
+	if (selection.kind !== "adapter") return { status: "off", reason: "no-storage" };
+	validateStoragePrefix(selection.prefix);
+	const context: StorageOpenContext = { agentDir, domain: "intercom" };
+	// Destructured: the broker's runtime-graph walker reads a bare `module` identifier as the CJS module object.
+	const { module: adapterModule } = selection;
+	const adapter = await loadStorageAdapter(adapterModule, context);
+	const hook = adapter.openDatabase?.bind(adapter);
+	if (hook === undefined) return { status: "off", reason: "domain-only-adapter" };
+	const options = await resolveAdapterOptions(selection, context);
+	const location = { adapter: adapterModule, location: adapterModule };
+	return {
+		status: "shared",
+		source: {
+			prefix: selection.prefix,
+			async openDatabase() {
+				try {
+					return await hook(options, context);
+				} catch (cause) {
+					if (cause instanceof StorageUnavailableError) throw cause;
+					throw new StorageUnavailableError(location, cause as Error);
+				}
+			},
+		},
+	};
+}
+
+/**
+ * A view of `database` whose `close` leaves it open. The wake-up bus borrows the stores' connection: a SQLite driver
+ * that waits for a lock synchronously (node:sqlite) would otherwise block this process on a second connection's
+ * write while one of the stores' transactions is suspended between statements.
+ */
+export function borrowedDatabase(database: SqliteDatabase): SqliteDatabase {
+	return {
+		descriptor: database.descriptor,
+		exec: (sql) => database.exec(sql),
+		run: (sql, ...params) => database.run(sql, ...params),
+		get: (sql, ...params) => database.get(sql, ...params),
+		all: (sql, ...params) => database.all(sql, ...params),
+		transaction: (callback) => database.transaction(callback),
+		close: async () => {},
+	};
+}
+
+/** Compose S1–S4 and the S5 known-session store, and the wake-up bus, on one connection. */
+export async function openIntercomBrokerState(
+	source: SharedDatabaseSource,
+	busOptions: Pick<DatabaseWakeUpBusOptions, "pollIntervalMs" | "schedule"> = {},
+): Promise<OpenedBrokerState> {
+	const database = await source.openDatabase();
+	try {
+		const options = { database, prefix: source.prefix ?? "" };
+		const leases = await openSqlBrokerLeaseStore(options);
+		const outbox = await openSqlOutboxStore(options);
+		const heldSends = await openSqlHeldSendStore(options);
+		const reservations = await openSqlRestoringReservationStore(options);
+		const pendingAsks = await openSqlPendingAskStore(options);
+		const sentMessages = await openSqlSentMessageAuthority(options);
+		const knownSessions = await openSqlKnownSessionStore(options);
+		const bus = await openDatabaseWakeUpBus({
+			openDatabase: async () => borrowedDatabase(database),
+			prefix: source.prefix,
+			...busOptions,
+		});
+		let closing: Promise<void> | undefined;
+		const state: IntercomBrokerStateStore = {
+			leases,
+			outbox,
+			heldSends,
+			reservations,
+			pendingAsks,
+			sentMessages,
+			knownSessions,
+			close() {
+				closing ??= database.close();
+				return closing;
+			},
+		};
+		return { state, bus };
+	} catch (cause) {
+		await database.close().catch(() => {});
+		throw cause;
+	}
+}

package/dist/builtin/intercom/broker/paths.ts

@@ -1,6 +1,12 @@
 import { homedir } from "os";
 import { join } from "path";
 
+/**
+ * The shared storage selection (`ATOMIC_STORAGE` JSON) the launching Atomic process hands its broker: its
+ * `--storage`/`--storage-file` or its own `ATOMIC_STORAGE`. It outranks the agent directory's settings.
+ */
+export const BROKER_STORAGE_ENV = "ATOMIC_STORAGE";
+
 function getHomeDir(): string {
   if (process.platform === "win32") {
     if (process.env.USERPROFILE) return process.env.USERPROFILE;

package/dist/builtin/intercom/broker/pending-send-registry.ts

@@ -122,6 +122,20 @@ export class PendingSendRegistry {
     return true;
   }
 
+  /**
+   * Wait up to `timeoutMs` more for this attempt's outcome: the broker is holding it for a recipient that is
+   * coming back (#3760). A pre-attemptId acknowledgement may extend only an ID's first active generation.
+   */
+  extend(messageId: string, attemptId: string | undefined, timeoutMs: number): boolean {
+    const attempt = this.attempts.get(messageId);
+    if (!attempt || (attemptId === undefined ? !attempt.legacyEligible : attempt.attemptId !== attemptId)) return false;
+    if (attempt.timer) clearTimeout(attempt.timer);
+    attempt.timer = setTimeout(() => {
+      this.reject(attempt, new Error("Send timeout"));
+    }, timeoutMs);
+    return true;
+  }
+
   /** Resolve a pre-attemptId response only for an ID's first active generation. */
   resolveLegacy(messageId: string, result: SendResultLike): boolean {
     const attempt = this.attempts.get(messageId);

package/dist/builtin/intercom/broker/recipient-receipts.ts

@@ -0,0 +1,93 @@
+import type net from "node:net";
+import { DELIVERED_MESSAGE_TTL_MS, RECIPIENT_RECEIPT_TIMEOUT_MS } from "../retry-policy.js";
+import type { BrokerMessage } from "../types.js";
+
+/** #3761: the sender is told this when the recipient did not confirm a save within the sender's wait. */
+export const UNCERTAIN_RECEIPT_REASON =
+	"The recipient did not confirm that it saved the message in time, so whether it arrived is uncertain: it may still save and show it. Check with the recipient before sending it again, because a new send is a new message.";
+
+export interface ReceiptSender {
+	readonly socket: net.Socket;
+	readonly attemptId?: string;
+}
+
+export interface AwaitedReceipt<Recipient> {
+	readonly messageId: string;
+	/** Stable recipient authority, so a reconnected recipient can confirm or receive the message again. */
+	readonly targetIdentity: string;
+	/** The frame written to the recipient, written again when it registers again before confirming. */
+	readonly frame: BrokerMessage;
+	/** Runs once, on the recipient's confirmation. `sender` is absent once the sender's wait has ended. */
+	readonly complete: (recipient: Recipient, sender: ReceiptSender | undefined) => void;
+	/** Runs when the sender's wait ends before the recipient confirms. */
+	readonly expire: (sender: ReceiptSender) => void;
+}
+
+interface PendingReceipt<Recipient> extends AwaitedReceipt<Recipient> {
+	sender: ReceiptSender | undefined;
+	senderTimer: ReturnType<typeof setTimeout> | undefined;
+	retentionTimer: ReturnType<typeof setTimeout>;
+}
+
+/**
+ * #3761: messages written to a recipient that confirms saves, held until that recipient says it saved them.
+ * A message stays held after the sender's wait ends, so a late confirmation still settles it and a
+ * re-registered recipient is sent it again; it is forgotten after the delivered-message retention.
+ */
+export class RecipientReceipts<Recipient> {
+	private readonly pending = new Map<string, PendingReceipt<Recipient>>();
+
+	constructor(
+		private readonly senderTimeoutMs = RECIPIENT_RECEIPT_TIMEOUT_MS,
+		private readonly retentionMs = DELIVERED_MESSAGE_TTL_MS,
+	) {}
+
+	await(receipt: AwaitedReceipt<Recipient>, sender: ReceiptSender): void {
+		this.drop(receipt.messageId);
+		const entry: PendingReceipt<Recipient> = {
+			...receipt,
+			sender,
+			senderTimer: undefined,
+			retentionTimer: setTimeout(() => {
+				if (this.pending.get(receipt.messageId) === entry) this.drop(receipt.messageId);
+			}, this.retentionMs),
+		};
+		entry.retentionTimer.unref?.();
+		entry.senderTimer = setTimeout(() => {
+			entry.senderTimer = undefined;
+			const waiting = entry.sender;
+			entry.sender = undefined;
+			if (waiting !== undefined && this.pending.get(receipt.messageId) === entry) entry.expire(waiting);
+		}, this.senderTimeoutMs);
+		this.pending.set(receipt.messageId, entry);
+	}
+
+	/** Settle a held message when the confirming recipient is the one it was written to. */
+	settle(messageId: string, targetIdentity: string, recipient: Recipient): boolean {
+		const entry = this.pending.get(messageId);
+		if (entry === undefined || entry.targetIdentity !== targetIdentity) return false;
+		const sender = entry.sender;
+		this.drop(messageId);
+		entry.complete(recipient, sender);
+		return true;
+	}
+
+	/** Frames still waiting for this recipient, in the order they were first written. */
+	framesFor(targetIdentity: string): BrokerMessage[] {
+		return [...this.pending.values()]
+			.filter((entry) => entry.targetIdentity === targetIdentity)
+			.map((entry) => entry.frame);
+	}
+
+	drop(messageId: string): void {
+		const entry = this.pending.get(messageId);
+		if (entry === undefined) return;
+		this.pending.delete(messageId);
+		if (entry.senderTimer !== undefined) clearTimeout(entry.senderTimer);
+		clearTimeout(entry.retentionTimer);
+	}
+
+	clear(): void {
+		for (const messageId of [...this.pending.keys()]) this.drop(messageId);
+	}
+}

package/dist/builtin/intercom/broker/restoring-recipients.ts

@@ -0,0 +1,263 @@
+import type net from "node:net";
+import { hasGroup } from "../group.js";
+import type { SessionInfo } from "../types.js";
+import { sessionGroups } from "./group-membership.js";
+
+/** How long the broker holds a send for a subagent that is being restored; matches its owner's own wait (#3760). */
+export const RESTORING_RECIPIENT_HOLD_MS = 20_000;
+/** A reservation covers the child's name plus its Atomic session id and its pre-restart Intercom id. */
+const MAX_RESERVED_TARGETS = 8;
+const MAX_HELD_SENDS = 256;
+
+/** A send the broker is holding for a subagent that is being restored. */
+export interface HeldSend {
+	readonly socket: net.Socket;
+	readonly senderId: string;
+	readonly messageId: string;
+	readonly attemptId?: string;
+	/** The original send frame, dispatched again once the hold ends. */
+	readonly clientMessage: { readonly type: string; readonly to: string } & Record<string, unknown>;
+}
+
+export type RestoringRecipientAdmission =
+	| { readonly kind: "none" }
+	| { readonly kind: "held"; readonly timeoutMs: number }
+	| { readonly kind: "refused"; readonly reason: string };
+
+/** One reservation as `RestoringRecipients.active` reports it; `gone` is the formatted gone refusal. */
+export interface RestoringReservationSnapshot {
+	readonly capability: string;
+	readonly ownerSessionId: string;
+	readonly childName: string;
+	readonly group: string;
+	readonly aliases: readonly string[];
+	readonly dormant: boolean;
+	readonly gone?: string;
+}
+
+interface Reservation {
+	readonly ownerSessionId: string;
+	readonly childName: string;
+	group: string;
+	/**
+	 * Made by a session that is only keeping the child addressable until the session that will really restore it
+	 * is back, such as the launching session of an interrupted workflow run. The restoring supervisor of the
+	 * same child and group takes it over with its held sends.
+	 */
+	dormant: boolean;
+	readonly targets: Set<string>;
+	/** Set once the owner reports the child gone; held and later senders get this refusal. */
+	gone?: string;
+	readonly held: Map<HeldSend, ReturnType<typeof setTimeout>>;
+}
+
+const normalizeTarget = (target: string): string => target.trim().toLowerCase();
+
+export function restoringRecipientRetryReason(childName: string): string {
+	return `Subagent \`${childName}\` is still being restored after the restart. The message was not sent; try again shortly.`;
+}
+
+export function goneRecipientReason(childName: string, reason: string): string {
+	return `Subagent \`${childName}\` did not survive the restart and is gone${reason.trim() ? `: ${reason.trim()}` : ""}. The message was not sent; launch a fresh subagent if its work is still needed.`;
+}
+
+/**
+ * Broker-side holds for background subagents that their owner is restoring after a restart (#3760).
+ *
+ * The owner reserves the child's targets (its name, Atomic session id and pre-restart Intercom id) under the
+ * supervisor capability it was issued for that child; only that capability names the child, and the extra
+ * targets are ids, never names. While a send by any sender in the child's group finds no
+ * live session, it is held instead of failing with `Session not found`. A hold ends in exactly one way: the
+ * child registers with that capability and every held send is dispatched again, once and in order; the owner
+ * reports the child gone and held and later senders get the gone refusal; or the hold times out and the sender
+ * is told to try again. Reservations end when their owner disconnects.
+ *
+ * A reservation can start before the restore does: a resumed session reserves its recorded children at session
+ * start, and the launching session of an interrupted workflow run reserves the children of the run's stages while
+ * the run waits for `/workflow resume`. The restoring supervisor's own reservation of the same child then takes
+ * the earlier one over, so a send is held from the first moment until the child is back, without a gap.
+ */
+export class RestoringRecipients {
+	private readonly reservations = new Map<string, Reservation>();
+
+	constructor(private readonly onExpired: (held: HeldSend, reason: string) => void) {}
+
+	/**
+	 * Reserve, or add targets to, the reservation for the child this capability authorizes. `group` is normalized.
+	 * A `dormant` reservation may be taken over by the supervisor that restores the child.
+	 */
+	reserve(
+		capability: string,
+		ownerSessionId: string,
+		childName: string,
+		group: string,
+		aliases: readonly string[],
+		options: { readonly dormant?: boolean } = {},
+	): void {
+		const dormant = options.dormant === true;
+		let reservation = this.reservations.get(capability);
+		if (reservation === undefined) {
+			reservation = {
+				ownerSessionId,
+				childName,
+				group,
+				dormant,
+				targets: new Set([normalizeTarget(childName)]),
+				held: new Map(),
+			};
+			// A newer restoration of the same child replaces its owner's earlier reservation, gone marker included,
+			// and a dormant owner's reservation of that child and group, whoever restores it.
+			for (const [other, previous] of this.reservations) {
+				if (normalizeTarget(previous.childName) !== normalizeTarget(childName)) continue;
+				if (previous.ownerSessionId !== ownerSessionId && !(previous.dormant && previous.group === group)) continue;
+				this.reservations.delete(other);
+				for (const target of previous.targets) reservation.targets.add(target);
+				for (const [held, timer] of previous.held) reservation.held.set(held, timer);
+			}
+			this.reservations.set(capability, reservation);
+		}
+		reservation.group = group;
+		reservation.dormant = dormant;
+		for (const alias of aliases) {
+			const target = normalizeTarget(alias);
+			if (target && reservation.targets.size < MAX_RESERVED_TARGETS) reservation.targets.add(target);
+		}
+	}
+
+	/**
+	 * Decide a send that found no live recipient. `canHold` is false for clients that cannot wait past their
+	 * ordinary acknowledgement timeout; they are told to try again instead of being held.
+	 */
+	admit(
+		target: string,
+		sender: { readonly id: string; readonly info: SessionInfo },
+		held: HeldSend,
+		canHold: boolean,
+	): RestoringRecipientAdmission {
+		const normalized = normalizeTarget(target);
+		const reservation = [...this.reservations.values()].find(
+			(candidate) =>
+				candidate.targets.has(normalized) &&
+				(candidate.ownerSessionId === sender.id || hasGroup(sessionGroups(sender.info), candidate.group)),
+		);
+		if (reservation === undefined) return { kind: "none" };
+		if (reservation.gone !== undefined) return { kind: "refused", reason: reservation.gone };
+		if (!canHold || reservation.held.size >= MAX_HELD_SENDS)
+			return { kind: "refused", reason: restoringRecipientRetryReason(reservation.childName) };
+		const timer = setTimeout(() => {
+			for (const candidate of this.reservations.values()) {
+				if (!candidate.held.delete(held)) continue;
+				this.onExpired(held, restoringRecipientRetryReason(candidate.childName));
+				return;
+			}
+		}, RESTORING_RECIPIENT_HOLD_MS);
+		timer.unref?.();
+		reservation.held.set(held, timer);
+		return { kind: "held", timeoutMs: RESTORING_RECIPIENT_HOLD_MS };
+	}
+
+	/**
+	 * The child registered under this capability: end its reservation and return the sends held for it, in the
+	 * order they arrived. Each is routed again as it was addressed, so it reaches the child only through a name or
+	 * id the child really registered with, never merely because the owner listed it as an alias.
+	 *
+	 * `registered` names the child that registered. A dormant reservation of that child and group that no
+	 * restoring supervisor took over (it restored the child without reserving first) ends with it too.
+	 */
+	consume(
+		capability: string,
+		registered?: { readonly childName: string; readonly groups: ReadonlySet<string> },
+	): HeldSend[] {
+		const held: HeldSend[] = [];
+		const reservation = this.reservations.get(capability);
+		if (reservation !== undefined && reservation.gone === undefined) {
+			this.reservations.delete(capability);
+			held.push(...this.take(reservation));
+		}
+		if (registered === undefined) return held;
+		const name = normalizeTarget(registered.childName);
+		for (const [other, orphan] of this.reservations) {
+			if (!orphan.dormant || orphan.gone !== undefined || normalizeTarget(orphan.childName) !== name) continue;
+			if (!hasGroup(registered.groups, orphan.group)) continue;
+			this.reservations.delete(other);
+			held.push(...this.take(orphan));
+		}
+		return held;
+	}
+
+	/**
+	 * The owner ended its reservation. With `goneReason` the reservation stays as a gone marker and the held
+	 * sends are returned with the gone refusal. Without it the reservation ends: the held sends are returned with
+	 * the owner's `refusal` when it gave one (the child is coming back elsewhere), otherwise for ordinary routing.
+	 * Undefined when this owner holds no such reservation.
+	 */
+	release(
+		capability: string,
+		ownerSessionId: string,
+		goneReason: string | undefined,
+		refusal?: string,
+	): { readonly held: HeldSend[]; readonly gone?: string; readonly refusal?: string } | undefined {
+		const reservation = this.reservations.get(capability);
+		if (reservation === undefined || reservation.ownerSessionId !== ownerSessionId) return undefined;
+		const held = this.take(reservation);
+		if (goneReason === undefined) {
+			this.reservations.delete(capability);
+			return refusal === undefined ? { held } : { held, refusal };
+		}
+		reservation.gone = goneRecipientReason(reservation.childName, goneReason);
+		return { held, gone: reservation.gone };
+	}
+
+	/** The owner disconnected: end its reservations and return their held sends for ordinary routing. */
+	dropOwner(ownerSessionId: string): HeldSend[] {
+		const released: HeldSend[] = [];
+		for (const [capability, reservation] of this.reservations) {
+			if (reservation.ownerSessionId !== ownerSessionId) continue;
+			this.reservations.delete(capability);
+			released.push(...this.take(reservation));
+		}
+		return released;
+	}
+
+	/** A sender disconnected: nobody is waiting for its held sends any more. */
+	dropSender(senderId: string): void {
+		for (const reservation of this.reservations.values()) {
+			for (const [held, timer] of reservation.held) {
+				if (held.senderId !== senderId) continue;
+				clearTimeout(timer);
+				reservation.held.delete(held);
+			}
+		}
+	}
+
+	/**
+	 * The reservations in force now, as their owners made them: the child's name, its other targets, and the gone
+	 * marker its owner left. Multi-node routing that starts after they were acknowledged mirrors them from here.
+	 */
+	active(): RestoringReservationSnapshot[] {
+		return [...this.reservations].map(([capability, reservation]) => {
+			const childTarget = normalizeTarget(reservation.childName);
+			return {
+				capability,
+				ownerSessionId: reservation.ownerSessionId,
+				childName: reservation.childName,
+				group: reservation.group,
+				aliases: [...reservation.targets].filter((target) => target !== childTarget),
+				dormant: reservation.dormant,
+				...(reservation.gone === undefined ? {} : { gone: reservation.gone }),
+			};
+		});
+	}
+
+	clear(): void {
+		for (const reservation of this.reservations.values()) this.take(reservation);
+		this.reservations.clear();
+	}
+
+	private take(reservation: Reservation): HeldSend[] {
+		const held = [...reservation.held.keys()];
+		for (const timer of reservation.held.values()) clearTimeout(timer);
+		reservation.held.clear();
+		return held;
+	}
+}