@bastani/atomic 0.9.45 → 0.9.46

package/dist/builtin/intercom/broker/multinode-router.ts

@@ -2,7 +2,11 @@ import { createHash, randomUUID } from "node:crypto";
 import { hostname } from "node:os";
 import { normalizeGroup } from "../group.js";
 import { isAgentRecipient, NON_AGENT_RECIPIENT_REFUSAL, TERMINAL_CHILD_ASK_REFUSAL } from "../recipient-purpose.js";
-import { DELIVERED_MESSAGE_TTL_MS, RECIPIENT_RECEIPT_TIMEOUT_MS } from "../retry-policy.js";
+import {
+	DELIVERED_MESSAGE_TTL_MS,
+	RECIPIENT_RECEIPT_TIMEOUT_MS,
+	RECIPIENT_SAVE_MAX_WAIT_MS,
+} from "../retry-policy.js";
 import {
 	resolveSessionTarget,
 	resolveSessionTargetByHostSessionId,
@@ -66,6 +70,12 @@ export const OFFLINE_SESSION_TARGET = "offline session";
 export const SHARED_STORAGE_UNAVAILABLE_REASON =
 	"Session not found on this node, and Intercom could not reach the shared storage database to look for it on other nodes. Try again once the database is available.";
 const MAX_HELD_SENDS = 256;
+/**
+ * #3983: how long the decision to keep a sender waiting may take before the sender is told to keep waiting while it
+ * finishes. Well inside the sender's own wait past the receipt wait (`SEND_RESPONSE_TIMEOUT_MS` minus
+ * `RECIPIENT_RECEIPT_TIMEOUT_MS`).
+ */
+const HOLD_CHECK_GRACE_MS = 500;
 /** A session that moved here may find its presence lease still held by the broker it just left. */
 const PRESENCE_HANDOVER_ATTEMPTS = 4;
 const PRESENCE_HANDOVER_RETRY_MS = 50;
@@ -180,6 +190,12 @@ interface Waiter {
 	/** Told `queued` already: `delivered` follows only while the sender is connected, and it never expires. */
 	readonly queued: boolean;
 	timer: ReturnType<typeof setTimeout> | undefined;
+	/** When the sender started waiting; a recipient's save reports never keep it waiting past the save cap (#3983). */
+	readonly waitingSince: number;
+	/** Ends the wait as uncertain at the save cap, whatever the store does (#3983). */
+	capTimer?: ReturnType<typeof setTimeout>;
+	/** The hold check whose outbox read is in flight; a wait that ends meanwhile reuses it (#3983). */
+	holdCheck?: Promise<void>;
 }
 
 interface TrackedHold {
@@ -218,6 +234,16 @@ interface RemoteTarget {
 	readonly hostBrokerId?: string;
 }
 
+function queuedFrame(messageId: string, attemptId: string | undefined, position: number): BrokerOutboundFrame {
+	return {
+		type: "queued",
+		messageId,
+		...(attemptId === undefined ? {} : { attemptId }),
+		target: OFFLINE_SESSION_TARGET,
+		position,
+	};
+}
+
 function returnAddressOf(identity: string): string | undefined {
 	try {
 		const parsed = JSON.parse(identity) as unknown;
@@ -404,7 +430,10 @@ export class MultiNodeRouter {
 		const sessionId = session.info.id;
 		this.serial(identity, async () => {
 			for (const inflight of claims) await this.state.outbox.releaseClaim(inflight.claim);
-			if (entry.lease !== undefined) await this.state.leases.release(entry.lease);
+			if (entry.lease !== undefined) {
+				await this.state.leases.release(entry.lease);
+				await this.wakeWaitingOrigins(identity);
+			}
 			await this.recordKnown(session, entry);
 			await this.state.pendingAsks.pruneSender(sessionId);
 			await this.state.pendingAsks.takeForTarget(sessionId);
@@ -426,6 +455,18 @@ export class MultiNodeRouter {
 		return true;
 	}
 
+	/**
+	 * #3983 `saving` from a local recipient: it is still saving a row this broker claimed for it. Recorded on the
+	 * row, so the sender's broker keeps its sender waiting. True when it matches such a row.
+	 */
+	saving(recipient: BrokerConnectedSession, messageId: string): boolean {
+		const inflight = this.inflight.get(messageId);
+		if (inflight === undefined || inflight.identity !== deliveryTargetIdentity(recipient)) return false;
+		const report = this.state.outbox.reportSaving?.(inflight.claim);
+		if (report !== undefined) void report.catch((error: Error) => this.report(error));
+		return true;
+	}
+
 	/**
 	 * Sessions other brokers host now (live presence leases), keyed by session id, for `list`: a session connected
 	 * through another node is a live peer exactly like a local one. Group visibility is the caller's, as for local
@@ -933,40 +974,83 @@ export class MultiNodeRouter {
 			targetIdentity: target.identity,
 			queued: offline,
 			timer: undefined,
+			waitingSince: Date.now(),
 		};
-		waiter.timer = setTimeout(
-			() => {
-				if (this.waiters.get(message.id) !== waiter) return;
-				this.dropWaiter(waiter);
-				if (!offline) {
-					this.host.write(request.socket, {
-						type: "delivery_failed",
-						messageId: message.id,
-						attemptId: request.attemptId,
-						reason: UNCERTAIN_RECEIPT_REASON,
-					});
-				}
-			},
-			offline ? DELIVERED_MESSAGE_TTL_MS : RECIPIENT_RECEIPT_TIMEOUT_MS,
-		);
-		waiter.timer.unref?.();
+		this.armWaiter(waiter, offline ? DELIVERED_MESSAGE_TTL_MS : RECIPIENT_RECEIPT_TIMEOUT_MS);
 		this.waiters.set(message.id, waiter);
 		if (offline) {
-			const listed = await this.state.outbox.listUndelivered(target.identity);
-			const index = listed.status === "listed" ? listed.rows.findIndex((entry) => entry.messageId === message.id) : -1;
-			this.host.write(request.socket, {
-				type: "queued",
-				messageId: message.id,
-				...(request.attemptId === undefined ? {} : { attemptId: request.attemptId }),
-				target: OFFLINE_SESSION_TARGET,
-				position: index + 1 || 1,
-			});
+			await this.writeQueued(request.socket, message.id, request.attemptId, target.identity);
 			return;
 		}
 		if (asksTarget) await this.writeQuestionTarget(request.socket, message.id, request.attemptId, target);
 		await this.bus.publish({ brokerId: target.hostBrokerId!, reason: "outbox", version: row.sequence });
+		// A clean quit recorded before this send's row was written is seen here; one recorded after it wakes this
+		// broker (`wakeWaitingOrigins`), so a send racing its recipient's quit is queued either way. The waiter's own
+		// receipt wait keeps running meanwhile, so slow reads here never hold the sender past it.
+		void this.queueIfRecipientLeft(waiter).catch((error: Error) => this.report(error));
 	}
 
+	private async writeQueued(
+		socket: BrokerConnectedSession["socket"],
+		messageId: string,
+		attemptId: string | undefined,
+		identity: string,
+	): Promise<void> {
+		this.host.write(socket, queuedFrame(messageId, attemptId, await this.queuePosition(identity, messageId)));
+	}
+
+	private async queuePosition(identity: string, messageId: string): Promise<number> {
+		const listed = await this.state.outbox.listUndelivered(identity);
+		const index = listed.status === "listed" ? listed.rows.findIndex((entry) => entry.messageId === messageId) : -1;
+		return index + 1 || 1;
+	}
+
+	/**
+	 * A send routed to a live recipient that has since quit, before any broker claimed its row: the row was never
+	 * written to the recipient and no broker hosts it now, so the sender is to be told `queued`, as for a send routed
+	 * after the quit, instead of waiting out a receipt that cannot come and hearing the outcome is uncertain. Presence
+	 * is read before the row, so a row still unclaimed was never written to any recipient. Only reads: the row's queue
+	 * position when that holds, otherwise undefined.
+	 */
+	private async positionIfRecipientLeft(waiter: Waiter): Promise<number | undefined> {
+		const presence = await this.state.leases.read({ kind: "presence", targetIdentity: waiter.targetIdentity });
+		if (presence.status === "invalid") return undefined;
+		if (presence.status === "found" && presence.lease.ageMs < BROKER_LEASE_STALE_AFTER_MS) return undefined;
+		const read = await this.state.outbox.read(waiter.messageId);
+		if (read.status !== "found" || read.row.state !== "queued" || read.row.claimAttempts !== 0) return undefined;
+		return this.queuePosition(waiter.targetIdentity, waiter.messageId);
+	}
+
+	/** Tell a still-waiting sender `queued`, at once: every read was made before. True when it was told. */
+	private settleQueued(waiter: Waiter, position: number): boolean {
+		if (waiter.queued || this.waiters.get(waiter.messageId) !== waiter) return false;
+		this.dropWaiter(waiter);
+		const queued: Waiter = { ...waiter, queued: true, timer: undefined, capTimer: undefined, holdCheck: undefined };
+		this.armWaiter(queued, DELIVERED_MESSAGE_TTL_MS);
+		this.waiters.set(queued.messageId, queued);
+		this.host.write(queued.socket, queuedFrame(queued.messageId, queued.attemptId, position));
+		return true;
+	}
+
+	/**
+	 * Off the receipt wait (a wake-up, the outcome poll, right after routing): the waiter's own timers keep running
+	 * while this reads, so `expireWaiter` still answers the sender in time, and an answer here after that is dropped.
+	 */
+	private async queueIfRecipientLeft(waiter: Waiter): Promise<void> {
+		if (waiter.queued || this.waiters.get(waiter.messageId) !== waiter) return;
+		const position = await this.positionIfRecipientLeft(waiter);
+		if (position !== undefined) this.settleQueued(waiter, position);
+	}
+
+	/** A recipient left: the brokers waiting on its undelivered rows re-read them, and queue the ones never claimed. */
+	private async wakeWaitingOrigins(identity: string): Promise<void> {
+		const listed = await this.state.outbox.listUndelivered(identity);
+		if (listed.status !== "listed") return;
+		for (const origin of new Set(listed.rows.map((row) => row.originBrokerId))) {
+			if (origin === this.brokerId) void this.checkOutcomes();
+			else await this.bus.publish({ brokerId: origin, reason: "outcome", version: 0 });
+		}
+	}
 	private async writeQuestionTarget(
 		socket: BrokerConnectedSession["socket"],
 		messageId: string,
@@ -1266,6 +1350,9 @@ export class MultiNodeRouter {
 							attemptId: waiter.attemptId,
 							reason: row.failureReason ?? "Session not found",
 						});
+					} else if (row.state === "queued" && row.claimAttempts === 0 && !waiter.queued) {
+						// Not awaited: its reads must not hold back the other senders' outcomes.
+						void this.queueIfRecipientLeft(waiter).catch((error: Error) => this.report(error));
 					}
 				}
 			} while (state.again && !this.closed);
@@ -1295,9 +1382,123 @@ export class MultiNodeRouter {
 		this.host.write(waiter.socket, { type: "delivered", messageId: waiter.messageId, attemptId: waiter.attemptId });
 	}
 
+	private armWaiter(waiter: Waiter, timeoutMs: number): void {
+		if (waiter.timer !== undefined) clearTimeout(waiter.timer);
+		waiter.timer = setTimeout(() => {
+			if (this.waiters.get(waiter.messageId) !== waiter) return;
+			if (waiter.queued) {
+				this.dropWaiter(waiter);
+				return;
+			}
+			// One hold check per waiter: a wait that ends while the previous check's read is still pending reuses it.
+			if (waiter.holdCheck !== undefined) return;
+			waiter.holdCheck = this.expireWaiter(waiter)
+				.catch((error: Error) => {
+					this.report(error);
+					this.expireUncertain(waiter);
+				})
+				.finally(() => {
+					waiter.holdCheck = undefined;
+				});
+		}, timeoutMs);
+		waiter.timer.unref?.();
+		// #3983: the save cap holds from the sender's own start, whatever the store does.
+		if (!waiter.queued && waiter.capTimer === undefined) {
+			waiter.capTimer = setTimeout(
+				() => this.expireUncertain(waiter),
+				Math.max(0, waiter.waitingSince + RECIPIENT_SAVE_MAX_WAIT_MS - Date.now()),
+			);
+			waiter.capTimer.unref?.();
+		}
+	}
+
+	/** The sender may wait this much longer under the save cap; 0 once the cap is reached. */
+	private capRemaining(waiter: Waiter): number {
+		return Math.max(0, waiter.waitingSince + RECIPIENT_SAVE_MAX_WAIT_MS - Date.now());
+	}
+
+	/**
+	 * #3983: the sender's receipt wait ended. A recipient on another broker that reported, within the last receipt
+	 * wait, that it is still saving the message keeps its sender waiting another wait (told `send_held`), within the
+	 * save cap; otherwise the sender hears that the outcome is uncertain, as before. The decision reads the outbox
+	 * row, which can wait behind other writes on the shared connection: a read slower than `HOLD_CHECK_GRACE_MS`
+	 * first tells the sender to keep waiting while it finishes (within the cap), and a read that has not answered
+	 * within one receipt wait counts as no evidence, so the sender hears the outcome is uncertain. A row no broker
+	 * claimed whose recipient has quit is queued for it instead; the reads deciding that run under the same grace and
+	 * deadline. Only one decision per waiter is ever in flight, and the cap ends the wait however the store behaves.
+	 */
+	private async expireWaiter(waiter: Waiter): Promise<void> {
+		const provisional = setTimeout(() => {
+			const timeoutMs = Math.min(RECIPIENT_RECEIPT_TIMEOUT_MS, this.capRemaining(waiter));
+			if (this.waiters.get(waiter.messageId) !== waiter || timeoutMs === 0) return;
+			this.armWaiter(waiter, timeoutMs);
+			this.holdSender(waiter, timeoutMs);
+		}, HOLD_CHECK_GRACE_MS);
+		provisional.unref?.();
+		let deadline: ReturnType<typeof setTimeout> | undefined;
+		const timedOut = new Promise<"timed-out">((resolveTimeout) => {
+			deadline = setTimeout(() => resolveTimeout("timed-out"), RECIPIENT_RECEIPT_TIMEOUT_MS);
+			deadline.unref?.();
+		});
+		const decide = async () => {
+			const read = await this.state.outbox.read(waiter.messageId);
+			const unclaimed = read.status === "found" && read.row.state === "queued" && read.row.claimAttempts === 0;
+			return { read, queuedAt: unclaimed ? await this.positionIfRecipientLeft(waiter) : undefined };
+		};
+		let decision: Awaited<ReturnType<typeof decide>> | "timed-out";
+		try {
+			decision = await Promise.race([decide(), timedOut]);
+		} finally {
+			clearTimeout(provisional);
+			clearTimeout(deadline);
+		}
+		if (this.waiters.get(waiter.messageId) !== waiter) return;
+		if (decision === "timed-out") {
+			this.expireUncertain(waiter);
+			return;
+		}
+		const { read, queuedAt } = decision;
+		if (read.status === "found" && read.row.state === "delivered") {
+			await this.finishDelivered(waiter);
+			return;
+		}
+		const remaining = this.capRemaining(waiter);
+		const savingAge = read.status === "found" && read.row.state === "claimed" ? read.row.savingAgeMs : undefined;
+		if (savingAge !== undefined && savingAge < RECIPIENT_RECEIPT_TIMEOUT_MS && remaining > 0) {
+			const timeoutMs = Math.min(RECIPIENT_RECEIPT_TIMEOUT_MS, remaining);
+			this.armWaiter(waiter, timeoutMs);
+			this.holdSender(waiter, timeoutMs);
+			return;
+		}
+		if (queuedAt !== undefined && this.settleQueued(waiter, queuedAt)) return;
+		this.expireUncertain(waiter);
+	}
+
+	private holdSender(waiter: Waiter, timeoutMs: number): void {
+		if (this.waiters.get(waiter.messageId) !== waiter) return;
+		this.host.write(waiter.socket, {
+			type: "send_held",
+			messageId: waiter.messageId,
+			...(waiter.attemptId === undefined ? {} : { attemptId: waiter.attemptId }),
+			timeoutMs,
+		});
+	}
+
+	private expireUncertain(waiter: Waiter): void {
+		if (this.waiters.get(waiter.messageId) !== waiter) return;
+		this.dropWaiter(waiter);
+		this.host.write(waiter.socket, {
+			type: "delivery_failed",
+			messageId: waiter.messageId,
+			attemptId: waiter.attemptId,
+			reason: UNCERTAIN_RECEIPT_REASON,
+		});
+	}
+
 	private dropWaiter(waiter: Waiter): void {
 		if (this.waiters.get(waiter.messageId) === waiter) this.waiters.delete(waiter.messageId);
 		if (waiter.timer !== undefined) clearTimeout(waiter.timer);
+		if (waiter.capTimer !== undefined) clearTimeout(waiter.capTimer);
 	}
 
 	// ---------------------------------------------------------------------------------------------------------

package/dist/builtin/intercom/broker/multinode-state-store.ts

@@ -151,6 +151,11 @@ export interface OutboxRow {
 	readonly claimAttempts: number;
 	/** Present only when `state` is `failed`; verbatim. */
 	readonly failureReason?: string;
+	/**
+	 * #3983: store-clock milliseconds since the claimed row's recipient last reported it is still saving it; absent
+	 * when it never reported, and on rows from a store that does not record it.
+	 */
+	readonly savingAgeMs?: number;
 }
 
 export interface OutboxClaim {
@@ -217,6 +222,11 @@ export interface OutboxStore {
 	 */
 	claim(request: OutboxClaimRequest): Promise<OutboxClaimResult>;
 	renewClaim(claim: OutboxClaim): Promise<OutboxRenewClaimResult>;
+	/**
+	 * #3983: the claim's recipient reports it is still saving the row, so the sender's broker keeps its sender
+	 * waiting. Optional: a store without it never extends a sender's wait.
+	 */
+	reportSaving?(claim: OutboxClaim): Promise<OutboxRenewClaimResult>;
 	/** Return the row to `queued`, for example when its recipient disconnected before acknowledging. */
 	releaseClaim(claim: OutboxClaim): Promise<OutboxReleaseClaimResult>;
 	/**

package/dist/builtin/intercom/broker/recipient-receipts.ts

@@ -1,5 +1,9 @@
 import type net from "node:net";
-import { DELIVERED_MESSAGE_TTL_MS, RECIPIENT_RECEIPT_TIMEOUT_MS } from "../retry-policy.js";
+import {
+	DELIVERED_MESSAGE_TTL_MS,
+	RECIPIENT_RECEIPT_TIMEOUT_MS,
+	RECIPIENT_SAVE_MAX_WAIT_MS,
+} from "../retry-policy.js";
 import type { BrokerMessage } from "../types.js";
 
 /** #3761: the sender is told this when the recipient did not confirm a save within the sender's wait. */
@@ -21,12 +25,19 @@ export interface AwaitedReceipt<Recipient> {
 	readonly complete: (recipient: Recipient, sender: ReceiptSender | undefined) => void;
 	/** Runs when the sender's wait ends before the recipient confirms. */
 	readonly expire: (sender: ReceiptSender) => void;
+	/**
+	 * Runs when a recipient still saving the message extends the sender's wait: tells the sender to keep waiting
+	 * for up to `timeoutMs` more (a `send_held` frame).
+	 */
+	readonly hold?: (sender: ReceiptSender, timeoutMs: number) => void;
 }
 
 interface PendingReceipt<Recipient> extends AwaitedReceipt<Recipient> {
 	sender: ReceiptSender | undefined;
 	senderTimer: ReturnType<typeof setTimeout> | undefined;
 	retentionTimer: ReturnType<typeof setTimeout>;
+	/** When the sender started waiting; a save that keeps reporting progress still ends its wait after the cap. */
+	readonly waitingSince: number;
 }
 
 /**
@@ -51,15 +62,38 @@ export class RecipientReceipts<Recipient> {
 			retentionTimer: setTimeout(() => {
 				if (this.pending.get(receipt.messageId) === entry) this.drop(receipt.messageId);
 			}, this.retentionMs),
+			waitingSince: Date.now(),
 		};
 		entry.retentionTimer.unref?.();
+		this.armSenderWait(entry, this.senderTimeoutMs);
+		this.pending.set(receipt.messageId, entry);
+	}
+
+	private armSenderWait(entry: PendingReceipt<Recipient>, timeoutMs: number): void {
+		if (entry.senderTimer !== undefined) clearTimeout(entry.senderTimer);
 		entry.senderTimer = setTimeout(() => {
 			entry.senderTimer = undefined;
 			const waiting = entry.sender;
 			entry.sender = undefined;
-			if (waiting !== undefined && this.pending.get(receipt.messageId) === entry) entry.expire(waiting);
-		}, this.senderTimeoutMs);
-		this.pending.set(receipt.messageId, entry);
+			if (waiting !== undefined && this.pending.get(entry.messageId) === entry) entry.expire(waiting);
+		}, timeoutMs);
+	}
+
+	/**
+	 * The recipient this message was written to reports it is still saving it (for example a pooled stage whose
+	 * journal commit waits behind other writers): the sender keeps waiting another full receipt wait instead of
+	 * hearing the outcome is uncertain. A recipient that dies stops reporting, so its wait still ends; and no save
+	 * holds a sender past `RECIPIENT_SAVE_MAX_WAIT_MS` (#3983).
+	 */
+	extend(messageId: string, targetIdentity: string): boolean {
+		const entry = this.pending.get(messageId);
+		if (entry === undefined || entry.targetIdentity !== targetIdentity || entry.sender === undefined) return false;
+		const remaining = entry.waitingSince + RECIPIENT_SAVE_MAX_WAIT_MS - Date.now();
+		if (remaining <= 0) return false;
+		const timeoutMs = Math.min(this.senderTimeoutMs, remaining);
+		this.armSenderWait(entry, timeoutMs);
+		entry.hold?.(entry.sender, timeoutMs);
+		return true;
 	}
 
 	/** Settle a held message when the confirming recipient is the one it was written to. */

package/dist/builtin/intercom/broker/send-handler.ts

@@ -6,7 +6,7 @@ import {
 	parseLegacyWorkflowStageTarget,
 	parseWorkflowStageTarget,
 } from "../workflow-stage-target.js";
-import { normalizeGroup } from "../group.js";
+import { normalizeGroup, workflowInvocationRefusalFor } from "../group.js";
 import { isMessage } from "./client-message-validation.js";
 import {
 	resolveSessionTarget,
@@ -541,7 +541,11 @@ export function handleBrokerSend(
         type: "delivery_failed",
         messageId: message.id,
         attemptId,
-        reason: "Target session is in a different intercom group",
+        // #4006: a live workflow stage is reached through its invocation; name the group to join,
+        // unless the sender is a workflow stage that joining could never authorize.
+        reason: liveWorkflowTarget !== undefined && workflowTarget !== undefined
+          ? workflowInvocationRefusalFor(fromSession.registrationGroup, `workflow:${workflowTarget.rootRunId}`)
+          : "Target session is in a different intercom group",
       });
       return;
     }
@@ -744,6 +748,9 @@ export function handleBrokerSend(
           expire: (waiting) => {
             write(waiting.socket, { type: "delivery_failed", messageId: message.id, attemptId: waiting.attemptId, reason: UNCERTAIN_RECEIPT_REASON });
           },
+          hold: (waiting, timeoutMs) => {
+            write(waiting.socket, { type: "send_held", messageId: message.id, attemptId: waiting.attemptId, timeoutMs });
+          },
         },
         senderWaiting,
       );

package/dist/builtin/intercom/broker/sql-outbox-store.ts

@@ -34,6 +34,8 @@ interface OutboxRecord extends SqliteRow {
 	readonly claim_token: string | null;
 	/** Store-clock milliseconds since the claim was taken or renewed; null when unclaimed. */
 	readonly claim_age: SqliteValue;
+	/** Store-clock milliseconds since the claim's recipient last reported it is still saving the row; null when never. */
+	readonly saving_age: SqliteValue;
 	readonly failure_reason: string | null;
 }
 
@@ -51,6 +53,9 @@ function toRow(record: OutboxRecord): OutboxRow {
 		state: record.state,
 		claimAttempts: Number(record.claim_attempts),
 		...(failureReason === null ? {} : { failureReason: JSON.parse(failureReason) as string }),
+		...(record.saving_age === null || record.saving_age === undefined
+			? {}
+			: { savingAgeMs: Number(record.saving_age) }),
 	};
 }
 
@@ -81,10 +86,22 @@ export async function openSqlOutboxStore(options: IntercomBrokerStateOptions): P
 		);
 		CREATE INDEX IF NOT EXISTS ${table}_target ON ${table}(target_identity, state, sequence);
 	`);
+	// #3983: added later; a table from an older broker gets the column, and an older broker ignores it.
+	const known = await database.all<{ name: string }>(`PRAGMA table_info(${table})`);
+	if (!known.some((column) => column.name === "saving_at")) {
+		try {
+			await database.exec(`ALTER TABLE ${table} ADD COLUMN saving_at INTEGER`);
+		} catch (error) {
+			// Another broker added it between the check and the change.
+			if (!/duplicate column/i.test(error instanceof Error ? `${error.message} ${String(error.cause ?? "")}` : ""))
+				throw error;
+		}
+	}
 
 	const columns = `sequence, message_id, target_identity, origin_broker_id, digest, frame_json, state,
 		claim_attempts, claimant_id, claim_token, failure_reason,
-		CASE WHEN claim_renewed_at IS NULL THEN NULL ELSE MAX(0, ${STORE_NOW} - claim_renewed_at) END AS claim_age`;
+		CASE WHEN claim_renewed_at IS NULL THEN NULL ELSE MAX(0, ${STORE_NOW} - claim_renewed_at) END AS claim_age,
+		CASE WHEN saving_at IS NULL THEN NULL ELSE MAX(0, ${STORE_NOW} - saving_at) END AS saving_age`;
 	const select = (executor: SqliteExecutor, messageId: string) =>
 		executor.get<OutboxRecord>(`SELECT ${columns} FROM ${table} WHERE message_id = ?`, messageId);
 	const undelivered = (executor: SqliteExecutor, targetIdentity: string) =>
@@ -121,7 +138,7 @@ export async function openSqlOutboxStore(options: IntercomBrokerStateOptions): P
 	): Promise<OutboxRow> => {
 		await executor.run(
 			`UPDATE ${table} SET state = ?, failure_reason = ?, settled_at = ${STORE_NOW},
-				claimant_id = NULL, claim_token = NULL, claim_renewed_at = NULL
+				claimant_id = NULL, claim_token = NULL, claim_renewed_at = NULL, saving_at = NULL
 			WHERE message_id = ?`,
 			state,
 			reason === null ? null : JSON.stringify(reason),
@@ -175,7 +192,7 @@ export async function openSqlOutboxStore(options: IntercomBrokerStateOptions): P
 					const token = randomUUID();
 					await executor.run(
 						`UPDATE ${table} SET state = 'claimed', claim_attempts = claim_attempts + 1, claimant_id = ?,
-							claim_token = ?, claim_renewed_at = ${STORE_NOW}
+							claim_token = ?, claim_renewed_at = ${STORE_NOW}, saving_at = NULL
 						WHERE message_id = ?`,
 						request.claimantId,
 						token,
@@ -201,10 +218,18 @@ export async function openSqlOutboxStore(options: IntercomBrokerStateOptions): P
 			});
 		},
 
+		reportSaving(claim) {
+			return withClaim(claim, async (executor) => {
+				await executor.run(`UPDATE ${table} SET saving_at = ${STORE_NOW} WHERE message_id = ?`, claim.messageId);
+				return { status: "renewed" } as const;
+			});
+		},
+
 		releaseClaim(claim) {
 			return withClaim(claim, async (executor) => {
 				await executor.run(
-					`UPDATE ${table} SET state = 'queued', claimant_id = NULL, claim_token = NULL, claim_renewed_at = NULL
+					`UPDATE ${table} SET state = 'queued', claimant_id = NULL, claim_token = NULL, claim_renewed_at = NULL,
+						saving_at = NULL
 					WHERE message_id = ?`,
 					claim.messageId,
 				);

package/dist/builtin/intercom/group.ts

@@ -123,3 +123,28 @@ export function resolveHomeGroup(
 	}
 	return DEFAULT_GROUP;
 }
+
+/**
+ * #4006: refusal for a sender outside a workflow invocation. It names the documented
+ * `workflow:<rootRunId>` group to join, never an internal identifier.
+ */
+export function workflowInvocationGroupRefusal(invocationGroup: string): string {
+	return `Target workflow run is in a different intercom group; join ${normalizeGroup(invocationGroup)} to send to its stages`;
+}
+
+/**
+ * Refusal for a workflow stage registered outside a run's invocation group: another run's stage,
+ * or an isolated subgroup stage of the same run. Joining never grants it control, so the reason
+ * must not tell it to join.
+ */
+export function workflowStageInvocationRefusal(senderRegistrationGroup: string, invocationGroup: string): string {
+	return `A workflow stage in ${normalizeGroup(senderRegistrationGroup)} cannot send to stages of ${normalizeGroup(invocationGroup)}; joining that group does not grant it control`;
+}
+
+/** The precise refusal for a sender registered in `senderRegistrationGroup` reaching stages of `invocationGroup`. */
+export function workflowInvocationRefusalFor(senderRegistrationGroup: string | undefined, invocationGroup: string): string {
+	const registration = normalizeGroup(senderRegistrationGroup);
+	return registration.startsWith("workflow:") && registration !== normalizeGroup(invocationGroup)
+		? workflowStageInvocationRefusal(registration, invocationGroup)
+		: workflowInvocationGroupRefusal(invocationGroup);
+}