@rebasepro/server-postgres 0.10.0 → 0.10.1-canary.14e53ae

package/src/services/pg-notify-listener.ts

@@ -0,0 +1,137 @@
+/**
+ * A dedicated, self-healing Postgres `LISTEN` connection.
+ *
+ * Every cross-instance feature in the backend needs the same thing: one
+ * connection *outside* the Drizzle pool that stays open, holds a `LISTEN`, and
+ * comes back on its own after the database or the network drops it. CDC needed
+ * it first; the channel bus needs it too. This is that connection, with the one
+ * behaviour that matters to callers preserved: the **first** connect is
+ * validated and rethrown, so a caller can fall back to a different strategy,
+ * while every later drop is repaired quietly in the background.
+ *
+ * `LISTEN` is session state, so this connection must not go through a
+ * transaction-mode pooler (PgBouncer): give it the direct database URL.
+ */
+
+import { Client as PgClient } from "pg";
+import { logger } from "@rebasepro/server";
+
+export interface PgNotifyListenerOptions {
+    /** Direct Postgres connection string (must bypass a transaction-mode pooler). */
+    connectionString: string;
+    /** NOTIFY channel to LISTEN on. Must be a plain identifier — it is interpolated. */
+    channel: string;
+    /** Called for every notification payload received. */
+    onPayload: (payload: string) => void | Promise<void>;
+    /** Prefix for log lines, e.g. `"[CDC]"`. */
+    logLabel: string;
+    /** Delay before a reconnect attempt. */
+    reconnectDelayMs?: number;
+}
+
+const DEFAULT_RECONNECT_DELAY_MS = 3000;
+/** Guards the identifier interpolated into `LISTEN`. */
+const SAFE_CHANNEL = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+export class PgNotifyListener {
+    private client?: PgClient;
+    private running = false;
+    private reconnectTimer?: ReturnType<typeof setTimeout>;
+
+    constructor(private readonly options: PgNotifyListenerOptions) {
+        if (!SAFE_CHANNEL.test(options.channel)) {
+            throw new Error(`Unsafe NOTIFY channel name "${options.channel}" — expected a plain SQL identifier.`);
+        }
+    }
+
+    /** Whether the listener is meant to be connected right now. */
+    get active(): boolean {
+        return this.running;
+    }
+
+    /**
+     * Connect and begin listening. Idempotent.
+     *
+     * Rejects if the *initial* connection or `LISTEN` fails, leaving the
+     * listener stopped — callers use that to degrade deliberately instead of
+     * running blind against a channel nothing is delivering.
+     */
+    async start(): Promise<void> {
+        if (this.running) return;
+        this.running = true;
+        try {
+            await this.connect({ initial: true });
+        } catch (err) {
+            this.running = false;
+            throw err;
+        }
+    }
+
+    /** Stop listening and release the connection. Idempotent. */
+    async stop(): Promise<void> {
+        this.running = false;
+        if (this.reconnectTimer) {
+            clearTimeout(this.reconnectTimer);
+            this.reconnectTimer = undefined;
+        }
+        if (this.client) {
+            try {
+                await this.client.end();
+            } catch { /* ignore close errors */ }
+            this.client = undefined;
+        }
+    }
+
+    private async connect({ initial = false }: { initial?: boolean } = {}): Promise<void> {
+        const { connectionString, channel, onPayload, logLabel } = this.options;
+        try {
+            const client = new PgClient({ connectionString });
+
+            client.on("error", (err) => {
+                logger.error(`❌ ${logLabel} LISTEN client error`, { detail: err.message });
+                this.scheduleReconnect();
+            });
+
+            client.on("end", () => {
+                if (this.running) {
+                    logger.warn(`⚠️ ${logLabel} LISTEN client disconnected unexpectedly.`);
+                    this.scheduleReconnect();
+                }
+            });
+
+            client.on("notification", (msg) => {
+                if (!msg.payload) return;
+                // A handler rejection must never surface as an unhandled
+                // rejection inside the pg client's event emitter.
+                Promise.resolve(onPayload(msg.payload)).catch((err) =>
+                    logger.error(`❌ ${logLabel} Error handling notification`, { error: err })
+                );
+            });
+
+            await client.connect();
+            await client.query(`LISTEN ${channel}`);
+            this.client = client;
+            logger.info(`📡 ${logLabel} Listening on channel "${channel}".`);
+        } catch (err) {
+            // Surface the initial failure so callers can choose to fall back;
+            // for reconnects, keep retrying quietly in the background.
+            if (initial) throw err;
+            logger.error(`❌ ${logLabel} Failed to connect LISTEN client`, { error: err });
+            this.scheduleReconnect();
+        }
+    }
+
+    private scheduleReconnect(): void {
+        if (!this.running || this.reconnectTimer) return;
+
+        this.reconnectTimer = setTimeout(async () => {
+            this.reconnectTimer = undefined;
+            if (!this.running) return;
+            if (this.client) {
+                try { await this.client.end(); } catch { /* ignore */ }
+                this.client = undefined;
+            }
+            await this.connect();
+        }, this.options.reconnectDelayMs ?? DEFAULT_RECONNECT_DELAY_MS);
+    }
+}

package/src/services/realtimeService.ts

@@ -4,7 +4,7 @@ import { Client as PgClient } from "pg";
 import { randomUUID } from "crypto";
 import { DataService } from "./dataService";
 
-import { FetchCollectionProps, ListenCollectionProps, ListenOneProps, DataDriver, CollectionUpdateMessage, SingleUpdateMessage, CollectionPatchMessage, WebSocketMessage, FilterValues, CollectionConfig, RebaseCallContext } from "@rebasepro/types";
+import { FetchCollectionProps, ListenCollectionProps, ListenOneProps, DataDriver, CollectionUpdateMessage, SingleUpdateMessage, CollectionPatchMessage, WebSocketMessage, FilterValues, CollectionConfig, RebaseCallContext, resolveClientListLimit } from "@rebasepro/types";
 import { NodePgDatabase } from "drizzle-orm/node-postgres";
 import { sql as drizzleSql } from "drizzle-orm";
 import { RealtimeProvider, CollectionSubscriptionConfig, SingleSubscriptionConfig } from "../interfaces";
@@ -16,6 +16,8 @@ import { sanitizeErrorForClient } from "../utils/pg-error-utils";
 import { CdcListener, type CdcChangeEvent } from "./cdc/CdcListener";
 import { deriveRowAddress, getPrimaryKeys, type PrimaryKeyInfo } from "./collection-helpers";
 import { ChannelHistoryStore, type ResolvedRetention } from "./channel-history";
+import { ChannelPresenceStore } from "./channel-presence";
+import { ChannelBus, ChannelBusFrame, MemoryChannelBus, frameByteLength } from "./channel-bus";
 import type { ChannelHistoryEntry, ChannelRetentionRule } from "@rebasepro/types";
 
 /** Channel name used for Postgres LISTEN/NOTIFY cross-instance realtime. */
@@ -76,8 +78,38 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
      * wait on each other.
      */
     private channelSendQueues = new Map<string, Promise<void>>();
+
+    /**
+     * Cross-instance transport for channel frames and presence.
+     *
+     * Defaults to the memory bus, which publishes nowhere — so a single-instance
+     * deployment runs the same fan-out it always did, with one resolved promise
+     * per broadcast for company. See `channel-bus/ChannelBus.ts`.
+     */
+    private bus: ChannelBus = new MemoryChannelBus();
+
+    /**
+     * The shared presence roster, present only when a real bus is active.
+     *
+     * Fan-out alone is not enough for presence: `presence_state` has to answer
+     * with everyone in the channel, and per-process maps can only answer for
+     * this replica's clients. See `channel-presence.ts`.
+     */
+    private presenceStore?: ChannelPresenceStore;
+
+    /** Sweeps roster rows left behind by instances that stopped heartbeating. */
+    private presenceSweepInterval?: ReturnType<typeof setInterval>;
+
+    /**
+     * Channels whose oversized ephemeral broadcasts have already been reported,
+     * so a hot channel logs the problem once rather than once per message.
+     */
+    private oversizedBroadcastWarned = new Set<string>();
+
     private presenceInterval?: ReturnType<typeof setInterval>;
     private static readonly PRESENCE_TIMEOUT_MS = 30000; // 30s
+    /** How often stale roster rows from other instances are reaped. */
+    private static readonly PRESENCE_SWEEP_INTERVAL_MS = 10000; // 10s
     private dataService: DataService;
     // Enhanced subscriptions storage with full request parameters
     private _subscriptions = new Map<string, {
@@ -307,15 +339,19 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
         for (const [channel, members] of this.channels.entries()) {
             if (members.has(clientId)) {
                 members.delete(clientId);
-                this.removePresence(clientId, channel);
+                this.removePresence(clientId, channel, { skipStore: true });
                 if (members.size === 0) this.channels.delete(channel);
             }
         }
 
         // Remove from all presence channels
         for (const [channel] of this.presence) {
-            this.removePresence(clientId, channel);
+            this.removePresence(clientId, channel, { skipStore: true });
         }
+
+        // One statement for every channel the client was in, rather than one
+        // per channel above — a disconnect is the common case, not a rare one.
+        void this.presenceStoreOp(() => this.presenceStore!.removeClient(clientId), "client removal");
     }
 
     private async handleMessage(clientId: string, message: WebSocketMessage, authContext?: SubscriptionAuthContext) {
@@ -391,6 +427,16 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                 return;
             }
 
+            // Bound the client-supplied limit with the SAME guarantee the REST
+            // ingress applies (`resolveClientListLimit`): clamp to the hard max
+            // and default an absent limit by mode. A subscription is re-fetched
+            // on every matching write, so an unbounded one is a DoS amplified
+            // per write — resolve it once and reuse for the stored request and
+            // the initial fetch.
+            const boundedLimit = resolveClientListLimit(request.limit, {
+                vectorSearch: !!request.vectorSearch
+            });
+
             // Store subscription with full request parameters and auth context for RLS
             this._subscriptions.set(subscriptionId, {
                 clientId,
@@ -400,7 +446,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                     filter: request.filter,
                     orderBy: request.orderBy,
                     order: request.order,
-                    limit: request.limit,
+                    limit: boundedLimit,
                     startAfter: request.startAfter as Record<string, unknown> | undefined,
                     databaseId: request.collection?.databaseId,
                     searchString: request.searchString
@@ -415,7 +461,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                     filter: request.filter,
                     orderBy: request.orderBy,
                     order: request.order,
-                    limit: request.limit,
+                    limit: boundedLimit,
                     startAfter: request.startAfter as Record<string, unknown> | undefined,
                     searchString: request.searchString
                 },
@@ -1088,6 +1134,11 @@ roles: activeAuth.roles },
         const retention = this.channelHistory?.retentionFor(channel);
         if (!retention) {
             this.fanOutBroadcast(clientId, channel, event, payload);
+            // Other instances get the same frame, but never before the clients
+            // on this one: the local fan-out above is synchronous and the
+            // publish is not, which is also what keeps the ephemeral path free
+            // of any await for a single-instance deployment.
+            this.publishBroadcast(clientId, channel, event, payload);
             return;
         }
 
@@ -1137,6 +1188,7 @@ roles: activeAuth.roles },
         }
 
         this.fanOutBroadcast(clientId, channel, event, payload, seq);
+        this.publishBroadcast(clientId, channel, event, payload, seq);
 
         try {
             await this.channelHistory!.prune(channel, retention);
@@ -1170,6 +1222,186 @@ roles: activeAuth.roles },
         }
     }
 
+    // =============================================================================
+    // Cross-Instance Channel Bus
+    // =============================================================================
+
+    /**
+     * Install the transport that carries channel frames between instances.
+     *
+     * Called once at boot. A bus that cannot start is reported and replaced with
+     * the memory bus: losing cross-instance fan-out degrades collaboration to
+     * what it was before this existed, whereas refusing to boot takes the whole
+     * backend down for it.
+     */
+    async configureChannelBus(bus: ChannelBus): Promise<void> {
+        if (bus.kind === "memory") {
+            this.bus = bus;
+            return;
+        }
+
+        try {
+            await bus.start((frame) => this.handleBusFrame(frame));
+        } catch (error) {
+            logger.warn(
+                `⚠️ [ChannelBus] Could not start the "${bus.kind}" channel bus — channel broadcast and presence ` +
+                "stay per-instance. Clients served by different replicas will not see each other.",
+                { error }
+            );
+            await bus.stop().catch(() => { /* best effort */ });
+            this.bus = new MemoryChannelBus();
+            return;
+        }
+
+        this.bus = bus;
+
+        // Presence needs shared *state*, not just shared fan-out — see
+        // `channel-presence.ts`. It comes up with the bus and only with it.
+        try {
+            const store = new ChannelPresenceStore(this.db, this.instanceId);
+            await store.ensureTables();
+            this.presenceStore = store;
+            this.ensurePresenceSweep();
+        } catch (error) {
+            logger.warn(
+                "⚠️ [ChannelBus] Could not create the shared presence table — presence rosters will only list " +
+                "clients connected to this instance (broadcast is unaffected).",
+                { error }
+            );
+            this.presenceStore = undefined;
+        }
+
+        logger.info(
+            `📡 [ChannelBus] Cross-instance channels active via ${bus.kind} (instanceId: ${this.instanceId}).`
+        );
+    }
+
+    /** Which transport is in use — `"memory"` means per-instance only. */
+    public getChannelBusKind(): ChannelBus["kind"] {
+        return this.bus.kind;
+    }
+
+    /**
+     * Send a broadcast to the other instances.
+     *
+     * Fire-and-forget by design: the clients on this instance have already been
+     * served, and a bus that is briefly unreachable must not turn a broadcast
+     * into an error for the sender.
+     */
+    private publishBroadcast(clientId: string, channel: string, event: string, payload: unknown, seq?: number): void {
+        if (this.bus.kind === "memory") return;
+
+        const frame: ChannelBusFrame = {
+            kind: "broadcast",
+            sid: this.instanceId,
+            channel,
+            event,
+            from: clientId,
+            ...(seq !== undefined ? { seq } : {}),
+            payload
+        };
+
+        // Postgres caps a NOTIFY payload at 8 KB. A retained message is already
+        // durable and addressable, so it travels as a pointer and each receiver
+        // reads the body back — the same shape as the entity path, which
+        // notifies an address and refetches the row.
+        if (frameByteLength(frame) > this.bus.maxFrameBytes) {
+            if (seq === undefined) {
+                this.reportOversizedBroadcast(clientId, channel);
+                return;
+            }
+            void this.publishFrame({
+                kind: "broadcast_ref",
+                sid: this.instanceId,
+                channel,
+                from: clientId,
+                seq
+            });
+            return;
+        }
+
+        void this.publishFrame(frame);
+    }
+
+    private async publishFrame(frame: ChannelBusFrame): Promise<void> {
+        try {
+            await this.bus.publish(frame);
+        } catch (error) {
+            logger.error("❌ [ChannelBus] Failed to publish frame — other instances did not receive it", {
+                detail: `${frame.kind} on "${frame.channel}"`,
+                error
+            });
+        }
+    }
+
+    /**
+     * Tell the sender that a message was delivered locally but nowhere else.
+     *
+     * Staying quiet here would be the worst option available: on one instance
+     * the app works, on two it works for half the users, and nothing in the
+     * logs connects the two. The fix is a one-liner in config — give the
+     * channel a retention rule and the message travels as a pointer instead —
+     * so the message says exactly that.
+     */
+    private reportOversizedBroadcast(clientId: string, channel: string): void {
+        const remedy =
+            `Add a retention rule for "${channel}" (realtime.channels) — retained messages travel by reference ` +
+            "and have no size limit.";
+
+        if (!this.oversizedBroadcastWarned.has(channel)) {
+            this.oversizedBroadcastWarned.add(channel);
+            logger.warn(
+                `⚠️ [ChannelBus] A broadcast on ephemeral channel "${channel}" exceeds the ` +
+                `${this.bus.maxFrameBytes}-byte limit of the ${this.bus.kind} bus and reached only this instance. ` +
+                remedy
+            );
+        }
+        this.sendError(
+            clientId,
+            `Broadcast on "${channel}" was too large to reach other instances. ${remedy}`,
+            undefined,
+            "CHANNEL_BUS_PAYLOAD_TOO_LARGE"
+        );
+    }
+
+    /**
+     * Deliver a frame published by another instance to this one's clients.
+     *
+     * Frames we published ourselves are dropped on arrival — the local fan-out
+     * happened before the publish — exactly as the entity-change handler skips
+     * its own `sid`.
+     */
+    private async handleBusFrame(frame: ChannelBusFrame): Promise<void> {
+        if (frame.sid === this.instanceId) return;
+
+        switch (frame.kind) {
+            case "broadcast":
+                this.fanOutBroadcast(frame.from ?? "", frame.channel, frame.event, frame.payload, frame.seq);
+                return;
+
+            case "broadcast_ref": {
+                // Nothing to read back for: skip the query rather than pay for
+                // a message no client here is waiting for.
+                if (!this.channels.get(frame.channel)?.size) return;
+
+                const entry = await this.channelHistory?.getBySeq(frame.channel, frame.seq);
+                if (!entry) {
+                    logger.warn(
+                        `⚠️ [ChannelBus] Message ${frame.seq} on "${frame.channel}" is no longer retained — ` +
+                        "clients on this instance will need to replay (channel_history) to catch up."
+                    );
+                    return;
+                }
+                this.fanOutBroadcast(frame.from ?? "", frame.channel, entry.event, entry.payload, entry.seq);
+                return;
+            }
+
+            case "presence_diff":
+                this.deliverPresenceDiff(frame.channel, frame.joins, frame.leaves);
+                return;
+        }
+    }
+
     // =============================================================================
     // Channel History
     // =============================================================================
@@ -1247,32 +1479,59 @@ roles: activeAuth.roles },
     // Presence
     // =============================================================================
 
-    /** Track presence in a channel */
+    /**
+     * Track presence in a channel.
+     *
+     * The client re-sends this every ~20s as a heartbeat against the 30s
+     * timeout, so most calls carry the state that is already recorded. Those
+     * refresh `last_seen` and stop there: re-announcing an unchanged state to
+     * every instance would put a bus message per client per heartbeat on the
+     * wire to tell everyone nothing happened.
+     */
     trackPresence(clientId: string, channel: string, state: Record<string, unknown>): void {
         if (!this.presence.has(channel)) {
             this.presence.set(channel, new Map());
         }
 
         const channelPresence = this.presence.get(channel)!;
+        const previous = channelPresence.get(clientId);
+        const changed = !previous || JSON.stringify(previous.state) !== JSON.stringify(state);
         channelPresence.set(clientId, { state,
 lastSeen: Date.now() });
 
+        // Refresh the shared roster on every heartbeat — that timestamp is what
+        // tells other instances this client is still here.
+        void this.presenceStoreOp(() => this.presenceStore!.track(channel, clientId, state), "track");
+
         // Broadcast join / state update to channel
-        this.broadcastPresenceDiff(channel, { [clientId]: state }, {});
+        this.deliverPresenceDiff(channel, { [clientId]: state }, {});
+        if (changed) {
+            this.publishPresenceDiff(channel, { [clientId]: state }, {});
+        }
 
         // Start cleanup interval if not running
         this.ensurePresenceCleanup();
     }
 
-    /** Remove presence from a channel */
-    removePresence(clientId: string, channel: string): void {
+    /**
+     * Remove presence from a channel.
+     *
+     * `skipStore` is for the socket-close path, which clears every channel at
+     * once and then deletes the client's rows in a single statement instead of
+     * one per channel.
+     */
+    removePresence(clientId: string, channel: string, options?: { skipStore?: boolean }): void {
         const channelPresence = this.presence.get(channel);
         if (!channelPresence) return;
 
         const entry = channelPresence.get(clientId);
         if (entry) {
             channelPresence.delete(clientId);
-            this.broadcastPresenceDiff(channel, {}, { [clientId]: entry.state });
+            this.deliverPresenceDiff(channel, {}, { [clientId]: entry.state });
+            this.publishPresenceDiff(channel, {}, { [clientId]: entry.state });
+            if (!options?.skipStore) {
+                void this.presenceStoreOp(() => this.presenceStore!.remove(channel, clientId), "remove");
+            }
         }
 
         if (channelPresence.size === 0) {
@@ -1280,17 +1539,50 @@ lastSeen: Date.now() });
         }
     }
 
-    /** Send full presence state to a specific client */
+    /**
+     * Send the full roster for a channel to one client.
+     *
+     * Answered from the shared table when there is one, because "who is in this
+     * document?" has a single answer that must not depend on which replica the
+     * asker happens to be connected to. Without a bus there is nothing to share
+     * and the local map *is* the roster — that path stays synchronous, which is
+     * what it always was.
+     */
     sendPresenceState(clientId: string, channel: string): void {
+        if (!this.presenceStore) {
+            this.sendPresenceStateMessage(clientId, channel, this.localPresences(channel));
+            return;
+        }
+
+        void this.presenceStore.roster(channel)
+            .then((presences) => {
+                this.sendPresenceStateMessage(clientId, channel, presences);
+            })
+            .catch((error) => {
+                // A roster the asker can act on beats none: fall back to the
+                // clients we can see rather than leaving the request unanswered.
+                logger.warn(`⚠️ [Presence] Could not read the shared roster for "${channel}" — answering with this instance's clients only.`, { error });
+                this.sendPresenceStateMessage(clientId, channel, this.localPresences(channel));
+            });
+    }
+
+    /** Presence of the clients connected to this instance. */
+    private localPresences(channel: string): Record<string, Record<string, unknown>> {
         const channelPresence = this.presence.get(channel);
         const presences: Record<string, Record<string, unknown>> = {};
-
         if (channelPresence) {
             for (const [id, { state }] of channelPresence) {
                 presences[id] = state;
             }
         }
+        return presences;
+    }
 
+    private sendPresenceStateMessage(
+        clientId: string,
+        channel: string,
+        presences: Record<string, Record<string, unknown>>
+    ): void {
         const ws = this.clients.get(clientId);
         if (ws && ws.readyState === WebSocket.OPEN) {
             ws.send(JSON.stringify({
@@ -1301,8 +1593,8 @@ lastSeen: Date.now() });
         }
     }
 
-    /** Broadcast presence diff (joins/leaves) to channel */
-    private broadcastPresenceDiff(
+    /** Deliver a presence diff to this instance's members of the channel. */
+    private deliverPresenceDiff(
         channel: string,
         joins: Record<string, Record<string, unknown>>,
         leaves: Record<string, Record<string, unknown>>
@@ -1325,6 +1617,26 @@ lastSeen: Date.now() });
         }
     }
 
+    /** Tell the other instances about a presence change. */
+    private publishPresenceDiff(
+        channel: string,
+        joins: Record<string, Record<string, unknown>>,
+        leaves: Record<string, Record<string, unknown>>
+    ): void {
+        if (this.bus.kind === "memory") return;
+        void this.publishFrame({ kind: "presence_diff", sid: this.instanceId, channel, joins, leaves });
+    }
+
+    /** Run a roster write when there is a roster, and never let it throw. */
+    private async presenceStoreOp(op: () => Promise<void>, label: string): Promise<void> {
+        if (!this.presenceStore) return;
+        try {
+            await op();
+        } catch (error) {
+            logger.warn(`⚠️ [Presence] Shared roster ${label} failed`, { error });
+        }
+    }
+
     /** Periodic cleanup for stale presences */
     private ensurePresenceCleanup(): void {
         if (this.presenceInterval) return;
@@ -1345,6 +1657,43 @@ lastSeen: Date.now() });
         }, 10000); // Check every 10s
     }
 
+    /**
+     * Reap roster rows whose owning instance stopped heartbeating.
+     *
+     * This is the cross-instance half of the sweep above, and it doubles as
+     * crash recovery: a pod that dies takes its clients with it but leaves
+     * their rows behind, and after one TTL window they look exactly like any
+     * other client that went quiet. The delete returns what it removed, so
+     * whichever instance wins the race is the one that announces the
+     * departures — once for the cluster, not once per replica.
+     */
+    private ensurePresenceSweep(): void {
+        if (this.presenceSweepInterval || !this.presenceStore) return;
+
+        this.presenceSweepInterval = setInterval(
+            () => void this.sweepStalePresence(),
+            RealtimeService.PRESENCE_SWEEP_INTERVAL_MS
+        );
+
+        // Never hold the process open for housekeeping.
+        (this.presenceSweepInterval as unknown as { unref?: () => void }).unref?.();
+    }
+
+    /** One pass of the stale-roster sweep. See {@link ensurePresenceSweep}. */
+    private async sweepStalePresence(): Promise<void> {
+        if (!this.presenceStore) return;
+        try {
+            const removed = await this.presenceStore.sweepStale(RealtimeService.PRESENCE_TIMEOUT_MS);
+            for (const row of removed) {
+                this.debugLog(`👻 [Presence] Reaped stale presence ${row.clientId} on "${row.channel}"`);
+                this.deliverPresenceDiff(row.channel, {}, { [row.clientId]: row.state });
+                this.publishPresenceDiff(row.channel, {}, { [row.clientId]: row.state });
+            }
+        } catch (error) {
+            logger.warn("⚠️ [Presence] Stale-roster sweep failed", { error });
+        }
+    }
+
     // =============================================================================
     // Lifecycle / Cleanup
     // =============================================================================
@@ -1383,10 +1732,30 @@ lastSeen: Date.now() });
             clearInterval(this.presenceInterval);
             this.presenceInterval = undefined;
         }
+        if (this.presenceSweepInterval) {
+            clearInterval(this.presenceSweepInterval);
+            this.presenceSweepInterval = undefined;
+        }
+        this.oversizedBroadcastWarned.clear();
+
+        // Drop this instance's roster rows now rather than leaving every other
+        // replica to wait out a TTL window on ghosts — a rolling deploy would
+        // otherwise show 30s of departed users on every restart.
+        if (this.presenceStore) {
+            try {
+                await this.presenceStore.removeInstance();
+            } catch (error) {
+                logger.warn("⚠️ [Presence] Could not clear this instance's roster rows on shutdown", { error });
+            }
+            this.presenceStore = undefined;
+        }
 
         // 4. Disconnect the dedicated LISTEN client(s)
         await this.stopListening();
         await this.stopCdc();
+        await this.bus.stop().catch((error) =>
+            logger.warn("⚠️ [ChannelBus] Error while stopping the channel bus", { error }));
+        this.bus = new MemoryChannelBus();
 
         // 5. Drop client references (don't close — server.close drains them)
         this.clients.clear();