@rebasepro/server-postgres 0.13.1-canary.gef9608c → 0.14.0

package/src/services/realtimeService.ts

@@ -4,7 +4,7 @@ import { Client as PgClient } from "pg";
 import { randomUUID } from "crypto";
 import { DataService } from "./dataService";
 
-import { ANONYMOUS_USER_ID, FetchCollectionProps, ListenCollectionProps, ListenOneProps, DataDriver, CollectionUpdateMessage, SingleUpdateMessage, CollectionPatchMessage, WebSocketMessage, FilterValues, LogicalCondition, CollectionConfig, RebaseCallContext, resolveClientListLimit } from "@rebasepro/types";
+import { ANONYMOUS_USER_ID, FetchCollectionProps, ListenCollectionProps, ListenOneProps, DataDriver, CollectionUpdateMessage, SingleUpdateMessage, CollectionPatchMessage, WebSocketMessage, FilterValues, LogicalCondition, CollectionConfig, RebaseCallContext, resolveClientListLimit, ListLimitError } from "@rebasepro/types";
 import { NodePgDatabase } from "drizzle-orm/node-postgres";
 import { sql as drizzleSql } from "drizzle-orm";
 import { RealtimeProvider, CollectionSubscriptionConfig, SingleSubscriptionConfig } from "../interfaces";
@@ -33,6 +33,37 @@ export interface SubscriptionAuthContext {
     roles: string[];
 }
 
+/** What a channel frame is asking to do. */
+export type ChannelAction = "join" | "broadcast" | "presence" | "history";
+
+/** Everything an authorizer is told about the frame it is asked to allow. */
+export interface ChannelAuthorizationRequest {
+    /** The channel the frame names, exactly as the client wrote it. */
+    channel: string;
+    action: ChannelAction;
+    /** The socket, not the principal — one user may hold several. */
+    clientId: string;
+    /** The socket's authenticated principal, or the anonymous one. */
+    user?: SubscriptionAuthContext;
+}
+
+/**
+ * The extension point for channel access rules.
+ *
+ * **This is deliberately not a product API yet.** The rule *language* — a
+ * config key, a per-pattern DSL, how it composes with `securityRules` — is an
+ * open design question (see `docs/channel-authorization.md`), and
+ * inventing one here would be inventing the answer. What exists is the single
+ * place every channel frame passes through, so that whatever shape the rules
+ * eventually take has exactly one seam to plug into and no arm of the switch
+ * can be forgotten.
+ *
+ * Returning `false` — or throwing — refuses the frame. It is consulted *after*
+ * the membership floor below, so an authorizer can only ever narrow access,
+ * never widen it.
+ */
+export type ChannelAuthorizer = (request: ChannelAuthorizationRequest) => boolean | Promise<boolean>;
+
 interface DataDriverWithData extends DataDriver {
     data: unknown;
 }
@@ -61,6 +92,8 @@ type StoredCollectionRequest = {
     startAfter?: Record<string, unknown>;
     databaseId?: string;
     searchString?: string;
+    /** Ask each row which declared search field matched — populates `_matches`. */
+    searchExplain?: boolean;
 };
 
 type RealTimeListenEntityProps = ListenOneProps & { subscriptionId: string };
@@ -72,6 +105,13 @@ type RealTimeListenEntityProps = ListenOneProps & { subscriptionId: string };
  * Implements the RealtimeProvider interface for database abstraction.
  */
 export class RealtimeService extends EventEmitter implements RealtimeProvider {
+    /**
+     * Declares to the multi-engine router that channel frames can be handled
+     * here. Read by `createRoutedRealtimeService`, which otherwise would have to
+     * guess — and guessed "the default provider", whichever engine that is.
+     */
+    public readonly supportsChannels = true;
+
     private clients = new Map<string, WebSocket>();
 
     // Broadcast channels: channel name → set of client IDs
@@ -129,6 +169,26 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
      */
     private oversizedBroadcastWarned = new Set<string>();
 
+    /**
+     * Optional narrowing on top of the membership floor — see
+     * {@link ChannelAuthorizer}. Unset by default, which leaves membership as
+     * the whole of the rule.
+     */
+    private channelAuthorizer?: ChannelAuthorizer;
+
+    /**
+     * Whether a notification from another instance has ever arrived.
+     *
+     * The entity LISTEN handler sees a foreign `sid` on every cross-instance
+     * change, which is proof that this deployment runs more than one pod — the
+     * one fact needed to tell "the memory bus is fine here" from "broadcast and
+     * presence silently reach a fraction of your users".
+     */
+    private foreignInstanceSeen = false;
+
+    /** So the multi-pod memory-bus warning is emitted once, not once per join. */
+    private memoryBusWarned = false;
+
     private presenceInterval?: ReturnType<typeof setInterval>;
     private static readonly PRESENCE_TIMEOUT_MS = 30000; // 30s
     /** How often stale roster rows from other instances are reaped. */
@@ -266,7 +326,8 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                 limit: config.limit,
                 startAfter: config.startAfter as Record<string, unknown> | undefined,
                 databaseId: config.databaseId,
-                searchString: config.searchString
+                searchString: config.searchString,
+                searchExplain: config.searchExplain
             }
         });
 
@@ -375,45 +436,19 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                 await this.handleUnsubscribe(clientId, message.subscriptionId!);
                 break;
 
-            // ── Broadcast Channels ──
+            // ── Broadcast Channels & Presence ──
+            //
+            // One arm for all of them, because every one has to pass the same
+            // gate and a switch with seven arms is a place to forget it once.
+            // See `handleChannelMessage`.
             case "join_channel":
-                this.joinChannel(clientId, payload?.channel as string);
-                break;
             case "leave_channel":
-                this.leaveChannel(clientId, payload?.channel as string);
-                break;
             case "broadcast":
-                this.broadcastToChannel(
-                    clientId,
-                    payload?.channel as string,
-                    payload?.event as string,
-                    payload?.payload
-                );
-                break;
             case "channel_history":
-                await this.handleChannelHistoryRequest(
-                    clientId,
-                    payload?.channel as string,
-                    payload?.sinceSeq as number | undefined,
-                    payload?.limit as number | undefined
-                );
-                break;
-
-            // ── Presence ──
             case "presence_track":
-                // Auto-join the channel so presence works without a separate join
-                this.joinChannel(clientId, payload?.channel as string);
-                this.trackPresence(
-                    clientId,
-                    payload?.channel as string,
-                    payload?.state as Record<string, unknown> ?? {}
-                );
-                break;
             case "presence_untrack":
-                this.removePresence(clientId, payload?.channel as string);
-                break;
             case "presence_state":
-                this.sendPresenceState(clientId, payload?.channel as string);
+                await this.handleChannelMessage(clientId, message.type, payload, authContext);
                 break;
 
             default:
@@ -435,15 +470,42 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                 return;
             }
 
+            // A vector search cannot be served here, and the parameter used to
+            // be read for one thing only — the limit default below — and then
+            // dropped: the stored request carries no `vectorSearch` and the
+            // refetch has no branch for one. So `.vectorSearch(…).listen()`
+            // delivered an ordinary `id DESC` listing, with no `_distance` and
+            // no error, forever. Refusing says what the silence did not.
+            if (request.vectorSearch) {
+                const msg =
+                    "Realtime subscriptions do not support vector search: a subscription is re-run on every " +
+                    "matching write, and nothing here computes distances. Use `.vectorSearch(...).find()` for " +
+                    "the query, and subscribe without it if you need live updates.";
+                logger.warn(`[RealtimeService] ${msg}`);
+                this.sendError(clientId, msg, subscriptionId, "VECTOR_SEARCH_NOT_LIVE");
+                return;
+            }
+
             // Bound the client-supplied limit with the SAME guarantee the REST
-            // ingress applies (`resolveClientListLimit`): clamp to the hard max
-            // and default an absent limit by mode. A subscription is re-fetched
-            // on every matching write, so an unbounded one is a DoS amplified
-            // per write — resolve it once and reuse for the stored request and
-            // the initial fetch.
-            const boundedLimit = resolveClientListLimit(request.limit, {
-                vectorSearch: !!request.vectorSearch
-            });
+            // ingress applies (`resolveClientListLimit`): default an absent
+            // limit by mode, refuse one above the ceiling. A subscription is
+            // re-fetched on every matching write, so an unbounded one is a DoS
+            // amplified per write — resolve it once and reuse for the stored
+            // request and the initial fetch.
+            //
+            // Refusing matters more here than on the REST route: a
+            // `collection_update` frame carries rows and nothing else — no
+            // `total`, no `hasMore` — so a subscriber handed a quietly smaller
+            // page has no way at all to learn it is not seeing the collection.
+            let boundedLimit: number;
+            try {
+                boundedLimit = resolveClientListLimit(request.limit);
+            } catch (e) {
+                if (!(e instanceof ListLimitError)) throw e;
+                logger.warn(`[RealtimeService] Refused subscription to '${request.path}': ${e.message}`);
+                this.sendError(clientId, e.message, subscriptionId, "INVALID_LIMIT");
+                return;
+            }
 
             // Store subscription with full request parameters and auth context for RLS
             this._subscriptions.set(subscriptionId, {
@@ -459,7 +521,8 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                     offset: request.offset,
                     startAfter: request.startAfter as Record<string, unknown> | undefined,
                     databaseId: request.collection?.databaseId,
-                    searchString: request.searchString
+                    searchString: request.searchString,
+                    searchExplain: request.searchExplain
                 },
                 authContext
             });
@@ -592,7 +655,39 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
     }
 
     /**
-     * Notify subscriptions for a specific path
+     * Notify subscriptions for a specific path.
+     *
+     * **A subscriber only ever receives rows re-read under its own scope.**
+     * `row` is used to decide *that* something changed, never to say *what* —
+     * every delivery below goes through a refetch that binds the subscription's
+     * own auth context.
+     *
+     * It used to be conditional. The CDC path already did the right thing: it
+     * discards the captured tuple and emits `{_rebase_invalidated: true}`, and
+     * that marker selected the refetch branch. But the marker is produced in
+     * exactly two places, and the *other* side of each branch here shipped the
+     * row it was handed straight to the socket. Two of the three entry paths
+     * took that side — every API mutation (`PostgresBackendDriver.save` passes
+     * the row it just wrote, read under the **writer's** scope) and the legacy
+     * cross-instance LISTEN handler (which re-reads on the owner connection,
+     * bypassing RLS altogether). Path matching was the only filter applied: the
+     * subscription's own `filter`/`logical` was never evaluated, and any
+     * `afterRead` redaction was the writer's rather than the reader's.
+     *
+     * A single-row subscription was the sharpest case. `subscribe_one` on a row
+     * RLS denies is accepted and answered `null`; the next update then pushed
+     * the full row with no later correction. The collection variant was merely
+     * papered over ~300 ms later by the debounced refetch — after the bytes had
+     * already reached the browser.
+     *
+     * The same defect was found and fixed on the Mongo driver in `065e2b615`
+     * (see `packages/server-mongo/test/realtime-authorization.test.ts`); this is
+     * the Postgres half, stated as one rule rather than three patched branches.
+     *
+     * The cost is the instant row-level patch that used to precede the refetch:
+     * cross-tab feedback now waits for the debounce. That is the price of not
+     * being able to know, without asking the database as this subscriber,
+     * whether this subscriber may see the row at all.
      */
     private async notifyPathUpdate(notifyPath: string, originalPath: string, id: string, row: Record<string, unknown> | null, _databaseId?: string) {
         this.debugLog(`📡 [RealtimeService] Notifying path: ${notifyPath} (original: ${originalPath})`);
@@ -627,21 +722,8 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
         for (const [subscriptionId, subscription] of webSocketSubscriptions) {
             try {
                 if (subscription.type === "single" && notifyPath === originalPath) {
-                    // Send row update directly (only for exact path matches)
-                    if (row && (row as Record<string, unknown>)?._rebase_invalidated) {
-                        this.debouncedSingleRefetch(subscriptionId, notifyPath, id, subscription);
-                    } else {
-                        this.sendSingleUpdate(subscription.clientId, subscriptionId, row);
-                    }
+                    this.debouncedSingleRefetch(subscriptionId, notifyPath, id, subscription);
                 } else if (subscription.type === "collection" && subscription.collectionRequest) {
-                    // Phase 1: Send instant row-level patch (no DB query)
-                    // This gives immediate cross-tab feedback
-                    if (!row || !(row as Record<string, unknown>)?._rebase_invalidated) {
-                        this.sendCollectionPatch(subscription.clientId, subscriptionId, id, row, notifyPath);
-                    }
-
-                    // Phase 2: Schedule a deferred full refetch for correctness
-                    // Handles filter/sort changes and ensures consistency
                     this.debouncedCollectionRefetch(subscriptionId, notifyPath, subscription);
                 }
             } catch (error) {
@@ -657,12 +739,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                 if (!callback) continue;
 
                 if (subscription.type === "single" && notifyPath === originalPath) {
-                    if (row && (row as Record<string, unknown>)?._rebase_invalidated) {
-                        this.debouncedSingleDriverRefetch(subscriptionId, notifyPath, id, subscription, callback);
-                    } else {
-                        // Call the callback directly with the row (only for exact path matches)
-                        callback(row);
-                    }
+                    this.debouncedSingleDriverRefetch(subscriptionId, notifyPath, id, subscription, callback);
                 } else if (subscription.type === "collection" && subscription.collectionRequest) {
                     // Debounce collection refetches for DataDriver subscriptions too
                     this.debouncedDriverRefetch(subscriptionId, notifyPath, subscription, callback);
@@ -747,7 +824,8 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                 limit: collectionRequest.limit,
                 offset: collectionRequest.offset,
                 startAfter: collectionRequest.startAfter,
-                searchString: collectionRequest.searchString
+                searchString: collectionRequest.searchString,
+                searchExplain: collectionRequest.searchExplain
             });
 
             // Always wrap in a transaction with session vars, defaulting to anonymous context if missing.
@@ -765,10 +843,15 @@ roles: ["anon"] };
                         collectionRequest.searchString,
                         {
                             filter: collectionRequest.filter as FilterValues<string>,
+                            // The subscription stored a group; the search branch
+                            // did not pass it on, so a filtered live search
+                            // widened to every row matching the text.
+                            logical: collectionRequest.logical,
                             orderBy: collectionRequest.orderBy,
                             order: collectionRequest.order,
                             limit: collectionRequest.limit,
-                            databaseId: collectionRequest.databaseId
+                            databaseId: collectionRequest.databaseId,
+                            searchExplain: collectionRequest.searchExplain
                         }
                     );
                 } else {
@@ -839,22 +922,28 @@ roles: activeAuth.roles },
             });
         }
 
-        // No driver — use dataService directly (no auth wrapping possible)
+        // No driver — use dataService directly (no auth wrapping possible).
+        // The `logical` group is carried here as well: this branch answers the
+        // same subscription as the one above, and a fallback that drops a
+        // condition returns *more* rows than the path it stands in for.
         if (collectionRequest.searchString) {
             return await this.dataService.searchRows(
                 notifyPath,
                 collectionRequest.searchString,
                 {
                     filter: collectionRequest.filter as FilterValues<string>,
+                    logical: collectionRequest.logical,
                     orderBy: collectionRequest.orderBy,
                     order: collectionRequest.order,
                     limit: collectionRequest.limit,
-                    databaseId: collectionRequest.databaseId
+                    databaseId: collectionRequest.databaseId,
+                    searchExplain: collectionRequest.searchExplain
                 }
             );
         }
         return await this.dataService.fetchCollection(notifyPath, {
             filter: collectionRequest.filter as FilterValues<string>,
+            logical: collectionRequest.logical,
             orderBy: collectionRequest.orderBy,
             order: collectionRequest.order,
             limit: collectionRequest.limit,
@@ -1023,23 +1112,6 @@ roles: activeAuth.roles },
      * columns and no address. The SDK holds no collection config to derive one
      * from, so this is the only place the mapping can come from.
      */
-    private sendCollectionPatch(
-        clientId: string,
-        subscriptionId: string,
-        id: string,
-        row: Record<string, unknown> | null,
-        notifyPath: string
-    ) {
-        const message: CollectionPatchMessage = {
-            type: "collection_patch",
-            subscriptionId,
-            id,
-            row: row,
-            pks: this.primaryKeysForPath(notifyPath)
-        };
-        this.sendMessage(clientId, message);
-    }
-
     /** The key columns of the collection at `path`, if they can be resolved. */
     private primaryKeysForPath(path: string): PrimaryKeyInfo[] | undefined {
         try {
@@ -1104,15 +1176,197 @@ roles: activeAuth.roles },
     // Broadcast Channels
     // =============================================================================
 
+    /**
+     * Install a channel authorizer — see {@link ChannelAuthorizer}.
+     *
+     * Nothing in the framework calls this yet: it is the seam a rules API will
+     * be built on, kept deliberately separate from the membership floor so the
+     * floor holds whether or not anyone uses it.
+     */
+    setChannelAuthorizer(authorizer: ChannelAuthorizer | undefined): void {
+        this.channelAuthorizer = authorizer;
+    }
+
+    /** Which action each channel frame is asking to perform. */
+    private static readonly CHANNEL_ACTIONS: Record<string, ChannelAction> = {
+        join_channel: "join",
+        broadcast: "broadcast",
+        channel_history: "history",
+        presence_track: "join",
+        presence_state: "presence"
+    };
+
+    /**
+     * The one door every channel frame comes through.
+     *
+     * Returns synchronously — and so dispatches synchronously — unless an
+     * authorizer is installed. That matters: a client sends `join_channel`,
+     * `presence_state` and `channel_history` back to back on connect, and the
+     * socket's message handler processes each frame up to its first `await`,
+     * so a gate that always yielded would let the reads overtake the join that
+     * is about to authorize them.
+     */
+    private handleChannelMessage(
+        clientId: string,
+        type: string,
+        payload: Record<string, unknown> | undefined,
+        authContext?: SubscriptionAuthContext
+    ): void | Promise<void> {
+        const channel = payload?.channel as string;
+
+        // Leaving and untracking only ever remove the caller's own state, so
+        // they need no permission — refusing them could only strand a client.
+        if (type === "leave_channel") {
+            this.leaveChannel(clientId, channel);
+            return;
+        }
+        if (type === "presence_untrack") {
+            this.removePresence(clientId, channel);
+            return;
+        }
+
+        const action = RealtimeService.CHANNEL_ACTIONS[type];
+        const allowed = this.authorizeChannelAction(clientId, channel, action, authContext);
+        if (allowed === false) return;
+        if (allowed === true) return this.dispatchChannelMessage(clientId, type, channel, payload);
+        return allowed.then((ok) => {
+            if (ok) return this.dispatchChannelMessage(clientId, type, channel, payload);
+        });
+    }
+
+    /** Perform an already-authorized channel frame. */
+    private dispatchChannelMessage(
+        clientId: string,
+        type: string,
+        channel: string,
+        payload: Record<string, unknown> | undefined
+    ): void | Promise<void> {
+        switch (type) {
+            case "join_channel":
+                this.joinChannel(clientId, channel);
+                return;
+            case "broadcast":
+                this.broadcastToChannel(clientId, channel, payload?.event as string, payload?.payload);
+                return;
+            case "channel_history":
+                return this.handleChannelHistoryRequest(
+                    clientId,
+                    channel,
+                    payload?.sinceSeq as number | undefined,
+                    payload?.limit as number | undefined
+                );
+            case "presence_track":
+                // Auto-join the channel so presence works without a separate join
+                this.joinChannel(clientId, channel);
+                this.trackPresence(clientId, channel, payload?.state as Record<string, unknown> ?? {});
+                return;
+            case "presence_state":
+                this.sendPresenceState(clientId, channel);
+                return;
+        }
+    }
+
+    /**
+     * Decide whether a client may perform an action on a channel.
+     *
+     * **Membership is the floor.** Reading a channel's presence roster, replaying
+     * its retained history and broadcasting into it all require that this client
+     * has joined it. That is a low bar — joining is open to anyone who can name
+     * the channel — but it is not the bar that was there before, which was none
+     * at all: `channel_history` and `presence_state` answered any socket about
+     * any channel, and a broadcast fanned out to members the sender had never
+     * joined. Two internal tables (`rebase.channel_presence`,
+     * `rebase.channel_messages`) are held outside RLS on the strength of this
+     * check, so it fails closed: an authorizer that throws refuses the frame.
+     *
+     * Anything richer than membership belongs in a {@link ChannelAuthorizer};
+     * this method is where it is consulted, and the only place.
+     */
+    private authorizeChannelAction(
+        clientId: string,
+        channel: string,
+        action: ChannelAction,
+        authContext?: SubscriptionAuthContext
+    ): boolean | Promise<boolean> {
+        // Joining is what establishes membership, so it cannot require it.
+        if (action !== "join" && !this.channels.get(channel)?.has(clientId)) {
+            this.denyChannelAction(clientId, channel, action, "not a member of the channel");
+            return false;
+        }
+
+        const authorizer = this.channelAuthorizer;
+        if (!authorizer) return true;
+
+        let verdict: boolean | Promise<boolean>;
+        try {
+            verdict = authorizer({ channel, action, clientId, user: authContext });
+        } catch (error) {
+            logger.error(`❌ [Channels] Authorizer threw for ${action} on "${channel}" — refusing`, { error });
+            this.denyChannelAction(clientId, channel, action, "channel authorization failed");
+            return false;
+        }
+
+        if (typeof verdict === "boolean") {
+            if (!verdict) this.denyChannelAction(clientId, channel, action, "refused by the channel authorizer");
+            return verdict;
+        }
+
+        return verdict.then(
+            (ok) => {
+                if (!ok) this.denyChannelAction(clientId, channel, action, "refused by the channel authorizer");
+                return ok;
+            },
+            (error) => {
+                logger.error(`❌ [Channels] Authorizer rejected for ${action} on "${channel}" — refusing`, { error });
+                this.denyChannelAction(clientId, channel, action, "channel authorization failed");
+                return false;
+            }
+        );
+    }
+
+    /** Tell the client why its channel frame went nowhere, and say so in the log. */
+    private denyChannelAction(clientId: string, channel: string, action: ChannelAction, reason: string): void {
+        this.debugLog(`🚫 [Channels] Refused ${action} on "${channel}" for ${clientId}: ${reason}`);
+        this.sendError(
+            clientId,
+            `Refused ${action} on channel "${channel}": ${reason}`,
+            undefined,
+            "CHANNEL_FORBIDDEN"
+        );
+    }
+
     /** Join a broadcast channel */
     joinChannel(clientId: string, channel: string): void {
         if (!this.channels.has(channel)) {
             this.channels.set(channel, new Set());
         }
         this.channels.get(channel)!.add(clientId);
+        this.warnIfMemoryBusOnMultiplePods();
         this.debugLog(`📡 [Broadcast] Client ${clientId} joined channel: ${channel}`);
     }
 
+    /**
+     * Say something the first time channels are used on a deployment that is
+     * demonstrably multi-pod while the bus is still the in-memory default.
+     *
+     * Every other warning in this subsystem covers a *configured* bus failing —
+     * the case where the operator already knew a bus mattered. The common
+     * misconfiguration is the opposite one: scaled to two replicas, never
+     * touched `realtime.bus`, and broadcast and presence quietly serve a
+     * fraction of the room. The evidence is already in the process, so use it.
+     */
+    private warnIfMemoryBusOnMultiplePods(): void {
+        if (this.memoryBusWarned) return;
+        if (this.bus.kind !== "memory" || !this.foreignInstanceSeen) return;
+        this.memoryBusWarned = true;
+        logger.warn(
+            "⚠️ [ChannelBus] Channels are in use with the in-memory bus, but notifications from another " +
+            "instance have been seen — this deployment runs more than one process. Broadcast and presence " +
+            "reach only the clients connected to this one. Set `realtime.bus` (or REBASE_REALTIME_BUS=postgres) " +
+            "to make channels cross-instance."
+        );
+    }
+
     /** Leave a broadcast channel */
     leaveChannel(clientId: string, channel: string): void {
         const members = this.channels.get(channel);
@@ -1817,7 +2071,9 @@ lastSeen: Date.now() });
             throw err;
         }
         this.cdcActive = true;
-        logger.info(
+        // The bootstrapper says the same thing one line later, in the
+        // vocabulary of the setting that produced it (REALTIME_CDC).
+        logger.debug(
             `📡 [RealtimeService] Database-level change capture ACTIVE — writes from ANY source now emit realtime events ` +
             `(${this.cdcTableMap.size} mapped table key(s)).`
         );
@@ -2068,6 +2324,11 @@ lastSeen: Date.now() });
                     // Skip our own notifications — already processed locally
                     if (sid === this.instanceId) return;
 
+                    // A foreign sid is proof of a second process. Nothing here
+                    // needs that fact, but the channel path does — see
+                    // `warnIfMemoryBusOnMultiplePods`.
+                    this.foreignInstanceSeen = true;
+
                     this.debugLog(`📡 [RealtimeService] Received cross-instance notification: path=${p}, id=${eid}, from=${sid}`);
 
                     // Refetch the row from the DB so row subscriptions

package/src/services/write-denial.ts

@@ -0,0 +1,55 @@
+import { and, sql, SQL } from "drizzle-orm";
+import { PgTable } from "drizzle-orm/pg-core";
+import { ApiError } from "@rebasepro/server";
+import { DrizzleClient } from "../interfaces";
+
+/**
+ * Explain a write that matched no rows.
+ *
+ * Row-level security filters UPDATE and DELETE through the policy's USING
+ * clause instead of raising: a denied write is reported by Postgres exactly
+ * like a successful one that happened to match nothing. Left unchecked, a
+ * caller cannot tell "denied" from "done" — the write returns 200/204 and the
+ * row is untouched. An agent handed a key with `orders:delete` and no delete
+ * policy is the case that makes it concrete: it deletes nothing, forever, and
+ * is told it worked every time.
+ *
+ * Re-reading the target over the *same* RLS-scoped handle separates the two
+ * cases. A visible row means the policy rejected the write (403); an invisible
+ * one means there is nothing there to write for this caller (404, matching what
+ * a GET would say). The re-read is bound by the caller's own policies, so it
+ * discloses nothing a plain read wouldn't.
+ *
+ * Only reached when zero rows matched, so the happy path pays nothing.
+ *
+ * It lives here, rather than beside its first caller, because every zero-row
+ * write has to answer the same question and answer it identically: the rule
+ * that a readable-but-unwritable target is a 403 is the contract, and a second
+ * copy of it is a second chance to get it wrong.
+ *
+ * @param handle     The RLS-scoped connection the write ran on — not a fresh
+ *                   one, or the re-read would answer for a different caller.
+ * @param table      The table the write targeted (the junction, for a link).
+ * @param conditions The write's own WHERE terms, reused verbatim.
+ * @param denied     Message for the 403: the target is there and was refused.
+ * @param missing    Message for the 404: there is nothing there for this caller.
+ */
+export async function explainZeroRowWrite(
+    handle: DrizzleClient,
+    table: PgTable,
+    conditions: SQL[],
+    denied: string,
+    missing: string
+): Promise<ApiError> {
+    const visible = await handle
+        .select({ present: sql<number>`1` })
+        .from(table)
+        .where(and(...conditions))
+        .limit(1);
+
+    if (visible.length > 0) {
+        return ApiError.forbidden(denied, "WRITE_DENIED");
+    }
+
+    return ApiError.notFound(missing);
+}