@rebasepro/server-postgres 0.10.1-canary.a54c057 → 0.10.1-canary.b1e3dbf

package/dist/services/realtimeService.d.ts

@@ -4,6 +4,7 @@ import { DataDriver, WebSocketMessage } from "@rebasepro/types";
 import { NodePgDatabase } from "drizzle-orm/node-postgres";
 import { RealtimeProvider, CollectionSubscriptionConfig, SingleSubscriptionConfig } from "../interfaces";
 import { PostgresCollectionRegistry } from "../collections/PostgresCollectionRegistry";
+import { ChannelBus } from "./channel-bus";
 import type { ChannelRetentionRule } from "@rebasepro/types";
 /**
  * Auth context stored per-subscription so real-time refetches respect RLS.
@@ -45,8 +46,33 @@ export declare class RealtimeService extends EventEmitter implements RealtimePro
      * wait on each other.
      */
     private channelSendQueues;
+    /**
+     * Cross-instance transport for channel frames and presence.
+     *
+     * Defaults to the memory bus, which publishes nowhere — so a single-instance
+     * deployment runs the same fan-out it always did, with one resolved promise
+     * per broadcast for company. See `channel-bus/ChannelBus.ts`.
+     */
+    private bus;
+    /**
+     * The shared presence roster, present only when a real bus is active.
+     *
+     * Fan-out alone is not enough for presence: `presence_state` has to answer
+     * with everyone in the channel, and per-process maps can only answer for
+     * this replica's clients. See `channel-presence.ts`.
+     */
+    private presenceStore?;
+    /** Sweeps roster rows left behind by instances that stopped heartbeating. */
+    private presenceSweepInterval?;
+    /**
+     * Channels whose oversized ephemeral broadcasts have already been reported,
+     * so a hot channel logs the problem once rather than once per message.
+     */
+    private oversizedBroadcastWarned;
     private presenceInterval?;
     private static readonly PRESENCE_TIMEOUT_MS;
+    /** How often stale roster rows from other instances are reaped. */
+    private static readonly PRESENCE_SWEEP_INTERVAL_MS;
     private dataService;
     private _subscriptions;
     private subscriptionCallbacks;
@@ -245,6 +271,44 @@ export declare class RealtimeService extends EventEmitter implements RealtimePro
     private persistAndFanOut;
     /** Deliver a broadcast frame to every member of a channel but the sender. */
     private fanOutBroadcast;
+    /**
+     * Install the transport that carries channel frames between instances.
+     *
+     * Called once at boot. A bus that cannot start is reported and replaced with
+     * the memory bus: losing cross-instance fan-out degrades collaboration to
+     * what it was before this existed, whereas refusing to boot takes the whole
+     * backend down for it.
+     */
+    configureChannelBus(bus: ChannelBus): Promise<void>;
+    /** Which transport is in use — `"memory"` means per-instance only. */
+    getChannelBusKind(): ChannelBus["kind"];
+    /**
+     * Send a broadcast to the other instances.
+     *
+     * Fire-and-forget by design: the clients on this instance have already been
+     * served, and a bus that is briefly unreachable must not turn a broadcast
+     * into an error for the sender.
+     */
+    private publishBroadcast;
+    private publishFrame;
+    /**
+     * Tell the sender that a message was delivered locally but nowhere else.
+     *
+     * Staying quiet here would be the worst option available: on one instance
+     * the app works, on two it works for half the users, and nothing in the
+     * logs connects the two. The fix is a one-liner in config — give the
+     * channel a retention rule and the message travels as a pointer instead —
+     * so the message says exactly that.
+     */
+    private reportOversizedBroadcast;
+    /**
+     * Deliver a frame published by another instance to this one's clients.
+     *
+     * Frames we published ourselves are dropped on arrival — the local fan-out
+     * happened before the publish — exactly as the entity-change handler skips
+     * its own `sid`.
+     */
+    private handleBusFrame;
     /**
      * Install retention rules and create the tables they need.
      *
@@ -266,16 +330,60 @@ export declare class RealtimeService extends EventEmitter implements RealtimePro
      */
     private handleChannelHistoryRequest;
     private sendChannelHistory;
-    /** Track presence in a channel */
+    /**
+     * Track presence in a channel.
+     *
+     * The client re-sends this every ~20s as a heartbeat against the 30s
+     * timeout, so most calls carry the state that is already recorded. Those
+     * refresh `last_seen` and stop there: re-announcing an unchanged state to
+     * every instance would put a bus message per client per heartbeat on the
+     * wire to tell everyone nothing happened.
+     */
     trackPresence(clientId: string, channel: string, state: Record<string, unknown>): void;
-    /** Remove presence from a channel */
-    removePresence(clientId: string, channel: string): void;
-    /** Send full presence state to a specific client */
+    /**
+     * Remove presence from a channel.
+     *
+     * `skipStore` is for the socket-close path, which clears every channel at
+     * once and then deletes the client's rows in a single statement instead of
+     * one per channel.
+     */
+    removePresence(clientId: string, channel: string, options?: {
+        skipStore?: boolean;
+    }): void;
+    /**
+     * Send the full roster for a channel to one client.
+     *
+     * Answered from the shared table when there is one, because "who is in this
+     * document?" has a single answer that must not depend on which replica the
+     * asker happens to be connected to. Without a bus there is nothing to share
+     * and the local map *is* the roster — that path stays synchronous, which is
+     * what it always was.
+     */
     sendPresenceState(clientId: string, channel: string): void;
-    /** Broadcast presence diff (joins/leaves) to channel */
-    private broadcastPresenceDiff;
+    /** Presence of the clients connected to this instance. */
+    private localPresences;
+    private sendPresenceStateMessage;
+    /** Deliver a presence diff to this instance's members of the channel. */
+    private deliverPresenceDiff;
+    /** Tell the other instances about a presence change. */
+    private publishPresenceDiff;
+    /** Run a roster write when there is a roster, and never let it throw. */
+    private presenceStoreOp;
     /** Periodic cleanup for stale presences */
     private ensurePresenceCleanup;
+    /**
+     * Reap roster rows whose owning instance stopped heartbeating.
+     *
+     * This is the cross-instance half of the sweep above, and it doubles as
+     * crash recovery: a pod that dies takes its clients with it but leaves
+     * their rows behind, and after one TTL window they look exactly like any
+     * other client that went quiet. The delete returns what it removed, so
+     * whichever instance wins the race is the one that announces the
+     * departures — once for the cluster, not once per replica.
+     */
+    private ensurePresenceSweep;
+    /** One pass of the stale-roster sweep. See {@link ensurePresenceSweep}. */
+    private sweepStalePresence;
     /**
      * Gracefully tear down all realtime resources.
      *

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rebasepro/server-postgres",
   "type": "module",
-  "version": "0.10.1-canary.a54c057",
+  "version": "0.10.1-canary.b1e3dbf",
   "description": "PostgreSQL data source backend implementation for Rebase with Drizzle ORM",
   "funding": {
     "url": "https://github.com/sponsors/rebaseco"
@@ -47,11 +47,11 @@
     "execa": "^9.6.1",
     "pg": "^8.21.0",
     "ws": "^8.21.0",
-    "@rebasepro/codegen": "0.10.1-canary.a54c057",
-    "@rebasepro/common": "0.10.1-canary.a54c057",
-    "@rebasepro/server": "0.10.1-canary.a54c057",
-    "@rebasepro/types": "0.10.1-canary.a54c057",
-    "@rebasepro/utils": "0.10.1-canary.a54c057"
+    "@rebasepro/codegen": "0.10.1-canary.b1e3dbf",
+    "@rebasepro/common": "0.10.1-canary.b1e3dbf",
+    "@rebasepro/types": "0.10.1-canary.b1e3dbf",
+    "@rebasepro/server": "0.10.1-canary.b1e3dbf",
+    "@rebasepro/utils": "0.10.1-canary.b1e3dbf"
   },
   "devDependencies": {
     "@hono/node-server": "^2.0.9",

package/src/PostgresBootstrapper.ts

@@ -37,6 +37,8 @@ import { buildCollectionsFromSchema, introspectSchema, readRlsStatus } from "./s
 import { buildDrizzleTablesFromSchema, buildDrizzleRelationsFromSchema } from "./schema/dynamic-tables";
 import { detectConnectionPosture, ensureAppRole, validatePolicyPgRoles, warnOnAnonymousGrants, REBASE_USER_ROLE, type RawSqlRunner } from "./security/rls-enforcement";
 import { provisionTriggerCdc, type CdcTableRef } from "./services/cdc/trigger-cdc";
+import { createChannelBus, resolveChannelBusSetting } from "./services/channel-bus";
+import { isChannelBusInstance } from "@rebasepro/types";
 
 export interface PostgresDriverConfig {
     connectionString?: string;
@@ -54,9 +56,13 @@ export interface PostgresDriverConfig {
      */
     introspectionSchema?: string;
     /**
-     * Realtime options. Currently only channel retention, which is opt-in:
-     * without rules here no channel keeps any history and broadcast stays
-     * fire-and-forget. See {@link ChannelRetentionRule}.
+     * Realtime options, both opt-in:
+     *
+     *  - `channels` — retention. Without rules no channel keeps any history and
+     *    broadcast stays fire-and-forget. See {@link ChannelRetentionRule}.
+     *  - `bus` — the cross-instance transport for channel broadcast and
+     *    presence. Defaults to in-process only, which is correct for a single
+     *    instance and wrong for two. See {@link ChannelBusConfig}.
      */
     realtime?: RealtimeChannelsConfig;
 }
@@ -337,6 +343,29 @@ export function createPostgresBootstrapper(pgConfig: PostgresDriverConfig): Back
             // Prefer DATABASE_DIRECT_URL to bypass PgBouncer for LISTEN/NOTIFY.
             const directUrl = process.env.DATABASE_DIRECT_URL || pgConfig.connectionString;
 
+            // ── Cross-instance channel bus ───────────────────────────────────
+            // Entity changes already span instances (CDC / LISTEN below).
+            // Channel broadcast and presence did not — they lived in per-process
+            // maps, so behind two replicas the clients of one were invisible to
+            // the other. Opt-in, and a no-op when left at "memory".
+            try {
+                const busSetting = resolveChannelBusSetting(pgConfig.realtime?.bus);
+                // A supplied instance is always installed; a named built-in only
+                // when it is not the in-process default, so the common case
+                // touches none of this machinery.
+                const wantsBus = isChannelBusInstance(busSetting) || busSetting.type !== "memory";
+                if (wantsBus) {
+                    await realtimeService.configureChannelBus(
+                        createChannelBus(busSetting, {
+                            db: schemaAwareDb as unknown as NodePgDatabase<Record<string, unknown>>,
+                            directUrl
+                        })
+                    );
+                }
+            } catch (err) {
+                logger.warn("⚠️ [ChannelBus] Could not configure the channel bus — channel broadcast and presence stay per-instance", { error: err });
+            }
+
             // Database-level Change Data Capture. When active, realtime events are
             // emitted for EVERY committed write — including ones that bypass the
             // Rebase API (psql, another service's cron, raw SQL, the Studio SQL

package/src/auth/ensure-tables.ts

@@ -107,6 +107,7 @@ export async function ensureAuthTablesExist(db: NodePgDatabase, collection?: Col
                 email_verification_sent_at TIMESTAMP WITH TIME ZONE,
                 is_anonymous BOOLEAN DEFAULT FALSE NOT NULL,
                 metadata JSONB DEFAULT '{}' NOT NULL,
+                tokens_valid_after TIMESTAMP WITH TIME ZONE,
                 created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
                 updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL
             )
@@ -209,17 +210,23 @@ export async function ensureAuthTablesExist(db: NodePgDatabase, collection?: Col
         `);
 
 
-        // Create refresh tokens table (includes user_agent, ip_address, and unique constraint)
+        // Create refresh tokens table. One row per TOKEN, grouped into a
+        // sign-in by session_id — deliberately without a uniqueness rule on
+        // (uid, user_agent, ip_address); see the schema module for why that
+        // constraint had to go.
         await db.execute(sql`
             CREATE TABLE IF NOT EXISTS ${sql.raw(refreshTokensTableName)} (
                 id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
                 uid ${sql.raw(userIdType)} NOT NULL REFERENCES ${sql.raw(usersTableName)}(id) ON DELETE CASCADE,
+                session_id TEXT NOT NULL DEFAULT gen_random_uuid()::text,
                 token_hash TEXT NOT NULL UNIQUE,
                 expires_at TIMESTAMP WITH TIME ZONE NOT NULL,
+                revoked BOOLEAN DEFAULT FALSE NOT NULL,
+                rotated_at TIMESTAMP WITH TIME ZONE,
+                session_started_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL,
                 user_agent TEXT,
                 ip_address TEXT,
-                created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
-                CONSTRAINT unique_device_session UNIQUE (uid, user_agent, ip_address)
+                created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
             )
         `);
 
@@ -348,6 +355,7 @@ export async function ensureAuthTablesExist(db: NodePgDatabase, collection?: Col
             "email_verification_sent_at TIMESTAMP WITH TIME ZONE",
             "is_anonymous BOOLEAN DEFAULT FALSE NOT NULL",
             "metadata JSONB DEFAULT '{}' NOT NULL",
+            "tokens_valid_after TIMESTAMP WITH TIME ZONE",
             "created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL",
             "updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL"
         ];
@@ -358,16 +366,26 @@ export async function ensureAuthTablesExist(db: NodePgDatabase, collection?: Col
             `);
         }
 
-        // ── Migration: reconcile refresh_tokens for the device-session upsert ──
-        // The refresh-token rotation does `INSERT ... ON CONFLICT (uid, user_agent,
-        // ip_address) DO UPDATE`, which requires the `unique_device_session`
-        // constraint. `CREATE TABLE IF NOT EXISTS` never adds it to a table that
-        // predates the constraint, so every login on such a database 500s with
-        // `42P10 no unique or exclusion constraint matching the ON CONFLICT`.
-        // Reconcile EVERY table named refresh_tokens, in whatever schema it lives,
-        // rather than only the derived name: a database provisioned by an older
-        // era can carry the table in a different schema than the one this run
-        // derives, and the login upsert then hits a table without the constraint.
+        // ── Migration: refresh_tokens become session-scoped, rotation-safe ──
+        // Two shapes are reconciled here, on EVERY table named refresh_tokens
+        // in whatever schema it lives (a database provisioned by an older era
+        // can carry the table in a different schema than the one this run
+        // derives, and auth would then read a table nobody migrated):
+        //
+        //   1. The new columns. A token is now a member of a session
+        //      (`session_id`) and is retained after rotation (`revoked`,
+        //      `rotated_at`) so a replayed token can be recognised instead of
+        //      looking like a forgery. `session_started_at` is carried across
+        //      rotations so `users.tokens_valid_after` cannot be outrun.
+        //   2. The removal of `unique_device_session`. It made (uid,
+        //      user_agent, ip_address) the identity of a session, which evicted
+        //      a second browser profile behind one NAT and churned rows as
+        //      phones changed networks. UA and IP are metadata now.
+        //
+        // Every existing row is adopted rather than dropped: it keeps its
+        // token_hash, gets a session of its own, and stays unrevoked — so the
+        // sessions live in browsers right now survive the upgrade rather than
+        // everyone being signed out by the fix for being signed out.
         try {
             const rtTables = await db.execute(sql`
                 SELECT table_schema, table_name
@@ -379,57 +397,53 @@ export async function ensureAuthTablesExist(db: NodePgDatabase, collection?: Col
             for (const { table_schema } of found) {
                 const qualified = `"${table_schema}"."refresh_tokens"`;
                 try {
-                    // Old tables may lack these columns entirely.
-                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} ADD COLUMN IF NOT EXISTS user_agent TEXT`);
-                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} ADD COLUMN IF NOT EXISTS ip_address TEXT`);
-                    // The upsert relies on '' (not NULL) so a device with no UA/IP collides.
-                    await db.execute(sql`UPDATE ${sql.raw(qualified)} SET user_agent = '' WHERE user_agent IS NULL`);
-                    await db.execute(sql`UPDATE ${sql.raw(qualified)} SET ip_address = '' WHERE ip_address IS NULL`);
-                    // Drop pre-existing duplicates that would block the unique constraint,
-                    // keeping one row per (uid, user_agent, ip_address).
+                    // Added nullable, then back-filled, then constrained: adding
+                    // `session_id NOT NULL DEFAULT gen_random_uuid()` in one step
+                    // would stamp every existing row with the SAME uuid on some
+                    // Postgres versions, silently merging every live session into
+                    // one that a single logout would then wipe.
+                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} ADD COLUMN IF NOT EXISTS session_id TEXT`);
+                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} ADD COLUMN IF NOT EXISTS revoked BOOLEAN DEFAULT FALSE NOT NULL`);
+                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} ADD COLUMN IF NOT EXISTS rotated_at TIMESTAMP WITH TIME ZONE`);
+                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} ADD COLUMN IF NOT EXISTS session_started_at TIMESTAMP WITH TIME ZONE`);
+
+                    // One session per pre-existing row: under the old model a row
+                    // WAS a device session, and there is no record of which rows
+                    // descended from the same sign-in.
                     await db.execute(sql`
-                        DELETE FROM ${sql.raw(qualified)} a
-                        USING ${sql.raw(qualified)} b
-                        WHERE a.ctid < b.ctid
-                          AND a.uid = b.uid
-                          AND a.user_agent = b.user_agent
-                          AND a.ip_address = b.ip_address
+                        UPDATE ${sql.raw(qualified)}
+                        SET session_id = gen_random_uuid()::text
+                        WHERE session_id IS NULL
                     `);
-                    // Log every unique constraint on the table so drift is visible.
-                    const uniques = await db.execute(sql`
-                        SELECT conname, pg_get_constraintdef(oid) AS def
-                        FROM pg_constraint
-                        WHERE conrelid = ${sql.raw(`'${qualified}'`)}::regclass AND contype = 'u'
+                    await db.execute(sql`
+                        UPDATE ${sql.raw(qualified)}
+                        SET session_started_at = COALESCE(created_at, NOW())
+                        WHERE session_started_at IS NULL
                     `);
-                    for (const u of uniques.rows as { conname: string; def: string }[]) {
-                        logger.info(`   ${qualified} unique: ${u.conname} → ${u.def}`);
-                    }
-                    // Does a usable unique constraint cover EXACTLY (uid, user_agent, ip_address)?
-                    // Checking by name alone is not enough: an older era may carry a
-                    // `unique_device_session` on different columns, which the ON CONFLICT
-                    // cannot infer, so login keeps 500ing with 42P10.
-                    const hasCorrect = (uniques.rows as { def: string }[]).some((u) => {
-                        const cols = (u.def.match(/\(([^)]*)\)/)?.[1] || "")
-                            .split(",").map((c) => c.trim().replace(/"/g, ""));
-                        return cols.length === 3 && cols.includes("uid") && cols.includes("user_agent") && cols.includes("ip_address");
-                    });
-                    if (hasCorrect) {
-                        logger.info(`✓ correct device-session unique already present on ${qualified}`);
-                    } else {
-                        // Remove a wrong-columned constraint squatting on the name, then add the right one.
-                        await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} DROP CONSTRAINT IF EXISTS unique_device_session`);
-                        await db.execute(sql`
-                            ALTER TABLE ${sql.raw(qualified)}
-                                ADD CONSTRAINT unique_device_session UNIQUE (uid, user_agent, ip_address)
-                        `);
-                        logger.info(`✅ Added correct unique_device_session constraint to ${qualified}`);
-                    }
+                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} ALTER COLUMN session_id SET DEFAULT gen_random_uuid()::text`);
+                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} ALTER COLUMN session_started_at SET DEFAULT NOW()`);
+                    // SET NOT NULL only once the back-fill above has definitely
+                    // run; on a table that somehow still holds a NULL this throws
+                    // and is caught below rather than failing the boot.
+                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} ALTER COLUMN session_id SET NOT NULL`);
+                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} ALTER COLUMN session_started_at SET NOT NULL`);
+
+                    await db.execute(sql`
+                        CREATE INDEX IF NOT EXISTS idx_refresh_tokens_session
+                        ON ${sql.raw(qualified)}(session_id)
+                    `);
+
+                    // The device-session constraint is now actively harmful: two
+                    // live tokens of one session (a rotation in flight) share a
+                    // uid, and usually a user agent and IP too.
+                    await db.execute(sql`ALTER TABLE ${sql.raw(qualified)} DROP CONSTRAINT IF EXISTS unique_device_session`);
+                    logger.info(`✅ refresh_tokens reconciled for session-scoped rotation: ${qualified}`);
                 } catch (perTableError: unknown) {
                     logger.warn(`⚠️  refresh_tokens reconcile failed for ${qualified}: ${perTableError instanceof Error ? perTableError.message : String(perTableError)}`);
                 }
             }
         } catch (migrationError: unknown) {
-            logger.warn(`⚠️  refresh_tokens device-session migration skipped: ${migrationError instanceof Error ? migrationError.message : String(migrationError)}`);
+            logger.warn(`⚠️  refresh_tokens session migration skipped: ${migrationError instanceof Error ? migrationError.message : String(migrationError)}`);
         }
 
         // ── Migration: Copy roles from legacy junction table to inline column ──