@rebasepro/server-postgres 0.10.0 → 0.10.1-canary.14e53ae

package/src/services/channel-bus/PostgresChannelBus.ts

@@ -0,0 +1,299 @@
+/**
+ * Channel bus over Postgres LISTEN/NOTIFY.
+ *
+ * Chosen because it needs nothing that a Rebase deployment does not already
+ * have — the same database, the same direct URL the CDC listener uses. Three
+ * properties of `NOTIFY` shape everything below:
+ *
+ *  - **8000 bytes per payload.** Presence and cursors fit with room to spare; a
+ *    scene snapshot does not. Rather than truncate or drop, an oversized frame
+ *    on a *retained* channel is published as a pointer — the body is already in
+ *    `rebase.channel_messages` with a sequence number, so the receiver reads it
+ *    back. That is the same trick the entity path uses (notify an address,
+ *    refetch the row), applied to a different table. On an ephemeral channel
+ *    there is nothing to point at, so the publish is refused loudly instead of
+ *    reaching some instances and not others.
+ *
+ *  - **A notify is a query on the primary database.** Not a slow one, but it
+ *    competes with the application's real queries, and that — not throughput —
+ *    is what actually limits this transport. Measured, it carried ~10k
+ *    cross-instance messages/second and stayed flat out to eight instances; what
+ *    it should not do is spend 10k queries/second of the database's budget on
+ *    cursor movement. Hence the batching below.
+ *
+ *  - **Delivery is best-effort.** Retained channels repair themselves through
+ *    the client's history replay, so a lost frame costs a live update rather
+ *    than correctness. That is what makes coalescing safe.
+ */
+
+import { sql } from "drizzle-orm";
+import { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { logger } from "@rebasepro/server";
+import { PgNotifyListener } from "../pg-notify-listener";
+import { ChannelBus, ChannelBusFrame, ChannelBusHandler, frameByteLength } from "./ChannelBus";
+
+/** NOTIFY channel carrying channel-bus frames. */
+export const CHANNEL_BUS_NOTIFY_CHANNEL = "rebase_channel_bus";
+
+/**
+ * Postgres refuses a NOTIFY payload of 8000 bytes or more. The margin below it
+ * is for nothing in particular — it is there so that a payload which passes this
+ * check cannot fail at the server for being a few bytes over.
+ */
+export const PG_NOTIFY_MAX_PAYLOAD_BYTES = 7500;
+
+/**
+ * How long a batching window stays open.
+ *
+ * Ten milliseconds is below the threshold where a human notices a cursor lag,
+ * and it is the difference between one query per message and one query per
+ * window under load. Set to 0 to disable coalescing entirely.
+ */
+export const DEFAULT_BATCH_WINDOW_MS = 10;
+
+/** JSON overhead per frame inside a batch: the wrapping array's comma. */
+const BATCH_SEPARATOR_BYTES = 1;
+/** JSON overhead of the batch envelope itself: `{"batch":[]}`. */
+const BATCH_ENVELOPE_BYTES = 12;
+
+interface PendingFrame {
+    frame: ChannelBusFrame;
+    bytes: number;
+    resolve: () => void;
+    reject: (error: unknown) => void;
+}
+
+export class PostgresChannelBus implements ChannelBus {
+    readonly kind = "postgres" as const;
+    readonly maxFrameBytes = PG_NOTIFY_MAX_PAYLOAD_BYTES;
+
+    private listener?: PgNotifyListener;
+    private readonly batchWindowMs: number;
+
+    /**
+     * Frames waiting for the current window to close.
+     *
+     * The window is opened by a publish that found none open, and that publish
+     * is sent *immediately* rather than joining a batch — see {@link publish}.
+     */
+    private pending: PendingFrame[] = [];
+    private pendingBytes = BATCH_ENVELOPE_BYTES;
+    private windowTimer?: ReturnType<typeof setTimeout>;
+    private stopped = false;
+
+    constructor(
+        private readonly db: NodePgDatabase<Record<string, unknown>>,
+        private readonly connectionString: string,
+        options: { batchWindowMs?: number } = {}
+    ) {
+        const configured = options.batchWindowMs;
+        this.batchWindowMs = typeof configured === "number" && configured >= 0
+            ? configured
+            : DEFAULT_BATCH_WINDOW_MS;
+    }
+
+    async start(handler: ChannelBusHandler): Promise<void> {
+        this.stopped = false;
+        this.listener = new PgNotifyListener({
+            connectionString: this.connectionString,
+            channel: CHANNEL_BUS_NOTIFY_CHANNEL,
+            logLabel: "[ChannelBus]",
+            onPayload: async (payload) => {
+                const frames = parseChannelBusPayload(payload);
+                if (!frames.length) {
+                    logger.warn("⚠️ [ChannelBus] Dropping unparseable payload.");
+                    return;
+                }
+                // In order: a batch preserves the sender's publish order, and a
+                // retained channel's consumers rely on it.
+                for (const frame of frames) await handler(frame);
+            }
+        });
+        await this.listener.start();
+    }
+
+    /**
+     * Publish, coalescing under load.
+     *
+     * The window is *leading edge*: a publish arriving when no window is open is
+     * sent straight away and opens one, so an idle channel pays no added latency
+     * at all. Frames arriving while it is open are collected and leave together
+     * when it closes. The effect is that cost tracks elapsed time rather than
+     * message count — one query per window instead of one per message — which is
+     * the same shape as the retention pruning throttle, for the same reason.
+     *
+     * The returned promise settles when the frame has actually left, not when it
+     * was queued, so the contract ("reaches the other instances, or rejects")
+     * still holds.
+     */
+    async publish(frame: ChannelBusFrame): Promise<void> {
+        if (this.batchWindowMs === 0 || this.stopped) {
+            await this.send([frame]);
+            return;
+        }
+
+        if (!this.windowTimer) {
+            this.openWindow();
+            await this.send([frame]);
+            return;
+        }
+
+        const bytes = frameByteLength(frame) + BATCH_SEPARATOR_BYTES;
+
+        // A batch is one NOTIFY payload, so the 8 KB ceiling applies to the
+        // whole batch. Send what we have rather than let the frame push it over.
+        if (this.pending.length && this.pendingBytes + bytes > this.maxFrameBytes) {
+            this.flush();
+        }
+
+        return new Promise<void>((resolve, reject) => {
+            this.pending.push({ frame, bytes, resolve, reject });
+            this.pendingBytes += bytes;
+        });
+    }
+
+    async stop(): Promise<void> {
+        this.stopped = true;
+        if (this.windowTimer) {
+            clearTimeout(this.windowTimer);
+            this.windowTimer = undefined;
+        }
+        // Anything still queued belongs to clients that are already waiting on
+        // it; dropping it on shutdown would be a silent loss where a flush costs
+        // one more query.
+        this.flush();
+        await this.listener?.stop();
+        this.listener = undefined;
+    }
+
+    private openWindow(): void {
+        this.windowTimer = setTimeout(() => {
+            this.windowTimer = undefined;
+            if (this.pending.length) {
+                // Still busy: send this window's frames and open the next one,
+                // so a sustained stream keeps costing one query per window.
+                this.flush();
+                this.openWindow();
+            }
+            // Otherwise leave it closed, so the next publish after a quiet
+            // moment goes out immediately.
+        }, this.batchWindowMs);
+
+        // Housekeeping must never hold the process open.
+        (this.windowTimer as unknown as { unref?: () => void }).unref?.();
+    }
+
+    /** Send everything queued and settle the promises waiting on it. */
+    private flush(): void {
+        if (!this.pending.length) return;
+
+        const batch = this.pending;
+        this.pending = [];
+        this.pendingBytes = BATCH_ENVELOPE_BYTES;
+
+        this.send(batch.map(p => p.frame))
+            .then(() => { for (const p of batch) p.resolve(); })
+            .catch((error) => { for (const p of batch) p.reject(error); });
+    }
+
+    /**
+     * One NOTIFY.
+     *
+     * A single frame goes out in the plain, unwrapped shape. That is not just
+     * economy: during a rolling deploy an instance running the previous build
+     * understands only that shape, and low-rate traffic — presence, the tail of
+     * a session — is exactly what is flowing while pods restart. Batching only
+     * appears under load, which shrinks the mixed-version window to almost
+     * nothing.
+     */
+    private async send(frames: ChannelBusFrame[]): Promise<void> {
+        if (!frames.length) return;
+        const payload = frames.length === 1
+            ? JSON.stringify(frames[0])
+            : JSON.stringify({ batch: frames });
+
+        await this.db.execute(sql`SELECT pg_notify(${CHANNEL_BUS_NOTIFY_CHANNEL}, ${payload})`);
+    }
+}
+
+/**
+ * Parse a bus payload into the frames it carries.
+ *
+ * Accepts both wire shapes — a bare frame and a `{ batch: [...] }` envelope —
+ * so an instance on the new build understands one on the old. Returns an empty
+ * array for anything unrecognisable: a malformed or future-versioned message
+ * must never take the listener down.
+ */
+export function parseChannelBusPayload(payload: string): ChannelBusFrame[] {
+    let parsed: unknown;
+    try {
+        parsed = JSON.parse(payload);
+    } catch {
+        return [];
+    }
+    if (!parsed || typeof parsed !== "object") return [];
+
+    const batch = (parsed as { batch?: unknown }).batch;
+    if (Array.isArray(batch)) {
+        return batch
+            .map(entry => coerceFrame(entry))
+            .filter((frame): frame is ChannelBusFrame => frame !== null);
+    }
+
+    const single = coerceFrame(parsed);
+    return single ? [single] : [];
+}
+
+/**
+ * Parse a single bus frame, returning null for anything that is not a frame we
+ * understand.
+ */
+export function parseChannelBusFrame(payload: string): ChannelBusFrame | null {
+    try {
+        return coerceFrame(JSON.parse(payload));
+    } catch {
+        return null;
+    }
+}
+
+function coerceFrame(value: unknown): ChannelBusFrame | null {
+    if (!value || typeof value !== "object") return null;
+
+    const obj = value as Record<string, unknown>;
+    const sid = typeof obj.sid === "string" ? obj.sid : undefined;
+    const channel = typeof obj.channel === "string" ? obj.channel : undefined;
+    if (!sid || !channel) return null;
+
+    switch (obj.kind) {
+        case "broadcast":
+            if (typeof obj.event !== "string") return null;
+            return {
+                kind: "broadcast",
+                sid,
+                channel,
+                event: obj.event,
+                from: typeof obj.from === "string" ? obj.from : undefined,
+                seq: typeof obj.seq === "number" ? obj.seq : undefined,
+                payload: obj.payload
+            };
+        case "broadcast_ref":
+            if (typeof obj.seq !== "number") return null;
+            return {
+                kind: "broadcast_ref",
+                sid,
+                channel,
+                from: typeof obj.from === "string" ? obj.from : undefined,
+                seq: obj.seq
+            };
+        case "presence_diff":
+            return {
+                kind: "presence_diff",
+                sid,
+                channel,
+                joins: (obj.joins ?? {}) as Record<string, Record<string, unknown>>,
+                leaves: (obj.leaves ?? {}) as Record<string, Record<string, unknown>>
+            };
+        default:
+            return null;
+    }
+}

package/src/services/channel-bus/index.ts

@@ -0,0 +1,123 @@
+/**
+ * Resolution of the channel bus from config, environment, or a supplied instance.
+ *
+ * Opt-in, like every other cross-cutting realtime switch here: with nothing
+ * configured a deployment gets the memory bus and behaves exactly as it did
+ * before this existed. Unlike `REALTIME_CDC=auto`, there is no "try it and see"
+ * default — a bus changes where messages go, and quietly turning on a Postgres
+ * NOTIFY per broadcast because a direct URL happened to be set is not a
+ * decision to make on the user's behalf.
+ *
+ * Two transports ship, and neither adds a service to a deployment. A third is
+ * not a code change here: `realtime.bus` also accepts an already-constructed
+ * {@link ChannelBus}, so a transport published as its own package plugs in
+ * without this file learning about it. See `@rebasepro/types` →
+ * `types/channel_bus.ts` for the contract such a package implements.
+ */
+
+import { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { isChannelBusInstance, type ChannelBus, type ChannelBusConfig, type ChannelBusSetting } from "@rebasepro/types";
+import { logger } from "@rebasepro/server";
+import { MemoryChannelBus } from "./ChannelBus";
+import { PostgresChannelBus } from "./PostgresChannelBus";
+
+export * from "./ChannelBus";
+export {
+    PostgresChannelBus,
+    CHANNEL_BUS_NOTIFY_CHANNEL,
+    PG_NOTIFY_MAX_PAYLOAD_BYTES,
+    DEFAULT_BATCH_WINDOW_MS,
+    parseChannelBusFrame,
+    parseChannelBusPayload
+} from "./PostgresChannelBus";
+
+export interface ChannelBusDeps {
+    db: NodePgDatabase<Record<string, unknown>>;
+    /**
+     * Direct (non-pooled) Postgres URL for the LISTEN client. `LISTEN` is
+     * session state, so behind PgBouncer in transaction mode this must be the
+     * database itself and not the pooler.
+     */
+    directUrl?: string;
+}
+
+/**
+ * Merge `REALTIME_CHANNEL_BUS` into the configured bus.
+ *
+ * The environment wins over a *named* built-in, so the transport can be changed
+ * per deployment without a rebuild — the same reason `REALTIME_CDC` is an env
+ * var. It does **not** win over a supplied instance: the env var can only name
+ * transports this package knows how to construct, so honouring it there would
+ * mean silently discarding the object the application handed us.
+ */
+export function resolveChannelBusSetting(configured?: ChannelBusSetting): ChannelBusSetting {
+    const raw = (process.env.REALTIME_CHANNEL_BUS || "").trim().toLowerCase();
+
+    if (isChannelBusInstance(configured)) {
+        if (raw && raw !== configured.kind) {
+            logger.warn(
+                `⚠️ [ChannelBus] REALTIME_CHANNEL_BUS="${raw}" is ignored because realtime.bus was given a ` +
+                `"${configured.kind}" transport instance directly. Remove one of the two to make the intent clear.`
+            );
+        }
+        return configured;
+    }
+
+    if (!raw) return configured ?? { type: "memory" };
+
+    if (raw !== "memory" && raw !== "postgres") {
+        logger.warn(
+            `⚠️ [ChannelBus] Unknown REALTIME_CHANNEL_BUS value "${raw}" — expected memory|postgres, or pass a ` +
+            "ChannelBus instance as realtime.bus for a transport that ships separately. Falling back to the " +
+            "configured bus."
+        );
+        return configured ?? { type: "memory" };
+    }
+
+    // Keep the configured options (an explicit connection string) when the env
+    // var only restates the type it was already set to.
+    if (configured?.type === raw) return configured;
+    return raw === "memory" ? { type: "memory" } : { type: "postgres" };
+}
+
+/**
+ * @deprecated Use {@link resolveChannelBusSetting}, which also accepts a
+ * supplied {@link ChannelBus} instance. Kept as a narrow alias so existing
+ * config-only callers keep their exact types.
+ */
+export function resolveChannelBusConfig(configured?: ChannelBusConfig): ChannelBusConfig {
+    return resolveChannelBusSetting(configured) as ChannelBusConfig;
+}
+
+/**
+ * Produce the bus a setting asks for.
+ *
+ * An instance is handed straight back — constructing it was the application's
+ * job, and this function has nothing to add. A named built-in that turns out to
+ * be unusable degrades to the memory bus, with the reason logged, rather than
+ * throwing: a misconfigured bus should cost a deployment its cross-instance
+ * fan-out, not its ability to boot.
+ */
+export function createChannelBus(setting: ChannelBusSetting, deps: ChannelBusDeps): ChannelBus {
+    if (isChannelBusInstance(setting)) return setting;
+
+    switch (setting.type) {
+        case "postgres": {
+            const connectionString = setting.connectionString || deps.directUrl;
+            if (!connectionString) {
+                logger.warn(
+                    "⚠️ [ChannelBus] realtime.bus is \"postgres\" but no direct database URL is available " +
+                    "(set DATABASE_DIRECT_URL or realtime.bus.connectionString) — channel broadcast and presence " +
+                    "stay per-instance."
+                );
+                return new MemoryChannelBus();
+            }
+            return new PostgresChannelBus(deps.db, connectionString, {
+                batchWindowMs: setting.batchWindowMs
+            });
+        }
+        case "memory":
+        default:
+            return new MemoryChannelBus();
+    }
+}

package/src/services/channel-history.ts

@@ -290,6 +290,41 @@ export class ChannelHistoryStore {
         return { messages, latestSeq };
     }
 
+    /**
+     * One retained message by its address.
+     *
+     * This is what makes the cross-instance pointer path work: a broadcast too
+     * large to travel inside a `pg_notify` payload is already stored here, so
+     * the notification carries `(channel, seq)` and each receiving instance
+     * reads the body back. Returns null when the message has since been pruned
+     * — a receiver that is that far behind has nothing useful to deliver, and
+     * the client's own `channel_history` replay is the repair path.
+     */
+    async getBySeq(channel: string, seq: number): Promise<ChannelHistoryEntry | null> {
+        const result = await this.db.execute(sql`
+            SELECT seq, event, payload, sender_id, created_at
+            FROM rebase.channel_messages
+            WHERE channel = ${channel} AND seq = ${seq}
+        `);
+
+        const row = result.rows[0] as {
+            seq: string | number;
+            event: string;
+            payload: unknown;
+            sender_id: string | null;
+            created_at: Date | string;
+        } | undefined;
+        if (!row) return null;
+
+        return {
+            seq: Number(row.seq),
+            event: row.event,
+            payload: row.payload,
+            senderId: row.sender_id ?? undefined,
+            at: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at)
+        };
+    }
+
     /**
      * Enforce a channel's retention bounds.
      *

package/src/services/channel-presence.ts

@@ -0,0 +1,148 @@
+/**
+ * The shared presence roster.
+ *
+ * Broadcast only ever needed *fan-out* to work across instances — a frame goes
+ * out, whoever is connected receives it. Presence needs more than that, because
+ * `presence_state` is a question ("who is in this document?") and a per-process
+ * `Map` can only answer for the clients that happen to share a replica with the
+ * asker. Two people editing the same scene through different pods would each
+ * see an empty room while broadcasting cursors at each other perfectly.
+ *
+ * So presence gets one row per tracked client, in Postgres, readable by every
+ * instance. Three consequences worth stating:
+ *
+ *  - **The table is the roster; the in-process map is a cache of our own
+ *    clients.** Reads answer from the table when this store is active, so the
+ *    answer is the same whichever instance is asked.
+ *
+ *  - **`last_seen` is the liveness signal, and it is already there.** The client
+ *    heartbeats presence every ~20 s against a 30 s window; the sweep that has
+ *    always reaped local stale entries now also reaps rows belonging to
+ *    instances that stopped writing — which is exactly what a crashed pod looks
+ *    like. Crash recovery is a property of the TTL, not a separate mechanism.
+ *
+ *  - **The sweep deletes with `RETURNING`.** Whichever instance wins the delete
+ *    is the one that announces the departures, so a stale client produces one
+ *    `presence_diff` for the cluster rather than one per replica.
+ */
+
+import { sql } from "drizzle-orm";
+import { NodePgDatabase } from "drizzle-orm/node-postgres";
+
+/** A tracked client, as any instance sees it. */
+export interface PresenceRow {
+    channel: string;
+    clientId: string;
+    state: Record<string, unknown>;
+}
+
+export class ChannelPresenceStore {
+    private tablesReady = false;
+
+    constructor(
+        private readonly db: NodePgDatabase<Record<string, unknown>>,
+        private readonly instanceId: string
+    ) {}
+
+    /** Create the roster table. Idempotent. */
+    async ensureTables(): Promise<void> {
+        if (this.tablesReady) return;
+
+        await this.db.execute(sql`CREATE SCHEMA IF NOT EXISTS rebase`);
+
+        // Keyed by (channel, client_id): a client id is globally unique, so the
+        // instance is a column rather than part of the identity — a client that
+        // reconnects onto another replica replaces its own row instead of
+        // appearing twice in the roster.
+        await this.db.execute(sql`
+            CREATE TABLE IF NOT EXISTS rebase.channel_presence (
+                channel TEXT NOT NULL,
+                client_id TEXT NOT NULL,
+                instance_id TEXT NOT NULL,
+                state JSONB NOT NULL DEFAULT '{}'::jsonb,
+                last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+                PRIMARY KEY (channel, client_id)
+            )
+        `);
+
+        // The sweep's access path; the roster read rides the primary key.
+        await this.db.execute(sql`
+            CREATE INDEX IF NOT EXISTS idx_channel_presence_last_seen
+            ON rebase.channel_presence (last_seen)
+        `);
+
+        this.tablesReady = true;
+    }
+
+    /** Record (or refresh) a client's presence. */
+    async track(channel: string, clientId: string, state: Record<string, unknown>): Promise<void> {
+        await this.db.execute(sql`
+            INSERT INTO rebase.channel_presence (channel, client_id, instance_id, state, last_seen)
+            VALUES (${channel}, ${clientId}, ${this.instanceId}, ${JSON.stringify(state ?? {})}::jsonb, NOW())
+            ON CONFLICT (channel, client_id) DO UPDATE
+            SET state = EXCLUDED.state,
+                instance_id = EXCLUDED.instance_id,
+                last_seen = NOW()
+        `);
+    }
+
+    /** Drop one client's presence in one channel. */
+    async remove(channel: string, clientId: string): Promise<void> {
+        await this.db.execute(sql`
+            DELETE FROM rebase.channel_presence
+            WHERE channel = ${channel} AND client_id = ${clientId}
+        `);
+    }
+
+    /** Drop a client from every channel — used when its socket closes. */
+    async removeClient(clientId: string): Promise<void> {
+        await this.db.execute(sql`
+            DELETE FROM rebase.channel_presence WHERE client_id = ${clientId}
+        `);
+    }
+
+    /** The global roster for a channel. */
+    async roster(channel: string): Promise<Record<string, Record<string, unknown>>> {
+        const result = await this.db.execute(sql`
+            SELECT client_id, state FROM rebase.channel_presence WHERE channel = ${channel}
+        `);
+
+        const presences: Record<string, Record<string, unknown>> = {};
+        for (const row of result.rows as Array<{ client_id: string; state: Record<string, unknown> | null }>) {
+            presences[row.client_id] = row.state ?? {};
+        }
+        return presences;
+    }
+
+    /**
+     * Reap rows this instance is not responsible for and that have gone quiet.
+     *
+     * Own rows are excluded because the in-process sweep already handles them —
+     * and handles them better, since it can tell "the socket is gone" from "the
+     * heartbeat is late". What is left is precisely the interesting case: rows
+     * written by an instance that is no longer writing.
+     *
+     * Returns what was removed, so the caller can announce it.
+     */
+    async sweepStale(ttlMs: number): Promise<PresenceRow[]> {
+        const result = await this.db.execute(sql`
+            DELETE FROM rebase.channel_presence
+            WHERE instance_id <> ${this.instanceId}
+              AND last_seen < NOW() - MAKE_INTERVAL(secs => ${ttlMs / 1000})
+            RETURNING channel, client_id, state
+        `);
+
+        return (result.rows as Array<{ channel: string; client_id: string; state: Record<string, unknown> | null }>)
+            .map(row => ({ channel: row.channel, clientId: row.client_id, state: row.state ?? {} }));
+    }
+
+    /**
+     * Remove every row this instance owns. Called on graceful shutdown so a
+     * rolling deploy does not leave a TTL window of ghosts in every roster.
+     */
+    async removeInstance(): Promise<void> {
+        await this.db.execute(sql`
+            DELETE FROM rebase.channel_presence WHERE instance_id = ${this.instanceId}
+        `);
+    }
+}