@rebasepro/server-postgres 0.10.1-canary.6f89f77 → 0.10.1-canary.7609565

package/src/services/cdc/CdcListener.ts

@@ -1,6 +1,6 @@
-import { Client as PgClient } from "pg";
 import { logger } from "@rebasepro/server";
 import { CDC_CHANNEL } from "./trigger-cdc";
+import { PgNotifyListener } from "../pg-notify-listener";
 
 /**
  * A single database change captured by the CDC triggers and delivered over the
@@ -54,22 +54,30 @@ export function parseCdcPayload(payload: string): CdcChangeEvent | null {
 /**
  * Dedicated Postgres LISTEN client for database-level CDC.
  *
- * Mirrors the resilience of the RealtimeService cross-instance LISTEN client: a
- * standalone `pg.Client` (outside the Drizzle pool) that stays connected, and
- * transparently reconnects on error/disconnect. Each backend instance runs one,
- * so every instance observes every committed change regardless of which
- * instance (or external process) made the write.
+ * A {@link PgNotifyListener} — a connection outside the Drizzle pool that stays
+ * open and repairs itself — plus the parsing that turns a `rebase_cdc` payload
+ * into a change event. Each backend instance runs one, so every instance
+ * observes every committed change regardless of which instance (or external
+ * process) made the write.
  */
 export class CdcListener {
-    private client?: PgClient;
-    private running = false;
-    private reconnectTimer?: ReturnType<typeof setTimeout>;
-    private static readonly RECONNECT_DELAY_MS = 3000;
-
-    constructor(
-        private readonly connectionString: string,
-        private readonly onEvent: (event: CdcChangeEvent) => void | Promise<void>
-    ) {}
+    private readonly listener: PgNotifyListener;
+
+    constructor(connectionString: string, onEvent: (event: CdcChangeEvent) => void | Promise<void>) {
+        this.listener = new PgNotifyListener({
+            connectionString,
+            channel: CDC_CHANNEL,
+            logLabel: "[CDC]",
+            onPayload: (payload) => {
+                const event = parseCdcPayload(payload);
+                if (!event) {
+                    logger.warn("⚠️ [CDC] Dropping unparseable change notification.");
+                    return;
+                }
+                return onEvent(event);
+            }
+        });
+    }
 
     /**
      * Connect and begin listening. Idempotent.
@@ -78,90 +86,18 @@ export class CdcListener {
      * established (or `LISTEN` is refused), this rejects so callers — notably
      * `REALTIME_CDC=auto` — can detect an unusable connection and fall back to
      * app-level realtime. Once the initial connection succeeds, later drops
-     * self-heal via {@link scheduleReconnect}.
+     * self-heal in the background.
      */
     async start(): Promise<void> {
-        if (this.running) {
+        if (this.listener.active) {
             logger.warn("⚠️ [CDC] CdcListener.start() called but already running. Ignoring.");
             return;
         }
-        this.running = true;
-        try {
-            await this.connect({ initial: true });
-        } catch (err) {
-            this.running = false;
-            throw err;
-        }
+        await this.listener.start();
     }
 
     /** Stop listening and release the connection. */
     async stop(): Promise<void> {
-        this.running = false;
-        if (this.reconnectTimer) {
-            clearTimeout(this.reconnectTimer);
-            this.reconnectTimer = undefined;
-        }
-        if (this.client) {
-            try {
-                await this.client.end();
-            } catch { /* ignore close errors */ }
-            this.client = undefined;
-        }
-    }
-
-    private async connect({ initial = false }: { initial?: boolean } = {}): Promise<void> {
-        try {
-            const client = new PgClient({ connectionString: this.connectionString });
-
-            client.on("error", (err) => {
-                logger.error("❌ [CDC] LISTEN client error", { detail: err.message });
-                this.scheduleReconnect();
-            });
-
-            client.on("end", () => {
-                if (this.running) {
-                    logger.warn("⚠️ [CDC] LISTEN client disconnected unexpectedly.");
-                    this.scheduleReconnect();
-                }
-            });
-
-            client.on("notification", (msg) => {
-                if (!msg.payload) return;
-                const event = parseCdcPayload(msg.payload);
-                if (!event) {
-                    logger.warn("⚠️ [CDC] Dropping unparseable change notification.");
-                    return;
-                }
-                // Never let a handler rejection escape into the pg client.
-                Promise.resolve(this.onEvent(event)).catch((err) =>
-                    logger.error("❌ [CDC] Error handling change event", { error: err })
-                );
-            });
-
-            await client.connect();
-            await client.query(`LISTEN ${CDC_CHANNEL}`);
-            this.client = client;
-            logger.info(`📡 [CDC] Listening for database changes on channel "${CDC_CHANNEL}".`);
-        } catch (err) {
-            // Surface the initial failure so callers can choose to fall back;
-            // for reconnects, keep retrying quietly in the background.
-            if (initial) throw err;
-            logger.error("❌ [CDC] Failed to connect LISTEN client", { error: err });
-            this.scheduleReconnect();
-        }
-    }
-
-    private scheduleReconnect(): void {
-        if (!this.running || this.reconnectTimer) return;
-
-        this.reconnectTimer = setTimeout(async () => {
-            this.reconnectTimer = undefined;
-            if (!this.running) return;
-            if (this.client) {
-                try { await this.client.end(); } catch { /* ignore */ }
-                this.client = undefined;
-            }
-            await this.connect();
-        }, CdcListener.RECONNECT_DELAY_MS);
+        await this.listener.stop();
     }
 }

package/src/services/channel-bus/ChannelBus.ts

@@ -0,0 +1,44 @@
+/**
+ * Runtime pieces of the channel bus that are not the contract itself.
+ *
+ * The interface, the frame shape and the implementer's contract live in
+ * `@rebasepro/types` (`types/channel_bus.ts`), so a transport shipped as its own
+ * package — a Redis one, say — depends on the contract and not on this database
+ * adapter. They are re-exported here for convenience: code already importing
+ * from the adapter should not have to know where the types are declared.
+ */
+
+import type { ChannelBusFrame } from "@rebasepro/types";
+
+export type {
+    ChannelBus,
+    ChannelBusFrame,
+    ChannelBusHandler,
+    ChannelBusConfig,
+    ChannelBusSetting
+} from "@rebasepro/types";
+export { isChannelBusInstance } from "@rebasepro/types";
+
+/**
+ * The default: no cross-instance delivery at all.
+ *
+ * This is what every deployment ran before the bus existed, and what a
+ * single-instance deployment should keep running — `publish` resolves without
+ * touching the network, so the broadcast path is the same handful of `ws.send`
+ * calls it always was.
+ */
+export class MemoryChannelBus {
+    readonly kind = "memory" as const;
+    readonly maxFrameBytes = Infinity;
+
+    async start(): Promise<void> { /* nothing to connect */ }
+
+    async publish(): Promise<void> { /* nowhere to publish to */ }
+
+    async stop(): Promise<void> { /* nothing to release */ }
+}
+
+/** Encoded size of a frame, for a transport's size check. */
+export function frameByteLength(frame: ChannelBusFrame): number {
+    return Buffer.byteLength(JSON.stringify(frame), "utf8");
+}

package/src/services/channel-bus/PostgresChannelBus.ts

@@ -0,0 +1,299 @@
+/**
+ * Channel bus over Postgres LISTEN/NOTIFY.
+ *
+ * Chosen because it needs nothing that a Rebase deployment does not already
+ * have — the same database, the same direct URL the CDC listener uses. Three
+ * properties of `NOTIFY` shape everything below:
+ *
+ *  - **8000 bytes per payload.** Presence and cursors fit with room to spare; a
+ *    scene snapshot does not. Rather than truncate or drop, an oversized frame
+ *    on a *retained* channel is published as a pointer — the body is already in
+ *    `rebase.channel_messages` with a sequence number, so the receiver reads it
+ *    back. That is the same trick the entity path uses (notify an address,
+ *    refetch the row), applied to a different table. On an ephemeral channel
+ *    there is nothing to point at, so the publish is refused loudly instead of
+ *    reaching some instances and not others.
+ *
+ *  - **A notify is a query on the primary database.** Not a slow one, but it
+ *    competes with the application's real queries, and that — not throughput —
+ *    is what actually limits this transport. Measured, it carried ~10k
+ *    cross-instance messages/second and stayed flat out to eight instances; what
+ *    it should not do is spend 10k queries/second of the database's budget on
+ *    cursor movement. Hence the batching below.
+ *
+ *  - **Delivery is best-effort.** Retained channels repair themselves through
+ *    the client's history replay, so a lost frame costs a live update rather
+ *    than correctness. That is what makes coalescing safe.
+ */
+
+import { sql } from "drizzle-orm";
+import { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { logger } from "@rebasepro/server";
+import { PgNotifyListener } from "../pg-notify-listener";
+import { ChannelBus, ChannelBusFrame, ChannelBusHandler, frameByteLength } from "./ChannelBus";
+
+/** NOTIFY channel carrying channel-bus frames. */
+export const CHANNEL_BUS_NOTIFY_CHANNEL = "rebase_channel_bus";
+
+/**
+ * Postgres refuses a NOTIFY payload of 8000 bytes or more. The margin below it
+ * is for nothing in particular — it is there so that a payload which passes this
+ * check cannot fail at the server for being a few bytes over.
+ */
+export const PG_NOTIFY_MAX_PAYLOAD_BYTES = 7500;
+
+/**
+ * How long a batching window stays open.
+ *
+ * Ten milliseconds is below the threshold where a human notices a cursor lag,
+ * and it is the difference between one query per message and one query per
+ * window under load. Set to 0 to disable coalescing entirely.
+ */
+export const DEFAULT_BATCH_WINDOW_MS = 10;
+
+/** JSON overhead per frame inside a batch: the wrapping array's comma. */
+const BATCH_SEPARATOR_BYTES = 1;
+/** JSON overhead of the batch envelope itself: `{"batch":[]}`. */
+const BATCH_ENVELOPE_BYTES = 12;
+
+interface PendingFrame {
+    frame: ChannelBusFrame;
+    bytes: number;
+    resolve: () => void;
+    reject: (error: unknown) => void;
+}
+
+export class PostgresChannelBus implements ChannelBus {
+    readonly kind = "postgres" as const;
+    readonly maxFrameBytes = PG_NOTIFY_MAX_PAYLOAD_BYTES;
+
+    private listener?: PgNotifyListener;
+    private readonly batchWindowMs: number;
+
+    /**
+     * Frames waiting for the current window to close.
+     *
+     * The window is opened by a publish that found none open, and that publish
+     * is sent *immediately* rather than joining a batch — see {@link publish}.
+     */
+    private pending: PendingFrame[] = [];
+    private pendingBytes = BATCH_ENVELOPE_BYTES;
+    private windowTimer?: ReturnType<typeof setTimeout>;
+    private stopped = false;
+
+    constructor(
+        private readonly db: NodePgDatabase<Record<string, unknown>>,
+        private readonly connectionString: string,
+        options: { batchWindowMs?: number } = {}
+    ) {
+        const configured = options.batchWindowMs;
+        this.batchWindowMs = typeof configured === "number" && configured >= 0
+            ? configured
+            : DEFAULT_BATCH_WINDOW_MS;
+    }
+
+    async start(handler: ChannelBusHandler): Promise<void> {
+        this.stopped = false;
+        this.listener = new PgNotifyListener({
+            connectionString: this.connectionString,
+            channel: CHANNEL_BUS_NOTIFY_CHANNEL,
+            logLabel: "[ChannelBus]",
+            onPayload: async (payload) => {
+                const frames = parseChannelBusPayload(payload);
+                if (!frames.length) {
+                    logger.warn("⚠️ [ChannelBus] Dropping unparseable payload.");
+                    return;
+                }
+                // In order: a batch preserves the sender's publish order, and a
+                // retained channel's consumers rely on it.
+                for (const frame of frames) await handler(frame);
+            }
+        });
+        await this.listener.start();
+    }
+
+    /**
+     * Publish, coalescing under load.
+     *
+     * The window is *leading edge*: a publish arriving when no window is open is
+     * sent straight away and opens one, so an idle channel pays no added latency
+     * at all. Frames arriving while it is open are collected and leave together
+     * when it closes. The effect is that cost tracks elapsed time rather than
+     * message count — one query per window instead of one per message — which is
+     * the same shape as the retention pruning throttle, for the same reason.
+     *
+     * The returned promise settles when the frame has actually left, not when it
+     * was queued, so the contract ("reaches the other instances, or rejects")
+     * still holds.
+     */
+    async publish(frame: ChannelBusFrame): Promise<void> {
+        if (this.batchWindowMs === 0 || this.stopped) {
+            await this.send([frame]);
+            return;
+        }
+
+        if (!this.windowTimer) {
+            this.openWindow();
+            await this.send([frame]);
+            return;
+        }
+
+        const bytes = frameByteLength(frame) + BATCH_SEPARATOR_BYTES;
+
+        // A batch is one NOTIFY payload, so the 8 KB ceiling applies to the
+        // whole batch. Send what we have rather than let the frame push it over.
+        if (this.pending.length && this.pendingBytes + bytes > this.maxFrameBytes) {
+            this.flush();
+        }
+
+        return new Promise<void>((resolve, reject) => {
+            this.pending.push({ frame, bytes, resolve, reject });
+            this.pendingBytes += bytes;
+        });
+    }
+
+    async stop(): Promise<void> {
+        this.stopped = true;
+        if (this.windowTimer) {
+            clearTimeout(this.windowTimer);
+            this.windowTimer = undefined;
+        }
+        // Anything still queued belongs to clients that are already waiting on
+        // it; dropping it on shutdown would be a silent loss where a flush costs
+        // one more query.
+        this.flush();
+        await this.listener?.stop();
+        this.listener = undefined;
+    }
+
+    private openWindow(): void {
+        this.windowTimer = setTimeout(() => {
+            this.windowTimer = undefined;
+            if (this.pending.length) {
+                // Still busy: send this window's frames and open the next one,
+                // so a sustained stream keeps costing one query per window.
+                this.flush();
+                this.openWindow();
+            }
+            // Otherwise leave it closed, so the next publish after a quiet
+            // moment goes out immediately.
+        }, this.batchWindowMs);
+
+        // Housekeeping must never hold the process open.
+        (this.windowTimer as unknown as { unref?: () => void }).unref?.();
+    }
+
+    /** Send everything queued and settle the promises waiting on it. */
+    private flush(): void {
+        if (!this.pending.length) return;
+
+        const batch = this.pending;
+        this.pending = [];
+        this.pendingBytes = BATCH_ENVELOPE_BYTES;
+
+        this.send(batch.map(p => p.frame))
+            .then(() => { for (const p of batch) p.resolve(); })
+            .catch((error) => { for (const p of batch) p.reject(error); });
+    }
+
+    /**
+     * One NOTIFY.
+     *
+     * A single frame goes out in the plain, unwrapped shape. That is not just
+     * economy: during a rolling deploy an instance running the previous build
+     * understands only that shape, and low-rate traffic — presence, the tail of
+     * a session — is exactly what is flowing while pods restart. Batching only
+     * appears under load, which shrinks the mixed-version window to almost
+     * nothing.
+     */
+    private async send(frames: ChannelBusFrame[]): Promise<void> {
+        if (!frames.length) return;
+        const payload = frames.length === 1
+            ? JSON.stringify(frames[0])
+            : JSON.stringify({ batch: frames });
+
+        await this.db.execute(sql`SELECT pg_notify(${CHANNEL_BUS_NOTIFY_CHANNEL}, ${payload})`);
+    }
+}
+
+/**
+ * Parse a bus payload into the frames it carries.
+ *
+ * Accepts both wire shapes — a bare frame and a `{ batch: [...] }` envelope —
+ * so an instance on the new build understands one on the old. Returns an empty
+ * array for anything unrecognisable: a malformed or future-versioned message
+ * must never take the listener down.
+ */
+export function parseChannelBusPayload(payload: string): ChannelBusFrame[] {
+    let parsed: unknown;
+    try {
+        parsed = JSON.parse(payload);
+    } catch {
+        return [];
+    }
+    if (!parsed || typeof parsed !== "object") return [];
+
+    const batch = (parsed as { batch?: unknown }).batch;
+    if (Array.isArray(batch)) {
+        return batch
+            .map(entry => coerceFrame(entry))
+            .filter((frame): frame is ChannelBusFrame => frame !== null);
+    }
+
+    const single = coerceFrame(parsed);
+    return single ? [single] : [];
+}
+
+/**
+ * Parse a single bus frame, returning null for anything that is not a frame we
+ * understand.
+ */
+export function parseChannelBusFrame(payload: string): ChannelBusFrame | null {
+    try {
+        return coerceFrame(JSON.parse(payload));
+    } catch {
+        return null;
+    }
+}
+
+function coerceFrame(value: unknown): ChannelBusFrame | null {
+    if (!value || typeof value !== "object") return null;
+
+    const obj = value as Record<string, unknown>;
+    const sid = typeof obj.sid === "string" ? obj.sid : undefined;
+    const channel = typeof obj.channel === "string" ? obj.channel : undefined;
+    if (!sid || !channel) return null;
+
+    switch (obj.kind) {
+        case "broadcast":
+            if (typeof obj.event !== "string") return null;
+            return {
+                kind: "broadcast",
+                sid,
+                channel,
+                event: obj.event,
+                from: typeof obj.from === "string" ? obj.from : undefined,
+                seq: typeof obj.seq === "number" ? obj.seq : undefined,
+                payload: obj.payload
+            };
+        case "broadcast_ref":
+            if (typeof obj.seq !== "number") return null;
+            return {
+                kind: "broadcast_ref",
+                sid,
+                channel,
+                from: typeof obj.from === "string" ? obj.from : undefined,
+                seq: obj.seq
+            };
+        case "presence_diff":
+            return {
+                kind: "presence_diff",
+                sid,
+                channel,
+                joins: (obj.joins ?? {}) as Record<string, Record<string, unknown>>,
+                leaves: (obj.leaves ?? {}) as Record<string, Record<string, unknown>>
+            };
+        default:
+            return null;
+    }
+}

package/src/services/channel-bus/index.ts

@@ -0,0 +1,123 @@
+/**
+ * Resolution of the channel bus from config, environment, or a supplied instance.
+ *
+ * Opt-in, like every other cross-cutting realtime switch here: with nothing
+ * configured a deployment gets the memory bus and behaves exactly as it did
+ * before this existed. Unlike `REALTIME_CDC=auto`, there is no "try it and see"
+ * default — a bus changes where messages go, and quietly turning on a Postgres
+ * NOTIFY per broadcast because a direct URL happened to be set is not a
+ * decision to make on the user's behalf.
+ *
+ * Two transports ship, and neither adds a service to a deployment. A third is
+ * not a code change here: `realtime.bus` also accepts an already-constructed
+ * {@link ChannelBus}, so a transport published as its own package plugs in
+ * without this file learning about it. See `@rebasepro/types` →
+ * `types/channel_bus.ts` for the contract such a package implements.
+ */
+
+import { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { isChannelBusInstance, type ChannelBus, type ChannelBusConfig, type ChannelBusSetting } from "@rebasepro/types";
+import { logger } from "@rebasepro/server";
+import { MemoryChannelBus } from "./ChannelBus";
+import { PostgresChannelBus } from "./PostgresChannelBus";
+
+export * from "./ChannelBus";
+export {
+    PostgresChannelBus,
+    CHANNEL_BUS_NOTIFY_CHANNEL,
+    PG_NOTIFY_MAX_PAYLOAD_BYTES,
+    DEFAULT_BATCH_WINDOW_MS,
+    parseChannelBusFrame,
+    parseChannelBusPayload
+} from "./PostgresChannelBus";
+
+export interface ChannelBusDeps {
+    db: NodePgDatabase<Record<string, unknown>>;
+    /**
+     * Direct (non-pooled) Postgres URL for the LISTEN client. `LISTEN` is
+     * session state, so behind PgBouncer in transaction mode this must be the
+     * database itself and not the pooler.
+     */
+    directUrl?: string;
+}
+
+/**
+ * Merge `REALTIME_CHANNEL_BUS` into the configured bus.
+ *
+ * The environment wins over a *named* built-in, so the transport can be changed
+ * per deployment without a rebuild — the same reason `REALTIME_CDC` is an env
+ * var. It does **not** win over a supplied instance: the env var can only name
+ * transports this package knows how to construct, so honouring it there would
+ * mean silently discarding the object the application handed us.
+ */
+export function resolveChannelBusSetting(configured?: ChannelBusSetting): ChannelBusSetting {
+    const raw = (process.env.REALTIME_CHANNEL_BUS || "").trim().toLowerCase();
+
+    if (isChannelBusInstance(configured)) {
+        if (raw && raw !== configured.kind) {
+            logger.warn(
+                `⚠️ [ChannelBus] REALTIME_CHANNEL_BUS="${raw}" is ignored because realtime.bus was given a ` +
+                `"${configured.kind}" transport instance directly. Remove one of the two to make the intent clear.`
+            );
+        }
+        return configured;
+    }
+
+    if (!raw) return configured ?? { type: "memory" };
+
+    if (raw !== "memory" && raw !== "postgres") {
+        logger.warn(
+            `⚠️ [ChannelBus] Unknown REALTIME_CHANNEL_BUS value "${raw}" — expected memory|postgres, or pass a ` +
+            "ChannelBus instance as realtime.bus for a transport that ships separately. Falling back to the " +
+            "configured bus."
+        );
+        return configured ?? { type: "memory" };
+    }
+
+    // Keep the configured options (an explicit connection string) when the env
+    // var only restates the type it was already set to.
+    if (configured?.type === raw) return configured;
+    return raw === "memory" ? { type: "memory" } : { type: "postgres" };
+}
+
+/**
+ * @deprecated Use {@link resolveChannelBusSetting}, which also accepts a
+ * supplied {@link ChannelBus} instance. Kept as a narrow alias so existing
+ * config-only callers keep their exact types.
+ */
+export function resolveChannelBusConfig(configured?: ChannelBusConfig): ChannelBusConfig {
+    return resolveChannelBusSetting(configured) as ChannelBusConfig;
+}
+
+/**
+ * Produce the bus a setting asks for.
+ *
+ * An instance is handed straight back — constructing it was the application's
+ * job, and this function has nothing to add. A named built-in that turns out to
+ * be unusable degrades to the memory bus, with the reason logged, rather than
+ * throwing: a misconfigured bus should cost a deployment its cross-instance
+ * fan-out, not its ability to boot.
+ */
+export function createChannelBus(setting: ChannelBusSetting, deps: ChannelBusDeps): ChannelBus {
+    if (isChannelBusInstance(setting)) return setting;
+
+    switch (setting.type) {
+        case "postgres": {
+            const connectionString = setting.connectionString || deps.directUrl;
+            if (!connectionString) {
+                logger.warn(
+                    "⚠️ [ChannelBus] realtime.bus is \"postgres\" but no direct database URL is available " +
+                    "(set DATABASE_DIRECT_URL or realtime.bus.connectionString) — channel broadcast and presence " +
+                    "stay per-instance."
+                );
+                return new MemoryChannelBus();
+            }
+            return new PostgresChannelBus(deps.db, connectionString, {
+                batchWindowMs: setting.batchWindowMs
+            });
+        }
+        case "memory":
+        default:
+            return new MemoryChannelBus();
+    }
+}

package/src/services/channel-history.ts

@@ -290,6 +290,41 @@ export class ChannelHistoryStore {
         return { messages, latestSeq };
     }
 
+    /**
+     * One retained message by its address.
+     *
+     * This is what makes the cross-instance pointer path work: a broadcast too
+     * large to travel inside a `pg_notify` payload is already stored here, so
+     * the notification carries `(channel, seq)` and each receiving instance
+     * reads the body back. Returns null when the message has since been pruned
+     * — a receiver that is that far behind has nothing useful to deliver, and
+     * the client's own `channel_history` replay is the repair path.
+     */
+    async getBySeq(channel: string, seq: number): Promise<ChannelHistoryEntry | null> {
+        const result = await this.db.execute(sql`
+            SELECT seq, event, payload, sender_id, created_at
+            FROM rebase.channel_messages
+            WHERE channel = ${channel} AND seq = ${seq}
+        `);
+
+        const row = result.rows[0] as {
+            seq: string | number;
+            event: string;
+            payload: unknown;
+            sender_id: string | null;
+            created_at: Date | string;
+        } | undefined;
+        if (!row) return null;
+
+        return {
+            seq: Number(row.seq),
+            event: row.event,
+            payload: row.payload,
+            senderId: row.sender_id ?? undefined,
+            at: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at)
+        };
+    }
+
     /**
      * Enforce a channel's retention bounds.
      *