@rebasepro/server-postgres 0.0.1-canary.4829d6e

package/src/services/cdc/CdcListener.ts

@@ -0,0 +1,167 @@
+import { Client as PgClient } from "pg";
+import { logger } from "@rebasepro/server";
+import { CDC_CHANNEL } from "./trigger-cdc";
+
+/**
+ * A single database change captured by the CDC triggers and delivered over the
+ * `rebase_cdc` NOTIFY channel.
+ */
+export interface CdcChangeEvent {
+    schema: string;
+    table: string;
+    op: "INSERT" | "UPDATE" | "DELETE";
+    /**
+     * The changed tuple (NEW for insert/update, OLD for delete). May be a
+     * partial identity-only object when the full row overflowed the pg_notify
+     * size cap — see {@link truncated}.
+     */
+    row: Record<string, unknown>;
+    /** True when the row was reduced to its identity because it was too large to notify. */
+    truncated?: boolean;
+}
+
+/**
+ * Parse a `rebase_cdc` NOTIFY payload. Returns `null` for anything malformed so
+ * a single bad message can never crash the listener.
+ */
+export function parseCdcPayload(payload: string): CdcChangeEvent | null {
+    let parsed: unknown;
+    try {
+        parsed = JSON.parse(payload);
+    } catch {
+        return null;
+    }
+    if (!parsed || typeof parsed !== "object") return null;
+
+    const obj = parsed as Record<string, unknown>;
+    const schema = typeof obj.schema === "string" ? obj.schema : undefined;
+    const table = typeof obj.table === "string" ? obj.table : undefined;
+    const op = obj.op;
+    if (!schema || !table) return null;
+    if (op !== "INSERT" && op !== "UPDATE" && op !== "DELETE") return null;
+
+    const row = obj.row && typeof obj.row === "object" ? (obj.row as Record<string, unknown>) : {};
+
+    return {
+        schema,
+        table,
+        op,
+        row,
+        truncated: obj.truncated === true
+    };
+}
+
+/**
+ * Dedicated Postgres LISTEN client for database-level CDC.
+ *
+ * Mirrors the resilience of the RealtimeService cross-instance LISTEN client: a
+ * standalone `pg.Client` (outside the Drizzle pool) that stays connected, and
+ * transparently reconnects on error/disconnect. Each backend instance runs one,
+ * so every instance observes every committed change regardless of which
+ * instance (or external process) made the write.
+ */
+export class CdcListener {
+    private client?: PgClient;
+    private running = false;
+    private reconnectTimer?: ReturnType<typeof setTimeout>;
+    private static readonly RECONNECT_DELAY_MS = 3000;
+
+    constructor(
+        private readonly connectionString: string,
+        private readonly onEvent: (event: CdcChangeEvent) => void | Promise<void>
+    ) {}
+
+    /**
+     * Connect and begin listening. Idempotent.
+     *
+     * The **initial** connection is validated synchronously: if it cannot be
+     * established (or `LISTEN` is refused), this rejects so callers — notably
+     * `REALTIME_CDC=auto` — can detect an unusable connection and fall back to
+     * app-level realtime. Once the initial connection succeeds, later drops
+     * self-heal via {@link scheduleReconnect}.
+     */
+    async start(): Promise<void> {
+        if (this.running) {
+            logger.warn("⚠️ [CDC] CdcListener.start() called but already running. Ignoring.");
+            return;
+        }
+        this.running = true;
+        try {
+            await this.connect({ initial: true });
+        } catch (err) {
+            this.running = false;
+            throw err;
+        }
+    }
+
+    /** Stop listening and release the connection. */
+    async stop(): Promise<void> {
+        this.running = false;
+        if (this.reconnectTimer) {
+            clearTimeout(this.reconnectTimer);
+            this.reconnectTimer = undefined;
+        }
+        if (this.client) {
+            try {
+                await this.client.end();
+            } catch { /* ignore close errors */ }
+            this.client = undefined;
+        }
+    }
+
+    private async connect({ initial = false }: { initial?: boolean } = {}): Promise<void> {
+        try {
+            const client = new PgClient({ connectionString: this.connectionString });
+
+            client.on("error", (err) => {
+                logger.error("❌ [CDC] LISTEN client error", { detail: err.message });
+                this.scheduleReconnect();
+            });
+
+            client.on("end", () => {
+                if (this.running) {
+                    logger.warn("⚠️ [CDC] LISTEN client disconnected unexpectedly.");
+                    this.scheduleReconnect();
+                }
+            });
+
+            client.on("notification", (msg) => {
+                if (!msg.payload) return;
+                const event = parseCdcPayload(msg.payload);
+                if (!event) {
+                    logger.warn("⚠️ [CDC] Dropping unparseable change notification.");
+                    return;
+                }
+                // Never let a handler rejection escape into the pg client.
+                Promise.resolve(this.onEvent(event)).catch((err) =>
+                    logger.error("❌ [CDC] Error handling change event", { error: err })
+                );
+            });
+
+            await client.connect();
+            await client.query(`LISTEN ${CDC_CHANNEL}`);
+            this.client = client;
+            logger.info(`📡 [CDC] Listening for database changes on channel "${CDC_CHANNEL}".`);
+        } catch (err) {
+            // Surface the initial failure so callers can choose to fall back;
+            // for reconnects, keep retrying quietly in the background.
+            if (initial) throw err;
+            logger.error("❌ [CDC] Failed to connect LISTEN client", { error: err });
+            this.scheduleReconnect();
+        }
+    }
+
+    private scheduleReconnect(): void {
+        if (!this.running || this.reconnectTimer) return;
+
+        this.reconnectTimer = setTimeout(async () => {
+            this.reconnectTimer = undefined;
+            if (!this.running) return;
+            if (this.client) {
+                try { await this.client.end(); } catch { /* ignore */ }
+                this.client = undefined;
+            }
+            await this.connect();
+        }, CdcListener.RECONNECT_DELAY_MS);
+    }
+}

package/src/services/cdc/trigger-cdc.ts

@@ -0,0 +1,169 @@
+import { logger } from "@rebasepro/server";
+import type { RawSqlRunner } from "../../security/rls-enforcement";
+
+/**
+ * Trigger-based Change Data Capture (CDC).
+ *
+ * The preferred CDC source is the write-ahead log (logical replication), which
+ * — like Supabase Realtime — sees *every* commit regardless of how it was made.
+ * When logical replication is unavailable (managed Postgres without
+ * `wal_level=logical`, no replication privilege, no `REPLICA IDENTITY`), this
+ * trigger-based fallback provides the same guarantee at the row level:
+ *
+ *   AFTER INSERT/UPDATE/DELETE trigger  →  pg_notify('rebase_cdc', payload)
+ *
+ * A single dedicated LISTEN client per backend instance consumes the channel
+ * (see {@link CdcListener}) and feeds the change into the existing
+ * `RealtimeService.notifyUpdate` pipeline, so subscribers see the change no
+ * matter what wrote it — psql, a cron in another service, raw Drizzle/SQL, or
+ * the Studio SQL editor.
+ *
+ * Provisioning runs from the framework's own bootstrap as the owner (server)
+ * context, alongside the RLS role provisioning. It is idempotent.
+ */
+
+/** Postgres NOTIFY channel carrying database-level change events. */
+export const CDC_CHANNEL = "rebase_cdc";
+
+/** Schema-qualified name of the generic trigger function. */
+export const CDC_TRIGGER_FUNCTION = "rebase.rebase_cdc_notify";
+
+/** Name of the per-table trigger (unqualified — triggers are namespaced by table). */
+export const CDC_TRIGGER_NAME = "rebase_cdc_trigger";
+
+/**
+ * pg_notify hard-caps payloads at 8000 bytes and *aborts the triggering
+ * statement* if the limit is exceeded. We stay comfortably under it and, for
+ * wide rows, fall back to an identity-only payload so CDC can never break a
+ * write. 7900 leaves headroom for the JSON envelope keys.
+ */
+const MAX_NOTIFY_BYTES = 7900;
+
+const quoteIdent = (name: string): string => `"${name.replace(/"/g, "\"\"")}"`;
+const quoteLiteral = (value: string): string => `'${value.replace(/'/g, "''")}'`;
+
+/**
+ * SQL that (re)creates the generic CDC trigger function. Safe to run repeatedly:
+ * `CREATE OR REPLACE` updates in place without dropping dependent triggers.
+ *
+ * The function emits `{ schema, table, op, row }`. The `row` is the full changed
+ * tuple (NEW for insert/update, OLD for delete) so the consumer can route it to
+ * a collection and extract the primary key. It is *not* trusted for delivery:
+ * the consumer marks the row invalidated and each subscriber re-reads it under
+ * its own RLS context, so a subscriber never receives a row it cannot read.
+ */
+export function buildCdcFunctionSql(): string {
+    return `
+CREATE SCHEMA IF NOT EXISTS rebase;
+
+CREATE OR REPLACE FUNCTION ${CDC_TRIGGER_FUNCTION}() RETURNS trigger
+LANGUAGE plpgsql AS $rebase_cdc$
+DECLARE
+    rec     jsonb;
+    payload text;
+BEGIN
+    IF (TG_OP = 'DELETE') THEN
+        rec := to_jsonb(OLD);
+    ELSE
+        rec := to_jsonb(NEW);
+    END IF;
+
+    payload := json_build_object(
+        'schema', TG_TABLE_SCHEMA,
+        'table',  TG_TABLE_NAME,
+        'op',     TG_OP,
+        'row',    rec
+    )::text;
+
+    -- Never let CDC abort the write: if the full row overflows the pg_notify
+    -- 8000-byte cap, emit an identity-only payload the consumer can still route
+    -- (and refetch the authoritative row from).
+    IF (octet_length(payload) > ${MAX_NOTIFY_BYTES}) THEN
+        payload := json_build_object(
+            'schema',    TG_TABLE_SCHEMA,
+            'table',     TG_TABLE_NAME,
+            'op',        TG_OP,
+            'row',       CASE WHEN rec ? 'id' THEN jsonb_build_object('id', rec->'id') ELSE '{}'::jsonb END,
+            'truncated', true
+        )::text;
+    END IF;
+
+    PERFORM pg_notify(${quoteLiteral(CDC_CHANNEL)}, payload);
+    RETURN NULL;
+END;
+$rebase_cdc$;
+`.trim();
+}
+
+/**
+ * SQL that (re)attaches the CDC trigger to a single table. `DROP ... IF EXISTS`
+ * before `CREATE` keeps it idempotent and picks up any function signature change.
+ */
+export function buildCdcTriggerSql(schema: string, table: string): string {
+    const qualified = `${quoteIdent(schema)}.${quoteIdent(table)}`;
+    return (
+        `DROP TRIGGER IF EXISTS ${quoteIdent(CDC_TRIGGER_NAME)} ON ${qualified};\n` +
+        `CREATE TRIGGER ${quoteIdent(CDC_TRIGGER_NAME)} ` +
+        `AFTER INSERT OR UPDATE OR DELETE ON ${qualified} ` +
+        `FOR EACH ROW EXECUTE FUNCTION ${CDC_TRIGGER_FUNCTION}();`
+    );
+}
+
+export interface CdcTableRef {
+    schema: string;
+    table: string;
+}
+
+export interface ProvisionResult {
+    /** Tables the trigger was successfully attached to. */
+    installed: CdcTableRef[];
+    /** Tables that could not be provisioned (e.g. not yet migrated), with the error. */
+    skipped: Array<CdcTableRef & { reason: string }>;
+}
+
+/**
+ * Idempotently install the CDC trigger function and per-table triggers.
+ *
+ * Runs as the owner (server) connection at bootstrap. A table that does not yet
+ * exist in the database (schema drift) is skipped with a warning rather than
+ * aborting the whole install, so one un-migrated collection cannot disable CDC
+ * for the rest.
+ */
+export async function provisionTriggerCdc(
+    run: RawSqlRunner,
+    tables: CdcTableRef[]
+): Promise<ProvisionResult> {
+    // 1. The shared trigger function (once).
+    await run(buildCdcFunctionSql());
+
+    // 2. One trigger per managed table. De-duplicate identical refs.
+    const seen = new Set<string>();
+    const installed: CdcTableRef[] = [];
+    const skipped: ProvisionResult["skipped"] = [];
+
+    for (const ref of tables) {
+        const key = `${ref.schema}.${ref.table}`;
+        if (seen.has(key)) continue;
+        seen.add(key);
+
+        try {
+            await run(buildCdcTriggerSql(ref.schema, ref.table));
+            installed.push(ref);
+        } catch (err) {
+            const reason = err instanceof Error ? err.message : String(err);
+            skipped.push({ ...ref, reason });
+            logger.warn(
+                `⚠️ [CDC] Could not attach change-capture trigger to "${key}" — ` +
+                `is the table migrated? Writes to it won't emit database-level events.`,
+                { detail: reason }
+            );
+        }
+    }
+
+    logger.info(
+        `📡 [CDC] Trigger-based change capture provisioned on ${installed.length} table(s)` +
+        (skipped.length ? ` (${skipped.length} skipped)` : "") + "."
+    );
+
+    return { installed, skipped };
+}

package/src/services/collection-helpers.ts

@@ -0,0 +1,151 @@
+import { PgTable, AnyPgColumn } from "drizzle-orm/pg-core";
+import { CollectionConfig, Property } from "@rebasepro/types";
+import { PostgresCollectionRegistry } from "../collections/PostgresCollectionRegistry";
+import { getTableName } from "@rebasepro/common";
+
+/**
+ * Shared helper functions for row operations.
+ * These are used by FetchService, PersistService, and RelationService.
+ *
+ * All functions that need collection/table lookups require an explicit
+ * `PostgresCollectionRegistry` instance — there is no global singleton.
+ */
+
+/**
+ * Interface for Drizzle column metadata introspection.
+ * Replaces unsafe `as Record<string, unknown>` double-cast chains.
+ */
+export interface DrizzleColumnMeta {
+    columnType?: string;
+    dataType?: string;
+    primary?: boolean;
+}
+
+/** Safely extract Drizzle column metadata from a column object. */
+export function getColumnMeta(col: AnyPgColumn): DrizzleColumnMeta {
+    const raw = col as unknown as Record<string | symbol, unknown>;
+    return {
+        columnType: typeof raw.columnType === "string" ? raw.columnType : undefined,
+        dataType: typeof raw.dataType === "string" ? raw.dataType : undefined,
+        primary: typeof raw.primary === "boolean" ? raw.primary : undefined
+    };
+}
+
+export function getCollectionByPath(collectionPath: string, registry: PostgresCollectionRegistry): CollectionConfig {
+    const collection = registry.getCollectionByPath(collectionPath);
+    if (!collection) {
+        const registered = registry.getCollections().map(c => c.slug).join(", ");
+        throw new Error(`Collection not found: ${collectionPath}. Registered collections: [${registered}]`);
+    }
+    return collection;
+}
+
+export function getTableForCollection(collection: CollectionConfig, registry: PostgresCollectionRegistry): PgTable<any> {
+    const tableName = getTableName(collection);
+    const table = registry.getTable(tableName);
+    if (!table) {
+        throw new Error(`Table not found for collection '${collection.slug}' (table: ${tableName})`);
+    }
+    return table;
+}
+
+export function getPrimaryKeys(collection: CollectionConfig, registry: PostgresCollectionRegistry): { fieldName: string; type: "string" | "number"; isUUID?: boolean }[] {
+    const table = getTableForCollection(collection, registry);
+
+    // Fallback to explicitly defined isId properties
+    if (collection.properties) {
+        const idProps = Object.entries(collection.properties)
+            .filter(([_, prop]) => "isId" in (prop as object) && Boolean((prop as { isId?: unknown }).isId))
+            .map(([key, prop]) => ({
+                fieldName: key,
+                type: prop.type === "number" ? "number" as const : "string" as const,
+                isUUID: (prop as { isId?: unknown }).isId === "uuid"
+            }));
+
+        if (idProps.length > 0) {
+            return idProps;
+        }
+    }
+
+    // Otherwise infer from Drizzle schema
+    const keys: { fieldName: string; type: "string" | "number"; isUUID?: boolean }[] = [];
+    for (const [key, colRaw] of Object.entries(table)) {
+        const col = colRaw as AnyPgColumn;
+        if (col && typeof col === "object" && "primary" in col && col.primary) {
+            const meta = getColumnMeta(col);
+            const type = col.dataType === "number" || meta.columnType === "PgSerial" || meta.columnType === "PgInteger" ? "number" : "string";
+            const isUUID = meta.columnType === "PgUUID";
+            keys.push({ fieldName: key,
+type,
+isUUID });
+        }
+    }
+
+    // Default to 'id' if no primary keys are found and it exists in the schema
+    // This maintains backwards compatibility
+    if (keys.length === 0 && "id" in table) {
+        const idCol = table["id" as keyof typeof table] as AnyPgColumn;
+        const idMeta = getColumnMeta(idCol);
+        const type = idCol.dataType === "number" || idMeta.columnType === "PgSerial" || idMeta.columnType === "PgInteger" ? "number" : "string";
+        const isUUID = idMeta.columnType === "PgUUID";
+        keys.push({ fieldName: "id",
+type,
+isUUID });
+    }
+
+    return keys;
+}
+
+export function parseIdValues(idValue: string | number, primaryKeys: { fieldName: string; type: "string" | "number"; isUUID?: boolean }[]): Record<string, string | number> {
+    const result: Record<string, string | number> = {};
+
+    if (primaryKeys.length === 0) {
+        return result;
+    }
+
+    if (primaryKeys.length === 1) {
+        const pk = primaryKeys[0];
+        if (pk.type === "number" && !pk.isUUID) {
+            const parsed = typeof idValue === "number" ? idValue : parseInt(String(idValue), 10);
+            if (isNaN(parsed)) {
+                throw new Error(`Invalid numeric ID: ${idValue}`);
+            }
+            result[pk.fieldName] = parsed;
+        } else {
+            result[pk.fieldName] = String(idValue);
+        }
+        return result;
+    }
+
+    // Composite key - split by :::
+    const parts = String(idValue).split(":::");
+    if (parts.length !== primaryKeys.length) {
+        throw new Error(`Composite ID parts mismatch. Expected ${primaryKeys.length}, got ${parts.length} for ID: ${idValue}`);
+    }
+
+    for (let i = 0; i < primaryKeys.length; i++) {
+        const pk = primaryKeys[i];
+        const val = parts[i];
+        if (pk.type === "number" && !pk.isUUID) {
+            const parsed = parseInt(val, 10);
+            if (isNaN(parsed)) {
+                throw new Error(`Invalid numeric ID component: ${val}`);
+            }
+            result[pk.fieldName] = parsed;
+        } else {
+            result[pk.fieldName] = val;
+        }
+    }
+
+    return result;
+}
+
+export function buildCompositeId(values: Record<string, unknown>, primaryKeys: { fieldName: string; type: "string" | "number"; isUUID?: boolean }[]): string {
+    if (primaryKeys.length === 0) {
+        return "";
+    }
+    if (primaryKeys.length === 1) {
+        return String(values[primaryKeys[0].fieldName] ?? "");
+    }
+    return primaryKeys.map(pk => String(values[pk.fieldName] ?? "")).join(":::");
+}