@rebasepro/server-postgres 0.9.1-canary.fd3754b → 0.10.0

package/src/PostgresBackendDriver.ts

@@ -17,6 +17,7 @@ import {
     RebaseData,
     RebaseSdkData,
     RestFetchService,
+    SaveManyProps,
     SaveProps,
     TableColumnInfo,
     TableForeignKeyInfo,
@@ -28,6 +29,7 @@ import {
 import { sql as drizzleSql } from "drizzle-orm";
 import { buildPropertyCallbacks, buildSdkData, resolveCollectionRelations, updateDateAutoValues } from "@rebasepro/common";
 import { PostgresCollectionRegistry } from "./collections/PostgresCollectionRegistry";
+import { deriveRowAddress } from "./services/collection-helpers";
 import { HistoryService } from "./history/HistoryService";
 import { mergeDeep } from "@rebasepro/utils";
 import { logger } from "@rebasepro/server";
@@ -109,6 +111,7 @@ export class PostgresBackendDriver implements DataDriver {
             executeSql: (...args: Parameters<NonNullable<DatabaseAdmin["executeSql"]>>) => this.executeSql(...args),
             fetchAvailableDatabases: () => this.fetchAvailableDatabases(),
             fetchAvailableRoles: () => this.fetchAvailableRoles(),
+            fetchApplicationRoles: () => this.fetchApplicationRoles(),
             fetchCurrentDatabase: () => this.fetchCurrentDatabase(),
             fetchUnmappedTables: (...args: Parameters<NonNullable<DatabaseAdmin["fetchUnmappedTables"]>>) => this.fetchUnmappedTables(...args),
             fetchTableMetadata: (...args: Parameters<NonNullable<DatabaseAdmin["fetchTableMetadata"]>>) => this.fetchTableMetadata(...args),
@@ -532,7 +535,8 @@ export class PostgresBackendDriver implements DataDriver {
                                                             id,
                                                             values,
                                                             collection,
-                                                            status
+                                                            status,
+                                                            upsert
                                                         }: SaveProps<M>): Promise<Record<string, unknown>> {
 
         const {
@@ -545,13 +549,26 @@ export class PostgresBackendDriver implements DataDriver {
         let updatedValues = values;
         const contextForCallback = this.buildCallContext();
 
-        // Fetch previous values for callbacks AND history recording
+        // Fetch previous values for callbacks AND history recording. Same walk
+        // as the saved row the callbacks receive (`fetchOneForRest`), so
+        // `values` and `previousValues` compare like with like — a Date on one
+        // side and its ISO string on the other reads as a change that never
+        // happened.
         let previousValuesForHistory: Partial<M> | undefined;
         if (status === "existing" && id) {
-            const existing = await this.dataService.fetchOne<M>(path, id, resolvedCollection?.databaseId);
-            if (existing) {
-                const { id: _existingId, ...existingValues } = existing;
-                previousValuesForHistory = existingValues as Partial<M>;
+            try {
+                const existing = await this.dataService.getFetchService()
+                    .fetchOneForRest(path, id, undefined, resolvedCollection?.databaseId);
+                if (existing) {
+                    const { id: _existingId, ...existingValues } = existing;
+                    previousValuesForHistory = existingValues as Partial<M>;
+                }
+            } catch (err) {
+                // Best-effort enrichment: callbacks and history run without
+                // previous values rather than the save failing on a read the
+                // write itself does not need (e.g. a collection whose key the
+                // registry cannot resolve).
+                logger.debug(`[save] Could not fetch previous values for "${path}"`, { detail: err instanceof Error ? err.message : String(err) });
             }
         }
 
@@ -616,7 +633,8 @@ export class PostgresBackendDriver implements DataDriver {
                 path,
                 updatedValues,
                 id,
-                resolvedCollection?.databaseId
+                resolvedCollection?.databaseId,
+                { upsert }
             );
 
             if (savedRow && (globalCallbacks?.afterRead || callbacks?.afterRead || propertyCallbacks?.afterRead)) {
@@ -649,8 +667,19 @@ export class PostgresBackendDriver implements DataDriver {
                 }
             }
 
-            const savedId = savedRow.id as string | number;
-            const { id: _savedId, ...savedValues } = savedRow;
+            // The row is exactly its columns, so its address is derived, not read
+            // off it: `savedRow.id` is undefined for every table whose key is not
+            // literally named `id`, and is ordinary data for a table that has such
+            // a column without it being the key.
+            const savedId = deriveRowAddress(
+                savedRow,
+                (resolvedCollection ?? collection) as CollectionConfig,
+                this.registry
+            );
+            // `values` are the row's columns — all of them. For an `id`-keyed table
+            // that includes `id`, which used to be stripped here because it was the
+            // synthesized address rather than the column it now is.
+            const savedValues = savedRow;
 
             if (globalCallbacks?.afterSave || callbacks?.afterSave || propertyCallbacks?.afterSave) {
                 // 1. Global callbacks first
@@ -695,7 +724,7 @@ export class PostgresBackendDriver implements DataDriver {
             if (this.historyService && resolvedCollection?.history) {
                 this.historyService.recordHistory({
                     tableName: path,
-                    id: savedId.toString(),
+                    id: savedId,
                     action: status === "new" ? "create" : "update",
                     values: savedValues as Record<string, unknown>,
                     previousValues: previousValuesForHistory as Record<string, unknown> | undefined,
@@ -707,14 +736,14 @@ export class PostgresBackendDriver implements DataDriver {
             if (this._deferNotifications) {
                 this._pendingNotifications.push({
                     path,
-                    id: savedId.toString(),
+                    id: savedId,
                     row: savedRow,
                     databaseId: resolvedCollection?.databaseId
                 });
             } else {
                 await this.realtimeService.notifyUpdate(
                     path,
-                    savedId.toString(),
+                    savedId,
                     savedRow,
                     resolvedCollection?.databaseId
                 );
@@ -764,13 +793,86 @@ export class PostgresBackendDriver implements DataDriver {
         }
     }
 
+    /**
+     * Write many rows through the same pipeline as {@link save}.
+     *
+     * The batch runs in one transaction of its own, so a failure part-way leaves
+     * nothing behind — the point of a batch is that a re-run starts from a known
+     * state. When this driver is already inside a transaction (the authenticated
+     * path, via `withTransaction`) the nested call becomes a savepoint, which is
+     * still atomic and still commits once.
+     *
+     * Rows are applied in order, so a batch that touches the same key twice ends
+     * with the last write winning, exactly as separate calls would.
+     */
+    async saveMany<M extends Record<string, unknown>>({
+                                                          path,
+                                                          rows,
+                                                          collection,
+                                                          upsert
+                                                      }: SaveManyProps<M>): Promise<Record<string, unknown>[]> {
+        return this.db.transaction(async (tx) => {
+            // Bind the whole batch to the transaction handle. Without this the
+            // rows would be written through `this.db` and survive a rollback.
+            const txDriver = new PostgresBackendDriver(
+                tx, this.realtimeService, this.registry, this.user, this.poolManager, this.historyService
+            );
+            txDriver.dataService = new DataService(tx, this.registry);
+            txDriver.client = this.client;
+            // Carry the caller's notification batching through, so a bulk write
+            // nested in an outer transaction still holds its events until commit.
+            txDriver._deferNotifications = this._deferNotifications;
+            txDriver._pendingNotifications = this._pendingNotifications;
+
+            const saved: Record<string, unknown>[] = [];
+
+            for (let i = 0; i < rows.length; i++) {
+                const values = rows[i];
+                const id = (values as Record<string, unknown>)?.id as string | number | undefined;
+                try {
+                    saved.push(await txDriver.save<M>({
+                        path,
+                        values,
+                        // No `id` argument, deliberately: passing one selects the
+                        // UPDATE path, and an import's rows usually carry a natural
+                        // key for a row that does not exist yet — which would 404 on
+                        // every one. Leaving the key inside `values` is what
+                        // single-row `create(data, id)` does, and it inserts.
+                        // Callers who want existing rows overwritten pass `upsert`.
+                        collection,
+                        status: "new",
+                        upsert
+                    }));
+                } catch (error) {
+                    // One bad row in ten thousand is impossible to find from a
+                    // message that only says the batch failed. Say which row, and
+                    // keep the original error as the cause so its status survives.
+                    const label = id !== undefined ? `id ${JSON.stringify(id)}` : "no id";
+                    throw Object.assign(
+                        new Error(`Row ${i} of ${rows.length} (${label}) failed: ${(error as Error)?.message ?? error}`, { cause: error }),
+                        {
+                            statusCode: (error as { statusCode?: number })?.statusCode,
+                            code: (error as { code?: string })?.code,
+                            name: (error as Error)?.name
+                        }
+                    );
+                }
+            }
+
+            return saved;
+        });
+    }
+
     async delete<M extends Record<string, unknown>>({
                                                               row,
                                                               collection
                                                           }: DeleteProps<M>): Promise<void> {
 
         const targetPath = row.path;
-        const targetRow: Record<string, unknown> = { id: row.id, ...(row.values ?? {}) };
+        // The callbacks' `row` is the row: its columns, nothing else. The address
+        // travels beside it as `id`, so merging it in here only ever invented an
+        // `id` field for tables that have no such column.
+        const targetRow: Record<string, unknown> = { ...(row.values ?? {}) };
 
         // Resolve from backend registry to restore callbacks lost during WebSocket serialization
         const {
@@ -1070,6 +1172,56 @@ export class PostgresBackendDriver implements DataDriver {
         return result.map((r: Record<string, unknown>) => r.rolname as string);
     }
 
+    /**
+     * Application-level roles actually in use in this project.
+     *
+     * Distinct from {@link fetchAvailableRoles}, which returns native
+     * PostgreSQL roles from `pg_roles` (`postgres`, `rebase_user`, …). Those
+     * are the roles the SQL editor can `SET ROLE` to. *These* are the strings
+     * held in the users table's `roles` column, injected per-transaction as
+     * `auth.roles()` and matched by `SecurityRule.roles`. Feeding the pg roles
+     * into a `SecurityRule.roles` field produces a condition no user can ever
+     * satisfy, so the two must not be conflated.
+     *
+     * Roles have no registry table — they were migrated out of
+     * `rebase.user_roles` onto an inline `roles TEXT[]` column — so the live
+     * set is derived from what is assigned. A role that is declared in a policy
+     * but held by nobody yet cannot be discovered here; callers that need it
+     * should union in the roles they already know about.
+     */
+    async fetchApplicationRoles(): Promise<string[]> {
+        // The users table lives in `rebase` for a default (public) setup, but
+        // follows the configured schema otherwise — locate it rather than
+        // assuming. The `roles` ARRAY column is what makes it the auth table.
+        const located = await this.executeSql(`
+            SELECT table_schema, table_name
+            FROM information_schema.columns
+            WHERE column_name = 'roles'
+              AND data_type = 'ARRAY'
+              AND table_name = 'users'
+              AND table_schema NOT IN ('information_schema', 'pg_catalog')
+            ORDER BY (table_schema = 'rebase') DESC, table_schema
+            LIMIT 1;
+        `);
+        if (located.length === 0) return [];
+
+        const schema = located[0].table_schema as string;
+        const table = located[0].table_name as string;
+        // Identifiers come from information_schema, not user input, but they
+        // are still interpolated — quote them so odd-but-legal names survive.
+        const qualified = `"${schema.replace(/"/g, "\"\"")}"."${table.replace(/"/g, "\"\"")}"`;
+
+        const rows = await this.executeSql(`
+            SELECT DISTINCT unnest(roles) AS role
+            FROM ${qualified}
+            WHERE roles IS NOT NULL
+            ORDER BY role;
+        `);
+        return rows
+            .map((r) => r.role as string)
+            .filter((r): r is string => typeof r === "string" && r.length > 0);
+    }
+
     async fetchCurrentDatabase(): Promise<string | undefined> {
         return this.poolManager?.defaultDatabaseName;
     }
@@ -1273,10 +1425,10 @@ export class AuthenticatedPostgresBackendDriver implements DataDriver {
         const pendingNotifications: PostgresBackendDriver["_pendingNotifications"] = [];
 
         const result = await this.delegate.db.transaction(async (tx) => {
-            let userId = this.user?.uid;
-            if (!userId) {
+            let uid = this.user?.uid;
+            if (!uid) {
                 logger.warn("[DataDriver] User ID (uid) is missing for authenticated delegate. Using 'anonymous'. User object", { detail: this.user });
-                userId = "anonymous";
+                uid = "anonymous";
             }
 
             const userRoles = this.user?.roles ?? [];
@@ -1295,7 +1447,7 @@ export class AuthenticatedPostgresBackendDriver implements DataDriver {
             //
             // Fails closed: if the switch cannot be performed, the transaction
             // aborts rather than falling back to an RLS-bypassing connection.
-            await applyAuthContext(tx, { userId, roles: userRoles }, this.delegate.rlsUserRole);
+            await applyAuthContext(tx, { uid, roles: userRoles }, this.delegate.rlsUserRole);
 
             const txEntityService = new DataService(tx, this.delegate.registry);
             const txDelegate = new PostgresBackendDriver(tx, this.delegate.realtimeService, this.delegate.registry, this.user, this.delegate.poolManager, this.delegate.historyService);
@@ -1334,7 +1486,7 @@ export class AuthenticatedPostgresBackendDriver implements DataDriver {
      */
     private injectAuthContext(unsubscribe: () => void): () => void {
         const authContext = {
-            userId: this.user?.uid || "anonymous",
+            uid: this.user?.uid || "anonymous",
             roles: this.user?.roles ?? []
         };
         const entries = Array.from(this.delegate.realtimeService.subscriptions.entries());
@@ -1362,6 +1514,19 @@ export class AuthenticatedPostgresBackendDriver implements DataDriver {
         return this.withTransaction((delegate) => delegate.save(props));
     }
 
+    /**
+     * One transaction for the whole batch, rather than one per row.
+     *
+     * This is the point of the method: `save` opens a transaction per call, so
+     * importing 10k rows through it means 10k transactions (and, over HTTP, 10k
+     * round trips). Here the RLS context is established once and every row lands
+     * or none does. Realtime notifications are already deferred to commit by
+     * `withTransaction`, so a batch does not flood subscribers mid-flight.
+     */
+    async saveMany<M extends Record<string, unknown>>(props: SaveManyProps<M>): Promise<Record<string, unknown>[]> {
+        return this.withTransaction((delegate) => delegate.saveMany(props));
+    }
+
     async delete<M extends Record<string, unknown>>(props: DeleteProps<M>): Promise<void> {
         return this.withTransaction((delegate) => delegate.delete(props));
     }

package/src/PostgresBootstrapper.ts

@@ -4,7 +4,7 @@
  * Implements the `BackendBootstrapper` interface for PostgreSQL.
  */
 
-import { getTableName, isTable, Relations, sql } from "drizzle-orm";
+import { Relations, sql } from "drizzle-orm";
 import { NodePgDatabase } from "drizzle-orm/node-postgres";
 import { PgEnum, PgTable } from "drizzle-orm/pg-core";
 import type { RebasePgTable } from "./types";
@@ -17,10 +17,12 @@ import {
     CollectionConfig,
     type HistoryConfig,
     InitializedDriver,
-    RealtimeProvider
+    RealtimeProvider,
+    type RealtimeChannelsConfig
 } from "@rebasepro/types";
 import { PostgresBackendDriver } from "./PostgresBackendDriver";
 import { RealtimeService } from "./services/realtimeService";
+import { buildCollectionRegistry } from "./collections/buildRegistry";
 import { DatabasePoolManager } from "./databasePoolManager";
 import { PostgresCollectionRegistry } from "./collections/PostgresCollectionRegistry";
 import { createEmailService, type EmailConfig, type EmailService, logger } from "@rebasepro/server";
@@ -33,7 +35,7 @@ import { ensureHistoryTableExists } from "./history/ensure-history-table";
 import { patchPgArrayNullSafety } from "./utils/pg-array-null-patch";
 import { buildCollectionsFromSchema, introspectSchema, readRlsStatus } from "./schema/introspect-runtime";
 import { buildDrizzleTablesFromSchema, buildDrizzleRelationsFromSchema } from "./schema/dynamic-tables";
-import { detectConnectionPosture, ensureAppRole, validatePolicyPgRoles, REBASE_USER_ROLE, type RawSqlRunner } from "./security/rls-enforcement";
+import { detectConnectionPosture, ensureAppRole, validatePolicyPgRoles, warnOnAnonymousGrants, REBASE_USER_ROLE, type RawSqlRunner } from "./security/rls-enforcement";
 import { provisionTriggerCdc, type CdcTableRef } from "./services/cdc/trigger-cdc";
 
 export interface PostgresDriverConfig {
@@ -51,6 +53,12 @@ export interface PostgresDriverConfig {
      * (BaaS mode). Defaults to `public`.
      */
     introspectionSchema?: string;
+    /**
+     * Realtime options. Currently only channel retention, which is opt-in:
+     * without rules here no channel keeps any history and broadcast stays
+     * fire-and-forget. See {@link ChannelRetentionRule}.
+     */
+    realtime?: RealtimeChannelsConfig;
 }
 
 /**
@@ -64,6 +72,15 @@ export interface PostgresDriverInternals {
     realtimeService: RealtimeService;
     driver: PostgresBackendDriver;
     poolManager?: DatabasePoolManager;
+    /**
+     * Attach CDC triggers to tables that did not exist when the driver
+     * bootstrapped. Only set when database-level capture is actually active.
+     *
+     * Auth owns its own tables and creates them later in boot, so at driver
+     * bootstrap they are legitimately missing and get skipped; without this
+     * they would stay uninstrumented until the next restart.
+     */
+    provisionCdcForTables?: (tables: CdcTableRef[]) => Promise<void>;
 }
 
 // Re-export from shared CLI error utilities
@@ -161,30 +178,17 @@ export function createPostgresBootstrapper(pgConfig: PostgresDriverConfig): Back
             }
 
             const activeCollections = introspectedCollections ?? collections;
-
-            // Create a fresh registry for this driver
-            const registry = new PostgresCollectionRegistry();
-            if (activeCollections) {
-                registry.registerMultiple(activeCollections);
-                logger.info(`📋 [PostgresRegistry] Registered ${registry.getCollections().length} collections: [${registry.getCollections().map(c => c.slug).join(", ")}]`);
-            }
-
             const schemaTables = introspectedTables ?? pgConfig.schema?.tables;
-
-            // Register tables
-            if (schemaTables) {
-                Object.values(schemaTables).forEach((table) => {
-                    if (isTable(table)) {
-                        const tableName = getTableName(table);
-                        registry.registerTable(table as PgTable, tableName);
-                    }
-                });
-            }
-
-            if (pgConfig.schema?.enums) registry.registerEnums(pgConfig.schema.enums as Record<string, PgEnum<[string, ...string[]]>>);
-
             const schemaRelations = introspectedRelations ?? (pgConfig.schema?.relations as Record<string, Relations> | undefined);
-            if (schemaRelations) registry.registerRelations(schemaRelations);
+
+            // Create a fresh registry for this driver. Registration order is
+            // load-bearing, so it lives in one place — see `buildCollectionRegistry`.
+            const registry = buildCollectionRegistry({
+                collections: activeCollections,
+                tables: schemaTables,
+                enums: pgConfig.schema?.enums as Record<string, PgEnum<[string, ...string[]]>> | undefined,
+                relations: schemaRelations
+            });
 
             // Patch Drizzle's PgArray columns to handle NULL values safely.
             // Drizzle's mapFromDriverValue crashes with "value.map is not a function"
@@ -303,6 +307,11 @@ export function createPostgresBootstrapper(pgConfig: PostgresDriverConfig): Back
                     registry.getCollections() as never,
                     driver.rlsUserRole ?? posture.role
                 );
+
+                // The same habit one surface over, and the dangerous direction:
+                // a rule that reads as "signed in only" but is true for every
+                // caller grants the data away rather than hiding it.
+                warnOnAnonymousGrants(registry.getCollections() as never);
             }
 
             // Ensure branch metadata table exists when branching is available
@@ -314,6 +323,16 @@ export function createPostgresBootstrapper(pgConfig: PostgresDriverConfig): Back
                 }
             }
 
+            // ── Channel history ──────────────────────────────────────────────
+            // Opt-in per channel pattern. With no rules this creates no tables
+            // and leaves broadcast on its original fire-and-forget path, so
+            // presence-only apps pay nothing for it.
+            try {
+                await realtimeService.configureChannelHistory(pgConfig.realtime?.channels);
+            } catch (err) {
+                logger.warn("⚠️ Could not initialize channel history tables — retained channels will not replay", { error: err });
+            }
+
             // ── Realtime change source ───────────────────────────────────────
             // Prefer DATABASE_DIRECT_URL to bypass PgBouncer for LISTEN/NOTIFY.
             const directUrl = process.env.DATABASE_DIRECT_URL || pgConfig.connectionString;
@@ -341,6 +360,7 @@ export function createPostgresBootstrapper(pgConfig: PostgresDriverConfig): Back
             const wantsCdc = cdcMode !== "off";
             const explicitCdc = cdcMode === "trigger" || cdcMode === "wal";
             let cdcEnabled = false;
+            let provisionCdcForTables: PostgresDriverInternals["provisionCdcForTables"];
 
             if (wantsCdc && !directUrl) {
                 const reason = "no direct database connection is available for the realtime LISTEN client (set DATABASE_DIRECT_URL)";
@@ -374,6 +394,11 @@ export function createPostgresBootstrapper(pgConfig: PostgresDriverConfig): Back
                     await provisionTriggerCdc(cdcRunSql, cdcTables);
                     await realtimeService.enableCdc(directUrl);
                     cdcEnabled = true;
+                    // Boot steps that create their own tables (auth) run after
+                    // this one and use it to instrument what they just created.
+                    provisionCdcForTables = async (tables) => {
+                        await provisionTriggerCdc(cdcRunSql, tables);
+                    };
                     logger.info(
                         `📡 [CDC] Realtime source = database-level change capture (mode: ${cdcMode === "wal" ? "wal→trigger" : "trigger"}). ` +
                         `All writes now emit realtime events regardless of origin.`
@@ -424,6 +449,14 @@ export function createPostgresBootstrapper(pgConfig: PostgresDriverConfig): Back
                     );
                     const missing: Array<{ slug: string; table: string }> = [];
                     for (const col of registeredCollections) {
+                        // Auth owns its table and creates it later in this same
+                        // boot (initializeAuth → ensureAuthTablesExist), so it is
+                        // legitimately absent right now. Reporting it as drift
+                        // tells the user to `db:push` a table that is about to
+                        // exist — and on an introspected database, one that the
+                        // database was never supposed to hold.
+                        if ((col as { auth?: { enabled?: boolean } }).auth?.enabled) continue;
+
                         const schemaName = "schema" in col && col.schema ? col.schema : "public";
                         const tableName = registry.hasTableForCollection(
                             col.table ?? col.slug
@@ -438,8 +471,10 @@ export function createPostgresBootstrapper(pgConfig: PostgresDriverConfig): Back
                         const checkName = resolvedTable ?? tableName;
                         const fullCheckName = schemaName === "public" ? checkName : `${schemaName}.${checkName}`;
                         if (!dbTables.has(fullCheckName)) {
+                            // Report what was actually looked up: an unqualified
+                            // "users" sends people hunting for public.users.
                             missing.push({ slug: col.slug,
-table: checkName });
+table: fullCheckName });
                         }
                     }
                     if (missing.length > 0) {
@@ -473,7 +508,8 @@ table: checkName });
                 registry,
                 realtimeService,
                 driver,
-                poolManager
+                poolManager,
+                provisionCdcForTables
             };
 
             return {
@@ -502,6 +538,29 @@ table: checkName });
             // ensureAuthTablesExist works with the collection abstraction — no Drizzle leakage.
             await ensureAuthTablesExist(db, authCollection);
 
+            // The driver bootstrapped before these tables existed, so CDC skipped
+            // them. Instrument them now, or writes to the user table emit no
+            // realtime events until the next restart.
+            if (authCollection && internals.provisionCdcForTables) {
+                const authSchema = "schema" in authCollection && typeof authCollection.schema === "string"
+                    ? authCollection.schema
+                    : "rebase";
+                const authTable = "table" in authCollection && typeof authCollection.table === "string"
+                    ? authCollection.table
+                    : authCollection.slug;
+                if (authTable) {
+                    try {
+                        await internals.provisionCdcForTables([{ schema: authSchema, table: authTable }]);
+                    } catch (err) {
+                        logger.warn(
+                            `⚠️ [CDC] Could not attach change-capture to the auth table "${authSchema}.${authTable}" — ` +
+                            "writes to it won't emit database-level events.",
+                            { detail: err instanceof Error ? err.message : String(err) }
+                        );
+                    }
+                }
+            }
+
             let emailService: EmailService | undefined;
             if (authConfig.email) {
                 emailService = createEmailService(authConfig.email as EmailConfig);