@rebasepro/server-postgres 0.12.1-canary.gf5f1d39 → 0.13.1-canary.g18cfeb7

package/src/schema/ensure-collection-tables.test.ts

@@ -87,16 +87,112 @@ describe("planning an additive schema ensure", () => {
         expect(sql).not.toMatch(/NOT NULL/i);
     });
 
-    it("leaves relation columns to a real migration rather than adding them without their key", () => {
-        const withRelation = {
-            ...posts,
-            properties: {
-                ...(posts as unknown as { properties: Record<string, unknown> }).properties,
-                author: { name: "Author", type: "reference", target: () => posts }
+    // ── Relations ────────────────────────────────────────────────────────────
+    //
+    // These columns were once skipped outright, on the reasoning that a bare
+    // column without its foreign key would disagree with what `db push` later
+    // generated. On a managed tenant nothing pushes afterwards, so the table
+    // arrived without the column its own collection reads: every insert 400ed
+    // with `column "author_id" does not exist`. The answer is to emit the key
+    // too, from the same planner `db push` uses.
+
+    const authors = {
+        name: "Authors",
+        slug: "authors",
+        properties: {
+            id: { name: "ID", type: "string", isId: "uuid" },
+            name: { name: "Name", type: "string" }
+        }
+    } as unknown as CollectionConfig;
+
+    const postsWithAuthor = {
+        ...posts,
+        properties: {
+            ...(posts as unknown as { properties: Record<string, unknown> }).properties,
+            author: { name: "Author", type: "reference", path: "authors" }
+        }
+    } as unknown as CollectionConfig;
+
+    const tags = {
+        name: "Tags",
+        slug: "tags",
+        properties: {
+            id: { name: "ID", type: "number", isId: "increment" },
+            name: { name: "Name", type: "string" }
+        }
+    } as unknown as CollectionConfig;
+
+    const postsWithTags = {
+        ...posts,
+        properties: {
+            ...(posts as unknown as { properties: Record<string, unknown> }).properties,
+            tags: {
+                name: "Tags",
+                type: "relation",
+                relation: { kind: "manyToMany", target: () => tags, relationName: "tags" }
             }
-        } as unknown as CollectionConfig;
-        const plan = planCollectionSchemaEnsure([withRelation], withTable("public.posts", ["id"]));
-        expect(plan.actions.some(a => a.target.endsWith(".author"))).toBe(false);
+        }
+    } as unknown as CollectionConfig;
+
+    it("adds a reference column, and the foreign key that makes it one", () => {
+        const plan = planCollectionSchemaEnsure([postsWithAuthor, authors], withTable("public.posts", ["id"]));
+
+        expect(plan.actions.some(a => a.kind === "add-column" && a.target === "public.posts.author")).toBe(true);
+        const fk = plan.actions.find(a => a.kind === "add-constraint");
+        expect(fk?.target).toBe("public.posts.posts_author_fkey");
+        expect(fk?.sql).toMatch(/REFERENCES "public"\."authors" \("id"\)/);
+    });
+
+    it("orders every constraint after the columns and tables it depends on", () => {
+        const plan = planCollectionSchemaEnsure([postsWithAuthor, authors], empty());
+        const kinds = plan.actions.map(a => a.kind);
+        expect(kinds.lastIndexOf("add-column")).toBeLessThan(kinds.indexOf("add-constraint"));
+        expect(kinds.lastIndexOf("create-table")).toBeLessThan(kinds.indexOf("add-constraint"));
+    });
+
+    it("skips a foreign key the database already has, since ADD CONSTRAINT has no IF NOT EXISTS", () => {
+        const existing: ExistingSchema = {
+            tables: new Map([
+                ["public.posts", new Set(["id", "author"])],
+                ["public.authors", new Set(["id", "name"])]
+            ]),
+            enums: new Set(["public.posts_status"]),
+            constraints: new Set(["public.posts.posts_author_fkey"])
+        };
+        const plan = planCollectionSchemaEnsure([postsWithAuthor, authors], existing);
+        expect(plan.actions.some(a => a.kind === "add-constraint")).toBe(false);
+    });
+
+    it("creates the junction table behind a many-to-many, keyed on both endpoints", () => {
+        const plan = planCollectionSchemaEnsure([postsWithTags, tags], empty());
+        const junction = plan.actions.find(a => a.kind === "create-table" && a.target === "public.posts_tags");
+
+        expect(junction).toBeDefined();
+        // The endpoint key types have to match the primary keys they reference:
+        // posts is a uuid, tags an auto-increment integer.
+        expect(junction!.sql).toMatch(/"post_id" UUID NOT NULL/);
+        expect(junction!.sql).toMatch(/"tag_id" INTEGER NOT NULL/);
+        expect(junction!.sql).toMatch(/PRIMARY KEY \("post_id", "tag_id"\)/);
+
+        const fks = plan.actions.filter(a => a.kind === "add-constraint").map(a => a.target);
+        expect(fks).toContain("public.posts_tags.posts_tags_post_id_fkey");
+        expect(fks).toContain("public.posts_tags.posts_tags_tag_id_fkey");
+    });
+
+    it("leaves an existing junction table alone", () => {
+        const existing: ExistingSchema = {
+            tables: new Map([
+                ["public.posts", new Set(["id", "title", "views", "status"])],
+                ["public.tags", new Set(["id", "name"])],
+                ["public.posts_tags", new Set(["post_id", "tag_id"])]
+            ]),
+            enums: new Set(["public.posts_status"]),
+            constraints: new Set([
+                "public.posts_tags.posts_tags_post_id_fkey",
+                "public.posts_tags.posts_tags_tag_id_fkey"
+            ])
+        };
+        expect(planCollectionSchemaEnsure([postsWithTags, tags], existing).actions).toEqual([]);
     });
 });
 

package/src/schema/ensure-collection-tables.ts

@@ -26,12 +26,22 @@
  * no-op.
  */
 import { type CollectionConfig, type Property, isPostgresCollectionConfig } from "@rebasepro/types";
-import { getTableName } from "@rebasepro/common";
+import { getTableName, relationalCollections } from "@rebasepro/common";
+import { logger } from "@rebasepro/server";
 import {
     getSqlColumnType,
     resolveColumnName,
-    isIdProperty
+    isIdProperty,
+    planRelationalColumns,
+    planJunctionTables,
+    quoteSqlLiteral
 } from "./generate-postgres-ddl-logic";
+import {
+    AUTH_USERS_COLUMNS,
+    authUsersColumnDefinition,
+    authUsersColumnSql,
+    isAuthCollection
+} from "./auth-users-columns";
 
 /**
  * The subset of a database handle this needs: run a statement, get rows back.
@@ -62,10 +72,18 @@ export interface ExistingSchema {
     tables: Map<string, Set<string>>;
     /** `schema.typename` of every enum type that already exists. */
     enums: Set<string>;
+    /**
+     * `schema.table.constraint` of every constraint that already exists.
+     *
+     * Optional so a caller that only cares about tables can still build one by
+     * hand; absent is read as "none known", which at worst re-attempts a
+     * constraint that then fails harmlessly as a duplicate.
+     */
+    constraints?: Set<string>;
 }
 
 export interface EnsureAction {
-    kind: "create-enum" | "create-table" | "add-column";
+    kind: "create-enum" | "create-table" | "add-column" | "add-constraint" | "rename-column";
     /** Qualified target, for logging: `public.posts` or `public.posts.title`. */
     target: string;
     sql: string;
@@ -75,6 +93,39 @@ export interface EnsurePlan {
     actions: EnsureAction[];
     /** Every statement, in dependency order. Empty when the schema is current. */
     statements: string[];
+    /**
+     * Relation columns this plan is about to create where the table already
+     * carries the same column under its pre-singularization name.
+     *
+     * The reason this is reported rather than silently handled: the ensure is
+     * additive, so it would add `category_id` beside a populated
+     * `categorie_id` and the relation would then read the new, empty one. No
+     * statement fails, no table is missing, and the only symptom is relations
+     * resolving to nothing — which is indistinguishable from having no data.
+     */
+    legacyForeignKeys: LegacyForeignKey[];
+}
+
+/** A relation column whose old and new spellings both plausibly apply. */
+export interface LegacyForeignKey {
+    /** `schema.table`. */
+    table: string;
+    /** The name the current rule derives, and what this plan would create. */
+    expected: string;
+    /** The name the old rule derived, which the table already has. */
+    legacy: string;
+}
+
+export interface EnsureOutcome extends EnsurePlan {
+    /**
+     * Constraints that could not be added — always non-fatal.
+     *
+     * A foreign key can only fail on data that already violates it, and the
+     * column it would police exists either way, so the collection still serves.
+     * Refusing to boot over one would turn a pre-existing data problem into an
+     * outage. Reported loudly instead.
+     */
+    failures: { target: string; error: string }[];
 }
 
 function schemaOf(collection: CollectionConfig): string {
@@ -114,11 +165,6 @@ function requiredEnums(collection: CollectionConfig): { name: string; values: st
     return out;
 }
 
-/** Single-quote escaping for an enum label. */
-function quoteLiteral(value: string): string {
-    return `'${value.replace(/'/g, "''")}'`;
-}
-
 /**
  * Decide what to add. Pure — the caller supplies what exists and runs the result.
  *
@@ -127,9 +173,15 @@ function quoteLiteral(value: string): string {
  * table may be the target of a relation), and nothing is emitted twice.
  */
 export function planCollectionSchemaEnsure(
-    collections: CollectionConfig[],
+    allCollections: CollectionConfig[],
     existing: ExistingSchema
 ): EnsurePlan {
+    // Boot receives every collection the bundle declares, including the ones
+    // served by another engine entirely. Creating a Postgres table for a
+    // Firestore collection is not a harmless extra: the app keeps reading
+    // documents from Firestore while an empty table with the same name accretes
+    // policies and shows up in every drift report.
+    const collections = relationalCollections(allCollections);
     const actions: EnsureAction[] = [];
     const plannedEnums = new Set<string>();
 
@@ -143,7 +195,7 @@ export function planCollectionSchemaEnsure(
             actions.push({
                 kind: "create-enum",
                 target: name,
-                sql: `CREATE TYPE "${schema}"."${typeName}" AS ENUM (${values.map(quoteLiteral).join(", ")});`
+                sql: `CREATE TYPE "${schema}"."${typeName}" AS ENUM (${values.map(quoteSqlLiteral).join(", ")});`
             });
         }
     }
@@ -164,17 +216,22 @@ export function planCollectionSchemaEnsure(
         );
         const idName = idEntry ? resolveColumnName(idEntry[0], idEntry[1] as Property) : "id";
         const idProp = idEntry?.[1] as Property | undefined;
-        let idDef: string;
-        if (idProp?.type === "number") {
-            idDef = `"${idName}" BIGSERIAL PRIMARY KEY`;
-        } else if (
-            idProp &&
-            idProp.type === "string" &&
-            (idProp as { isId?: unknown }).isId === "uuid"
-        ) {
-            idDef = `"${idName}" UUID PRIMARY KEY DEFAULT gen_random_uuid()`;
-        } else {
-            idDef = `"${idName}" TEXT PRIMARY KEY`;
+        // Derived through the generator's own type mapping, not re-decided here.
+        // This branch used to emit BIGSERIAL for a numeric id while `db push`
+        // emitted INTEGER GENERATED BY DEFAULT AS IDENTITY, so the same project
+        // got an int8 key when the runtime brought the schema up and an int4 key
+        // when a human pushed it. Two consequences, both real: node-postgres
+        // hands back int8 as a *string*, so a collection declaring
+        // `type: "number"` served `"1"` instead of `1` on the managed path only;
+        // and every foreign key and junction column pointing at it stayed
+        // INTEGER on both paths, which is a truncation waiting for the sequence
+        // to pass 2^31. The agreement test now pins this.
+        const idType = idProp
+            ? getSqlColumnType(idEntry![0], idProp, collection, collections)
+            : "TEXT";
+        let idDef = `"${idName}" ${idType} PRIMARY KEY`;
+        if (idProp?.type === "string" && (idProp as { isId?: unknown }).isId === "uuid") {
+            idDef += " DEFAULT gen_random_uuid()";
         }
         actions.push({
             kind: "create-table",
@@ -183,35 +240,189 @@ export function planCollectionSchemaEnsure(
         });
     }
 
+    // 2b. Junction tables behind many-to-many relations. No collection declares
+    //     them, so the walk above never sees them — and until they existed, an
+    //     m2m write had nowhere to land and the junction's derived RLS had
+    //     nothing to attach to.
+    const junctions = planJunctionTables(collections);
+    for (const junction of junctions) {
+        const key = `${junction.schema}.${junction.table}`;
+        if (existing.tables.has(key) || created.has(key)) continue;
+        created.add(key);
+        actions.push({ kind: "create-table", target: key, sql: junction.createTable });
+    }
+
     // 3. Missing columns, on both brand-new and pre-existing tables.
+    const legacyForeignKeys: LegacyForeignKey[] = [];
+
+    /**
+     * Move a relation column that is only missing because it was renamed.
+     *
+     * Returns true when it handled the column, so the caller skips the ordinary
+     * ADD. Adding here would be the wrong move and a quiet one: the data is in
+     * the old column, `ADD COLUMN` creates the new one empty beside it, every
+     * statement succeeds, and the relation reads the empty one. A rename is
+     * metadata-only in Postgres, keeps the values, and carries the column's
+     * indexes and constraints with it.
+     *
+     * Only ever reached when the new name is absent and the old name is
+     * present, so there is nothing to overwrite and nothing to choose between.
+     */
+    const renameLegacyColumn = (
+        key: string,
+        schema: string,
+        table: string,
+        column: string,
+        legacyName: string | undefined
+    ): boolean => {
+        const present = existing.tables.get(key);
+        if (!legacyName || !present) return false;
+        if (present.has(column) || !present.has(legacyName)) return false;
+
+        legacyForeignKeys.push({ table: key, expected: column, legacy: legacyName });
+        actions.push({
+            kind: "rename-column",
+            target: `${key}.${column}`,
+            sql: `ALTER TABLE "${schema}"."${table}" RENAME COLUMN "${legacyName}" TO "${column}";`
+        });
+        return true;
+    };
+
+    const addColumn = (
+        key: string,
+        schema: string,
+        table: string,
+        column: string,
+        definition: string
+    ): void => {
+        const present = existing.tables.get(key);
+        if (present?.has(column)) return;
+        actions.push({
+            kind: "add-column",
+            target: `${key}.${column}`,
+            sql: `ALTER TABLE "${schema}"."${table}" ADD COLUMN IF NOT EXISTS "${column}" ${definition};`
+        });
+    };
+
     for (const collection of collections) {
         const key = qualified(collection);
         const schema = schemaOf(collection);
         const table = getTableName(collection);
-        const present = existing.tables.get(key) ?? new Set<string>();
+        // A table this run is creating has no rows yet, so the constraints
+        // `db push` writes are free to apply. On a table that already exists
+        // they are not: `SET NOT NULL` is checked against live rows and a UNIQUE
+        // would fail on existing duplicates, and this module runs unattended
+        // against customer data with nobody reading a diff. So the constraints
+        // are emitted for the fresh case — which is the whole managed-runtime
+        // path, and the one that diverged from `db push` — and withheld for the
+        // adopted one. `rebase db push` remains how an existing table gets them.
+        const fresh = created.has(key);
+        const auth = isAuthCollection(collection);
         for (const [propName, prop] of Object.entries(collection.properties ?? {})) {
             const p = prop as Property;
             if (isIdProperty(propName, p, collection)) continue;
-            // A relation's own column is emitted by the DDL generator with a
-            // foreign key; adding a bare column here would create the column
-            // without the constraint and make the generator's later output
-            // disagree with the database. Left to a real migration.
+            // Relation and reference columns are planned from the shared
+            // relational planner below, which derives the column name, type and
+            // foreign key the same way `db push` does. Deriving them here as
+            // plain columns is what once produced a column with no constraint.
             if (p.type === "reference" || p.type === "relation") continue;
+
             const column = resolveColumnName(propName, p);
-            if (present.has(column)) continue;
-            const type = getSqlColumnType(propName, p, collection, collections);
-            actions.push({
-                kind: "add-column",
-                target: `${key}.${column}`,
-                // Never NOT NULL: an existing table with rows cannot take a
-                // non-null column without a default, and inventing one would be
-                // guessing at the customer's data.
-                sql: `ALTER TABLE "${schema}"."${table}" ADD COLUMN IF NOT EXISTS "${column}" ${type};`
-            });
+            // On an auth collection, the columns auth itself reads and writes
+            // have exactly one definition, wherever the table is created from —
+            // see `auth-users-columns`. Anything else on that collection is an
+            // ordinary user-declared field and is generated like any other.
+            const authDefinition = auth ? authUsersColumnDefinition(column) : undefined;
+            if (authDefinition) {
+                addColumn(key, schema, table, column, authDefinition);
+                continue;
+            }
+
+            // Assembled in the generator's order — type, UNIQUE, DEFAULT,
+            // NOT NULL — so the two produce byte-identical column definitions
+            // and the agreement test can compare them directly instead of
+            // checking that a column merely exists, which is how BIGSERIAL-vs-
+            // INTEGER and every missing constraint went unnoticed.
+            let definition = getSqlColumnType(propName, p, collection, collections);
+            if (fresh && p.validation?.unique) definition += " UNIQUE";
+            // Not gated on `fresh`: a default binds future writes only, so it is
+            // safe on a live table, and a column added without it would take the
+            // value the application forgot to send rather than `now()`.
+            const autoValue = (p as { autoValue?: string }).autoValue;
+            if (p.type === "date" && (autoValue === "on_create" || autoValue === "on_update")) {
+                definition += " DEFAULT now()";
+            }
+            if (fresh && p.validation?.required) definition += " NOT NULL";
+            addColumn(key, schema, table, column, definition);
+        }
+
+        // The auth columns the collection never mentions. The scaffold's users
+        // collection describes 12 of the 14 auth reads and writes, so planning
+        // only from properties left `is_anonymous` and `tokens_valid_after` to
+        // `ensureAuthTablesExist` — which does create them, but only because
+        // that function happens to run later in the same boot. Planning them
+        // here makes this path self-contained and identical to `db push`, so
+        // neither depends on the other having run.
+        if (auth) {
+            const declared = new Set(
+                Object.entries(collection.properties ?? {})
+                    .map(([name, prop]) => resolveColumnName(name, prop as Property))
+            );
+            for (const spec of AUTH_USERS_COLUMNS) {
+                if (declared.has(spec.column)) continue;
+                addColumn(key, schema, table, spec.column, authUsersColumnSql(spec));
+            }
         }
     }
 
-    return { actions, statements: actions.map(a => a.sql) };
+    // 3b. A junction that already existed, but is short a column. One created
+    //     above already carries both — unlike a collection table, whose CREATE
+    //     declares only the identity column — so re-listing them would log two
+    //     no-op statements and inflate the count of changes applied.
+    for (const junction of junctions) {
+        const key = `${junction.schema}.${junction.table}`;
+        if (created.has(key)) continue;
+        for (const column of junction.columns) {
+            if (renameLegacyColumn(key, junction.schema, junction.table, column.name, column.legacyName)) continue;
+            addColumn(key, junction.schema, junction.table, column.name, column.type);
+        }
+    }
+
+    // 3c. The columns relation and reference properties own.
+    for (const relational of planRelationalColumns(collections)) {
+        const relKey = `${relational.schema}.${relational.table}`;
+        if (renameLegacyColumn(relKey, relational.schema, relational.table, relational.column, relational.legacyColumn)) continue;
+        addColumn(
+            relKey,
+            relational.schema,
+            relational.table,
+            relational.column,
+            relational.type
+        );
+    }
+
+    // 4. Foreign keys, last: the tables and columns on both ends have to exist
+    //    first, and a constraint is the one thing here that can fail on data
+    //    rather than on schema, so nothing else depends on it.
+    const knownConstraints = existing.constraints ?? new Set<string>();
+    const plannedConstraints = new Set<string>();
+    const foreignKeys = [
+        ...planRelationalColumns(collections).map(r => r.foreignKey),
+        ...junctions.flatMap(j => j.foreignKeys)
+    ];
+    for (const fk of foreignKeys) {
+        if (!fk) continue;
+        const name = `${fk.schema}.${fk.table}.${fk.constraintName}`;
+        if (knownConstraints.has(name) || plannedConstraints.has(name)) continue;
+        plannedConstraints.add(name);
+        actions.push({
+            kind: "add-constraint",
+            target: `${fk.schema}.${fk.table}.${fk.constraintName}`,
+            sql: fk.sql
+        });
+    }
+
+    return { actions, statements: actions.map(a => a.sql), legacyForeignKeys };
 }
 
 /** Read what the database has, for the schemas the collections live in. */
@@ -250,7 +461,23 @@ export async function readExistingSchema(
     );
     for (const row of enumRows) enums.add(`${row.schema}.${row.name}`);
 
-    return { tables, enums };
+    // `ADD CONSTRAINT` has no IF NOT EXISTS, so an existing foreign key is
+    // skipped by name rather than guarded in SQL.
+    const constraints = new Set<string>();
+    const { rows: constraintRows } = await client.query<{
+        schema: string;
+        table: string;
+        name: string;
+    }>(
+        `SELECT n.nspname AS schema, c.relname AS table, con.conname AS name
+         FROM pg_constraint con
+         JOIN pg_class c ON con.conrelid = c.oid
+         JOIN pg_namespace n ON c.relnamespace = n.oid
+         WHERE n.nspname IN (${inList})`
+    );
+    for (const row of constraintRows) constraints.add(`${row.schema}.${row.table}.${row.name}`);
+
+    return { tables, enums, constraints };
 }
 
 /**
@@ -265,8 +492,14 @@ export async function ensureCollectionTables(
     client: Queryable,
     collections: CollectionConfig[],
     log?: (message: string) => void
-): Promise<EnsurePlan> {
-    const schemas = Array.from(new Set(collections.map(schemaOf)));
+): Promise<EnsureOutcome> {
+    // Junctions live alongside the collections that declare them, so their
+    // schema has to be read too — otherwise an existing junction reads as
+    // missing and its constraints as unplanned.
+    const schemas = Array.from(new Set([
+        ...collections.map(schemaOf),
+        ...planJunctionTables(collections).map(j => j.schema)
+    ]));
     for (const schema of schemas) {
         assertSafeIdentifier(schema, "schema name");
         if (schema !== "public") {
@@ -276,10 +509,26 @@ export async function ensureCollectionTables(
 
     const existing = await readExistingSchema(client, schemas);
     const plan = planCollectionSchemaEnsure(collections, existing);
+    const failures: { target: string; error: string }[] = [];
+
+    // Reported, not warned: this is a rename the ensure is about to perform, and
+    // the operator should be able to see in the log why a column changed name.
+    // The interesting case is the one that no longer happens — before this,
+    // ensure added the new column empty beside the populated old one, every
+    // statement succeeded, and the relation read the empty one.
+    for (const legacy of plan.legacyForeignKeys) {
+        const message =
+            `Renaming "${legacy.table}"."${legacy.legacy}" to "${legacy.expected}". The old name is ` +
+            "the one Rebase derived for this relation before it singularized properly; the column " +
+            "keeps its data, indexes and constraints. To keep the old name instead, set " +
+            `\`localKey: "${legacy.legacy}"\` on the relation and this will stop.`;
+        logger.info(`[schema] ${message}`);
+        log?.(message);
+    }
 
     if (plan.actions.length === 0) {
         log?.("Schema is up to date; nothing to create.");
-        return plan;
+        return { ...plan, failures };
     }
 
     for (const action of plan.actions) {
@@ -287,11 +536,19 @@ export async function ensureCollectionTables(
             await client.query(action.sql);
             log?.(`${action.kind}: ${action.target}`);
         } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            // A foreign key is the only action that can fail on the customer's
+            // data rather than on the schema. The column it polices is already
+            // there, so the collection serves either way — record it and carry
+            // on rather than crash-looping the deployment.
+            if (action.kind === "add-constraint") {
+                failures.push({ target: action.target, error: message });
+                continue;
+            }
             throw new Error(
-                `Failed to ${action.kind} ${action.target}: ` +
-                `${err instanceof Error ? err.message : String(err)}\n  ${action.sql}`
+                `Failed to ${action.kind} ${action.target}: ${message}\n  ${action.sql}`
             );
         }
     }
-    return plan;
+    return { ...plan, failures };
 }