@rebasepro/server-postgres 0.13.1-canary.gef9608c → 0.14.0

package/src/schema/ensure-collection-tables.ts

@@ -28,6 +28,18 @@
 import { type CollectionConfig, type Property, isPostgresCollectionConfig } from "@rebasepro/types";
 import { getTableName, relationalCollections } from "@rebasepro/common";
 import { logger } from "@rebasepro/server";
+import {
+    assertSearchIsPostgresOnly,
+    buildSearchColumnSpec,
+    searchExtensionStatements,
+    searchHelperFunctions,
+    searchIndexStatements,
+    searchColumnStamps,
+    SEARCH_STAMP_PREFIX,
+    SEARCH_TEXT_FN,
+    SEARCH_UNACCENT_FN,
+    type SearchColumnSpec
+} from "./search-column";
 import {
     getSqlColumnType,
     resolveColumnName,
@@ -36,6 +48,12 @@ import {
     planJunctionTables,
     quoteSqlLiteral
 } from "./generate-postgres-ddl-logic";
+import {
+    AUTH_USERS_COLUMNS,
+    authUsersColumnDefinition,
+    authUsersColumnSql,
+    isAuthCollection
+} from "./auth-users-columns";
 
 /**
  * The subset of a database handle this needs: run a statement, get rows back.
@@ -74,10 +92,19 @@ export interface ExistingSchema {
      * constraint that then fails harmlessly as a duplicate.
      */
     constraints?: Set<string>;
+    /**
+     * `schema.table.column` → that column's comment, for the columns that have
+     * one. This is where a generated search column's fingerprint lives, so it
+     * is the only evidence that a `search` block has changed since the column
+     * was built. Absent is read as "no column is stamped", which plans a stamp
+     * and reports nothing as drifted.
+     */
+    columnComments?: Map<string, string>;
 }
 
 export interface EnsureAction {
-    kind: "create-enum" | "create-table" | "add-column" | "add-constraint" | "rename-column";
+    kind: "create-enum" | "create-table" | "add-column" | "add-constraint" | "rename-column"
+        | "create-extension" | "create-function" | "create-index" | "comment-column";
     /** Qualified target, for logging: `public.posts` or `public.posts.title`. */
     target: string;
     sql: string;
@@ -98,6 +125,47 @@ export interface EnsurePlan {
      * resolving to nothing — which is indistinguishable from having no data.
      */
     legacyForeignKeys: LegacyForeignKey[];
+    /**
+     * Generated search columns whose `search` block has changed since they were
+     * built. Reported, never planned into `actions` — see
+     * {@link SearchColumnDrift} for why applying it is not this path's call.
+     */
+    searchDrift: SearchColumnDrift[];
+    /**
+     * Generated search columns that exist but carry no fingerprint — created
+     * before this check existed, or by `search.sql` on an older CLI. The plan
+     * stamps them so the *next* change is detectable; whether they match the
+     * current block cannot be known, which is what the caller reports.
+     */
+    searchAdopted: { table: string; column: string }[];
+}
+
+/**
+ * A generated search column built from a `search` block that has since changed.
+ *
+ * Reported instead of applied because the two ways to apply it are both worse
+ * than stopping. `ALTER COLUMN … SET EXPRESSION` exists only on PG17+ and
+ * rewrites the table either way; `DROP COLUMN` + `ADD COLUMN` rewrites it under
+ * an ACCESS EXCLUSIVE lock and rebuilds the GIN index. This module runs
+ * unattended against live customer data with nobody reading a diff — the same
+ * reason it withholds `SET NOT NULL` from an adopted table — so a multi-minute
+ * outage is not a decision it may take on its own.
+ *
+ * Not applying it silently is not an option either: that is the bug this
+ * detection exists for. A collection that added a field, flipped `unaccent` or
+ * raised a weight kept indexing the *old* set forever, and the only symptom was
+ * searches returning nothing for content plainly in the row.
+ */
+export interface SearchColumnDrift {
+    /** `schema.table`. */
+    table: string;
+    column: string;
+    /** The fingerprint recorded on the column. */
+    found: string;
+    /** The fingerprint the current `search` block computes. */
+    expected: string;
+    /** The statements that would rebuild the column, for the operator to run. */
+    rebuild: string[];
 }
 
 /** A relation column whose old and new spellings both plausibly apply. */
@@ -112,14 +180,16 @@ export interface LegacyForeignKey {
 
 export interface EnsureOutcome extends EnsurePlan {
     /**
-     * Constraints that could not be added — always non-fatal.
+     * Actions that could not be applied and are non-fatal by nature.
      *
-     * A foreign key can only fail on data that already violates it, and the
-     * column it would police exists either way, so the collection still serves.
-     * Refusing to boot over one would turn a pre-existing data problem into an
-     * outage. Reported loudly instead.
+     * Two kinds qualify. A foreign key can only fail on data that already
+     * violates it, and the column it would police exists either way, so the
+     * collection still serves; refusing to boot over one would turn a
+     * pre-existing data problem into an outage. A column comment is the search
+     * fingerprint, which needs table ownership — losing it costs drift
+     * detection on the next boot, not the deployment. Both are reported loudly.
      */
-    failures: { target: string; error: string }[];
+    failures: { kind: EnsureAction["kind"]; target: string; error: string }[];
 }
 
 function schemaOf(collection: CollectionConfig): string {
@@ -175,6 +245,10 @@ export function planCollectionSchemaEnsure(
     // Firestore collection is not a harmless extra: the app keeps reading
     // documents from Firestore while an empty table with the same name accretes
     // policies and shows up in every drift report.
+    // Before the filter, deliberately: a `search` block on a collection this
+    // engine does not store would otherwise be dropped here without a word.
+    assertSearchIsPostgresOnly(allCollections);
+
     const collections = relationalCollections(allCollections);
     const actions: EnsureAction[] = [];
     const plannedEnums = new Set<string>();
@@ -194,6 +268,34 @@ export function planCollectionSchemaEnsure(
         }
     }
 
+    // 1b. Search support, for collections that declared a `search` block.
+    //
+    //     Before the tables, because a generated column's expression is
+    //     resolved when the column is created: a table whose search column
+    //     calls `rebase_search_text` cannot be added before that function
+    //     exists. Both forms are idempotent, so a boot against a database that
+    //     already has them plans nothing.
+    const searchSpecs = collections
+        .map(c => buildSearchColumnSpec(c))
+        .filter((spec): spec is SearchColumnSpec => spec !== undefined);
+
+    const plannedExtensions = new Set<string>();
+    for (const spec of searchSpecs) {
+        for (const statement of searchExtensionStatements(spec)) {
+            if (plannedExtensions.has(statement)) continue;
+            plannedExtensions.add(statement);
+            actions.push({ kind: "create-extension", target: statement.replace(/^CREATE EXTENSION IF NOT EXISTS |;$/g, ""), sql: statement });
+        }
+    }
+    const plannedFunctions = new Set<string>();
+    for (const spec of searchSpecs) {
+        for (const statement of searchHelperFunctions(spec)) {
+            if (plannedFunctions.has(statement)) continue;
+            plannedFunctions.add(statement);
+            actions.push({ kind: "create-function", target: statement.includes("unaccent") ? SEARCH_UNACCENT_FN : SEARCH_TEXT_FN, sql: statement });
+        }
+    }
+
     // 2. Missing tables. Only the identity column is created here; every other
     //    column is added by step 3, so a new table and an existing table that
     //    gained a field travel the exact same code path. One way to build a
@@ -210,17 +312,22 @@ export function planCollectionSchemaEnsure(
         );
         const idName = idEntry ? resolveColumnName(idEntry[0], idEntry[1] as Property) : "id";
         const idProp = idEntry?.[1] as Property | undefined;
-        let idDef: string;
-        if (idProp?.type === "number") {
-            idDef = `"${idName}" BIGSERIAL PRIMARY KEY`;
-        } else if (
-            idProp &&
-            idProp.type === "string" &&
-            (idProp as { isId?: unknown }).isId === "uuid"
-        ) {
-            idDef = `"${idName}" UUID PRIMARY KEY DEFAULT gen_random_uuid()`;
-        } else {
-            idDef = `"${idName}" TEXT PRIMARY KEY`;
+        // Derived through the generator's own type mapping, not re-decided here.
+        // This branch used to emit BIGSERIAL for a numeric id while `db push`
+        // emitted INTEGER GENERATED BY DEFAULT AS IDENTITY, so the same project
+        // got an int8 key when the runtime brought the schema up and an int4 key
+        // when a human pushed it. Two consequences, both real: node-postgres
+        // hands back int8 as a *string*, so a collection declaring
+        // `type: "number"` served `"1"` instead of `1` on the managed path only;
+        // and every foreign key and junction column pointing at it stayed
+        // INTEGER on both paths, which is a truncation waiting for the sequence
+        // to pass 2^31. The agreement test now pins this.
+        const idType = idProp
+            ? getSqlColumnType(idEntry![0], idProp, collection, collections)
+            : "TEXT";
+        let idDef = `"${idName}" ${idType} PRIMARY KEY`;
+        if (idProp?.type === "string" && (idProp as { isId?: unknown }).isId === "uuid") {
+            idDef += " DEFAULT gen_random_uuid()";
         }
         actions.push({
             kind: "create-table",
@@ -282,17 +389,14 @@ export function planCollectionSchemaEnsure(
         schema: string,
         table: string,
         column: string,
-        type: string
+        definition: string
     ): void => {
         const present = existing.tables.get(key);
         if (present?.has(column)) return;
         actions.push({
             kind: "add-column",
             target: `${key}.${column}`,
-            // Never NOT NULL: an existing table with rows cannot take a
-            // non-null column without a default, and inventing one would be
-            // guessing at the customer's data.
-            sql: `ALTER TABLE "${schema}"."${table}" ADD COLUMN IF NOT EXISTS "${column}" ${type};`
+            sql: `ALTER TABLE "${schema}"."${table}" ADD COLUMN IF NOT EXISTS "${column}" ${definition};`
         });
     };
 
@@ -300,6 +404,16 @@ export function planCollectionSchemaEnsure(
         const key = qualified(collection);
         const schema = schemaOf(collection);
         const table = getTableName(collection);
+        // A table this run is creating has no rows yet, so the constraints
+        // `db push` writes are free to apply. On a table that already exists
+        // they are not: `SET NOT NULL` is checked against live rows and a UNIQUE
+        // would fail on existing duplicates, and this module runs unattended
+        // against customer data with nobody reading a diff. So the constraints
+        // are emitted for the fresh case — which is the whole managed-runtime
+        // path, and the one that diverged from `db push` — and withheld for the
+        // adopted one. `rebase db push` remains how an existing table gets them.
+        const fresh = created.has(key);
+        const auth = isAuthCollection(collection);
         for (const [propName, prop] of Object.entries(collection.properties ?? {})) {
             const p = prop as Property;
             if (isIdProperty(propName, p, collection)) continue;
@@ -308,7 +422,112 @@ export function planCollectionSchemaEnsure(
             // foreign key the same way `db push` does. Deriving them here as
             // plain columns is what once produced a column with no constraint.
             if (p.type === "reference" || p.type === "relation") continue;
-            addColumn(key, schema, table, resolveColumnName(propName, p), getSqlColumnType(propName, p, collection, collections));
+
+            const column = resolveColumnName(propName, p);
+            // On an auth collection, the columns auth itself reads and writes
+            // have exactly one definition, wherever the table is created from —
+            // see `auth-users-columns`. Anything else on that collection is an
+            // ordinary user-declared field and is generated like any other.
+            const authDefinition = auth ? authUsersColumnDefinition(column) : undefined;
+            if (authDefinition) {
+                addColumn(key, schema, table, column, authDefinition);
+                continue;
+            }
+
+            // Assembled in the generator's order — type, UNIQUE, DEFAULT,
+            // NOT NULL — so the two produce byte-identical column definitions
+            // and the agreement test can compare them directly instead of
+            // checking that a column merely exists, which is how BIGSERIAL-vs-
+            // INTEGER and every missing constraint went unnoticed.
+            let definition = getSqlColumnType(propName, p, collection, collections);
+            if (fresh && p.validation?.unique) definition += " UNIQUE";
+            // Not gated on `fresh`: a default binds future writes only, so it is
+            // safe on a live table, and a column added without it would take the
+            // value the application forgot to send rather than `now()`.
+            const autoValue = (p as { autoValue?: string }).autoValue;
+            if (p.type === "date" && (autoValue === "on_create" || autoValue === "on_update")) {
+                definition += " DEFAULT now()";
+            }
+            if (fresh && p.validation?.required) definition += " NOT NULL";
+            addColumn(key, schema, table, column, definition);
+        }
+
+        // The auth columns the collection never mentions. The scaffold's users
+        // collection describes 12 of the 14 auth reads and writes, so planning
+        // only from properties left `is_anonymous` and `tokens_valid_after` to
+        // `ensureAuthTablesExist` — which does create them, but only because
+        // that function happens to run later in the same boot. Planning them
+        // here makes this path self-contained and identical to `db push`, so
+        // neither depends on the other having run.
+        if (auth) {
+            const declared = new Set(
+                Object.entries(collection.properties ?? {})
+                    .map(([name, prop]) => resolveColumnName(name, prop as Property))
+            );
+            for (const spec of AUTH_USERS_COLUMNS) {
+                if (declared.has(spec.column)) continue;
+                addColumn(key, schema, table, spec.column, authUsersColumnSql(spec));
+            }
+        }
+    }
+
+    // 3aa. The generated search columns.
+    //
+    //      Adding a STORED generated column rewrites the table, which on a large
+    //      one is not free — but it is the same additive shape as every other
+    //      column here, and the alternative (leaving it out until someone runs a
+    //      migration) is a declared `search` block that silently does nothing.
+    //
+    //      Changing one is not additive, and `ADD COLUMN IF NOT EXISTS` is a
+    //      no-op against a column that is already there — which is why a `search`
+    //      block that gained a field, flipped `unaccent` or moved a weight used
+    //      to be inert forever, on every path, with nothing logged. Each column
+    //      therefore carries a fingerprint of the expression it was built from
+    //      (in its comment), and a mismatch is reported rather than applied.
+    const searchDrift: SearchColumnDrift[] = [];
+    const searchAdopted: { table: string; column: string }[] = [];
+    for (const spec of searchSpecs) {
+        const key = `${spec.schema}.${spec.table}`;
+        const definitions: Record<string, string> = {
+            [spec.column]: `tsvector GENERATED ALWAYS AS (${spec.expression}) STORED`
+        };
+        if (spec.fuzzy) {
+            definitions[spec.fuzzy.column] = `text GENERATED ALWAYS AS (${spec.fuzzy.expression}) STORED`;
+        }
+
+        for (const stamp of searchColumnStamps(spec)) {
+            const definition = definitions[stamp.column];
+            const exists = existing.tables.get(key)?.has(stamp.column) === true;
+            const recorded = existing.columnComments?.get(`${key}.${stamp.column}`);
+
+            if (exists && recorded?.startsWith(SEARCH_STAMP_PREFIX) && recorded !== stamp.fingerprint) {
+                searchDrift.push({
+                    table: key,
+                    column: stamp.column,
+                    found: recorded,
+                    expected: stamp.fingerprint,
+                    rebuild: [
+                        `ALTER TABLE "${spec.schema}"."${spec.table}" DROP COLUMN "${stamp.column}";`,
+                        `ALTER TABLE "${spec.schema}"."${spec.table}" ADD COLUMN "${stamp.column}" ${definition};`,
+                        stamp.sql
+                    ]
+                });
+                // The old stamp is the only evidence of what the column holds;
+                // overwriting it here would erase the drift instead of fixing it.
+                continue;
+            }
+
+            addColumn(key, spec.schema, spec.table, stamp.column, definition);
+            if (exists && recorded === undefined) {
+                searchAdopted.push({ table: key, column: stamp.column });
+            }
+            if (recorded !== stamp.fingerprint) {
+                actions.push({
+                    kind: "comment-column",
+                    target: `${key}.${stamp.column}`,
+                    sql: stamp.sql
+                });
+            }
         }
     }
 
@@ -359,7 +578,24 @@ export function planCollectionSchemaEnsure(
         });
     }
 
-    return { actions, statements: actions.map(a => a.sql), legacyForeignKeys };
+    // 5. Search indexes, after everything — the column has to exist, and this is
+    //    the one step that runs against a populated table for real work.
+    //
+    //    CONCURRENTLY: a plain CREATE INDEX takes a lock that blocks writes for
+    //    the duration of the build, which on a live table is an outage. Each
+    //    statement here is issued on its own, outside any transaction, which is
+    //    the condition CONCURRENTLY requires.
+    for (const spec of searchSpecs) {
+        for (const statement of searchIndexStatements(spec)) {
+            actions.push({
+                kind: "create-index",
+                target: `${spec.schema}.${spec.table}`,
+                sql: statement.replace("CREATE INDEX IF NOT EXISTS", "CREATE INDEX CONCURRENTLY IF NOT EXISTS")
+            });
+        }
+    }
+
+    return { actions, statements: actions.map(a => a.sql), legacyForeignKeys, searchDrift, searchAdopted };
 }
 
 /** Read what the database has, for the schemas the collections live in. */
@@ -414,7 +650,78 @@ export async function readExistingSchema(
     );
     for (const row of constraintRows) constraints.add(`${row.schema}.${row.table}.${row.name}`);
 
-    return { tables, enums, constraints };
+    // Column comments, which is where a generated search column records the
+    // expression it was built from. `objsubid > 0` is what makes a row a
+    // *column* comment rather than the table's own.
+    const columnComments = new Map<string, string>();
+    const { rows: commentRows } = await client.query<{
+        schema: string;
+        table: string;
+        column: string;
+        comment: string | null;
+    }>(
+        `SELECT n.nspname AS schema, c.relname AS table, a.attname AS column, d.description AS comment
+         FROM pg_description d
+         JOIN pg_class c ON d.objoid = c.oid
+         JOIN pg_namespace n ON c.relnamespace = n.oid
+         JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = d.objsubid
+         WHERE d.objsubid > 0 AND n.nspname IN (${inList})`
+    );
+    for (const row of commentRows) {
+        if (row.comment == null) continue;
+        columnComments.set(`${row.schema}.${row.table}.${row.column}`, row.comment);
+    }
+
+    return { tables, enums, constraints, columnComments };
+}
+
+/**
+ * What to tell an operator whose `search` block no longer matches its column.
+ *
+ * Every line here is doing work: naming the collection is not enough, because
+ * the symptom (a search that finds nothing) points at the data, not the schema;
+ * and the remediation has to be exact, because it is a table rewrite the
+ * operator is being asked to schedule rather than discover.
+ */
+function searchDriftMessage(drift: SearchColumnDrift[]): string {
+    const blocks = drift.map(d =>
+        `  "${d.table}"."${d.column}" was generated from a different \`search\` block ` +
+        `(recorded ${d.found}, current ${d.expected}).\n` +
+        d.rebuild.map(s => `      ${s}`).join("\n")
+    );
+    return (
+        "The `search` block changed after its generated column was created, and Postgres cannot alter a " +
+        "generated expression in place.\n" +
+        "Rebase will not rebuild it for you: dropping and re-adding a STORED generated column rewrites the whole " +
+        "table under an ACCESS EXCLUSIVE lock and rebuilds its GIN index, which is an outage this unattended path " +
+        "may not schedule on your behalf.\n" +
+        "Until it is rebuilt the column keeps indexing the previous fields, weights and language — searches for " +
+        "anything added since return nothing, which reads from outside as \"no such row\".\n" +
+        "Run these (or revert the block to what the column was built from), then boot again:\n" +
+        blocks.join("\n") +
+        "\n  The GIN index is dropped with the column and recreated concurrently on the next boot."
+    );
+}
+
+/**
+ * The missing-pgvector explanation, appended to the error that reveals it.
+ *
+ * A `{ type: "vector" }` property compiles to `VECTOR(n)`, and nothing in the
+ * OSS pipeline installs pgvector — not this ensure, not `db push`, not the
+ * scaffold's `postgres:18-alpine`, which does not ship it. Installing an
+ * extension on someone's database is a decision with a deployment behind it
+ * (image, superuser, cloud allow-list), so this path stays a refusal; what it
+ * must not stay is a bare `type "vector" does not exist` on a crash-looping
+ * pod, which names nothing the reader can act on.
+ */
+function vectorExtensionHint(message: string): string {
+    if (!/type "(vector|halfvec|sparsevec)" does not exist/i.test(message)) return "";
+    return (
+        "\n  pgvector is not installed on this database, and Rebase does not install it: it is a server extension, " +
+        "so it needs an image that ships it (e.g. `pgvector/pgvector:pg18` — the scaffold's `postgres:18-alpine` " +
+        "does not) and a role allowed to run `CREATE EXTENSION vector;`. Install it once, then boot again. " +
+        "Note also that Rebase creates no ANN index for a vector column, so `vectorSearch` is an exact scan."
+    );
 }
 
 /**
@@ -446,7 +753,7 @@ export async function ensureCollectionTables(
 
     const existing = await readExistingSchema(client, schemas);
     const plan = planCollectionSchemaEnsure(collections, existing);
-    const failures: { target: string; error: string }[] = [];
+    const failures: EnsureOutcome["failures"] = [];
 
     // Reported, not warned: this is a rename the ensure is about to perform, and
     // the operator should be able to see in the log why a column changed name.
@@ -463,6 +770,29 @@ export async function ensureCollectionTables(
         log?.(message);
     }
 
+    // Before anything is applied: a `search` block that changed after its column
+    // was generated cannot be honoured by an additive plan, and serving the old
+    // index while the config describes a new one is the silent failure this
+    // check exists to end. Refusing is the loud half — boot is fatal on purpose
+    // (see `ensureCollectionSchema` in the server's boot) and the message
+    // carries the exact statements that resolve it.
+    if (plan.searchDrift.length > 0) {
+        throw new Error(searchDriftMessage(plan.searchDrift));
+    }
+
+    // Said once per column, at the moment the stamp is applied: from here on a
+    // change is detected, but whether *this* column matches the block it is
+    // being stamped with is not knowable — it predates the stamp.
+    for (const adopted of plan.searchAdopted) {
+        const message =
+            `Adopting the existing generated column "${adopted.table}"."${adopted.column}" and recording what the ` +
+            "current `search` block would generate. Any later change to that block will be detected and refused; a " +
+            "change made *before* this version was deployed cannot be, so if search has been missing content, " +
+            `rebuild the column once: ALTER TABLE "${adopted.table.split(".").join('"."')}" DROP COLUMN "${adopted.column}"; and boot again.`;
+        logger.info(`[schema] ${message}`);
+        log?.(message);
+    }
+
     if (plan.actions.length === 0) {
         log?.("Schema is up to date; nothing to create.");
         return { ...plan, failures };
@@ -478,12 +808,18 @@ export async function ensureCollectionTables(
             // data rather than on the schema. The column it polices is already
             // there, so the collection serves either way — record it and carry
             // on rather than crash-looping the deployment.
-            if (action.kind === "add-constraint") {
-                failures.push({ target: action.target, error: message });
+            //
+            // A comment is metadata about a column that was just created
+            // successfully, and it can only fail on ownership (COMMENT requires
+            // owning the table, which an adopted table may not grant). Losing
+            // the stamp costs drift detection on the next boot; it must not cost
+            // the deployment.
+            if (action.kind === "add-constraint" || action.kind === "comment-column") {
+                failures.push({ kind: action.kind, target: action.target, error: message });
                 continue;
             }
             throw new Error(
-                `Failed to ${action.kind} ${action.target}: ${message}\n  ${action.sql}`
+                `Failed to ${action.kind} ${action.target}: ${message}${vectorExtensionHint(message)}\n  ${action.sql}`
             );
         }
     }