@rebasepro/server-postgres 0.17.1 → 0.17.2-canary.g4d21a53

package/src/schema/carved-out-migration.ts

@@ -0,0 +1,404 @@
+/**
+ * Keep the objects Rebase carved out of Atlas's view out of the migrations
+ * Atlas writes.
+ *
+ * `db push` protects them with `--exclude`. `atlas migrate diff` has no such
+ * flag — measured on the pinned 1.2.3, `--exclude` is rejected outright with
+ * `unknown flag`, and the `exclude` attribute of an `atlas.hcl` `env` block is
+ * accepted and then silently ignored on that path. So the diff sees the search
+ * column that the migration directory's own appended DDL created when it
+ * replayed, does not see it in `schema.sql`, and plans a drop:
+ *
+ *     ALTER TABLE "public"."talents" DROP COLUMN "search_vector";
+ *
+ * That statement is removed here, after Atlas has written the file and before
+ * anything is appended to it — so the input is always Atlas's own plain output,
+ * never the dollar-quoted helper bodies that get appended afterwards.
+ *
+ * The drop does not arrive alone. Atlas folds every change to one table into a
+ * single statement, so a real edit in the same run produces
+ *
+ *     ALTER TABLE "public"."talents"
+ *         DROP COLUMN "search_vector", DROP COLUMN "interests", ADD COLUMN "headline" text NULL;
+ *
+ * and dropping the whole statement would silently throw away a migration the
+ * project asked for. The removal is therefore per *clause*.
+ *
+ * Fail-safe throughout: a statement that mentions a carved-out object but does
+ * not match a shape this understands is left exactly as it is and reported as
+ * `unhandled`, so the caller can say so out loud. A spurious drop the developer
+ * can see beats a migration quietly rewritten wrong.
+ */
+
+/** Anything Postgres accepts in an unquoted identifier past ASCII. */
+const IDENT = "(?:\"(?:[^\"]|\"\")*\"|[A-Za-z_\\u0080-\\uffff][\\w$\\u0080-\\uffff]*)";
+const QUALIFIED = `${IDENT}(?:\\s*\\.\\s*${IDENT})*`;
+
+const ALTER_TABLE_RE = new RegExp(`^ALTER\\s+TABLE\\s+(?:ONLY\\s+)?(${QUALIFIED})\\s+([\\s\\S]+)$`, "i");
+const DROP_COLUMN_RE = new RegExp(`^DROP\\s+(?:COLUMN\\s+)?(?:IF\\s+EXISTS\\s+)?(${IDENT})$`, "i");
+const DROP_INDEX_RE = new RegExp(`^DROP\\s+INDEX\\s+(?:CONCURRENTLY\\s+)?(?:IF\\s+EXISTS\\s+)?(${QUALIFIED})$`, "i");
+
+/**
+ * Does this statement destroy anything at all?
+ *
+ * The gate on reporting. A statement that merely *names* a carved-out object —
+ * `COMMENT ON COLUMN … "search_vector"` — is not a problem and must not be
+ * reported as one, because an unrewritable drop stops the caller outright.
+ */
+const MENTIONS_DROP = /\bDROP\b/i;
+
+/** One `schema.table.object` exclude pattern, taken apart. */
+export interface CarvedOutObject {
+    schema: string;
+    table: string;
+    object: string;
+}
+
+export interface StripResult {
+    /** The migration text with the carved-out clauses gone. */
+    sql: string;
+    /** What was removed, as SQL fragments, for reporting. */
+    removed: string[];
+    /**
+     * Statements naming a carved-out object that this could not rewrite with
+     * confidence. Left untouched in `sql`.
+     */
+    unhandled: string[];
+    /** True when nothing but comments and whitespace survives. */
+    empty: boolean;
+}
+
+/**
+ * Split `schema.table.object` patterns. Anything not in three parts is
+ * ignored rather than guessed at — the two-part form is the one Atlas itself
+ * silently mis-reads, and repeating that mistake here would be worse than
+ * doing nothing.
+ */
+export function parseExcludePatterns(patterns: string[]): CarvedOutObject[] {
+    const parsed: CarvedOutObject[] = [];
+    for (const pattern of patterns) {
+        const parts = pattern.split(".");
+        if (parts.length !== 3) continue;
+        if (parts.some((p) => p.length === 0)) continue;
+        parsed.push({ schema: parts[0], table: parts[1], object: parts[2] });
+    }
+    return parsed;
+}
+
+/** `"public"` and `public` are the same name; `"Public"` is not `public`. */
+function unquoteIdentifier(raw: string): string {
+    const trimmed = raw.trim();
+    if (trimmed.startsWith("\"") && trimmed.endsWith("\"") && trimmed.length >= 2) {
+        return trimmed.slice(1, -1).replace(/""/g, "\"");
+    }
+    // An unquoted identifier is folded to lower case by Postgres.
+    return trimmed.toLowerCase();
+}
+
+/** Split a dotted identifier at its top-level dots, respecting quoting. */
+function splitQualifiedName(raw: string): string[] | null {
+    const parts: string[] = [];
+    let current = "";
+    let inQuote = false;
+    for (let i = 0; i < raw.length; i++) {
+        const ch = raw[i];
+        if (inQuote) {
+            if (ch === "\"") {
+                if (raw[i + 1] === "\"") { current += "\"\""; i++; continue; }
+                inQuote = false;
+            }
+            current += ch;
+            continue;
+        }
+        if (ch === "\"") { inQuote = true; current += ch; continue; }
+        if (ch === ".") { parts.push(current); current = ""; continue; }
+        current += ch;
+    }
+    if (inQuote) return null;
+    parts.push(current);
+    return parts.map(unquoteIdentifier);
+}
+
+/**
+ * One statement located in the source text, with the comment lines that
+ * introduce it. Atlas writes `-- Modify "talents" table` above each statement;
+ * removing the statement without its comment leaves a caption over nothing.
+ */
+interface LocatedStatement {
+    /** Offset of the first character of the leading comment block. */
+    start: number;
+    /** Offset just past the terminating semicolon. */
+    end: number;
+    /** Offset of the first character of the statement itself. */
+    bodyStart: number;
+    /** The statement without its leading comments or trailing semicolon. */
+    body: string;
+}
+
+/**
+ * Walk the script and locate every statement. Quote-, comment- and
+ * dollar-aware: the appended halves of a migration are not passed here, but a
+ * splitter that shreds a `DO $tag$ ... $tag$` block is a trap worth not
+ * leaving behind — it is how the derived-names renderer broke.
+ *
+ * Returns null if the text ends inside a quote or a dollar-quoted body, or
+ * trails off in a statement with no semicolon. Either means this cannot be
+ * reasoned about safely.
+ */
+function locateStatements(sql: string): LocatedStatement[] | null {
+    const statements: LocatedStatement[] = [];
+    let i = 0;
+    let statementStart = 0;
+    let sawCode = false;
+
+    const pushStatement = (endExclusive: number) => {
+        const leading = sql.slice(statementStart, endExclusive);
+        // The leading run of comment/blank lines belongs to this statement.
+        let consumed = 0;
+        for (const line of leading.split("\n")) {
+            const trimmed = line.trim();
+            if (trimmed === "" || trimmed.startsWith("--")) {
+                consumed += line.length + 1;
+                continue;
+            }
+            break;
+        }
+        const bodyStart = statementStart + consumed;
+        statements.push({
+            start: statementStart,
+            end: endExclusive,
+            bodyStart,
+            body: sql.slice(bodyStart, endExclusive).replace(/;\s*$/, "").trim()
+        });
+    };
+
+    while (i < sql.length) {
+        const ch = sql[i];
+
+        if (ch === "-" && sql[i + 1] === "-") {
+            const nl = sql.indexOf("\n", i);
+            i = nl === -1 ? sql.length : nl + 1;
+            continue;
+        }
+        if (ch === "/" && sql[i + 1] === "*") {
+            const close = sql.indexOf("*/", i + 2);
+            if (close === -1) return null;
+            i = close + 2;
+            sawCode = true;
+            continue;
+        }
+        if (ch === "'" || ch === "\"") {
+            const quote = ch;
+            i++;
+            let closed = false;
+            while (i < sql.length) {
+                if (sql[i] === quote) {
+                    if (sql[i + 1] === quote) { i += 2; continue; }
+                    i++;
+                    closed = true;
+                    break;
+                }
+                i++;
+            }
+            if (!closed) return null;
+            sawCode = true;
+            continue;
+        }
+        if (ch === "$") {
+            const tag = /^\$(?:[A-Za-z_\u0080-\uffff][\w\u0080-\uffff]*)?\$/.exec(sql.slice(i));
+            if (tag) {
+                const close = sql.indexOf(tag[0], i + tag[0].length);
+                if (close === -1) return null;
+                i = close + tag[0].length;
+                sawCode = true;
+                continue;
+            }
+        }
+        if (ch === ";") {
+            pushStatement(i + 1);
+            i++;
+            statementStart = i;
+            sawCode = false;
+            continue;
+        }
+        if (!/\s/.test(ch)) sawCode = true;
+        i++;
+    }
+
+    // Trailing text with no semicolon: only comments and whitespace is fine.
+    if (sawCode) return null;
+    return statements;
+}
+
+/** Split an ALTER TABLE action list at its top-level commas. */
+function splitClauses(actions: string): string[] | null {
+    const clauses: string[] = [];
+    let depth = 0;
+    let current = "";
+    let i = 0;
+    while (i < actions.length) {
+        const ch = actions[i];
+        if (ch === "'" || ch === "\"") {
+            const quote = ch;
+            current += ch;
+            i++;
+            let closed = false;
+            while (i < actions.length) {
+                current += actions[i];
+                if (actions[i] === quote) {
+                    if (actions[i + 1] === quote) { current += actions[i + 1]; i += 2; continue; }
+                    i++;
+                    closed = true;
+                    break;
+                }
+                i++;
+            }
+            if (!closed) return null;
+            continue;
+        }
+        if (ch === "(") { depth++; current += ch; i++; continue; }
+        if (ch === ")") { depth--; current += ch; i++; continue; }
+        if (ch === "," && depth === 0) { clauses.push(current); current = ""; i++; continue; }
+        current += ch;
+        i++;
+    }
+    if (depth !== 0) return null;
+    clauses.push(current);
+    return clauses.map((c) => c.trim()).filter((c) => c.length > 0);
+}
+
+/**
+ * Remove every clause and statement that would drop one of `patterns`.
+ *
+ * `patterns` are the `schema.table.object` strings the CLI already builds for
+ * Atlas's `--exclude`, so the diff path and the apply path protect exactly the
+ * same set by construction.
+ */
+export function stripCarvedOutStatements(migrationSql: string, patterns: string[]): StripResult {
+    const carved = parseExcludePatterns(patterns);
+    if (carved.length === 0) {
+        return { sql: migrationSql, removed: [], unhandled: [], empty: isBlank(migrationSql) };
+    }
+
+    const located = locateStatements(migrationSql);
+    if (located === null) {
+        // Nothing can be trusted about the offsets, so nothing moves. Worth
+        // stopping the caller over only if one of ours is being dropped in
+        // there; otherwise this file has nothing to do with us.
+        const atStake = MENTIONS_DROP.test(migrationSql)
+            && carved.some((c) => migrationSql.includes(c.object));
+        return {
+            sql: migrationSql,
+            removed: [],
+            unhandled: atStake ? [migrationSql.trim()] : [],
+            empty: isBlank(migrationSql)
+        };
+    }
+
+    const removed: string[] = [];
+    const unhandled: string[] = [];
+    // Edits as [start, end) → replacement, applied back-to-front so the
+    // offsets of the ones still pending stay valid.
+    const edits: { start: number; end: number; replacement: string }[] = [];
+
+    for (const statement of located) {
+        if (!carved.some((c) => statement.body.includes(c.object))) continue;
+
+        const dropIndex = DROP_INDEX_RE.exec(statement.body);
+        if (dropIndex) {
+            const parts = splitQualifiedName(dropIndex[1]);
+            const name = parts?.[parts.length - 1];
+            const schema = parts && parts.length > 1 ? parts[parts.length - 2] : undefined;
+            if (name !== undefined && carved.some((c) =>
+                c.object === name && (schema === undefined || c.schema === schema))) {
+                removed.push(statement.body);
+                edits.push({ start: statement.start, end: statement.end, replacement: "" });
+            } else {
+                // A near-miss on the name — some other index whose name
+                // contains ours. Left alone, and still a drop, so still
+                // reported.
+                unhandled.push(statement.body);
+            }
+            continue;
+        }
+
+        // Anything else naming a carved-out object is only our business if it
+        // destroys something. `COMMENT ON COLUMN … "search_vector"` does not.
+        const alter = ALTER_TABLE_RE.exec(statement.body);
+        if (!alter) {
+            if (MENTIONS_DROP.test(statement.body)) unhandled.push(statement.body);
+            continue;
+        }
+
+        const target = splitQualifiedName(alter[1]);
+        if (!target) {
+            if (MENTIONS_DROP.test(statement.body)) unhandled.push(statement.body);
+            continue;
+        }
+        const table = target[target.length - 1];
+        const schema = target.length > 1 ? target[target.length - 2] : undefined;
+
+        const clauses = splitClauses(alter[2]);
+        if (!clauses) {
+            if (MENTIONS_DROP.test(statement.body)) unhandled.push(statement.body);
+            continue;
+        }
+
+        const survivors: string[] = [];
+        let removedHere = 0;
+        let unhandledHere = false;
+        for (const clause of clauses) {
+            const drop = DROP_COLUMN_RE.exec(clause);
+            const column = drop ? unquoteIdentifier(drop[1]) : undefined;
+            const isCarved = column !== undefined && carved.some((c) =>
+                c.object === column &&
+                c.table === table &&
+                (schema === undefined || c.schema === schema));
+            if (isCarved) {
+                removed.push(`ALTER TABLE ${alter[1]}: ${clause}`);
+                removedHere++;
+                continue;
+            }
+            // A clause naming a carved-out object in some other shape is not
+            // something to guess at — if it destroys it.
+            if (MENTIONS_DROP.test(clause) && carved.some((c) => clause.includes(c.object))) {
+                unhandledHere = true;
+            }
+            survivors.push(clause);
+        }
+
+        if (unhandledHere) unhandled.push(statement.body);
+        if (removedHere === 0) continue;
+
+        if (survivors.length === 0) {
+            edits.push({ start: statement.start, end: statement.end, replacement: "" });
+        } else {
+            edits.push({
+                start: statement.bodyStart,
+                end: statement.end,
+                replacement: `ALTER TABLE ${alter[1]} ${survivors.join(", ")};`
+            });
+        }
+    }
+
+    let sql = migrationSql;
+    for (const edit of edits.sort((a, b) => b.start - a.start)) {
+        sql = sql.slice(0, edit.start) + edit.replacement + sql.slice(edit.end);
+    }
+    if (edits.length > 0) sql = tidy(sql);
+
+    return { sql, removed, unhandled, empty: isBlank(sql) };
+}
+
+/** Only comments and whitespace left — nothing a migration would do. */
+function isBlank(sql: string): boolean {
+    return sql
+        .split("\n")
+        .every((line) => line.trim() === "" || line.trim().startsWith("--"));
+}
+
+/** Collapse the blank runs an excision leaves behind. */
+function tidy(sql: string): string {
+    const trimmed = sql.replace(/\n{3,}/g, "\n\n").replace(/^\s*\n/, "").trimEnd();
+    return trimmed.length === 0 ? "" : `${trimmed}\n`;
+}

package/src/schema/ensure-collection-tables.ts

@@ -25,7 +25,12 @@
  * Because of that, this is safe to run on every boot, and re-running it is a
  * no-op.
  */
-import { type CollectionConfig, type Property, isPostgresCollectionConfig } from "@rebasepro/types";
+import {
+    declaredDatabaseExtensions,
+    isPostgresCollectionConfig,
+    type CollectionConfig,
+    type Property
+} from "@rebasepro/types";
 import { getTableName, relationalCollections } from "@rebasepro/common";
 import { logger, isConcurrentDdlRace, isDuplicateObjectRace } from "@rebasepro/server";
 import {
@@ -48,7 +53,15 @@ import {
     planJunctionTables,
     quoteSqlLiteral
 } from "./generate-postgres-ddl-logic";
-import { buildVectorIndexPlan, vectorIndexStatement, type SkippedVectorIndex } from "./vector-index";
+import {
+    buildVectorColumnSpecs,
+    buildVectorIndexPlan,
+    vectorExtensionDeclared,
+    vectorExtensionHint,
+    vectorExtensionStatement,
+    vectorIndexStatement,
+    type SkippedVectorIndex
+} from "./vector-index";
 import { buildCollectionIndexPlan, collectionIndexStatement } from "./collection-index";
 import {
     AUTH_USERS_COLUMNS,
@@ -147,6 +160,18 @@ export type ConstraintPolicy = "additive" | "converge";
 export interface EnsureOptions {
     /** Defaults to `additive`. See {@link ConstraintPolicy}. */
     constraints?: ConstraintPolicy;
+    /**
+     * Server extensions the project's databases gave Rebase leave to install —
+     * `declaredDatabaseExtensions()`. Absent means none, which is a refusal and
+     * is the right default for a planner given no configuration at all.
+     *
+     * Explicit rather than read from the resource registry in here, because
+     * this function is pure and several callers plan against fixtures: reaching
+     * for a process-wide registry would make the plan depend on whatever some
+     * other module happened to import. `ensureCollectionTables` is the boundary
+     * that reads the world.
+     */
+    databaseExtensions?: readonly string[];
 }
 
 export interface EnsureAction {
@@ -397,12 +422,28 @@ export function planCollectionSchemaEnsure(
         .filter((spec): spec is SearchColumnSpec => spec !== undefined);
 
     const plannedExtensions = new Set<string>();
+    const planExtension = (statement: string): void => {
+        if (plannedExtensions.has(statement)) return;
+        plannedExtensions.add(statement);
+        actions.push({ kind: "create-extension", target: statement.replace(/^CREATE EXTENSION IF NOT EXISTS |;$/g, ""), sql: statement });
+    };
     for (const spec of searchSpecs) {
-        for (const statement of searchExtensionStatements(spec)) {
-            if (plannedExtensions.has(statement)) continue;
-            plannedExtensions.add(statement);
-            actions.push({ kind: "create-extension", target: statement.replace(/^CREATE EXTENSION IF NOT EXISTS |;$/g, ""), sql: statement });
-        }
+        for (const statement of searchExtensionStatements(spec)) planExtension(statement);
+    }
+
+    // pgvector, for collections that declared a `vector` property — and only
+    // when a database gave leave to install it. Unlike the search extensions
+    // above, which are contrib modules every Postgres carries, pgvector is a
+    // separate build behind an image, a grant and a provider allow-list, so
+    // whether Rebase may install it is not Rebase's to decide. See
+    // `DatabaseOptions.extensions`.
+    //
+    // Boot has to make the same call `rebase db push` makes, or the two produce
+    // different databases from one commit — which is the rule
+    // `contracts/derived-names.txt` enforces.
+    if (vectorExtensionDeclared(options.databaseExtensions)
+        && collections.some(c => buildVectorColumnSpecs(c, resolveColumnName).length > 0)) {
+        planExtension(vectorExtensionStatement());
     }
     const plannedFunctions = new Set<string>();
     for (const spec of searchSpecs) {
@@ -1009,29 +1050,6 @@ function searchDriftMessage(drift: SearchColumnDrift[]): string {
     );
 }
 
-/**
- * The missing-pgvector explanation, appended to the error that reveals it.
- *
- * A `{ type: "vector" }` property compiles to `VECTOR(n)`, and nothing in the
- * OSS pipeline installs pgvector — not this ensure, not `db push`. Installing
- * an extension on someone's database is a decision with a deployment behind it
- * (image, superuser, cloud allow-list), so this path stays a refusal; what it
- * must not stay is a bare `type "vector" does not exist` on a crash-looping
- * pod, which names nothing the reader can act on.
- *
- * The scaffold now ships `pgvector/pgvector:pg18`, so this is reached by a
- * project pointed at a database someone else provisioned — which is exactly
- * the case where naming the extension and the image is worth the words.
- */
-function vectorExtensionHint(message: string): string {
-    if (!/type "(vector|halfvec|sparsevec)" does not exist/i.test(message)) return "";
-    return (
-        "\n  pgvector is not installed on this database, and Rebase does not install it: it is a server extension, " +
-        "so it needs an image that ships it (the scaffold's `pgvector/pgvector:pg18` does; a stock `postgres:18` " +
-        "does not) and a role allowed to run `CREATE EXTENSION vector;`. Install it once, then boot again. " +
-        "Rebase then creates an ANN index for the column automatically — see the `index` option on the property."
-    );
-}
 
 /**
  * Read what the database looks like, for the schemas a set of collections
@@ -1065,7 +1083,8 @@ export async function readSchemaFactsFor(
 export async function ensureCollectionTables(
     client: Queryable,
     collections: CollectionConfig[],
-    log?: (message: string) => void
+    log?: (message: string) => void,
+    options: EnsureOptions = {}
 ): Promise<EnsureOutcome> {
     // Junctions live alongside the collections that declare them, so their
     // schema has to be read too — otherwise an existing junction reads as
@@ -1082,7 +1101,15 @@ export async function ensureCollectionTables(
     }
 
     const existing = await readExistingSchema(client, schemas);
-    const plan = planCollectionSchemaEnsure(collections, existing);
+    // This is the boundary, so this is where the world is read: the planner
+    // itself takes the permission as an argument and never reaches for the
+    // registry. By the time boot gets here `loadBundleResourceGraph` has
+    // evaluated the project's `resources.ts`, so the declarations are there —
+    // and a caller that already knows them can still say so.
+    const plan = planCollectionSchemaEnsure(collections, existing, {
+        ...options,
+        databaseExtensions: options.databaseExtensions ?? declaredDatabaseExtensions()
+    });
     const failures: EnsureOutcome["failures"] = [];
 
     // Reported, not warned: this is a rename the ensure is about to perform, and