@rebasepro/server-postgres 0.16.0 → 0.16.1-canary.g0d7af95

package/src/schema/generate-postgres-ddl-logic.ts

@@ -15,6 +15,12 @@ import {
     searchIndexNames,
     type SearchColumnSpec
 } from "./search-column";
+import {
+    buildVectorIndexPlan,
+    vectorIndexStatements,
+    type VectorIndexPlan
+} from "./vector-index";
+import { buildCollectionIndexSpecs, collectionIndexStatements } from "./collection-index";
 import { REBASE_SCHEMA } from "@rebasepro/types";
 
 // --- Helper Functions ---
@@ -719,7 +725,24 @@ export const generatePostgresDdl = async (
                 indexStatements.push(...searchIndexStatements(searchSpec));
             }
 
-            // Backwards compatibility: add default id primary key if missing
+            // ANN indexes for vector columns. Emitted with the other indexes
+            // rather than inline, because `CREATE INDEX` is a statement and a
+            // column definition is not — and because a column too wide for
+            // pgvector to index still needs its column.
+            const vectorPlan: VectorIndexPlan = buildVectorIndexPlan(collection, resolveColumnName);
+            indexStatements.push(...vectorIndexStatements(vectorPlan));
+            for (const skip of vectorPlan.skipped) {
+                indexStatements.push(`-- No ANN index on "${skip.schema}"."${skip.table}"."${skip.column}": ${skip.reason}`);
+            }
+
+            // The collection's own `indexes:` block. Last of the three index
+            // producers, and the only one the developer wrote deliberately
+            // rather than getting as a side effect of another feature.
+            indexStatements.push(...collectionIndexStatements(
+                buildCollectionIndexSpecs(collection, resolveColumnName)
+            ));
+
+            // The implicit primary key, for a collection that declares none.
             const hasPk = columns.some(c => c.includes("PRIMARY KEY"));
             if (!hasPk) {
                 columns.unshift('  "id" TEXT PRIMARY KEY');
@@ -747,16 +770,20 @@ export const generatePostgresDdl = async (
         }
     }
 
-    if (fkStatements.length > 0) {
-        ddl += "-- Foreign Key Constraints\n";
-        ddl += fkStatements.join("\n") + "\n\n";
-    }
-
+    // Indexes before foreign keys. Today every FK targets a primary key created
+    // inline, so the order is safe by accident; the moment a declared
+    // `unique: true` index can back an FK target, a constraint emitted first
+    // would reference an index that does not exist yet.
     if (indexStatements.length > 0) {
         ddl += "-- Indexes\n";
         ddl += indexStatements.join("\n") + "\n\n";
     }
 
+    if (fkStatements.length > 0) {
+        ddl += "-- Foreign Key Constraints\n";
+        ddl += fkStatements.join("\n") + "\n\n";
+    }
+
     if (policyStatements.length > 0) {
         ddl += "-- Row Level Security Policies\n";
         ddl += policyStatements.join("");

package/src/schema/generate-postgres-ddl.ts

@@ -2,7 +2,6 @@ import { promises as fsPromises } from "fs";
 import * as fs from "fs";
 import path from "path";
 import { pathToFileURL } from "url";
-import chokidar from "chokidar";
 import {
     generatePostgresDdl,
     generatePostgresPoliciesDdl,
@@ -77,7 +76,7 @@ const runGeneration = async (collectionsFilePath?: string, outputPath?: string)
     }
 };
 
-const main = () => {
+const main = async () => {
     const collectionsFilePathArg = process.argv.find(arg => arg.startsWith("--collections="));
     const collectionsFilePath = collectionsFilePathArg ? collectionsFilePathArg.split("=")[1] : process.argv[2];
 
@@ -96,6 +95,18 @@ const main = () => {
 
     if (watch) {
         out(`Watching for changes in ${resolvedPath}...`);
+        // Imported here rather than at module scope, and this is not a style
+        // choice: chokidar is needed only by `--watch`, which is a
+        // schema-authoring path that never runs inside the runtime image. A
+        // top-level import puts it on the boot path of the published driver
+        // bundle, and the image installs a hand-listed set of runtime
+        // dependencies that does not include it — so the whole driver failed to
+        // load with "Cannot find package 'chokidar'", and every self-hosted
+        // container answered 500 with a stack trace about a file watcher.
+        //
+        // Same reasoning the image already applies to @ariga/atlas: an
+        // authoring-only dependency does not belong on a boot path.
+        const { default: chokidar } = await import("chokidar");
         const watcher = chokidar.watch(resolvedPath, {
             persistent: true,
             ignoreInitial: false

package/src/schema/generate-schema-commit.ts

@@ -0,0 +1,242 @@
+/**
+ * Everything a schema change has to write, as file contents.
+ *
+ * A live schema editor that only edits the collection source produces a repo
+ * that does not build: `backend/src/schema.generated.ts` is a committed
+ * artifact, and a stale one has broken every deploy at least once. So the unit
+ * of a schema change is not a file, it is a **commit** — and this module
+ * produces one, without touching a disk, a database or a network.
+ *
+ * Pure on purpose. The risky half of "commit, then apply" is generating a
+ * correct commit; keeping it a function from collections to file contents is
+ * what lets that half be tested by building a database from the result and
+ * comparing it to the one the change describes.
+ *
+ * ## Where the migration comes from, and why not from Atlas
+ *
+ * `rebase db generate` mints migrations by running Atlas over the generated
+ * `schema.sql`. Atlas is an external binary, it wants a dev database, and it
+ * maintains an `atlas.sum` integrity file whose hash this module would have to
+ * reproduce byte-for-byte to stay valid.
+ *
+ * None of that is necessary here, because of what the editor is allowed to do.
+ * `classify-change.ts` refuses anything the boot-time ensure path cannot
+ * express, which leaves only additive statements — and those are computable as
+ * a plain difference between two ensure plans:
+ *
+ *     plan(after, nothing) − plan(before, nothing)
+ *
+ * Both plans are pure functions of the collections, every statement is
+ * idempotent, and the difference is exactly what the change adds. No diff
+ * engine, no database, no binary.
+ *
+ * The statements are *returned* rather than written into a migration file. A
+ * project provisioned by boot-ensure needs no migration at all — its
+ * collections are the schema — while a project provisioned by migrations needs
+ * the file to carry an Atlas hash, which only Atlas can mint. Writing a
+ * migration this module cannot make valid would be worse than handing the
+ * statements to a caller who knows which kind of project it is.
+ */
+import { DEFAULT_COMMIT_PATHS, type CollectionConfig, type SchemaCommitPaths } from "@rebasepro/types";
+import {
+    generatePostgresDdl,
+    generatePostgresPoliciesDdl,
+    generatePostgresSearchDdl
+} from "./generate-postgres-ddl-logic";
+import { generateSchema } from "./generate-drizzle-schema-logic";
+import {
+    planCollectionSchemaEnsure,
+    type EnsureOptions,
+    type ExistingSchema,
+    type WithheldConstraint
+} from "./ensure-collection-tables";
+import { classifyCollectionChanges, type ClassifiedChanges } from "./classify-change";
+
+/**
+ * Re-exported from the shared kernel. `@rebasepro/server` derives these for a
+ * project in a subdirectory and cannot import a driver to do it, so the shape
+ * and the defaults live in `@rebasepro/types`.
+ */
+export { DEFAULT_COMMIT_PATHS, type SchemaCommitPaths } from "@rebasepro/types";
+
+export interface SchemaCommitFile {
+    path: string;
+    contents: string;
+}
+
+export interface SchemaCommitInput {
+    /** What the running database was built from. */
+    before: CollectionConfig[];
+    /** What the editor is proposing. */
+    after: CollectionConfig[];
+    /**
+     * Files the caller has already produced — in practice the rewritten
+     * collection source from the AST editor. Carried through unchanged so the
+     * commit is complete in one object.
+     */
+    sourceFiles?: SchemaCommitFile[];
+    paths?: Partial<SchemaCommitPaths>;
+    /**
+     * What the database this change is destined for actually has.
+     *
+     * Supplied by the live editor, which read it a moment ago; omitted by the
+     * pure callers, which have no database. It decides two things a plan cannot
+     * know from the collections alone — whether a table holds rows, and which
+     * values an enum type already carries — and both are the difference between
+     * a statement that applies and one that is rejected.
+     */
+    existing?: ExistingSchema;
+}
+
+export interface SchemaCommit {
+    /** Every file the commit writes, source and generated alike. */
+    files: SchemaCommitFile[];
+    /**
+     * The additive statements this change adds, in dependency order.
+     *
+     * Empty when the change needs no DDL. Not written to a migration file —
+     * see the module comment.
+     */
+    statements: string[];
+    classified: ClassifiedChanges;
+    /** A commit message describing the change in the terms a reader wants. */
+    message: string;
+    /**
+     * Constraints this change asks for that the statements do not carry, and
+     * why. Empty for almost every change; when it is not, it is the thing the
+     * person confirming needs to read before they confirm.
+     */
+    withheldConstraints: WithheldConstraint[];
+}
+
+export class SchemaCommitError extends Error {
+    constructor(message: string, readonly classified: ClassifiedChanges) {
+        super(message);
+        this.name = "SchemaCommitError";
+    }
+}
+
+/** An `ExistingSchema` describing a database that has nothing in it. */
+const nothing = () => ({ tables: new Map<string, Set<string>>(), enums: new Set<string>() });
+
+/**
+ * The statements that take `before` to `after`.
+ *
+ * Both sides are planned against the *same* database and the difference is
+ * taken by exact statement text. That works because the planner is
+ * deterministic: the same collections against the same schema produce the same
+ * strings, so anything in the second plan and absent from the first is what
+ * this change adds — and nothing else. Planning both sides is what keeps
+ * pre-existing drift, which belongs to neither side of the edit, out of the
+ * statements this change gets credited with.
+ *
+ * ## Why `existing` matters more than it looks
+ *
+ * Planned against `nothing()`, every table reads as one this plan is creating,
+ * and the planner is then free to attach constraints that only hold on a table
+ * with no rows: a new required property comes out as
+ * `ADD COLUMN "x" TEXT NOT NULL`, which is right for a fresh table and fails
+ * against a live one holding rows. Those statements would be generated,
+ * committed, and then rejected by the very database they were written for.
+ *
+ * So a caller holding a real database passes it, and gets statements that
+ * describe that database. `nothing()` stays the default for the pure uses —
+ * generating a commit for inspection, and the tests that compare two plans —
+ * where there is no database to describe.
+ */
+export function additiveStatements(
+    before: CollectionConfig[],
+    after: CollectionConfig[],
+    existing: ExistingSchema = nothing(),
+    options: EnsureOptions = {}
+): string[] {
+    const previous = new Set(planCollectionSchemaEnsure(before, existing, options).statements);
+    return planCollectionSchemaEnsure(after, existing, options).statements
+        .filter(statement => !previous.has(statement));
+}
+
+/** A commit message that says what changed rather than that something did. */
+export function commitMessage(classified: ClassifiedChanges): string {
+    const { changes } = classified;
+    if (changes.length === 0) return "chore(schema): no change";
+
+    const collections = [...new Set(changes.map(change => change.collection))].sort();
+    const added = changes.filter(c => c.kind === "add-collection").map(c => c.collection);
+    const properties = changes.filter(c => c.kind === "add-property");
+
+    let subject: string;
+    if (added.length === 1 && changes.length === 1) {
+        subject = `add the ${added[0]} collection`;
+    } else if (properties.length === 1 && changes.length === 1) {
+        subject = `add ${properties[0].property} to ${properties[0].collection}`;
+    } else if (collections.length === 1) {
+        subject = `${changes.length} change(s) to ${collections[0]}`;
+    } else {
+        subject = `${changes.length} change(s) across ${collections.length} collections`;
+    }
+
+    const body = changes.map(change => `- ${change.detail}`).join("\n");
+    return `feat(schema): ${subject}\n\n${body}\n`;
+}
+
+/**
+ * Build the commit.
+ *
+ * Refuses when the change is not applicable — a commit describing a schema the
+ * ensure path will not produce is a commit that makes the repository lie about
+ * the database. The classification travels on the error so a caller can show
+ * exactly which change was the problem.
+ */
+export async function generateSchemaCommit(input: SchemaCommitInput): Promise<SchemaCommit> {
+    const paths = { ...DEFAULT_COMMIT_PATHS, ...input.paths };
+    const existing = input.existing ?? nothing();
+    // `converge` because every statement this produces is shown to somebody
+    // before it runs. See `ConstraintPolicy` for why the unattended boot does
+    // not get the same latitude.
+    const options: EnsureOptions = { constraints: "converge" };
+    const classified = classifyCollectionChanges(input.before, input.after, input.existing);
+
+    if (!classified.applicable) {
+        const blocking = classified.changes.filter(change => change.verdict !== "safe");
+        throw new SchemaCommitError(
+            `This change cannot be applied to a running database:\n` +
+            blocking.map(change =>
+                `  • ${change.detail}${change.remedy ? `\n    ${change.remedy}` : ""}`
+            ).join("\n"),
+            classified
+        );
+    }
+
+    const [schema, ddl, policies, search] = await Promise.all([
+        generateSchema(input.after),
+        generatePostgresDdl(input.after),
+        Promise.resolve(generatePostgresPoliciesDdl(input.after)),
+        Promise.resolve(generatePostgresSearchDdl(input.after))
+    ]);
+
+    const generated: SchemaCommitFile[] = [
+        { path: paths.schemaFile, contents: schema },
+        { path: paths.ddlFile, contents: ddl },
+        { path: paths.policiesFile, contents: policies },
+        { path: paths.searchFile, contents: search }
+    ];
+
+    // Both sides planned once, here, rather than through `additiveStatements` —
+    // which would plan `after` a second time for the withheld constraints. The
+    // planner is pure, so a second call is only wasted work rather than a
+    // correctness problem, but this runs on every `/plan` keystroke in the panel.
+    const previous = planCollectionSchemaEnsure(input.before, existing, options);
+    const next = planCollectionSchemaEnsure(input.after, existing, options);
+    const already = new Set(previous.statements);
+
+    return {
+        files: [...(input.sourceFiles ?? []), ...generated],
+        statements: next.statements.filter(statement => !already.has(statement)),
+        classified,
+        message: commitMessage(classified),
+        // From the `after` plan alone rather than differenced against `before`:
+        // a constraint that was already unenforceable is still something the
+        // person confirming this change should see named.
+        withheldConstraints: next.withheldConstraints
+    };
+}

package/src/schema/generated-schema-staleness.ts

@@ -44,7 +44,14 @@ export interface LegacyForeignKeyName {
  * which is not staleness — it is a table the generator has not been asked about.
  */
 function tableBlock(source: string, table: string): string {
-    const start = source.indexOf(`pgTable("${table}"`);
+    // Both forms the generator emits. A collection with `schema: "rebase"` — the
+    // auth users collection, in every scaffold — is written as
+    // `rebaseSchema.table("users", …)`, so matching only `pgTable(` made this
+    // blind to every table outside `public`: the block came back empty, and an
+    // empty block reads as "a table the generator has not been asked about".
+    // The rename check therefore never inspected them.
+    const literal = table.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const start = source.search(new RegExp(`(?:pgTable|\\.table)\\(\\s*["']${literal}["']`));
     if (start === -1) return "";
     // Generated files put every table in its own `export const`, so the next one
     // is the end of this block. No brace counting, nothing to get wrong.
@@ -169,3 +176,109 @@ export function describeLegacyForeignKeyNames(found: LegacyForeignKeyName[]): st
         .map(f => `  • ${f.table}.${f.legacy} → ${f.current}  (${f.relation})`)
         .join("\n");
 }
+
+// ── The other way a generated schema goes stale ──────────────────────────────
+//
+// The rename above is the subtle one: nothing the developer owns changed. The
+// ordinary one is the opposite — a collection or a relation was added and the
+// generator was not re-run, so the checked-in module simply does not describe
+// part of the schema. It fails later and elsewhere: relation validation, or a
+// query against a table the module has never heard of.
+//
+// Still not "is this byte-for-byte what the generator would emit now". This asks
+// only about *names*, which is what the rest of the system reads out of the
+// file, so reformatting the generator cannot make it red.
+
+/** Something the collections derive that the generated schema does not declare. */
+export interface MissingGeneratedName {
+    /** The table it belongs to, or is. */
+    table: string;
+    /** The column, when a column is what is missing. */
+    column?: string;
+    /** `<collection>` or `<collection>.<relation>`, for the message. */
+    source: string;
+}
+
+/**
+ * Tables and derived foreign-key columns the collections name and the generated
+ * schema does not.
+ *
+ * @param generatedSource contents of `backend/src/schema.generated.ts`
+ * @param collections     the project's collections, as this release reads them
+ */
+export function findMissingGeneratedNames(
+    generatedSource: string,
+    collections: CollectionConfig[]
+): MissingGeneratedName[] {
+    const missing: MissingGeneratedName[] = [];
+    const seen = new Set<string>();
+
+    const report = (table: string, source: string, column?: string): void => {
+        const key = `${table}.${column ?? ""}`;
+        if (seen.has(key)) return;
+        seen.add(key);
+        missing.push({ table, source, ...(column ? { column } : {}) });
+    };
+
+    const requireColumn = (table: string, column: string | undefined, source: string): void => {
+        if (!table || !column) return;
+        const block = tableBlock(generatedSource, table);
+        // A table that is absent is reported once, as a table. Listing each of
+        // its columns as separately missing would bury the one fact that
+        // explains all of them.
+        if (!block) return report(table, source);
+        if (!declaresColumn(block, column)) report(table, source, column);
+    };
+
+    for (const collection of relationalCollections(collections)) {
+        const sourceTable = getTableName(collection);
+        if (!tableBlock(generatedSource, sourceTable)) {
+            report(sourceTable, collection.slug);
+            continue;
+        }
+
+        for (const [name, relation] of Object.entries(resolveCollectionRelations(collection))) {
+            const at = `${collection.slug}.${name}`;
+
+            let target: CollectionConfig | undefined;
+            try {
+                target = relation.target?.();
+            } catch {
+                // Its own defect, reported at boot by `validate-relations`.
+                continue;
+            }
+
+            switch (relation.kind) {
+                case "belongsTo":
+                    requireColumn(sourceTable, relation.localKey, at);
+                    break;
+
+                case "hasOne":
+                case "hasMany":
+                    if (target) requireColumn(getTableName(target), relation.foreignKeyOnTarget, at);
+                    break;
+
+                case "manyToMany":
+                    requireColumn(relation.through.table, relation.through.sourceColumn, at);
+                    requireColumn(relation.through.table, relation.through.targetColumn, at);
+                    break;
+
+                default:
+                    // `via` joins are written by hand; there is no derived name
+                    // to be missing.
+                    break;
+            }
+        }
+    }
+
+    return missing;
+}
+
+/** One-line summary for a log or a CLI notice. */
+export function describeMissingGeneratedNames(missing: MissingGeneratedName[]): string {
+    return missing
+        .map(m => (m.column
+            ? `  • ${m.table}.${m.column} is not declared  (${m.source})`
+            : `  • table ${m.table} is not declared  (${m.source})`))
+        .join("\n");
+}