@rebasepro/server-postgres 0.16.0 → 0.16.1-canary.g2d1aec8

package/src/schema/generate-schema-commit.ts

@@ -0,0 +1,242 @@
+/**
+ * Everything a schema change has to write, as file contents.
+ *
+ * A live schema editor that only edits the collection source produces a repo
+ * that does not build: `backend/src/schema.generated.ts` is a committed
+ * artifact, and a stale one has broken every deploy at least once. So the unit
+ * of a schema change is not a file, it is a **commit** — and this module
+ * produces one, without touching a disk, a database or a network.
+ *
+ * Pure on purpose. The risky half of "commit, then apply" is generating a
+ * correct commit; keeping it a function from collections to file contents is
+ * what lets that half be tested by building a database from the result and
+ * comparing it to the one the change describes.
+ *
+ * ## Where the migration comes from, and why not from Atlas
+ *
+ * `rebase db generate` mints migrations by running Atlas over the generated
+ * `schema.sql`. Atlas is an external binary, it wants a dev database, and it
+ * maintains an `atlas.sum` integrity file whose hash this module would have to
+ * reproduce byte-for-byte to stay valid.
+ *
+ * None of that is necessary here, because of what the editor is allowed to do.
+ * `classify-change.ts` refuses anything the boot-time ensure path cannot
+ * express, which leaves only additive statements — and those are computable as
+ * a plain difference between two ensure plans:
+ *
+ *     plan(after, nothing) − plan(before, nothing)
+ *
+ * Both plans are pure functions of the collections, every statement is
+ * idempotent, and the difference is exactly what the change adds. No diff
+ * engine, no database, no binary.
+ *
+ * The statements are *returned* rather than written into a migration file. A
+ * project provisioned by boot-ensure needs no migration at all — its
+ * collections are the schema — while a project provisioned by migrations needs
+ * the file to carry an Atlas hash, which only Atlas can mint. Writing a
+ * migration this module cannot make valid would be worse than handing the
+ * statements to a caller who knows which kind of project it is.
+ */
+import { DEFAULT_COMMIT_PATHS, type CollectionConfig, type SchemaCommitPaths } from "@rebasepro/types";
+import {
+    generatePostgresDdl,
+    generatePostgresPoliciesDdl,
+    generatePostgresSearchDdl
+} from "./generate-postgres-ddl-logic";
+import { generateSchema } from "./generate-drizzle-schema-logic";
+import {
+    planCollectionSchemaEnsure,
+    type EnsureOptions,
+    type ExistingSchema,
+    type WithheldConstraint
+} from "./ensure-collection-tables";
+import { classifyCollectionChanges, type ClassifiedChanges } from "./classify-change";
+
+/**
+ * Re-exported from the shared kernel. `@rebasepro/server` derives these for a
+ * project in a subdirectory and cannot import a driver to do it, so the shape
+ * and the defaults live in `@rebasepro/types`.
+ */
+export { DEFAULT_COMMIT_PATHS, type SchemaCommitPaths } from "@rebasepro/types";
+
+export interface SchemaCommitFile {
+    path: string;
+    contents: string;
+}
+
+export interface SchemaCommitInput {
+    /** What the running database was built from. */
+    before: CollectionConfig[];
+    /** What the editor is proposing. */
+    after: CollectionConfig[];
+    /**
+     * Files the caller has already produced — in practice the rewritten
+     * collection source from the AST editor. Carried through unchanged so the
+     * commit is complete in one object.
+     */
+    sourceFiles?: SchemaCommitFile[];
+    paths?: Partial<SchemaCommitPaths>;
+    /**
+     * What the database this change is destined for actually has.
+     *
+     * Supplied by the live editor, which read it a moment ago; omitted by the
+     * pure callers, which have no database. It decides two things a plan cannot
+     * know from the collections alone — whether a table holds rows, and which
+     * values an enum type already carries — and both are the difference between
+     * a statement that applies and one that is rejected.
+     */
+    existing?: ExistingSchema;
+}
+
+export interface SchemaCommit {
+    /** Every file the commit writes, source and generated alike. */
+    files: SchemaCommitFile[];
+    /**
+     * The additive statements this change adds, in dependency order.
+     *
+     * Empty when the change needs no DDL. Not written to a migration file —
+     * see the module comment.
+     */
+    statements: string[];
+    classified: ClassifiedChanges;
+    /** A commit message describing the change in the terms a reader wants. */
+    message: string;
+    /**
+     * Constraints this change asks for that the statements do not carry, and
+     * why. Empty for almost every change; when it is not, it is the thing the
+     * person confirming needs to read before they confirm.
+     */
+    withheldConstraints: WithheldConstraint[];
+}
+
+export class SchemaCommitError extends Error {
+    constructor(message: string, readonly classified: ClassifiedChanges) {
+        super(message);
+        this.name = "SchemaCommitError";
+    }
+}
+
+/** An `ExistingSchema` describing a database that has nothing in it. */
+const nothing = () => ({ tables: new Map<string, Set<string>>(), enums: new Set<string>() });
+
+/**
+ * The statements that take `before` to `after`.
+ *
+ * Both sides are planned against the *same* database and the difference is
+ * taken by exact statement text. That works because the planner is
+ * deterministic: the same collections against the same schema produce the same
+ * strings, so anything in the second plan and absent from the first is what
+ * this change adds — and nothing else. Planning both sides is what keeps
+ * pre-existing drift, which belongs to neither side of the edit, out of the
+ * statements this change gets credited with.
+ *
+ * ## Why `existing` matters more than it looks
+ *
+ * Planned against `nothing()`, every table reads as one this plan is creating,
+ * and the planner is then free to attach constraints that only hold on a table
+ * with no rows: a new required property comes out as
+ * `ADD COLUMN "x" TEXT NOT NULL`, which is right for a fresh table and fails
+ * against a live one holding rows. Those statements would be generated,
+ * committed, and then rejected by the very database they were written for.
+ *
+ * So a caller holding a real database passes it, and gets statements that
+ * describe that database. `nothing()` stays the default for the pure uses —
+ * generating a commit for inspection, and the tests that compare two plans —
+ * where there is no database to describe.
+ */
+export function additiveStatements(
+    before: CollectionConfig[],
+    after: CollectionConfig[],
+    existing: ExistingSchema = nothing(),
+    options: EnsureOptions = {}
+): string[] {
+    const previous = new Set(planCollectionSchemaEnsure(before, existing, options).statements);
+    return planCollectionSchemaEnsure(after, existing, options).statements
+        .filter(statement => !previous.has(statement));
+}
+
+/** A commit message that says what changed rather than that something did. */
+export function commitMessage(classified: ClassifiedChanges): string {
+    const { changes } = classified;
+    if (changes.length === 0) return "chore(schema): no change";
+
+    const collections = [...new Set(changes.map(change => change.collection))].sort();
+    const added = changes.filter(c => c.kind === "add-collection").map(c => c.collection);
+    const properties = changes.filter(c => c.kind === "add-property");
+
+    let subject: string;
+    if (added.length === 1 && changes.length === 1) {
+        subject = `add the ${added[0]} collection`;
+    } else if (properties.length === 1 && changes.length === 1) {
+        subject = `add ${properties[0].property} to ${properties[0].collection}`;
+    } else if (collections.length === 1) {
+        subject = `${changes.length} change(s) to ${collections[0]}`;
+    } else {
+        subject = `${changes.length} change(s) across ${collections.length} collections`;
+    }
+
+    const body = changes.map(change => `- ${change.detail}`).join("\n");
+    return `feat(schema): ${subject}\n\n${body}\n`;
+}
+
+/**
+ * Build the commit.
+ *
+ * Refuses when the change is not applicable — a commit describing a schema the
+ * ensure path will not produce is a commit that makes the repository lie about
+ * the database. The classification travels on the error so a caller can show
+ * exactly which change was the problem.
+ */
+export async function generateSchemaCommit(input: SchemaCommitInput): Promise<SchemaCommit> {
+    const paths = { ...DEFAULT_COMMIT_PATHS, ...input.paths };
+    const existing = input.existing ?? nothing();
+    // `converge` because every statement this produces is shown to somebody
+    // before it runs. See `ConstraintPolicy` for why the unattended boot does
+    // not get the same latitude.
+    const options: EnsureOptions = { constraints: "converge" };
+    const classified = classifyCollectionChanges(input.before, input.after, input.existing);
+
+    if (!classified.applicable) {
+        const blocking = classified.changes.filter(change => change.verdict !== "safe");
+        throw new SchemaCommitError(
+            `This change cannot be applied to a running database:\n` +
+            blocking.map(change =>
+                `  • ${change.detail}${change.remedy ? `\n    ${change.remedy}` : ""}`
+            ).join("\n"),
+            classified
+        );
+    }
+
+    const [schema, ddl, policies, search] = await Promise.all([
+        generateSchema(input.after),
+        generatePostgresDdl(input.after),
+        Promise.resolve(generatePostgresPoliciesDdl(input.after)),
+        Promise.resolve(generatePostgresSearchDdl(input.after))
+    ]);
+
+    const generated: SchemaCommitFile[] = [
+        { path: paths.schemaFile, contents: schema },
+        { path: paths.ddlFile, contents: ddl },
+        { path: paths.policiesFile, contents: policies },
+        { path: paths.searchFile, contents: search }
+    ];
+
+    // Both sides planned once, here, rather than through `additiveStatements` —
+    // which would plan `after` a second time for the withheld constraints. The
+    // planner is pure, so a second call is only wasted work rather than a
+    // correctness problem, but this runs on every `/plan` keystroke in the panel.
+    const previous = planCollectionSchemaEnsure(input.before, existing, options);
+    const next = planCollectionSchemaEnsure(input.after, existing, options);
+    const already = new Set(previous.statements);
+
+    return {
+        files: [...(input.sourceFiles ?? []), ...generated],
+        statements: next.statements.filter(statement => !already.has(statement)),
+        classified,
+        message: commitMessage(classified),
+        // From the `after` plan alone rather than differenced against `before`:
+        // a constraint that was already unenforceable is still something the
+        // person confirming this change should see named.
+        withheldConstraints: next.withheldConstraints
+    };
+}

package/src/schema/vector-index.ts

@@ -0,0 +1,278 @@
+/**
+ * The one place a `{ type: "vector" }` property becomes an ANN index.
+ *
+ * Without an index, pgvector answers `ORDER BY embedding <=> $1` by computing
+ * the distance to every row and sorting — exact, and linear. That is the right
+ * answer at ten thousand rows and the wrong one at a million, which is why the
+ * default here is to build an index rather than to leave the column bare.
+ *
+ * ## The operator class is not a detail
+ *
+ * An index is built for exactly one operator class, and the planner uses it
+ * only for the operator that class implements. `vector_cosine_ops` answers
+ * `<=>` and nothing else; a query asking for `<->` against a cosine index gets
+ * a sequential scan and no error. So the default indexed distance here is
+ * `cosine`, because `cosine` is what `vectorSearch` measures with when the
+ * caller does not say — see `DrizzleConditionBuilder.buildVectorSearch`. The
+ * two defaults have to agree, and this comment is the reason they do.
+ *
+ * ## Why 2000 dimensions is a ceiling and not an error
+ *
+ * pgvector cannot index a `vector` wider than 2000 dimensions with either
+ * method. A 3072-dimension embedding (`text-embedding-3-large`) is therefore
+ * storable and searchable but not indexable. Refusing the boot over that would
+ * make a working configuration unbootable; silently indexing it is impossible.
+ * So the column is created, the index is skipped, and the reason is reported.
+ *
+ * Like `search-column.ts`, this module exists so that the DDL generator and the
+ * boot-time ensure render the *same* specification rather than describing the
+ * same index twice, differently. `contracts/derived-names.txt` records the
+ * names both produce, and CI fails if they diverge.
+ */
+import type { CollectionConfig, Property, VectorDistance, VectorIndexConfig } from "@rebasepro/types";
+import { isPostgresCollectionConfig } from "@rebasepro/types";
+import { getTableName } from "@rebasepro/common";
+import { toPostgresIdentifier } from "@rebasepro/utils";
+
+/**
+ * The widest `vector` pgvector will build an HNSW or IVFFlat index over.
+ * Storage and exact search are unaffected by this limit.
+ */
+export const MAX_INDEXABLE_VECTOR_DIMENSIONS = 2000;
+
+/** The distance indexed when a property does not name one. */
+export const DEFAULT_VECTOR_DISTANCE: VectorDistance = "cosine";
+
+/** The index method used when a property does not name one. */
+export const DEFAULT_VECTOR_INDEX_METHOD = "hnsw" as const;
+
+/**
+ * Operator class per distance. These strings are part of the database contract:
+ * they appear in `CREATE INDEX`, so renaming one renames an index.
+ */
+const OPERATOR_CLASS: Record<VectorDistance, string> = {
+    cosine: "vector_cosine_ops",
+    l2: "vector_l2_ops",
+    inner_product: "vector_ip_ops"
+};
+
+/** Short, stable tag per distance, used to name the index. */
+const DISTANCE_TAG: Record<VectorDistance, string> = {
+    cosine: "cosine",
+    l2: "l2",
+    inner_product: "ip"
+};
+
+export interface VectorIndexSpec {
+    schema: string;
+    table: string;
+    column: string;
+    indexName: string;
+    method: "hnsw" | "ivfflat";
+    distance: VectorDistance;
+    operatorClass: string;
+    /** Rendered into `WITH (...)`; empty when every parameter is defaulted. */
+    parameters: Array<[string, number]>;
+}
+
+/** A vector column that will not be indexed, and why. */
+export interface SkippedVectorIndex {
+    schema: string;
+    table: string;
+    column: string;
+    dimensions: number;
+    reason: string;
+}
+
+export interface VectorIndexPlan {
+    specs: VectorIndexSpec[];
+    skipped: SkippedVectorIndex[];
+}
+
+export class VectorIndexConfigError extends Error {
+    constructor(message: string) {
+        super(message);
+        this.name = "VectorIndexConfigError";
+    }
+}
+
+const isVectorProperty = (prop: unknown): prop is Property & {
+    type: "vector";
+    dimensions: number;
+    index?: VectorIndexConfig | false;
+} => !!prop && typeof prop === "object" && (prop as { type?: string }).type === "vector";
+
+const asDistances = (config: VectorIndexConfig, label: string): VectorDistance[] => {
+    const raw = config.distance ?? DEFAULT_VECTOR_DISTANCE;
+    const list = Array.isArray(raw) ? raw : [raw];
+    if (list.length === 0) {
+        throw new VectorIndexConfigError(
+            `${label}: \`index.distance\` is an empty array. Name at least one distance, or set \`index: false\` to create no index.`
+        );
+    }
+    const seen = new Set<VectorDistance>();
+    for (const distance of list) {
+        if (!(distance in OPERATOR_CLASS)) {
+            throw new VectorIndexConfigError(
+                `${label}: \`index.distance\` is "${distance}", which is not a pgvector distance. Use ${Object.keys(OPERATOR_CLASS).map(d => `"${d}"`).join(", ")}.`
+            );
+        }
+        if (seen.has(distance)) {
+            throw new VectorIndexConfigError(`${label}: \`index.distance\` lists "${distance}" twice.`);
+        }
+        seen.add(distance);
+    }
+    return list;
+};
+
+const assertPositiveInteger = (value: number | undefined, key: string, label: string): void => {
+    if (value === undefined) return;
+    if (!Number.isInteger(value) || value <= 0) {
+        throw new VectorIndexConfigError(
+            `${label}: \`index.${key}\` is ${JSON.stringify(value)}. It must be a positive integer.`
+        );
+    }
+};
+
+/**
+ * Index parameters for one method. Parameters belonging to the *other* method
+ * are rejected rather than ignored, because a silently dropped `lists` on an
+ * HNSW index reads, from the config, exactly like a tuned index.
+ */
+const parametersFor = (
+    method: "hnsw" | "ivfflat",
+    config: VectorIndexConfig,
+    label: string
+): Array<[string, number]> => {
+    assertPositiveInteger(config.m, "m", label);
+    assertPositiveInteger(config.efConstruction, "efConstruction", label);
+    assertPositiveInteger(config.lists, "lists", label);
+
+    if (method === "hnsw") {
+        if (config.lists !== undefined) {
+            throw new VectorIndexConfigError(
+                `${label}: \`index.lists\` only applies to \`method: "ivfflat"\`. Remove it, or switch the method.`
+            );
+        }
+        const params: Array<[string, number]> = [];
+        if (config.m !== undefined) params.push(["m", config.m]);
+        if (config.efConstruction !== undefined) params.push(["ef_construction", config.efConstruction]);
+        return params;
+    }
+
+    for (const key of ["m", "efConstruction"] as const) {
+        if (config[key] !== undefined) {
+            throw new VectorIndexConfigError(
+                `${label}: \`index.${key}\` only applies to \`method: "hnsw"\`. Remove it, or switch the method.`
+            );
+        }
+    }
+    return config.lists !== undefined ? [["lists", config.lists]] : [];
+};
+
+/**
+ * Every ANN index a collection's vector properties call for.
+ *
+ * `resolveColumn` is passed in rather than imported so that this module stays
+ * free of the DDL generator, which imports *it*. Both callers hand it the same
+ * `resolveColumnName`, and a contract test asserts the names agree.
+ */
+export const buildVectorIndexPlan = (
+    collection: CollectionConfig,
+    resolveColumn: (propName: string, prop?: Property | null) => string
+): VectorIndexPlan => {
+    const specs: VectorIndexSpec[] = [];
+    const skipped: SkippedVectorIndex[] = [];
+    const properties = collection.properties ?? {};
+
+    const table = getTableName(collection);
+    const schema = isPostgresCollectionConfig(collection) && collection.schema ? collection.schema : "public";
+
+    for (const [propName, prop] of Object.entries(properties)) {
+        if (!isVectorProperty(prop)) continue;
+        if (prop.index === false) continue;
+
+        const label = `${collection.slug}.${propName}`;
+        const column = resolveColumn(propName, prop as Property);
+        const config: VectorIndexConfig = prop.index ?? {};
+        const method = config.method ?? DEFAULT_VECTOR_INDEX_METHOD;
+
+        if (method !== "hnsw" && method !== "ivfflat") {
+            throw new VectorIndexConfigError(
+                `${label}: \`index.method\` is "${method}". Use "hnsw" or "ivfflat".`
+            );
+        }
+
+        // Read before the dimension gate so a malformed block is reported even
+        // on a column too wide to index — the config is wrong either way.
+        const distances = asDistances(config, label);
+        const parameters = parametersFor(method, config, label);
+
+        if (!Number.isInteger(prop.dimensions) || prop.dimensions <= 0) {
+            throw new VectorIndexConfigError(
+                `${label}: \`dimensions\` is ${JSON.stringify(prop.dimensions)}. It must be a positive integer.`
+            );
+        }
+
+        if (prop.dimensions > MAX_INDEXABLE_VECTOR_DIMENSIONS) {
+            skipped.push({
+                schema,
+                table,
+                column,
+                dimensions: prop.dimensions,
+                reason:
+                    `pgvector cannot index a vector wider than ${MAX_INDEXABLE_VECTOR_DIMENSIONS} dimensions, and ` +
+                    `${label} declares ${prop.dimensions}. The column works and \`vectorSearch\` still answers, as an ` +
+                    `exact scan. To index it, reduce the dimensions (many embedding models support a shorter output) ` +
+                    `or set \`index: false\` to state that the scan is intended.`
+            });
+            continue;
+        }
+
+        for (const distance of distances) {
+            specs.push({
+                schema,
+                table,
+                column,
+                // Distance is in the name because one column may carry an index
+                // per distance, and two indexes cannot share a name. The method
+                // is in it because switching methods is a different index, not
+                // a redefinition of the same one — `CREATE INDEX IF NOT EXISTS`
+                // would otherwise keep the old one and report success.
+                indexName: toPostgresIdentifier(`${table}_${column}_${method}_${DISTANCE_TAG[distance]}`),
+                method,
+                distance,
+                operatorClass: OPERATOR_CLASS[distance],
+                parameters
+            });
+        }
+    }
+
+    return { specs, skipped };
+};
+
+/**
+ * The `CREATE INDEX` for one spec.
+ *
+ * `CONCURRENTLY` is deliberately absent, for the same reason it is absent from
+ * `searchIndexStatements`: this form is replayed as part of a migration, where
+ * a concurrent build is not allowed. The boot-time ensure rewrites it — see
+ * `ensureCollectionTables`.
+ */
+export const vectorIndexStatement = (spec: VectorIndexSpec): string => {
+    const params = spec.parameters.length
+        ? ` WITH (${spec.parameters.map(([key, value]) => `${key} = ${value}`).join(", ")})`
+        : "";
+    return (
+        `CREATE INDEX IF NOT EXISTS "${spec.indexName}" ON "${spec.schema}"."${spec.table}" ` +
+        `USING ${spec.method} ("${spec.column}" ${spec.operatorClass})${params};`
+    );
+};
+
+/** Every statement for a plan, in a stable order. */
+export const vectorIndexStatements = (plan: VectorIndexPlan): string[] =>
+    plan.specs.map(vectorIndexStatement);
+
+/** The index names a plan creates — what the derived-names contract records. */
+export const vectorIndexNames = (plan: VectorIndexPlan): string[] =>
+    plan.specs.map(spec => spec.indexName);