@rebasepro/server-postgres 0.17.3 → 0.18.1

package/src/schema/introspect-db-types.ts

@@ -1,56 +0,0 @@
-/**
- * The PostgreSQL type → Rebase property type mapping.
- *
- * Split out of `introspect-db-logic` so that the structural analysis can use it
- * without importing the generator, which imports the analysis. Re-exported from
- * `introspect-db-logic` so existing callers keep their import path.
- */
-
-/**
- * Map a PostgreSQL data type to a Rebase property type.
- */
-export function mapPgType(dataType: string): string {
-    const dt = dataType.toLowerCase();
-
-    // Interval MUST be checked before numeric ("interval" contains "int")
-    if (dt === "interval") return "string";
-
-    // Array types MUST be checked before numeric ("_int4" contains "int")
-    if (dt === "array" || dt.startsWith("_")) return "array";
-
-    // Numeric types
-    if (
-        dt.includes("int") || // integer, smallint, bigint
-        dt.includes("numeric") ||
-        dt.includes("decimal") ||
-        dt.includes("serial") || // serial, bigserial
-        dt === "real" ||
-        dt === "float4" ||
-        dt === "float8" ||
-        dt === "double precision" ||
-        dt === "money"
-    ) {
-        return "number";
-    }
-
-    // Boolean
-    if (dt.includes("bool")) return "boolean";
-
-    // Date / Time
-    if (dt.includes("time") || dt.includes("date")) return "date";
-
-    // JSON
-    if (dt === "json" || dt === "jsonb") return "map";
-
-    // Binary
-    if (dt === "bytea") return "binary";
-
-    // Network types
-    if (dt === "inet" || dt === "cidr" || dt === "macaddr" || dt === "macaddr8") return "string";
-
-    // UUID
-    if (dt === "uuid") return "string";
-
-    // Text/varchar/char — default to string
-    return "string";
-}

package/src/schema/introspect-db.ts

@@ -1,257 +0,0 @@
-import chalk from "chalk";
-import fs from "fs";
-import path from "path";
-import pg from "pg";
-import arg from "arg";
-import * as dotenv from "dotenv";
-import readline from "readline";
-
-import {
-    buildTablesMap,
-    buildEnumMap,
-    generateCollectionFile,
-    generateIndexContent,
-    mergeIndexContent,
-    safeHostFromUrl
-} from "./introspect-db-logic";
-import { detectCollectionBuilder } from "./introspect-db-project";
-import { countRowsUpTo, readSchemaMetadata } from "./introspect-db-queries";
-import { classifyTables, lookupCandidates, LOOKUP_MAX_ROWS } from "./introspect-db-structure";
-import { parseCheckConstraints } from "./introspect-db-constraints";
-import { out, outWarn, outError } from "../cli-output";
-
-async function main() {
-    const args = arg(
-        {
-            "--output": String,
-            "--collections": String,
-            "--force": Boolean,
-            "--schema": String,
-            "--data-inference": Boolean,
-            "--no-data-inference": Boolean,
-            "-o": "--output",
-            "-c": "--collections",
-            "-f": "--force"
-        },
-        { permissive: true }
-    );
-
-    const cwd = process.cwd();
-    const isBackendDir = path.basename(cwd) === "backend";
-    const defaultOutDir = isBackendDir
-        ? path.resolve(cwd, "..", "config", "collections")
-        : path.resolve(cwd, "config", "collections");
-
-    const outDir = args["--output"] || args["--collections"] || defaultOutDir;
-    const force = args["--force"] || false;
-    const pgSchema = args["--schema"] || "public";
-
-    if (!fs.existsSync(outDir)) {
-        fs.mkdirSync(outDir, { recursive: true });
-    }
-
-    // Load env
-    const envPaths = [
-        process.env.DOTENV_CONFIG_PATH,
-        path.resolve(process.cwd(), ".env"),
-        path.resolve(process.cwd(), "../.env"),
-        path.resolve(process.cwd(), "../../.env")
-    ].filter(Boolean) as string[];
-
-    for (const p of envPaths) {
-        if (fs.existsSync(p)) {
-            dotenv.config({ path: p, quiet: true });
-            break;
-        }
-    }
-
-    const databaseUrl = process.env.DATABASE_URL || process.env.ADMIN_CONNECTION_STRING;
-    if (!databaseUrl) {
-        outError(chalk.red("✗ DATABASE_URL is not set. Make sure your .env file is configured."));
-        process.exit(1);
-    }
-
-    const client = new pg.Client({ connectionString: databaseUrl });
-
-    try {
-        await client.connect();
-    } catch (err) {
-        outError(chalk.red(`✗ Failed to connect to database: ${err instanceof Error ? err.message : String(err)}`));
-        outError(chalk.gray("  Check your DATABASE_URL and ensure the database is reachable."));
-        process.exit(1);
-    }
-
-    // Log the host portion safely — handle URLs without "@"
-    const hostPart = safeHostFromUrl(databaseUrl);
-    out(chalk.gray(`Connected to database: ${hostPart}`));
-    out(chalk.gray(`Introspecting schema '${pgSchema}'...`));
-
-    try {
-        const metadata = await readSchemaMetadata(client, pgSchema);
-        const enumMap = buildEnumMap(metadata.enumValues);
-        const tablesMap = buildTablesMap(metadata.tables, metadata.columns, metadata.pks, metadata.fks);
-        const fks = metadata.fks;
-
-        // Only tables that could structurally be a code list are counted, and
-        // each count stops at the threshold — see `countRowsUpTo`. Introspection
-        // runs against a database it does not own, so "cheap on a table of any
-        // size" is a requirement, not an optimization.
-        for (const table of lookupCandidates(metadata, tablesMap)) {
-            try {
-                metadata.rowCounts[table] = await countRowsUpTo(client, pgSchema, table, LOOKUP_MAX_ROWS);
-            } catch (err) {
-                // A table this run cannot read is simply not classified as a code
-                // list; everything else about it still generates.
-                out(chalk.gray(`  (skipped row count for ${table}: ${err instanceof Error ? err.message : String(err)})`));
-            }
-        }
-
-        const classifications = classifyTables(metadata, tablesMap);
-        const checkFacts = parseCheckConstraints(metadata.checks);
-
-        // Which builder the generated files may import — read from the manifests
-        // above `outDir`, because a project without `@rebasepro/cms-types` cannot
-        // resolve that import and has no `admin` block to write into either.
-        const builder = detectCollectionBuilder(outDir);
-        if (builder === "annotation") {
-            outWarn(chalk.yellow(
-                "⚠ Neither @rebasepro/cms-types nor @rebasepro/common is declared above " +
-                `${outDir}.`
-            ));
-            outWarn(chalk.gray(
-                "  Generating plain type annotations, which widen the property keys to `string`: " +
-                "`propertiesOrder` and friends will not be checked. Add @rebasepro/common to the " +
-                "config package to turn that checking on."
-            ));
-        }
-        const joinTables = new Set(
-            Array.from(classifications.values())
-                .filter((c) => c.role === "junction")
-                .map((c) => c.table)
-        );
-
-        const roleCount = (role: string) =>
-            Array.from(classifications.values()).filter((c) => c.role === role).length;
-
-        out(chalk.blue(`Found ${tablesMap.size} tables.`));
-        out(chalk.gray(
-            `  ${roleCount("entity")} entities, ${joinTables.size} join tables (folded into relations), ` +
-            `${roleCount("lookup")} code lists, ${roleCount("owned-child")} owned by another table (hidden from navigation).`
-        ));
-
-        let runDataInference = false;
-        if (args["--no-data-inference"]) {
-            runDataInference = false;
-        } else if (args["--data-inference"] !== undefined) {
-            runDataInference = args["--data-inference"];
-        } else if (!process.stdin.isTTY) {
-            // No terminal to answer the question below (scaffolding scripts, CI,
-            // `rebase init --introspect`) — asking would hang forever.
-            out(chalk.gray("Skipping data inference (non-interactive run; pass --data-inference to enable)."));
-        } else {
-            const rl = readline.createInterface({
-                input: process.stdin,
-                output: process.stdout
-            });
-            const answer = await new Promise<string>((resolve) => rl.question(chalk.yellow("? Do you want to run comprehensive data inference on sampled rows to auto-detect types, formats, constraints, and UI configurations? (y/N) "), resolve));
-            runDataInference = answer.trim().toLowerCase() === "y";
-            rl.close();
-        }
-
-        if (runDataInference) {
-            out(chalk.gray("Sampling database rows for data inference..."));
-        }
-
-        // Generate Collections
-        const generatedFiles: string[] = [];
-        const skippedFiles: string[] = [];
-
-        const tablesToProcess = Array.from(tablesMap.entries()).filter(([tableName]) => !joinTables.has(tableName));
-
-        const BATCH_SIZE = 10;
-        for (let i = 0; i < tablesToProcess.length; i += BATCH_SIZE) {
-            const batch = tablesToProcess.slice(i, i + BATCH_SIZE);
-
-            await Promise.all(batch.map(async ([tableName, meta]) => {
-                // ── File overwrite protection ──────────────────────────────
-                const filePath = path.join(outDir, `${tableName}.ts`);
-                if (fs.existsSync(filePath) && !force) {
-                    skippedFiles.push(tableName);
-                    return;
-                }
-
-                let sampleData: Record<string, unknown>[] | undefined = undefined;
-                if (runDataInference) {
-                    try {
-                        const { rows } = await client.query(`SELECT * FROM "${pgSchema}"."${tableName}" LIMIT 100`);
-                        sampleData = rows;
-                    } catch (err) {
-                        outError(chalk.yellow(`⚠ Failed to sample data for table ${tableName}: ${err instanceof Error ? err.message : String(err)}`));
-                    }
-                }
-
-                const fileContent = generateCollectionFile(
-                    tableName,
-                    meta,
-                    fks,
-                    joinTables,
-                    tablesMap,
-                    enumMap,
-                    sampleData,
-                    { metadata, classifications, checkFacts, builder }
-                );
-
-                fs.writeFileSync(filePath, fileContent, "utf-8");
-                generatedFiles.push(tableName);
-                out(chalk.green(`  ✓ ${filePath}`));
-            }));
-        }
-
-        // Generate index.ts (sorted alphabetically for deterministic output)
-        if (generatedFiles.length > 0) {
-            const indexPath = path.join(outDir, "index.ts");
-
-            if (fs.existsSync(indexPath) && !force) {
-                // Merge: read existing index, add new exports that don't already exist
-                const existing = fs.readFileSync(indexPath, "utf-8");
-                const merged = mergeIndexContent(existing, generatedFiles);
-                fs.writeFileSync(indexPath, merged, "utf-8");
-            } else {
-                // --force replaces collections derived from the database, but the
-                // directory can also hold hand-written ones with no table in the
-                // introspected schema (the auth users collection lives in
-                // "rebase"). The backend discovers the whole directory, so an
-                // index listing only introspected tables would silently drop them
-                // from the admin UI while the API still served them.
-                const siblings = fs.readdirSync(outDir)
-                    .filter(f => f.endsWith(".ts") && f !== "index.ts")
-                    .map(f => f.replace(/\.ts$/, ""));
-                const allFiles = [...new Set([...generatedFiles, ...siblings])];
-                const indexContent = generateIndexContent(allFiles);
-                fs.writeFileSync(indexPath, indexContent, "utf-8");
-            }
-            out(chalk.green(`  ✓ ${indexPath}`));
-        }
-
-        out("");
-        if (skippedFiles.length > 0) {
-            out(chalk.yellow(`⚠ Skipped ${skippedFiles.length} existing file(s): ${skippedFiles.join(", ")}`));
-            out(chalk.gray("  Use --force to overwrite existing files."));
-            out("");
-        }
-        out(chalk.bold.green(`✓ Introspected ${tablesMap.size} tables — generated ${generatedFiles.length} collection(s).`));
-        out(chalk.gray(`  Review the generated files in ${outDir} and customize properties as needed.`));
-        out("");
-
-    } catch (e) {
-        outError(chalk.red(`✗ Error introspecting database: ${e instanceof Error ? e.message : String(e)}`));
-        process.exit(1);
-    } finally {
-        await client.end();
-    }
-}
-
-main().catch((err) => {
-    outError(String(err));
-    process.exit(1);
-});

package/src/schema/introspect-runtime.test.ts

@@ -1,260 +0,0 @@
-import { describe, expect, it } from "@jest/globals";
-import { resolveCollectionRelations } from "@rebasepro/common";
-import type { CollectionConfig } from "@rebasepro/types";
-
-import { buildCollectionsFromSchema, introspectSchema, readRlsStatus, IntrospectedSchema, Queryable } from "./introspect-runtime";
-import { buildTablesMap, identifyJoinTables, TableColumn, ForeignKeyRow, PrimaryKeyRow } from "./introspect-db-logic";
-
-function column(overrides: Partial<TableColumn> & { table_name: string; column_name: string }): TableColumn {
-    return {
-        data_type: "text",
-        udt_name: "text",
-        is_nullable: "YES",
-        column_default: null,
-        atttypmod: null,
-        ...overrides
-    };
-}
-
-function schemaOf(
-    columns: TableColumn[],
-    pks: PrimaryKeyRow[],
-    fks: ForeignKeyRow[] = [],
-    enums: Map<string, string[]> = new Map()
-): IntrospectedSchema {
-    const tableNames = [...new Set(columns.map((c) => c.table_name))].map((table_name) => ({ table_name }));
-    const tablesMap = buildTablesMap(tableNames, columns, pks, fks);
-    return { tablesMap, enumMap: enums, joinTables: identifyJoinTables(tablesMap) };
-}
-
-describe("buildCollectionsFromSchema", () => {
-    it("derives a collection per table with humanized names and the pg schema", () => {
-        const schema = schemaOf(
-            [
-                column({ table_name: "blog_posts", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
-                column({ table_name: "blog_posts", column_name: "title", is_nullable: "NO" })
-            ],
-            [{ table_name: "blog_posts", column_name: "id" }]
-        );
-
-        const [collection] = buildCollectionsFromSchema(schema, "public");
-
-        expect(collection.slug).toBe("blog_posts");
-        expect(collection.table).toBe("blog_posts");
-        expect((collection as { schema?: string }).schema).toBe("public");
-        expect(collection.name).toBe("Blog Posts");
-        expect(collection.singularName).toBe("Blog Post");
-    });
-
-    it("maps a uuid primary key to isId uuid and a serial one to increment", () => {
-        const schema = schemaOf(
-            [
-                column({ table_name: "users", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
-                column({ table_name: "orders", column_name: "id", data_type: "integer", udt_name: "int4", is_nullable: "NO" })
-            ],
-            [
-                { table_name: "users", column_name: "id" },
-                { table_name: "orders", column_name: "id" }
-            ]
-        );
-
-        const collections = buildCollectionsFromSchema(schema, "public");
-        const users = collections.find((c) => c.slug === "users")!;
-        const orders = collections.find((c) => c.slug === "orders")!;
-
-        expect((users.properties as any).id.isId).toBe("uuid");
-        expect((orders.properties as any).id.isId).toBe("increment");
-    });
-
-    it("marks NOT NULL columns without a default as required, and leaves others optional", () => {
-        const schema = schemaOf(
-            [
-                column({ table_name: "posts", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
-                column({ table_name: "posts", column_name: "title", is_nullable: "NO" }),
-                column({ table_name: "posts", column_name: "subtitle", is_nullable: "YES" }),
-                column({ table_name: "posts", column_name: "status", is_nullable: "NO", column_default: "'draft'" })
-            ],
-            [{ table_name: "posts", column_name: "id" }]
-        );
-
-        const props = buildCollectionsFromSchema(schema, "public")[0].properties as any;
-
-        expect(props.title.validation).toEqual({ required: true });
-        expect(props.subtitle.validation).toBeUndefined();
-        // A default means the database fills it in, so the API must not demand it.
-        expect(props.status.validation).toBeUndefined();
-    });
-
-    it("expands a pg enum column into enum entries", () => {
-        const schema = schemaOf(
-            [
-                column({ table_name: "posts", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
-                column({ table_name: "posts", column_name: "status", data_type: "USER-DEFINED", udt_name: "post_status" })
-            ],
-            [{ table_name: "posts", column_name: "id" }],
-            [],
-            new Map([["post_status", ["draft", "in_review"]]])
-        );
-
-        const props = buildCollectionsFromSchema(schema, "public")[0].properties as any;
-
-        expect(props.status.type).toBe("string");
-        expect(props.status.enum).toEqual([
-            { id: "draft", label: "Draft" },
-            { id: "in_review", label: "In Review" }
-        ]);
-    });
-
-    it("turns a foreign key into an owning relation and drops the raw fk column", () => {
-        const schema = schemaOf(
-            [
-                column({ table_name: "authors", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
-                column({ table_name: "posts", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
-                column({ table_name: "posts", column_name: "author_id", data_type: "uuid", udt_name: "uuid" })
-            ],
-            [
-                { table_name: "authors", column_name: "id" },
-                { table_name: "posts", column_name: "id" }
-            ],
-            [{ table_name: "posts", column_name: "author_id", foreign_table_name: "authors", foreign_column_name: "id" }]
-        );
-
-        const posts = buildCollectionsFromSchema(schema, "public").find((c) => c.slug === "posts")!;
-        const props = posts.properties as any;
-
-        expect(props.author_id).toBeUndefined();
-        expect(props.author.name).toBe("Author");
-        expect(props.author.type).toBe("relation");
-        // The descriptor is nested under `relation`, carries `kind`, and its
-        // `target` is a thunk — see the resolvability test below for why none of
-        // those three are stylistic.
-        expect(props.author.relation).toMatchObject({ kind: "belongsTo", localKey: "author_id" });
-        expect(typeof props.author.relation.target).toBe("function");
-        expect(props.author.relation.target().slug).toBe("authors");
-    });
-
-    /**
-     * The shape has to be one the resolver actually reads.
-     *
-     * This path emitted `cardinality` / `direction` / a bare-string `target`
-     * flat on the property — the shape `Relation` stopped accepting, and the
-     * same drift `introspect-emits-valid-relations.test.ts` caught in the
-     * generated-file path and this one was missed by. Nothing threw:
-     * `resolveCollectionRelations` reads `property.relation`, found none, and
-     * reported a collection with no relations at all.
-     *
-     * The visible cost was writes. `assertKnownWriteFields` learns an owning
-     * relation's FK column from the *resolved* relation's `localKey`, and the FK
-     * column is deliberately absent from `properties` ("surfaces as a
-     * relation"), so with nothing resolving, `POST /api/data/orders` with
-     * `product_id` came back 400 `has no field 'product_id'` — the column was
-     * simultaneously the only way to set the relation and not a known field.
-     *
-     * Asserting the literal object is what let it drift, so this asserts
-     * through the resolver instead: a future reshape has to keep it resolvable,
-     * not merely keep the keys someone once wrote down.
-     */
-    it("emits a relation the resolver can read, so the fk column is writable", () => {
-        const schema = schemaOf(
-            [
-                column({ table_name: "products", column_name: "id", data_type: "integer", udt_name: "int4", is_nullable: "NO" }),
-                column({ table_name: "orders", column_name: "id", data_type: "integer", udt_name: "int4", is_nullable: "NO" }),
-                column({ table_name: "orders", column_name: "product_id", data_type: "integer", udt_name: "int4" })
-            ],
-            [
-                { table_name: "products", column_name: "id" },
-                { table_name: "orders", column_name: "id" }
-            ],
-            [{ table_name: "orders", column_name: "product_id", foreign_table_name: "products", foreign_column_name: "id" }]
-        );
-
-        const orders = buildCollectionsFromSchema(schema, "public").find((c) => c.slug === "orders")!;
-        const resolved = resolveCollectionRelations(orders as unknown as CollectionConfig);
-
-        expect(Object.keys(resolved)).toEqual(["product"]);
-        expect(resolved.product.kind).toBe("belongsTo");
-        // `localKey` is the field `assertKnownWriteFields` adds to the known set,
-        // which is what makes `product_id` writable. The write itself is covered
-        // end-to-end by `scripts/smoke-baas.ts` against a real database.
-        expect((resolved.product as { localKey: string }).localKey).toBe("product_id");
-    });
-
-    it("skips join tables — they are an edge between collections, not a collection", () => {
-        const schema = schemaOf(
-            [
-                column({ table_name: "posts", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
-                column({ table_name: "tags", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
-                column({ table_name: "posts_tags", column_name: "post_id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
-                column({ table_name: "posts_tags", column_name: "tag_id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" })
-            ],
-            [
-                { table_name: "posts", column_name: "id" },
-                { table_name: "tags", column_name: "id" },
-                { table_name: "posts_tags", column_name: "post_id" },
-                { table_name: "posts_tags", column_name: "tag_id" }
-            ],
-            [
-                { table_name: "posts_tags", column_name: "post_id", foreign_table_name: "posts", foreign_column_name: "id" },
-                { table_name: "posts_tags", column_name: "tag_id", foreign_table_name: "tags", foreign_column_name: "id" }
-            ]
-        );
-
-        const slugs = buildCollectionsFromSchema(schema, "public").map((c) => c.slug);
-
-        expect(slugs).toEqual(expect.arrayContaining(["posts", "tags"]));
-        expect(slugs).not.toContain("posts_tags");
-    });
-});
-
-describe("introspectSchema", () => {
-    it("scopes every query to the requested schema", async () => {
-        const calls: unknown[][] = [];
-        const client: Queryable = {
-            query: async (_text: string, values?: unknown[]) => {
-                calls.push(values ?? []);
-                return { rows: [] as any[] };
-            }
-        };
-
-        await introspectSchema(client, "analytics");
-
-        expect(calls.length).toBeGreaterThan(0);
-        for (const values of calls) {
-            expect(values).toEqual(["analytics"]);
-        }
-    });
-});
-
-describe("readRlsStatus", () => {
-    it("reports RLS state and policy count per table", async () => {
-        const client: Queryable = {
-            query: async () => ({
-                rows: [
-                    { table: "posts", rls_enabled: true, policy_count: "2" },
-                    { table: "secrets", rls_enabled: false, policy_count: "0" },
-                    { table: "locked", rls_enabled: true, policy_count: 0 }
-                ] as never[]
-            })
-        };
-
-        const status = await readRlsStatus(client, "public");
-
-        // Unprotected: no authorization model, so baas must not serve it.
-        expect(status.get("secrets")).toEqual({ table: "secrets", rlsEnabled: false, policyCount: 0 });
-        // Protected and policied.
-        expect(status.get("posts")).toEqual({ table: "posts", rlsEnabled: true, policyCount: 2 });
-        // RLS on, no policies — legal, and returns nothing at all.
-        expect(status.get("locked")).toEqual({ table: "locked", rlsEnabled: true, policyCount: 0 });
-    });
-
-    it("scopes the lookup to the requested schema", async () => {
-        const calls: unknown[][] = [];
-        const client: Queryable = {
-            query: async (_t: string, v?: unknown[]) => { calls.push(v ?? []); return { rows: [] as never[] }; }
-        };
-
-        await readRlsStatus(client, "analytics");
-
-        expect(calls[0]).toEqual(["analytics"]);
-    });
-});