@rebasepro/server-postgres 0.0.1-canary.4829d6e

package/src/schema/introspect-db.ts

@@ -0,0 +1,266 @@
+import chalk from "chalk";
+import fs from "fs";
+import path from "path";
+import pg from "pg";
+import arg from "arg";
+import * as dotenv from "dotenv";
+import readline from "readline";
+
+import {
+    TableRow,
+    TableColumn,
+    EnumValue,
+    PrimaryKeyRow,
+    ForeignKeyRow,
+    buildTablesMap,
+    buildEnumMap,
+    identifyJoinTables,
+    generateCollectionFile,
+    generateIndexContent,
+    mergeIndexContent,
+    safeHostFromUrl
+} from "./introspect-db-logic";
+import { logger } from "@rebasepro/server";
+
+async function main() {
+    const args = arg(
+        {
+            "--output": String,
+            "--collections": String,
+            "--force": Boolean,
+            "--schema": String,
+            "--data-inference": Boolean,
+            "-o": "--output",
+            "-c": "--collections",
+            "-f": "--force"
+        },
+        { permissive: true }
+    );
+
+    const cwd = process.cwd();
+    const isBackendDir = path.basename(cwd) === "backend";
+    const defaultOutDir = isBackendDir
+        ? path.resolve(cwd, "..", "config", "collections")
+        : path.resolve(cwd, "config", "collections");
+
+    const outDir = args["--output"] || args["--collections"] || defaultOutDir;
+    const force = args["--force"] || false;
+    const pgSchema = args["--schema"] || "public";
+
+    if (!fs.existsSync(outDir)) {
+        fs.mkdirSync(outDir, { recursive: true });
+    }
+
+    // Load env
+    const envPaths = [
+        process.env.DOTENV_CONFIG_PATH,
+        path.resolve(process.cwd(), ".env"),
+        path.resolve(process.cwd(), "../.env"),
+        path.resolve(process.cwd(), "../../.env")
+    ].filter(Boolean) as string[];
+
+    for (const p of envPaths) {
+        if (fs.existsSync(p)) {
+            dotenv.config({ path: p });
+            break;
+        }
+    }
+
+    const databaseUrl = process.env.DATABASE_URL || process.env.ADMIN_CONNECTION_STRING;
+    if (!databaseUrl) {
+        logger.error(chalk.red("✗ DATABASE_URL is not set. Make sure your .env file is configured."));
+        process.exit(1);
+    }
+
+    const client = new pg.Client({ connectionString: databaseUrl });
+
+    try {
+        await client.connect();
+    } catch (err) {
+        logger.error(chalk.red(`✗ Failed to connect to database: ${err instanceof Error ? err.message : String(err)}`));
+        logger.error(chalk.gray("  Check your DATABASE_URL and ensure the database is reachable."));
+        process.exit(1);
+    }
+
+    // Log the host portion safely — handle URLs without "@"
+    const hostPart = safeHostFromUrl(databaseUrl);
+    logger.info(chalk.gray(`Connected to database: ${hostPart}`));
+    logger.info(chalk.gray(`Introspecting schema '${pgSchema}'...`));
+
+    try {
+        // 1. Get Tables
+        const { rows: tables } = await client.query<TableRow>(`
+            SELECT table_name
+            FROM information_schema.tables
+            WHERE table_schema = $1 AND table_type = 'BASE TABLE'
+              AND table_name NOT LIKE 'drizzle_%'
+              AND table_name NOT LIKE 'rebase_%'
+            ORDER BY table_name
+        `, [pgSchema]);
+
+        // 2. Get Columns
+        const { rows: columns } = await client.query<TableColumn>(`
+            SELECT 
+                c.table_name, 
+                c.column_name, 
+                c.data_type, 
+                c.udt_name, 
+                c.is_nullable, 
+                c.column_default,
+                (SELECT a.atttypmod FROM pg_attribute a 
+                 JOIN pg_class pc ON a.attrelid = pc.oid 
+                 WHERE pc.relname = c.table_name 
+                   AND a.attname = c.column_name 
+                   AND pc.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = c.table_schema)) as atttypmod
+            FROM information_schema.columns c
+            WHERE c.table_schema = $1
+        `, [pgSchema]);
+
+        // 2b. Get Enum Types and their values
+        const { rows: enumValues } = await client.query<EnumValue>(`
+            SELECT t.typname AS enum_name,
+                   e.enumlabel AS enum_value,
+                   e.enumsortorder AS sort_order
+            FROM pg_type t
+            JOIN pg_enum e ON t.oid = e.enumtypid
+            JOIN pg_namespace n ON t.typnamespace = n.oid
+            WHERE n.nspname = $1
+            ORDER BY t.typname, e.enumsortorder
+        `, [pgSchema]);
+
+        // Build a map: enum_name -> ordered list of values
+        const enumMap = buildEnumMap(enumValues);
+
+        // 3. Get Primary Keys
+        const { rows: pks } = await client.query<PrimaryKeyRow>(`
+            SELECT t.relname as table_name, a.attname as column_name
+            FROM   pg_index i
+            JOIN   pg_attribute a ON a.attrelid = i.indrelid
+                                AND a.attnum = ANY(i.indkey)
+            JOIN   pg_class t ON t.oid = i.indrelid
+            JOIN   pg_namespace n ON n.oid = t.relnamespace
+            WHERE  i.indisprimary AND n.nspname = $1
+        `, [pgSchema]);
+
+        // 4. Get Foreign Keys
+        const { rows: fks } = await client.query<ForeignKeyRow>(`
+            SELECT
+                tc.table_name, 
+                kcu.column_name, 
+                ccu.table_name AS foreign_table_name,
+                ccu.column_name AS foreign_column_name 
+            FROM 
+                information_schema.table_constraints AS tc 
+                JOIN information_schema.key_column_usage AS kcu
+                  ON tc.constraint_name = kcu.constraint_name
+                  AND tc.table_schema = kcu.table_schema
+                JOIN information_schema.constraint_column_usage AS ccu
+                  ON ccu.constraint_name = tc.constraint_name
+                  AND ccu.table_schema = tc.table_schema
+            WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema = $1
+        `, [pgSchema]);
+
+        const tablesMap = buildTablesMap(tables, columns, pks, fks);
+        const joinTables = identifyJoinTables(tablesMap);
+
+        logger.info(chalk.blue(`Found ${tablesMap.size} tables (including ${joinTables.size} detected join tables).`));
+
+        let runDataInference = false;
+        if (args["--data-inference"] !== undefined) {
+            runDataInference = args["--data-inference"];
+        } else {
+            const rl = readline.createInterface({
+                input: process.stdin,
+                output: process.stdout
+            });
+            const answer = await new Promise<string>((resolve) => rl.question(chalk.yellow("? Do you want to run comprehensive data inference on sampled rows to auto-detect types, formats, constraints, and UI configurations? (y/N) "), resolve));
+            runDataInference = answer.trim().toLowerCase() === "y";
+            rl.close();
+        }
+
+        if (runDataInference) {
+            logger.info(chalk.gray("Sampling database rows for data inference..."));
+        }
+
+        // Generate Collections
+        const generatedFiles: string[] = [];
+        const skippedFiles: string[] = [];
+
+        const tablesToProcess = Array.from(tablesMap.entries()).filter(([tableName]) => !joinTables.has(tableName));
+
+        const BATCH_SIZE = 10;
+        for (let i = 0; i < tablesToProcess.length; i += BATCH_SIZE) {
+            const batch = tablesToProcess.slice(i, i + BATCH_SIZE);
+
+            await Promise.all(batch.map(async ([tableName, meta]) => {
+                // ── File overwrite protection ──────────────────────────────
+                const filePath = path.join(outDir, `${tableName}.ts`);
+                if (fs.existsSync(filePath) && !force) {
+                    skippedFiles.push(tableName);
+                    return;
+                }
+
+                let sampleData: Record<string, unknown>[] | undefined = undefined;
+                if (runDataInference) {
+                    try {
+                        const { rows } = await client.query(`SELECT * FROM "${pgSchema}"."${tableName}" LIMIT 100`);
+                        sampleData = rows;
+                    } catch (err) {
+                        logger.error(chalk.yellow(`⚠ Failed to sample data for table ${tableName}: ${err instanceof Error ? err.message : String(err)}`));
+                    }
+                }
+
+                const fileContent = generateCollectionFile(
+                    tableName,
+                    meta,
+                    fks,
+                    joinTables,
+                    tablesMap,
+                    enumMap,
+                    sampleData
+                );
+
+                fs.writeFileSync(filePath, fileContent, "utf-8");
+                generatedFiles.push(tableName);
+                logger.info(chalk.green(`  ✓ ${filePath}`));
+            }));
+        }
+
+        // Generate index.ts (sorted alphabetically for deterministic output)
+        if (generatedFiles.length > 0) {
+            const indexPath = path.join(outDir, "index.ts");
+
+            if (fs.existsSync(indexPath) && !force) {
+                // Merge: read existing index, add new exports that don't already exist
+                const existing = fs.readFileSync(indexPath, "utf-8");
+                const merged = mergeIndexContent(existing, generatedFiles);
+                fs.writeFileSync(indexPath, merged, "utf-8");
+            } else {
+                const indexContent = generateIndexContent(generatedFiles);
+                fs.writeFileSync(indexPath, indexContent, "utf-8");
+            }
+            logger.info(chalk.green(`  ✓ ${indexPath}`));
+        }
+
+        logger.info("");
+        if (skippedFiles.length > 0) {
+            logger.info(chalk.yellow(`⚠ Skipped ${skippedFiles.length} existing file(s): ${skippedFiles.join(", ")}`));
+            logger.info(chalk.gray("  Use --force to overwrite existing files."));
+            logger.info("");
+        }
+        logger.info(chalk.bold.green(`✓ Introspected ${tablesMap.size} tables — generated ${generatedFiles.length} collection(s).`));
+        logger.info(chalk.gray(`  Review the generated files in ${outDir} and customize properties as needed.`));
+        logger.info("");
+
+    } catch (e) {
+        logger.error(chalk.red(`✗ Error introspecting database: ${e instanceof Error ? e.message : String(e)}`));
+        process.exit(1);
+    } finally {
+        await client.end();
+    }
+}
+
+main().catch((err) => {
+    logger.error(String(err));
+    process.exit(1);
+});

package/src/schema/introspect-runtime.test.ts

@@ -0,0 +1,212 @@
+import { describe, expect, it } from "@jest/globals";
+
+import { buildCollectionsFromSchema, introspectSchema, readRlsStatus, IntrospectedSchema, Queryable } from "./introspect-runtime";
+import { buildTablesMap, identifyJoinTables, TableColumn, ForeignKeyRow, PrimaryKeyRow } from "./introspect-db-logic";
+
+function column(overrides: Partial<TableColumn> & { table_name: string; column_name: string }): TableColumn {
+    return {
+        data_type: "text",
+        udt_name: "text",
+        is_nullable: "YES",
+        column_default: null,
+        atttypmod: null,
+        ...overrides
+    };
+}
+
+function schemaOf(
+    columns: TableColumn[],
+    pks: PrimaryKeyRow[],
+    fks: ForeignKeyRow[] = [],
+    enums: Map<string, string[]> = new Map()
+): IntrospectedSchema {
+    const tableNames = [...new Set(columns.map((c) => c.table_name))].map((table_name) => ({ table_name }));
+    const tablesMap = buildTablesMap(tableNames, columns, pks, fks);
+    return { tablesMap, enumMap: enums, joinTables: identifyJoinTables(tablesMap) };
+}
+
+describe("buildCollectionsFromSchema", () => {
+    it("derives a collection per table with humanized names and the pg schema", () => {
+        const schema = schemaOf(
+            [
+                column({ table_name: "blog_posts", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
+                column({ table_name: "blog_posts", column_name: "title", is_nullable: "NO" })
+            ],
+            [{ table_name: "blog_posts", column_name: "id" }]
+        );
+
+        const [collection] = buildCollectionsFromSchema(schema, "public");
+
+        expect(collection.slug).toBe("blog_posts");
+        expect(collection.table).toBe("blog_posts");
+        expect((collection as { schema?: string }).schema).toBe("public");
+        expect(collection.name).toBe("Blog Posts");
+        expect(collection.singularName).toBe("Blog Post");
+    });
+
+    it("maps a uuid primary key to isId uuid and a serial one to increment", () => {
+        const schema = schemaOf(
+            [
+                column({ table_name: "users", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
+                column({ table_name: "orders", column_name: "id", data_type: "integer", udt_name: "int4", is_nullable: "NO" })
+            ],
+            [
+                { table_name: "users", column_name: "id" },
+                { table_name: "orders", column_name: "id" }
+            ]
+        );
+
+        const collections = buildCollectionsFromSchema(schema, "public");
+        const users = collections.find((c) => c.slug === "users")!;
+        const orders = collections.find((c) => c.slug === "orders")!;
+
+        expect((users.properties as any).id.isId).toBe("uuid");
+        expect((orders.properties as any).id.isId).toBe("increment");
+    });
+
+    it("marks NOT NULL columns without a default as required, and leaves others optional", () => {
+        const schema = schemaOf(
+            [
+                column({ table_name: "posts", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
+                column({ table_name: "posts", column_name: "title", is_nullable: "NO" }),
+                column({ table_name: "posts", column_name: "subtitle", is_nullable: "YES" }),
+                column({ table_name: "posts", column_name: "status", is_nullable: "NO", column_default: "'draft'" })
+            ],
+            [{ table_name: "posts", column_name: "id" }]
+        );
+
+        const props = buildCollectionsFromSchema(schema, "public")[0].properties as any;
+
+        expect(props.title.validation).toEqual({ required: true });
+        expect(props.subtitle.validation).toBeUndefined();
+        // A default means the database fills it in, so the API must not demand it.
+        expect(props.status.validation).toBeUndefined();
+    });
+
+    it("expands a pg enum column into enum entries", () => {
+        const schema = schemaOf(
+            [
+                column({ table_name: "posts", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
+                column({ table_name: "posts", column_name: "status", data_type: "USER-DEFINED", udt_name: "post_status" })
+            ],
+            [{ table_name: "posts", column_name: "id" }],
+            [],
+            new Map([["post_status", ["draft", "in_review"]]])
+        );
+
+        const props = buildCollectionsFromSchema(schema, "public")[0].properties as any;
+
+        expect(props.status.type).toBe("string");
+        expect(props.status.enum).toEqual([
+            { id: "draft", label: "Draft" },
+            { id: "in_review", label: "In Review" }
+        ]);
+    });
+
+    it("turns a foreign key into an owning relation and drops the raw fk column", () => {
+        const schema = schemaOf(
+            [
+                column({ table_name: "authors", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
+                column({ table_name: "posts", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
+                column({ table_name: "posts", column_name: "author_id", data_type: "uuid", udt_name: "uuid" })
+            ],
+            [
+                { table_name: "authors", column_name: "id" },
+                { table_name: "posts", column_name: "id" }
+            ],
+            [{ table_name: "posts", column_name: "author_id", foreign_table_name: "authors", foreign_column_name: "id" }]
+        );
+
+        const posts = buildCollectionsFromSchema(schema, "public").find((c) => c.slug === "posts")!;
+        const props = posts.properties as any;
+
+        expect(props.author_id).toBeUndefined();
+        expect(props.author).toEqual({
+            name: "Author",
+            type: "relation",
+            target: "authors",
+            cardinality: "one",
+            direction: "owning",
+            localKey: "author_id"
+        });
+    });
+
+    it("skips join tables — they are an edge between collections, not a collection", () => {
+        const schema = schemaOf(
+            [
+                column({ table_name: "posts", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
+                column({ table_name: "tags", column_name: "id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
+                column({ table_name: "posts_tags", column_name: "post_id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" }),
+                column({ table_name: "posts_tags", column_name: "tag_id", data_type: "uuid", udt_name: "uuid", is_nullable: "NO" })
+            ],
+            [
+                { table_name: "posts", column_name: "id" },
+                { table_name: "tags", column_name: "id" },
+                { table_name: "posts_tags", column_name: "post_id" },
+                { table_name: "posts_tags", column_name: "tag_id" }
+            ],
+            [
+                { table_name: "posts_tags", column_name: "post_id", foreign_table_name: "posts", foreign_column_name: "id" },
+                { table_name: "posts_tags", column_name: "tag_id", foreign_table_name: "tags", foreign_column_name: "id" }
+            ]
+        );
+
+        const slugs = buildCollectionsFromSchema(schema, "public").map((c) => c.slug);
+
+        expect(slugs).toEqual(expect.arrayContaining(["posts", "tags"]));
+        expect(slugs).not.toContain("posts_tags");
+    });
+});
+
+describe("introspectSchema", () => {
+    it("scopes every query to the requested schema", async () => {
+        const calls: unknown[][] = [];
+        const client: Queryable = {
+            query: async (_text: string, values?: unknown[]) => {
+                calls.push(values ?? []);
+                return { rows: [] as any[] };
+            }
+        };
+
+        await introspectSchema(client, "analytics");
+
+        expect(calls.length).toBeGreaterThan(0);
+        for (const values of calls) {
+            expect(values).toEqual(["analytics"]);
+        }
+    });
+});
+
+describe("readRlsStatus", () => {
+    it("reports RLS state and policy count per table", async () => {
+        const client: Queryable = {
+            query: async () => ({
+                rows: [
+                    { table: "posts", rls_enabled: true, policy_count: "2" },
+                    { table: "secrets", rls_enabled: false, policy_count: "0" },
+                    { table: "locked", rls_enabled: true, policy_count: 0 }
+                ] as never[]
+            })
+        };
+
+        const status = await readRlsStatus(client, "public");
+
+        // Unprotected: no authorization model, so baas must not serve it.
+        expect(status.get("secrets")).toEqual({ table: "secrets", rlsEnabled: false, policyCount: 0 });
+        // Protected and policied.
+        expect(status.get("posts")).toEqual({ table: "posts", rlsEnabled: true, policyCount: 2 });
+        // RLS on, no policies — legal, and returns nothing at all.
+        expect(status.get("locked")).toEqual({ table: "locked", rlsEnabled: true, policyCount: 0 });
+    });
+
+    it("scopes the lookup to the requested schema", async () => {
+        const calls: unknown[][] = [];
+        const client: Queryable = {
+            query: async (_t: string, v?: unknown[]) => { calls.push(v ?? []); return { rows: [] as never[] }; }
+        };
+
+        await readRlsStatus(client, "analytics");
+
+        expect(calls[0]).toEqual(["analytics"]);
+    });
+});