@rebasepro/server-postgres 0.13.1-canary.gf57a27e → 0.14.0

package/src/schema/introspect-db-project.ts

@@ -0,0 +1,78 @@
+/**
+ * What the project being introspected into can actually import.
+ *
+ * The generator is pure — `introspect-db-logic.ts` reads no files — so the one
+ * question it cannot answer for itself lives here: which `defineCollection`, if
+ * any, will resolve from the directory the collections are written to.
+ */
+import fs from "fs";
+import path from "path";
+
+import { ADMIN_TYPES_PACKAGE, COMMON_PACKAGE, type CollectionBuilder } from "./introspect-db-logic";
+
+/** How far up to look before giving up, if no project root announces itself. */
+const MAX_LEVELS = 8;
+
+/** Every dependency name a manifest declares, in any of the three fields. */
+function declaredDependencies(manifestPath: string): Set<string> {
+    const names = new Set<string>();
+    let parsed: unknown;
+    try {
+        parsed = JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
+    } catch {
+        // A malformed or unreadable package.json is not this function's problem
+        // to report — it answers "can this import resolve", and the answer is no.
+        return names;
+    }
+    if (typeof parsed !== "object" || parsed === null) return names;
+    const manifest = parsed as Record<string, unknown>;
+    for (const field of ["dependencies", "devDependencies", "peerDependencies"]) {
+        const block = manifest[field];
+        if (typeof block === "object" && block !== null) {
+            for (const name of Object.keys(block)) names.add(name);
+        }
+    }
+    return names;
+}
+
+/**
+ * Which `defineCollection` to generate against, for collections written to `outDir`.
+ *
+ * **The detection is the package manifests above the output directory**, unioned from
+ * `outDir` up to and including the project root — the first ancestor holding a
+ * `rebase.json`, or `MAX_LEVELS` up if there is none.
+ *
+ * That is the rule Node itself applies: a file in `config/collections` resolves a bare
+ * specifier through `config/node_modules`, then `<project>/node_modules`, and so on up.
+ * Reading the manifests along that same path answers the only question that matters —
+ * *will this import resolve in the project I am writing into* — from the state on disk
+ * at the moment of generation. The alternatives are all proxies for it: `rebase.json`'s
+ * `apps` block says a CMS scaffold declared an admin app, and a `frontend/` directory
+ * says one was scaffolded, but neither is what the compiler consults, and either can be
+ * true of a project whose `config` package does not depend on `@rebasepro/admin-types`.
+ *
+ * Ambiguity resolves towards the admin panel: a project that declares both packages has
+ * a panel, and `@rebasepro/admin-types` is the flavour that keeps the `admin` block.
+ */
+export function detectCollectionBuilder(outDir: string): CollectionBuilder {
+    let dir = path.resolve(outDir);
+    const declared = new Set<string>();
+
+    for (let level = 0; level < MAX_LEVELS; level++) {
+        const manifest = path.join(dir, "package.json");
+        if (fs.existsSync(manifest)) {
+            for (const name of declaredDependencies(manifest)) declared.add(name);
+        }
+        // The project root, and the last level worth reading: anything above it
+        // belongs to whatever the project happens to be checked out inside.
+        if (fs.existsSync(path.join(dir, "rebase.json"))) break;
+
+        const parent = path.dirname(dir);
+        if (parent === dir) break;
+        dir = parent;
+    }
+
+    if (declared.has(ADMIN_TYPES_PACKAGE)) return "admin-types";
+    if (declared.has(COMMON_PACKAGE)) return "common";
+    return "annotation";
+}

package/src/schema/introspect-db.ts

@@ -14,10 +14,11 @@ import {
     mergeIndexContent,
     safeHostFromUrl
 } from "./introspect-db-logic";
+import { detectCollectionBuilder } from "./introspect-db-project";
 import { countRowsUpTo, readSchemaMetadata } from "./introspect-db-queries";
 import { classifyTables, lookupCandidates, LOOKUP_MAX_ROWS } from "./introspect-db-structure";
 import { parseCheckConstraints } from "./introspect-db-constraints";
-import { logger } from "@rebasepro/server";
+import { out, outWarn, outError } from "../cli-output";
 
 async function main() {
     const args = arg(
@@ -59,14 +60,14 @@ async function main() {
 
     for (const p of envPaths) {
         if (fs.existsSync(p)) {
-            dotenv.config({ path: p });
+            dotenv.config({ path: p, quiet: true });
             break;
         }
     }
 
     const databaseUrl = process.env.DATABASE_URL || process.env.ADMIN_CONNECTION_STRING;
     if (!databaseUrl) {
-        logger.error(chalk.red("✗ DATABASE_URL is not set. Make sure your .env file is configured."));
+        outError(chalk.red("✗ DATABASE_URL is not set. Make sure your .env file is configured."));
         process.exit(1);
     }
 
@@ -75,15 +76,15 @@ async function main() {
     try {
         await client.connect();
     } catch (err) {
-        logger.error(chalk.red(`✗ Failed to connect to database: ${err instanceof Error ? err.message : String(err)}`));
-        logger.error(chalk.gray("  Check your DATABASE_URL and ensure the database is reachable."));
+        outError(chalk.red(`✗ Failed to connect to database: ${err instanceof Error ? err.message : String(err)}`));
+        outError(chalk.gray("  Check your DATABASE_URL and ensure the database is reachable."));
         process.exit(1);
     }
 
     // Log the host portion safely — handle URLs without "@"
     const hostPart = safeHostFromUrl(databaseUrl);
-    logger.info(chalk.gray(`Connected to database: ${hostPart}`));
-    logger.info(chalk.gray(`Introspecting schema '${pgSchema}'...`));
+    out(chalk.gray(`Connected to database: ${hostPart}`));
+    out(chalk.gray(`Introspecting schema '${pgSchema}'...`));
 
     try {
         const metadata = await readSchemaMetadata(client, pgSchema);
@@ -101,12 +102,28 @@ async function main() {
             } catch (err) {
                 // A table this run cannot read is simply not classified as a code
                 // list; everything else about it still generates.
-                logger.info(chalk.gray(`  (skipped row count for ${table}: ${err instanceof Error ? err.message : String(err)})`));
+                out(chalk.gray(`  (skipped row count for ${table}: ${err instanceof Error ? err.message : String(err)})`));
             }
         }
 
         const classifications = classifyTables(metadata, tablesMap);
         const checkFacts = parseCheckConstraints(metadata.checks);
+
+        // Which builder the generated files may import — read from the manifests
+        // above `outDir`, because a project without `@rebasepro/admin-types` cannot
+        // resolve that import and has no `admin` block to write into either.
+        const builder = detectCollectionBuilder(outDir);
+        if (builder === "annotation") {
+            outWarn(chalk.yellow(
+                "⚠ Neither @rebasepro/admin-types nor @rebasepro/common is declared above " +
+                `${outDir}.`
+            ));
+            outWarn(chalk.gray(
+                "  Generating plain type annotations, which widen the property keys to `string`: " +
+                "`propertiesOrder` and friends will not be checked. Add @rebasepro/common to the " +
+                "config package to turn that checking on."
+            ));
+        }
         const joinTables = new Set(
             Array.from(classifications.values())
                 .filter((c) => c.role === "junction")
@@ -116,8 +133,8 @@ async function main() {
         const roleCount = (role: string) =>
             Array.from(classifications.values()).filter((c) => c.role === role).length;
 
-        logger.info(chalk.blue(`Found ${tablesMap.size} tables.`));
-        logger.info(chalk.gray(
+        out(chalk.blue(`Found ${tablesMap.size} tables.`));
+        out(chalk.gray(
             `  ${roleCount("entity")} entities, ${joinTables.size} join tables (folded into relations), ` +
             `${roleCount("lookup")} code lists, ${roleCount("owned-child")} owned by another table (hidden from navigation).`
         ));
@@ -130,7 +147,7 @@ async function main() {
         } else if (!process.stdin.isTTY) {
             // No terminal to answer the question below (scaffolding scripts, CI,
             // `rebase init --introspect`) — asking would hang forever.
-            logger.info(chalk.gray("Skipping data inference (non-interactive run; pass --data-inference to enable)."));
+            out(chalk.gray("Skipping data inference (non-interactive run; pass --data-inference to enable)."));
         } else {
             const rl = readline.createInterface({
                 input: process.stdin,
@@ -142,7 +159,7 @@ async function main() {
         }
 
         if (runDataInference) {
-            logger.info(chalk.gray("Sampling database rows for data inference..."));
+            out(chalk.gray("Sampling database rows for data inference..."));
         }
 
         // Generate Collections
@@ -169,7 +186,7 @@ async function main() {
                         const { rows } = await client.query(`SELECT * FROM "${pgSchema}"."${tableName}" LIMIT 100`);
                         sampleData = rows;
                     } catch (err) {
-                        logger.error(chalk.yellow(`⚠ Failed to sample data for table ${tableName}: ${err instanceof Error ? err.message : String(err)}`));
+                        outError(chalk.yellow(`⚠ Failed to sample data for table ${tableName}: ${err instanceof Error ? err.message : String(err)}`));
                     }
                 }
 
@@ -181,12 +198,12 @@ async function main() {
                     tablesMap,
                     enumMap,
                     sampleData,
-                    { metadata, classifications, checkFacts }
+                    { metadata, classifications, checkFacts, builder }
                 );
 
                 fs.writeFileSync(filePath, fileContent, "utf-8");
                 generatedFiles.push(tableName);
-                logger.info(chalk.green(`  ✓ ${filePath}`));
+                out(chalk.green(`  ✓ ${filePath}`));
             }));
         }
 
@@ -213,21 +230,21 @@ async function main() {
                 const indexContent = generateIndexContent(allFiles);
                 fs.writeFileSync(indexPath, indexContent, "utf-8");
             }
-            logger.info(chalk.green(`  ✓ ${indexPath}`));
+            out(chalk.green(`  ✓ ${indexPath}`));
         }
 
-        logger.info("");
+        out("");
         if (skippedFiles.length > 0) {
-            logger.info(chalk.yellow(`⚠ Skipped ${skippedFiles.length} existing file(s): ${skippedFiles.join(", ")}`));
-            logger.info(chalk.gray("  Use --force to overwrite existing files."));
-            logger.info("");
+            out(chalk.yellow(`⚠ Skipped ${skippedFiles.length} existing file(s): ${skippedFiles.join(", ")}`));
+            out(chalk.gray("  Use --force to overwrite existing files."));
+            out("");
         }
-        logger.info(chalk.bold.green(`✓ Introspected ${tablesMap.size} tables — generated ${generatedFiles.length} collection(s).`));
-        logger.info(chalk.gray(`  Review the generated files in ${outDir} and customize properties as needed.`));
-        logger.info("");
+        out(chalk.bold.green(`✓ Introspected ${tablesMap.size} tables — generated ${generatedFiles.length} collection(s).`));
+        out(chalk.gray(`  Review the generated files in ${outDir} and customize properties as needed.`));
+        out("");
 
     } catch (e) {
-        logger.error(chalk.red(`✗ Error introspecting database: ${e instanceof Error ? e.message : String(e)}`));
+        outError(chalk.red(`✗ Error introspecting database: ${e instanceof Error ? e.message : String(e)}`));
         process.exit(1);
     } finally {
         await client.end();
@@ -235,6 +252,6 @@ async function main() {
 }
 
 main().catch((err) => {
-    logger.error(String(err));
+    outError(String(err));
     process.exit(1);
 });

package/src/schema/introspect-runtime.ts

@@ -28,6 +28,7 @@ import {
     getIconForTable
 } from "./introspect-db-logic";
 import { humanize } from "./introspect-db-naming";
+import { firstFreeKey, toWireKey } from "@rebasepro/utils";
 
 export interface IntrospectedSchema {
     tablesMap: Map<string, TableMeta>;
@@ -169,6 +170,7 @@ function buildProperties(
     enumMap: Map<string, string[]>
 ): Record<string, Record<string, unknown>> {
     const properties: Record<string, Record<string, unknown>> = {};
+    const takenKeys = new Set<string>();
 
     for (const col of meta.columns) {
         const isPk = meta.pks.includes(col.column_name);
@@ -188,7 +190,17 @@ function buildProperties(
             type: propType
         };
 
-        const key = col.column_name;
+        // The wire name, with `columnName` above carrying the column. The two
+        // are different names for different things — this used to serve the
+        // column, so a runtime-introspected collection answered `user_id` while
+        // every authored one beside it answered `displayName`.
+        //
+        // `user_id` and `userId` as two real columns camel-case to one key, so
+        // the first free candidate wins and the second falls back to its own
+        // column name, then to a numbered tail. Never dropped, and stable for a
+        // given database: columns arrive in ordinal order.
+        const key = firstFreeKey([toWireKey(col.column_name), col.column_name], takenKeys);
+        takenKeys.add(key);
 
         if (isPk) {
             property.isId = idKindFor(col, propType);
@@ -239,7 +251,7 @@ function buildRelations(
         if (!targetSlug) continue;
 
         // Strip the conventional _id suffix: author_id -> author
-        let key = fk.column_name.replace(/_id$/, "");
+        let key = toWireKey(fk.column_name.replace(/_id$/, ""));
         if (meta.pks.includes(fk.column_name) && key === fk.column_name) {
             // The FK is also the PK and its name doesn't imply a relation (e.g.
             // "id"), so naming the relation after the column would collide with

package/src/schema/search-column.ts

@@ -58,6 +58,7 @@ import {
     DEFAULT_SEARCH_WEIGHT,
     DEFAULT_FUZZY_THRESHOLD
 } from "@rebasepro/types";
+import { createHash } from "node:crypto";
 import { getTableName } from "@rebasepro/common";
 import { toSnakeCase, toPostgresIdentifier } from "@rebasepro/utils";
 
@@ -460,6 +461,90 @@ export const searchIndexStatements = (spec: SearchColumnSpec): string[] => {
     return statements;
 };
 
+// ── Telling a changed `search` block from an unchanged one ──────────────────
+
+/**
+ * Marker on the comment of every generated search column this module creates.
+ *
+ * Versioned because the fingerprint below is only comparable against itself: a
+ * future change to how it is computed has to read as "not stamped by this
+ * version" rather than as drift on every existing column.
+ */
+export const SEARCH_STAMP_PREFIX = "rebase:search:v1:";
+
+/**
+ * A stable fingerprint of one generated column's expression.
+ *
+ * Why a stamp rather than reading the expression back: Postgres stores a
+ * generated column's expression *parsed*, and hands it back deparsed — casts
+ * made explicit, identifiers requoted, schema qualifications added or dropped
+ * according to `search_path`. Comparing that text to the text we generated
+ * would report drift on wording, and this comparison decides whether a boot
+ * refuses, so a false positive is an outage. The stamp is written by the same
+ * code that writes the column, so equality means what it says.
+ */
+export const searchExpressionFingerprint = (expression: string): string =>
+    `${SEARCH_STAMP_PREFIX}${createHash("sha256").update(expression).digest("hex").slice(0, 16)}`;
+
+/** One generated column, with the fingerprint that identifies its expression. */
+export interface SearchColumnStamp {
+    column: string;
+    /** The expression the column is generated from. */
+    expression: string;
+    fingerprint: string;
+    /** `COMMENT ON COLUMN …`, which is where the fingerprint is recorded. */
+    sql: string;
+}
+
+/**
+ * The stamps for a spec's generated columns — one per column, never shared.
+ *
+ * Per column on purpose: turning `fuzzy` on adds a second column and changes
+ * nothing about the first, and a spec-wide fingerprint would report the
+ * untouched `tsvector` column as drifted and refuse a boot over a change that
+ * is purely additive.
+ */
+export const searchColumnStamps = (spec: SearchColumnSpec): SearchColumnStamp[] => {
+    const stamp = (column: string, expression: string): SearchColumnStamp => {
+        const fingerprint = searchExpressionFingerprint(expression);
+        return {
+            column,
+            expression,
+            fingerprint,
+            sql: `COMMENT ON COLUMN "${spec.schema}"."${spec.table}"."${column}" IS ${quote(fingerprint)};`
+        };
+    };
+    const stamps = [stamp(spec.column, spec.expression)];
+    if (spec.fuzzy) stamps.push(stamp(spec.fuzzy.column, spec.fuzzy.expression));
+    return stamps;
+};
+
+/**
+ * The same drift check as the boot ensure, for the SQL file.
+ *
+ * Needed because {@link searchColumnStamps} would otherwise *launder* drift on
+ * the migration path: `ADD COLUMN IF NOT EXISTS` does nothing to a column that
+ * exists, so a re-generated `search.sql` would stamp a stale column with the
+ * new block's fingerprint and the next boot would find them in agreement.
+ * Guarding first means the file refuses instead — `rebase db push` is attended,
+ * and the operator reading the failure is the person who changed the block.
+ */
+export const searchStampGuards = (spec: SearchColumnSpec): string[] =>
+    searchColumnStamps(spec).map(stamp => {
+        const relation = quote(`"${spec.schema}"."${spec.table}"`);
+        return `DO $rebase_search$
+DECLARE recorded text;
+BEGIN
+    SELECT col_description(a.attrelid, a.attnum) INTO recorded
+    FROM pg_attribute a
+    WHERE a.attrelid = ${relation}::regclass AND a.attname = ${quote(stamp.column)} AND NOT a.attisdropped;
+    IF recorded LIKE ${quote(`${SEARCH_STAMP_PREFIX}%`)} AND recorded <> ${quote(stamp.fingerprint)} THEN
+        RAISE EXCEPTION 'Rebase: the search block for ${spec.schema}.${spec.table} changed after the generated column "${stamp.column}" was built (recorded %, expected ${stamp.fingerprint}). Postgres cannot alter a generated expression in place. Drop the column and re-apply this file — it rewrites the table and rebuilds the index: ALTER TABLE ${relation.slice(1, -1)} DROP COLUMN "${stamp.column}";', recorded;
+    END IF;
+END
+$rebase_search$;`;
+    });
+
 /**
  * The index names the spec creates.
  *

package/src/security/policy-drift.test.ts

@@ -18,9 +18,50 @@ function collection(slug: string): CollectionConfig {
     } as unknown as CollectionConfig;
 }
 
-/** Stands in for pg_policies. */
-function dbWith(rows: Record<string, unknown>[]): Queryable {
-    return { query: async () => ({ rows: rows as never[] }) };
+/** A collection whose write rule is declared RESTRICTIVE, so the DDL says `AS RESTRICTIVE`. */
+function restrictiveCollection(slug: string): CollectionConfig {
+    return {
+        name: slug,
+        slug,
+        table: slug,
+        schema: "public",
+        properties: { id: { name: "ID", type: "string", isId: "uuid" } },
+        securityRules: [
+            { operation: "select", access: "public" },
+            { operations: ["insert", "update", "delete"], roles: ["admin"], mode: "restrictive" }
+        ]
+    } as unknown as CollectionConfig;
+}
+
+/**
+ * Stands in for the two catalogue reads: `pg_policies`, then `pg_class`.
+ *
+ * The second one is dispatched on the query text rather than call order, so a
+ * test that stops caring about ordering does not silently start feeding policy
+ * rows to the RLS-status read — which would report every table as RLS-off,
+ * since a policy row has no `relrowsecurity` column and `undefined` is falsy.
+ *
+ * `rlsOff` names tables whose RLS switch is off. Anything not listed is on,
+ * which is what a healthy `db push` leaves behind.
+ */
+function dbWith(rows: Record<string, unknown>[], rlsOff: string[] = []): Queryable {
+    return {
+        query: async (text: string, values?: unknown[]) => {
+            if (!text.includes("pg_class")) return { rows: rows as never[] };
+            const wanted = (values?.[0] as string[] | undefined) ?? [];
+            return {
+                rows: wanted.map((qualified) => {
+                    const [schemaname, tablename] = qualified.split(".");
+                    return {
+                        schemaname,
+                        tablename,
+                        relrowsecurity: !rlsOff.includes(tablename),
+                        relforcerowsecurity: !rlsOff.includes(tablename)
+                    };
+                }) as never[]
+            };
+        }
+    };
 }
 
 /** A pg_policies row matching an expected policy, clauses and all. */
@@ -30,6 +71,7 @@ function liveRow(p: PolicyRef, overrides: Record<string, unknown> = {}) {
         // Postgres rewrites the text it stores; only presence is compared.
         qual: p.hasUsing ? "(rewritten by postgres)" : null,
         with_check: p.hasWithCheck ? "(rewritten by postgres)" : null,
+        permissive: p.mode ?? "PERMISSIVE",
         ...overrides
     };
 }
@@ -56,6 +98,65 @@ describe("checkPolicyDrift", () => {
         expect(formatPolicyDrift(drift)).toBe("");
     });
 
+    it("flags a table whose RLS switch is off, though every policy still matches", async () => {
+        // The whole point: DISABLE ROW LEVEL SECURITY leaves pg_policies
+        // untouched, so name, roles, command and clause presence all agree and
+        // every other check here passes. Before this category the report was
+        // empty on a table Postgres was filtering nothing on.
+        const cols = [collection("authors")];
+        const expected = parseExpectedPolicies(generatePostgresPoliciesDdl(cols));
+
+        const drift = await checkPolicyDrift(
+            dbWith(expected.map((p) => liveRow(p)), ["authors"]),
+            cols
+        );
+
+        expect(drift.missing).toHaveLength(0);
+        expect(drift.diverged).toHaveLength(0);
+        expect(drift.orphaned).toHaveLength(0);
+        expect(drift.rlsDisabled).toEqual([{ schema: "public", table: "authors", forced: false }]);
+        expect(hasDrift(drift)).toBe(true);
+        expect(formatPolicyDrift(drift)).toContain("RLS DISABLED");
+    });
+
+    it("flags a RESTRICTIVE rule the database stores as PERMISSIVE", async () => {
+        // A gate that should be ANDed in is now ORed in — it widens access
+        // instead of narrowing it, with identical name, roles, command and
+        // clauses. Only the `AS` clause differs, and it used to be discarded by
+        // the DDL parser and never selected from pg_policies.
+        const cols = [restrictiveCollection("ledger")];
+        const expected = parseExpectedPolicies(generatePostgresPoliciesDdl(cols));
+        const restrictive = expected.filter((p) => p.mode === "RESTRICTIVE");
+        // Guards the test itself: if the generator stops emitting AS RESTRICTIVE
+        // this must fail loudly rather than pass by having nothing to compare.
+        expect(restrictive.length).toBeGreaterThan(0);
+
+        const live = expected.map((p) =>
+            p.name === restrictive[0].name ? liveRow(p, { permissive: "PERMISSIVE" }) : liveRow(p)
+        );
+
+        const drift = await checkPolicyDrift(dbWith(live), cols);
+
+        expect(drift.diverged).toHaveLength(1);
+        expect(drift.diverged[0].differences.join(" ")).toContain("mode: expected RESTRICTIVE");
+        expect(formatPolicyDrift(drift)).toContain("widens access");
+    });
+
+    it("does not invent drift when either side's mode is unreadable", async () => {
+        // An older server or a driver that hands back a shape we do not know
+        // must not turn into a wall of false positives.
+        const cols = [collection("authors")];
+        const expected = parseExpectedPolicies(generatePostgresPoliciesDdl(cols));
+
+        const drift = await checkPolicyDrift(
+            dbWith(expected.map((p) => liveRow(p, { permissive: null }))),
+            cols
+        );
+
+        expect(drift.diverged).toHaveLength(0);
+        expect(hasDrift(drift)).toBe(false);
+    });
+
     it("flags a policy the database never received as missing", async () => {
         const drift = await checkPolicyDrift(dbWith([]), [collection("authors")]);