@rebasepro/server-postgres 0.12.1-canary.gcfa333c → 0.12.1-canary.gddbd489

package/src/schema/introspect-db-types.ts

@@ -0,0 +1,56 @@
+/**
+ * The PostgreSQL type → Rebase property type mapping.
+ *
+ * Split out of `introspect-db-logic` so that the structural analysis can use it
+ * without importing the generator, which imports the analysis. Re-exported from
+ * `introspect-db-logic` so existing callers keep their import path.
+ */
+
+/**
+ * Map a PostgreSQL data type to a Rebase property type.
+ */
+export function mapPgType(dataType: string): string {
+    const dt = dataType.toLowerCase();
+
+    // Interval MUST be checked before numeric ("interval" contains "int")
+    if (dt === "interval") return "string";
+
+    // Array types MUST be checked before numeric ("_int4" contains "int")
+    if (dt === "array" || dt.startsWith("_")) return "array";
+
+    // Numeric types
+    if (
+        dt.includes("int") || // integer, smallint, bigint
+        dt.includes("numeric") ||
+        dt.includes("decimal") ||
+        dt.includes("serial") || // serial, bigserial
+        dt === "real" ||
+        dt === "float4" ||
+        dt === "float8" ||
+        dt === "double precision" ||
+        dt === "money"
+    ) {
+        return "number";
+    }
+
+    // Boolean
+    if (dt.includes("bool")) return "boolean";
+
+    // Date / Time
+    if (dt.includes("time") || dt.includes("date")) return "date";
+
+    // JSON
+    if (dt === "json" || dt === "jsonb") return "map";
+
+    // Binary
+    if (dt === "bytea") return "binary";
+
+    // Network types
+    if (dt === "inet" || dt === "cidr" || dt === "macaddr" || dt === "macaddr8") return "string";
+
+    // UUID
+    if (dt === "uuid") return "string";
+
+    // Text/varchar/char — default to string
+    return "string";
+}

package/src/schema/introspect-db.ts

@@ -7,19 +7,16 @@ import * as dotenv from "dotenv";
 import readline from "readline";
 
 import {
-    TableRow,
-    TableColumn,
-    EnumValue,
-    PrimaryKeyRow,
-    ForeignKeyRow,
     buildTablesMap,
     buildEnumMap,
-    identifyJoinTables,
     generateCollectionFile,
     generateIndexContent,
     mergeIndexContent,
     safeHostFromUrl
 } from "./introspect-db-logic";
+import { countRowsUpTo, readSchemaMetadata } from "./introspect-db-queries";
+import { classifyTables, lookupCandidates, LOOKUP_MAX_ROWS } from "./introspect-db-structure";
+import { parseCheckConstraints } from "./introspect-db-constraints";
 import { logger } from "@rebasepro/server";
 
 async function main() {
@@ -89,82 +86,41 @@ async function main() {
     logger.info(chalk.gray(`Introspecting schema '${pgSchema}'...`));
 
     try {
-        // 1. Get Tables
-        const { rows: tables } = await client.query<TableRow>(`
-            SELECT table_name
-            FROM information_schema.tables
-            WHERE table_schema = $1 AND table_type = 'BASE TABLE'
-              AND table_name NOT LIKE 'drizzle_%'
-              AND table_name NOT LIKE 'rebase_%'
-            ORDER BY table_name
-        `, [pgSchema]);
-
-        // 2. Get Columns
-        const { rows: columns } = await client.query<TableColumn>(`
-            SELECT 
-                c.table_name, 
-                c.column_name, 
-                c.data_type, 
-                c.udt_name, 
-                c.is_nullable, 
-                c.column_default,
-                (SELECT a.atttypmod FROM pg_attribute a 
-                 JOIN pg_class pc ON a.attrelid = pc.oid 
-                 WHERE pc.relname = c.table_name 
-                   AND a.attname = c.column_name 
-                   AND pc.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = c.table_schema)) as atttypmod
-            FROM information_schema.columns c
-            WHERE c.table_schema = $1
-        `, [pgSchema]);
-
-        // 2b. Get Enum Types and their values
-        const { rows: enumValues } = await client.query<EnumValue>(`
-            SELECT t.typname AS enum_name,
-                   e.enumlabel AS enum_value,
-                   e.enumsortorder AS sort_order
-            FROM pg_type t
-            JOIN pg_enum e ON t.oid = e.enumtypid
-            JOIN pg_namespace n ON t.typnamespace = n.oid
-            WHERE n.nspname = $1
-            ORDER BY t.typname, e.enumsortorder
-        `, [pgSchema]);
-
-        // Build a map: enum_name -> ordered list of values
-        const enumMap = buildEnumMap(enumValues);
-
-        // 3. Get Primary Keys
-        const { rows: pks } = await client.query<PrimaryKeyRow>(`
-            SELECT t.relname as table_name, a.attname as column_name
-            FROM   pg_index i
-            JOIN   pg_attribute a ON a.attrelid = i.indrelid
-                                AND a.attnum = ANY(i.indkey)
-            JOIN   pg_class t ON t.oid = i.indrelid
-            JOIN   pg_namespace n ON n.oid = t.relnamespace
-            WHERE  i.indisprimary AND n.nspname = $1
-        `, [pgSchema]);
+        const metadata = await readSchemaMetadata(client, pgSchema);
+        const enumMap = buildEnumMap(metadata.enumValues);
+        const tablesMap = buildTablesMap(metadata.tables, metadata.columns, metadata.pks, metadata.fks);
+        const fks = metadata.fks;
+
+        // Only tables that could structurally be a code list are counted, and
+        // each count stops at the threshold — see `countRowsUpTo`. Introspection
+        // runs against a database it does not own, so "cheap on a table of any
+        // size" is a requirement, not an optimization.
+        for (const table of lookupCandidates(metadata, tablesMap)) {
+            try {
+                metadata.rowCounts[table] = await countRowsUpTo(client, pgSchema, table, LOOKUP_MAX_ROWS);
+            } catch (err) {
+                // A table this run cannot read is simply not classified as a code
+                // list; everything else about it still generates.
+                logger.info(chalk.gray(`  (skipped row count for ${table}: ${err instanceof Error ? err.message : String(err)})`));
+            }
+        }
 
-        // 4. Get Foreign Keys
-        const { rows: fks } = await client.query<ForeignKeyRow>(`
-            SELECT
-                tc.table_name, 
-                kcu.column_name, 
-                ccu.table_name AS foreign_table_name,
-                ccu.column_name AS foreign_column_name 
-            FROM 
-                information_schema.table_constraints AS tc 
-                JOIN information_schema.key_column_usage AS kcu
-                  ON tc.constraint_name = kcu.constraint_name
-                  AND tc.table_schema = kcu.table_schema
-                JOIN information_schema.constraint_column_usage AS ccu
-                  ON ccu.constraint_name = tc.constraint_name
-                  AND ccu.table_schema = tc.table_schema
-            WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema = $1
-        `, [pgSchema]);
+        const classifications = classifyTables(metadata, tablesMap);
+        const checkFacts = parseCheckConstraints(metadata.checks);
+        const joinTables = new Set(
+            Array.from(classifications.values())
+                .filter((c) => c.role === "junction")
+                .map((c) => c.table)
+        );
 
-        const tablesMap = buildTablesMap(tables, columns, pks, fks);
-        const joinTables = identifyJoinTables(tablesMap);
+        const roleCount = (role: string) =>
+            Array.from(classifications.values()).filter((c) => c.role === role).length;
 
-        logger.info(chalk.blue(`Found ${tablesMap.size} tables (including ${joinTables.size} detected join tables).`));
+        logger.info(chalk.blue(`Found ${tablesMap.size} tables.`));
+        logger.info(chalk.gray(
+            `  ${roleCount("entity")} entities, ${joinTables.size} join tables (folded into relations), ` +
+            `${roleCount("lookup")} code lists, ${roleCount("owned-child")} owned by another table (hidden from navigation).`
+        ));
 
         let runDataInference = false;
         if (args["--no-data-inference"]) {
@@ -224,7 +180,8 @@ async function main() {
                     joinTables,
                     tablesMap,
                     enumMap,
-                    sampleData
+                    sampleData,
+                    { metadata, classifications, checkFacts }
                 );
 
                 fs.writeFileSync(filePath, fileContent, "utf-8");

package/src/security/policy-drift.test.ts

@@ -158,15 +158,23 @@ describe("checkPolicyDrift", () => {
     it("also flags the tautology in a WITH CHECK clause", async () => {
         const cols = [collection("posts")];
         const expected = parseExpectedPolicies(generatePostgresPoliciesDdl(cols));
+        const withCheck = expected.filter((p) => p.hasWithCheck);
+        // Asserted rather than assumed: if the generator ever stopped emitting a
+        // WITH CHECK for these rules the fixture would carry no tautology at
+        // all, and every assertion below would be vacuously satisfied.
+        expect(withCheck.length).toBeGreaterThan(0);
+
         const live = expected.map((p) => liveRow(p, {
             with_check: p.hasWithCheck ? "(auth.uid() IS NOT NULL)" : null
         }));
 
         const drift = await checkPolicyDrift(dbWith(live), cols);
 
-        const flagged = drift.insecure.some((i) => /WITH CHECK/.test(i.reason));
-        // Only assert when the fixture actually had a WITH CHECK policy to carry it.
-        if (expected.some((p) => p.hasWithCheck)) expect(flagged).toBe(true);
+        // Every WITH CHECK policy carries the tautology and no USING clause
+        // does, so anything flagged here came from the WITH CHECK scan — the
+        // half that a check reading only `qual` would miss entirely.
+        expect(drift.insecure.map((i) => i.policy.name)).toEqual(withCheck.map((p) => p.name));
+        expect(drift.insecure.every((i) => /WITH CHECK/.test(i.reason))).toBe(true);
     });
 
     it("parses roles when the driver returns the raw {a,b} text form", async () => {

package/src/services/FetchService.ts

@@ -1,10 +1,10 @@
-import { and, asc, count, desc, eq, getTableName, gt, lt, or, SQL, TableRelationalConfig, TablesRelationalConfig } from "drizzle-orm";
+import { and, asc, count, desc, eq, getTableColumns, getTableName, gt, lt, or, SQL, TableRelationalConfig, TablesRelationalConfig } from "drizzle-orm";
 import { AnyPgColumn, PgTable } from "drizzle-orm/pg-core";
 import { CollectionConfig, FilterValues, ResolvedRelation, LogicalCondition, isManyToMany } from "@rebasepro/types";
 import type { VectorSearchParams } from "@rebasepro/types";
 import { resolveCollectionRelations, findRelation, createRelationRef, createRelationRefWithData } from "@rebasepro/common";
 import { generateForeignKeyName } from "@rebasepro/utils";
-import { DrizzleConditionBuilder, type FilterCompilationOptions } from "../utils/drizzle-conditions";
+import { DrizzleConditionBuilder, getUnknownFilterFieldsMode, type FilterCompilationOptions } from "../utils/drizzle-conditions";
 import {
     getCollectionByPath,
     getTableForCollection,
@@ -12,6 +12,7 @@ import {
     requirePrimaryKeys,
     deriveRowAddress,
     parseIdValues,
+    idCanAddressTable,
     buildCompositeId,
     COMPOSITE_ID_SEPARATOR
 } from "./collection-helpers";
@@ -20,9 +21,10 @@ import { RelationService } from "./RelationService";
 import { RelationalQueryBuilder } from "drizzle-orm/pg-core/query-builders/query";
 import { DrizzleClient } from "../interfaces";
 import { PostgresCollectionRegistry } from "../collections/PostgresCollectionRegistry";
-import { toCmsRow, toRestRow, isJunctionRelation } from "./row-pipeline";
+import { toFlatRow, toRestRow, isJunctionRelation } from "./row-pipeline";
 import { isNestedPath, resolveNestedPath, type NestedPathHop } from "./nested-path";
-import { logger } from "@rebasepro/server";
+import { ApiError, logger } from "@rebasepro/server";
+import { reachedDatabase } from "../utils/pg-error-utils";
 
 /** Type-safe accessor for Drizzle's relational query API via dynamic table name */
 type DbQueryAccessor = Record<string, RelationalQueryBuilder<any, any>> | undefined;
@@ -129,12 +131,10 @@ export class FetchService {
         if (direct) return direct;
 
         // Owning relation, resolved: the relation names its own local key.
-        if (collection) {
-            const relation = resolveCollectionRelations(collection)[orderBy];
-            if (relation?.kind === "belongsTo") {
-                const foreignKey = columnAt(relation.localKey);
-                if (foreignKey) return foreignKey;
-            }
+        const declaredRelation = collection ? resolveCollectionRelations(collection)[orderBy] : undefined;
+        if (declaredRelation?.kind === "belongsTo") {
+            const foreignKey = columnAt(declaredRelation.localKey);
+            if (foreignKey) return foreignKey;
         }
 
         // No collection in hand — the two shapes an owning relation's key takes
@@ -145,7 +145,61 @@ export class FetchService {
             if (foreignKey) return foreignKey;
         }
 
-        return undefined;
+        // Nothing resolved. Returning `undefined` is exactly what the docblock
+        // above describes: the caller drops the ORDER BY and hands back rows in
+        // whatever order Postgres pleases, while the requester believes they
+        // are sorted. `?orderBy=titel` answered 200 with unsorted data and no
+        // hint that the sort had been ignored.
+        //
+        // A *filter* naming a field that does not exist is already refused for
+        // precisely this reason — it "used to widen results silently". An
+        // unresolvable sort field is the same drift between a query and the
+        // schema, so it answers the same way and honours the same switch: one
+        // knob, because a deployment that wants the lenient behaviour wants it
+        // for both.
+        const collectionName = collection?.slug ?? collection?.name;
+        const onCollection = collectionName ? ` on collection '${collectionName}'` : "";
+
+        // A declared to-many relation is a different mistake from a typo, and
+        // saying "unknown field" about a field the collection plainly declares
+        // sends the reader looking for a spelling error that is not there.
+        // There is simply no single value per row to order by — `posts.tags` is
+        // a set — so no ORDER BY exists to write, with or without a typo.
+        if (declaredRelation && declaredRelation.kind !== "belongsTo") {
+            throw ApiError.badRequest(
+                `Cannot sort by '${orderBy}'${onCollection}: it is a to-many relation ` +
+                `(${declaredRelation.kind}), which has no single value per row to order by.`,
+                "ORDER_BY_FIELD_NOT_SORTABLE",
+                { field: orderBy, kind: declaredRelation.kind, ...(collectionName && { collection: collectionName }) }
+            );
+        }
+
+        if (getUnknownFilterFieldsMode() === "warn") {
+            logger.warn(
+                `Sorting by field '${orderBy}'${onCollection}, but it does not exist in the table — ` +
+                "the ORDER BY was dropped and these rows are unsorted."
+            );
+            return undefined;
+        }
+
+        let validFields: string[] = [];
+        try {
+            validFields = Object.keys(getTableColumns(table)).sort();
+        } catch {
+            // A table stand-in without Drizzle's column symbols — the message
+            // is worth less without the list, but not worth failing over.
+        }
+
+        throw ApiError.badRequest(
+            `Unknown orderBy field '${orderBy}'${onCollection}` +
+            (validFields.length > 0 ? `. Valid fields: ${validFields.join(", ")}` : ""),
+            "UNKNOWN_ORDER_BY_FIELD",
+            {
+                field: orderBy,
+                ...(collectionName && { collection: collectionName }),
+                ...(validFields.length > 0 && { validFields })
+            }
+        );
     }
 
     /**
@@ -536,6 +590,11 @@ idColumn };
             throw new Error(`ID field '${idInfo.fieldName}' not found in table for collection '${collectionPath}'`);
         }
 
+        // An address the key columns cannot hold names no row — the same answer
+        // as a well-formed id nobody has. Asking Postgres instead raises 22P02
+        // and aborts the transaction around this read.
+        if (!idCanAddressTable(id, table, idInfoArray)) return undefined;
+
         const parsedIdObj = parseIdValues(id, idInfoArray);
         const parsedId = parsedIdObj[idInfo.fieldName];
 
@@ -555,7 +614,7 @@ idColumn };
 
                 if (!row) return undefined;
 
-                const flatRow = toCmsRow(row, collection, this.registry);
+                const flatRow = toFlatRow(row, collection, this.registry);
 
                 // Post-fetch joinPath relations that Drizzle's `with` can't express
                 await this.resolveJoinPathRelations<M>(flatRow, collection, collectionPath, parsedId, databaseId);
@@ -566,6 +625,7 @@ idColumn };
                     logger.error(`[FetchService] ResolvedRelation inference error for collection '${collectionPath}': ${e.message}`);
                     logger.error("Hint: This usually means a relation in your drizzle schema is missing a reciprocal 'one()' or 'many()' definition. Run 'rebase schema generate' to fix this.");
                 }
+                if (reachedDatabase(e)) throw e;
                 logger.warn(`[FetchService] db.query.findFirst failed for ${collectionPath}, falling back to db.select`, { error: e });
             }
         }
@@ -682,7 +742,7 @@ idColumn };
                 const results = await qb.findMany(queryOpts as Parameters<NonNullable<typeof qb>["findMany"]>[0]);
 
                 const rows = (results as Record<string, unknown>[]).map(row =>
-                    toCmsRow(row, collection, this.registry)
+                    toFlatRow(row, collection, this.registry)
                 );
 
                 return rows;
@@ -691,6 +751,7 @@ idColumn };
                     logger.error(`[FetchService] ResolvedRelation inference error for collection '${collectionPath}': ${e.message}`);
                     logger.error("Hint: This usually means a relation in your drizzle schema is missing a reciprocal 'one()' or 'many()' definition. Run 'rebase schema generate' to fix this.");
                 }
+                if (reachedDatabase(e)) throw e;
                 logger.warn(`[FetchService] db.query.findMany failed for ${collectionPath}, falling back to db.select`, { error: e });
             }
         }
@@ -785,7 +846,7 @@ _distance: vectorMeta.distanceSelect }).from(table).$dynamic()
     /**
      * Fallback path used when db.query is unavailable.
      *
-     * The primary path runs the results through `toCmsRow`, which maps
+     * The primary path runs the results through `toFlatRow`, which maps
      * relations from what drizzle already nested — no query per row. This one
      * has no nesting to read, so it resolves relations itself, in batches.
      *
@@ -942,6 +1003,7 @@ relatedTo: hop });
         collectionPath: string,
         options: {
             filter?: FilterValues<Extract<keyof M, string>>;
+            logical?: LogicalCondition;
             searchString?: string;
             databaseId?: string;
         } = {}
@@ -973,6 +1035,13 @@ relatedTo: hop });
             if (filterConditions.length > 0) allConditions.push(...filterConditions);
         }
 
+        if (options.logical) {
+            const logicalCondition = DrizzleConditionBuilder.buildLogicalConditions(
+                options.logical, table, effectivePath, this.filterContext(effectivePath, table)
+            );
+            if (logicalCondition) allConditions.push(logicalCondition);
+        }
+
         if (allConditions.length > 0) {
             const finalCondition = DrizzleConditionBuilder.combineConditionsWithAnd(allConditions);
             if (finalCondition) query = query.where(finalCondition);
@@ -1043,6 +1112,8 @@ relatedTo: hop });
         collectionPath: string,
         options: {
             filter?: FilterValues<Extract<keyof M, string>>;
+            /** An `or(...)`/`and(...)` group, applied alongside `filter`. */
+            logical?: LogicalCondition;
             orderBy?: string;
             order?: "desc" | "asc";
             limit?: number;
@@ -1111,6 +1182,7 @@ relatedTo: hop }, include
                     logger.error(`[FetchService] ResolvedRelation inference error for collection '${collectionPath}': ${e.message}`);
                     logger.error("Hint: This usually means a relation in your drizzle schema is missing a reciprocal 'one()' or 'many()' definition. Run 'rebase schema generate' to fix this.");
                 }
+                if (reachedDatabase(e)) throw e;
                 logger.warn(`[fetchCollectionForRest] db.query.findMany failed for ${collectionPath}, falling back`, { error: e });
             }
         }
@@ -1184,6 +1256,9 @@ relatedTo: hop }, include
         const idInfo = idInfoArray[0];
         const idField = table[idInfo.fieldName as keyof typeof table] as AnyPgColumn;
 
+        // See `fetchOne`: an unaddressable id is a 404, not a database error.
+        if (!idCanAddressTable(id, table, idInfoArray)) return null;
+
         const parsedIdObj = parseIdValues(id, idInfoArray);
         const parsedId = parsedIdObj[idInfo.fieldName];
 
@@ -1217,6 +1292,7 @@ relatedTo: hop }, include
                     logger.error(`[FetchService] ResolvedRelation inference error for collection '${collectionPath}': ${e.message}`);
                     logger.error("Hint: This usually means a relation in your drizzle schema is missing a reciprocal 'one()' or 'many()' definition. Run 'rebase schema generate' to fix this.");
                 }
+                if (reachedDatabase(e)) throw e;
                 logger.warn(`[fetchOneForRest] db.query.findFirst failed for ${collectionPath}, falling back`, { error: e });
             }
         }

package/src/services/PersistService.ts

@@ -1,7 +1,7 @@
 import { eq, and, sql, SQL } from "drizzle-orm";
 import { AnyPgColumn, PgTable } from "drizzle-orm/pg-core";
 // import { NodePgDatabase } from "drizzle-orm/node-postgres";
-import { CollectionConfig, Properties, ResolvedRelation, type ResolvedManyToMany, isManyToMany } from "@rebasepro/types";
+import { CollectionConfig, Properties, ResolvedRelation, type ResolvedManyToMany, isManyToMany, hasForeignKeyOnTarget } from "@rebasepro/types";
 import { getTableName, resolveCollectionRelations } from "@rebasepro/common";
 import { DrizzleConditionBuilder } from "../utils/drizzle-conditions";
 import {
@@ -240,16 +240,30 @@ export class PersistService {
                     throw ApiError.notFound(`No row "${id}" in "${collectionPath}" to update.`);
                 }
             } else {
-                // One-to-many create: stamp the parent's id onto the child's FK.
+                // One-to-many create: stamp the parent's key onto the child's FK.
                 const targetColumnName = this.resolveParentForeignKeyColumn(hop);
 
                 if (targetColumnName) {
-                    const parsedParentId = parentIdForWrite();
+                    // Not necessarily the id in the path: a link with a
+                    // `sourceKey` is pointed at a column on the parent row, and
+                    // stamping the id would create a child that belongs to
+                    // nobody — a row the read path would never return again.
+                    const parentKeyValue = hasForeignKeyOnTarget(hop.relation)
+                        ? await this.relationService.parentKeyValue(hop.parentCollection, hop.relation, hop.parentId)
+                        : parentIdForWrite();
+
+                    if (parentKeyValue === undefined) {
+                        throw ApiError.badRequest(
+                            `Cannot create under "${collectionPath}": the parent row has no value in ` +
+                            `\`sourceKey\`, so the new row has nothing to point at.`
+                        );
+                    }
+
                     const existingValue = (effectiveValues as Record<string, unknown>)[targetColumnName];
-                    if (existingValue !== undefined && existingValue !== null && existingValue !== parsedParentId) {
-                        logger.warn(`Overriding provided value '${existingValue}' for FK '${targetColumnName}' with path parent id '${parsedParentId}'.`);
+                    if (existingValue !== undefined && existingValue !== null && existingValue !== parentKeyValue) {
+                        logger.warn(`Overriding provided value '${existingValue}' for FK '${targetColumnName}' with path parent key '${parentKeyValue}'.`);
                     }
-                    (effectiveValues as Record<string, unknown>)[targetColumnName] = parsedParentId;
+                    (effectiveValues as Record<string, unknown>)[targetColumnName] = parentKeyValue;
                 }
             }
         }