@rebasepro/server-postgres 0.13.1-canary.gef9608c → 0.14.0

package/src/services/FetchService.ts

@@ -2,8 +2,8 @@ import { and, asc, count, desc, eq, getTableColumns, getTableName, gt, lt, or, S
 import { AnyPgColumn, PgTable } from "drizzle-orm/pg-core";
 import { CollectionConfig, FilterValues, ResolvedRelation, LogicalCondition, isManyToMany } from "@rebasepro/types";
 import type { VectorSearchParams } from "@rebasepro/types";
-import { resolveCollectionRelations, findRelation, createRelationRef, createRelationRefWithData } from "@rebasepro/common";
-import { generateForeignKeyName } from "@rebasepro/utils";
+import { resolveCollectionRelations, findRelation, fieldKeyForColumn, createRelationRef, createRelationRefWithData } from "@rebasepro/common";
+import { generateForeignKeyName, toWireKey } from "@rebasepro/utils";
 import { DrizzleConditionBuilder, getUnknownFilterFieldsMode, type FilterCompilationOptions } from "../utils/drizzle-conditions";
 import {
     getCollectionByPath,
@@ -22,6 +22,7 @@ import { RelationalQueryBuilder } from "drizzle-orm/pg-core/query-builders/query
 import { DrizzleClient } from "../interfaces";
 import { PostgresCollectionRegistry } from "../collections/PostgresCollectionRegistry";
 import { toFlatRow, toRestRow, isJunctionRelation } from "./row-pipeline";
+import { visibleColumnProjection, hiddenColumnsOption } from "../schema/search-column";
 import { isNestedPath, resolveNestedPath, type NestedPathHop } from "./nested-path";
 import { ApiError, logger } from "@rebasepro/server";
 import { reachedDatabase } from "../utils/pg-error-utils";
@@ -119,6 +120,31 @@ export class FetchService {
      * and skips rows rather than erroring. The guesses stay, last, for a
      * caller that hands over no collection to resolve against.
      */
+    /**
+     * The ORDER BY target, which may be relevance rather than a column.
+     *
+     * `_score` is only meaningful for a collection that declared a `search`
+     * block *and* for a request that carried a search string — ranking rows
+     * against no query ranks them all at zero. Outside those two conditions it
+     * is an unknown field and gets the same 400 as any other typo, which is the
+     * behaviour that matters: a sort that is silently dropped returns 200 with
+     * rows in arbitrary order, and paging over that repeats and skips rows.
+     */
+    static readonly SCORE_FIELD = "_score";
+
+    private resolveOrderTarget(
+        table: PgTable<any>,
+        orderBy: string,
+        collection?: CollectionConfig,
+        searchString?: string
+    ): AnyPgColumn | SQL | undefined {
+        if (orderBy === FetchService.SCORE_FIELD && collection && searchString) {
+            const rank = DrizzleConditionBuilder.buildSearchRankExpression(searchString, table, collection);
+            if (rank) return rank;
+        }
+        return this.resolveOrderByField(table, orderBy, collection);
+    }
+
     private resolveOrderByField(
         table: PgTable<any>,
         orderBy: string,
@@ -133,14 +159,21 @@ export class FetchService {
         // Owning relation, resolved: the relation names its own local key.
         const declaredRelation = collection ? resolveCollectionRelations(collection)[orderBy] : undefined;
         if (declaredRelation?.kind === "belongsTo") {
-            const foreignKey = columnAt(declaredRelation.localKey);
+            // `localKey` is the column; the table is keyed by the wire name.
+            const foreignKey = columnAt(fieldKeyForColumn(collection, declaredRelation.localKey));
             if (foreignKey) return foreignKey;
         }
 
-        // No collection in hand — the two shapes an owning relation's key takes
-        // by default (e.g. `project` → `project_id`, `userProfile` →
-        // `user_profile_id`).
-        for (const guess of [`${orderBy}_id`, generateForeignKeyName(orderBy)]) {
+        // No collection in hand — the shapes an owning relation's key takes by
+        // default (e.g. `project` → `projectId`, `userProfile` →
+        // `userProfileId`), then the snake forms for a project that authored the
+        // property under its column name.
+        for (const guess of [
+            `${orderBy}Id`,
+            toWireKey(generateForeignKeyName(orderBy)),
+            `${orderBy}_id`,
+            generateForeignKeyName(orderBy)
+        ]) {
             const foreignKey = columnAt(guess);
             if (foreignKey) return foreignKey;
         }
@@ -300,6 +333,14 @@ export class FetchService {
                         );
                     }
                 } catch (e) {
+                    // A relation that failed to load is not a relation that is absent.
+                    // Without this the request answers 200 with the field quietly
+                    // missing — and because a Postgres error poisons the surrounding
+                    // transaction, every later relation in the same request is
+                    // swallowed too, so one failure becomes a response missing
+                    // several fields. Same guard the four other catches in this file
+                    // already use.
+                    if (reachedDatabase(e)) throw e;
                     logger.warn(`Could not resolve joinPath relation '${key}'`, { error: e });
                 }
             });
@@ -378,6 +419,14 @@ export class FetchService {
                     }
                 }
             } catch (e) {
+                // A relation that failed to load is not a relation that is absent.
+                // Without this the request answers 200 with the field quietly
+                // missing — and because a Postgres error poisons the surrounding
+                // transaction, every later relation in the same request is
+                // swallowed too, so one failure becomes a response missing
+                // several fields. Same guard the four other catches in this file
+                // already use.
+                if (reachedDatabase(e)) throw e;
                 logger.warn(`Could not batch resolve joinPath relation '${key}' for REST`, { error: e });
             }
         }
@@ -407,6 +456,15 @@ export class FetchService {
     ): Record<string, unknown> {
         const queryOpts: Record<string, unknown> = {};
 
+        // Same exclusion the `db.select` fallback applies, in the shape the
+        // relational query builder takes. Both paths serve the same request, so
+        // a row must not carry the search column down one and not the other.
+        const hidden = hiddenColumnsOption(
+            getTableColumns(table),
+            this.registry.getCollectionByPath(collectionPath) ?? undefined
+        );
+        if (hidden) queryOpts.columns = hidden;
+
         if (withConfig) queryOpts.with = withConfig;
 
         // Build where conditions
@@ -417,7 +475,7 @@ export class FetchService {
         if (options.searchString) {
             const collection = getCollectionByPath(collectionPath, this.registry);
             const searchConditions = DrizzleConditionBuilder.buildSearchConditions(
-                options.searchString, collection.properties, table
+                options.searchString, collection.properties, table, collection
             );
             if (searchConditions.length === 0) {
                 // Return options that will produce empty results
@@ -451,7 +509,7 @@ export class FetchService {
         const orderExpressions: unknown[] = [];
         if (options.orderBy) {
             const collection = getCollectionByPath(collectionPath, this.registry);
-            const orderByField = this.resolveOrderByField(table, options.orderBy, collection);
+            const orderByField = this.resolveOrderTarget(table, options.orderBy, collection, options.searchString);
             if (orderByField) {
                 orderExpressions.push(options.order === "asc" ? asc(orderByField) : desc(orderByField));
             }
@@ -485,6 +543,21 @@ export class FetchService {
         const cursor = options.startAfter;
 
         if (options.orderBy) {
+            // Relevance is computed per query, not stored, so there is no value
+            // on the cursor row to compare a later page against — and two
+            // requests with different search strings would produce scores that
+            // are not on the same scale at all. Refusing is the only honest
+            // answer: a dropped cursor condition silently repeats and skips
+            // rows, which is precisely what paging exists to prevent.
+            if (options.orderBy === FetchService.SCORE_FIELD) {
+                throw ApiError.badRequest(
+                    "Cursor pagination (`startAfter`) cannot be combined with `orderBy: \"_score\"`. " +
+                    "Relevance is computed per query rather than stored, so it cannot key a cursor. " +
+                    "Use `limit`/`offset` for relevance-ordered pages, or order by a column.",
+                    "SCORE_CURSOR_UNSUPPORTED",
+                    { field: FetchService.SCORE_FIELD }
+                );
+            }
             const collection = collectionPath ? getCollectionByPath(collectionPath, this.registry) : undefined;
             const orderByField = this.resolveOrderByField(table, options.orderBy, collection);
             if (orderByField) {
@@ -607,9 +680,12 @@ idColumn };
             try {
                 const withConfig = this.buildWithConfig(collection);
 
+                const hidden = hiddenColumnsOption(getTableColumns(table), collection);
+
                 const row = await qb.findFirst({
                     where: eq(idField, parsedId),
-                    with: withConfig
+                    with: withConfig,
+                    ...(hidden ? { columns: hidden } : {})
                 } as Parameters<NonNullable<typeof qb>["findFirst"]>[0]);
 
                 if (!row) return undefined;
@@ -631,8 +707,9 @@ idColumn };
         }
 
         // Fallback: db.select + N+1 relation loading
+        const visibleOne = visibleColumnProjection(getTableColumns(table), collection);
         const result = await this.db
-            .select()
+            .select(visibleOne as never)
             .from(table)
             .where(eq(idField, parsedId))
             .limit(1);
@@ -673,6 +750,14 @@ idColumn };
                                 values[key] = createRelationRef(e.id, e.path);
                             }
                         } catch (e) {
+                            // A relation that failed to load is not a relation that is absent.
+                            // Without this the request answers 200 with the field quietly
+                            // missing — and because a Postgres error poisons the surrounding
+                            // transaction, every later relation in the same request is
+                            // swallowed too, so one failure becomes a response missing
+                            // several fields. Same guard the four other catches in this file
+                            // already use.
+                            if (reachedDatabase(e)) throw e;
                             logger.warn(`Could not resolve one-to-one relation property: ${key}`, { error: e });
                         }
                     }
@@ -700,6 +785,7 @@ idColumn };
             offset?: number;
             startAfter?: Record<string, unknown>;
             searchString?: string;
+            searchExplain?: boolean;
             databaseId?: string;
             vectorSearch?: VectorSearchParams;
             logical?: LogicalCondition;
@@ -763,17 +849,41 @@ idColumn };
             vectorMeta = DrizzleConditionBuilder.buildVectorSearchConditions(table, options.vectorSearch);
         }
 
+        // A generated search column is an index in column form; `SELECT *`
+        // would ship it to every caller. The projection is undefined — and the
+        // SQL therefore unchanged — for any table without one.
+        const visible = visibleColumnProjection(getTableColumns(table), collection);
+
+        // Relevance, alongside the row, exactly as `_distance` rides along with
+        // a vector search. Present only when the collection opted in and the
+        // request carried a search string, so a caller can order by it, show
+        // it, or blend it with a score of their own.
+        const rankSelect = options.searchString
+            ? DrizzleConditionBuilder.buildSearchRankExpression(options.searchString, table, collection)
+            : undefined;
+
+        // Only when asked: a `ts_headline` per declared field per row.
+        const matchesSelect = options.searchString && options.searchExplain
+            ? DrizzleConditionBuilder.buildSearchMatchesExpression(options.searchString, table, collection)
+            : undefined;
+
         let query = vectorMeta
-            ? this.db.select({ table_row: table,
+            ? this.db.select({ table_row: (visible ?? table) as never,
 _distance: vectorMeta.distanceSelect }).from(table).$dynamic()
-            : this.db.select().from(table).$dynamic();
+            : rankSelect
+                ? this.db.select({
+                    table_row: (visible ?? table) as never,
+                    _score: rankSelect,
+                    ...(matchesSelect ? { _matches: matchesSelect } : {})
+                }).from(table).$dynamic()
+                : (visible ? this.db.select(visible as never).from(table).$dynamic() : this.db.select().from(table).$dynamic());
         const allConditions: SQL[] = [];
 
         if (scopeCondition) allConditions.push(scopeCondition);
 
         if (options.searchString) {
             const searchConditions = DrizzleConditionBuilder.buildSearchConditions(
-                options.searchString, collection.properties, table
+                options.searchString, collection.properties, table, collection
             );
             if (searchConditions.length === 0) return [];
             allConditions.push(DrizzleConditionBuilder.combineConditionsWithOr(searchConditions)!);
@@ -804,7 +914,7 @@ _distance: vectorMeta.distanceSelect }).from(table).$dynamic()
         if (vectorMeta) {
             orderExpressions.push(asc(vectorMeta.orderBy));
         } else if (options.orderBy) {
-            const orderByField = this.resolveOrderByField(table, options.orderBy, collection);
+            const orderByField = this.resolveOrderTarget(table, options.orderBy, collection, options.searchString);
             if (orderByField) {
                 orderExpressions.push(options.order === "asc" ? asc(orderByField) : desc(orderByField));
             }
@@ -838,7 +948,15 @@ _distance: vectorMeta.distanceSelect }).from(table).$dynamic()
                 ...r.table_row,
                 _distance: typeof r._distance === "number" ? r._distance : parseFloat(String(r._distance))
             }))
-            : rawResults as Record<string, unknown>[];
+            // Same nested shape, unwrapped the same way, when a relevance
+            // score was selected instead.
+            : rankSelect
+                ? (rawResults as { table_row: Record<string, unknown>; _score: unknown; _matches?: unknown }[]).map(r => ({
+                    ...r.table_row,
+                    _score: typeof r._score === "number" ? r._score : parseFloat(String(r._score)),
+                    ...(matchesSelect ? { _matches: r._matches ?? [] } : {})
+                }))
+                : rawResults as Record<string, unknown>[];
 
         return this.processRowResults<M>(results, collection, collectionPath, idInfo, options.databaseId, false, idInfoArray);
     }
@@ -911,6 +1029,14 @@ _distance: vectorMeta.distanceSelect }).from(table).$dynamic()
                         }
                     });
                 } catch (e) {
+                    // A relation that failed to load is not a relation that is absent.
+                    // Without this the request answers 200 with the field quietly
+                    // missing — and because a Postgres error poisons the surrounding
+                    // transaction, every later relation in the same request is
+                    // swallowed too, so one failure becomes a response missing
+                    // several fields. Same guard the four other catches in this file
+                    // already use.
+                    if (reachedDatabase(e)) throw e;
                     logger.warn(`Could not batch load one-to-one relation property: ${key}`, { error: e });
                 }
             }
@@ -938,6 +1064,14 @@ _distance: vectorMeta.distanceSelect }).from(table).$dynamic()
                         );
                     });
                 } catch (e) {
+                    // A relation that failed to load is not a relation that is absent.
+                    // Without this the request answers 200 with the field quietly
+                    // missing — and because a Postgres error poisons the surrounding
+                    // transaction, every later relation in the same request is
+                    // swallowed too, so one failure becomes a response missing
+                    // several fields. Same guard the four other catches in this file
+                    // already use.
+                    if (reachedDatabase(e)) throw e;
                     logger.warn(`Could not batch load many relation property: ${key}`, { error: e });
                 }
             }
@@ -994,10 +1128,21 @@ relatedTo: hop });
         searchString: string,
         options: {
             filter?: FilterValues<Extract<keyof M, string>>;
+            /**
+             * An `or(...)`/`and(...)` group, applied alongside `filter`.
+             *
+             * `fetchRowsWithConditions` has always applied one; it was missing
+             * from this signature, so a realtime search subscription carrying a
+             * group could not pass it on and served every row matching the text
+             * that RLS allowed.
+             */
+            logical?: LogicalCondition;
             orderBy?: string;
             order?: "desc" | "asc";
             limit?: number;
             databaseId?: string;
+            /** Ask each row which declared search field matched. */
+            searchExplain?: boolean;
         } = {}
     ): Promise<Record<string, unknown>[]> {
         return this.fetchRowsWithConditions<M>(collectionPath, {
@@ -1034,7 +1179,7 @@ relatedTo: hop });
 
         if (options.searchString) {
             const searchConditions = DrizzleConditionBuilder.buildSearchConditions(
-                options.searchString, collection.properties, table
+                options.searchString, collection.properties, table, collection
             );
             if (searchConditions.length === 0) return 0;
             allConditions.push(DrizzleConditionBuilder.combineConditionsWithOr(searchConditions)!);
@@ -1226,6 +1371,14 @@ relatedTo: hop }, include
                     }
                 }
             } catch (e) {
+                // A relation that failed to load is not a relation that is absent.
+                // Without this the request answers 200 with the field quietly
+                // missing — and because a Postgres error poisons the surrounding
+                // transaction, every later relation in the same request is
+                // swallowed too, so one failure becomes a response missing
+                // several fields. Same guard the four other catches in this file
+                // already use.
+                if (reachedDatabase(e)) throw e;
                 logger.warn(`[include] Failed to batch load one-to-one '${key}'`, { error: e });
             }
         }
@@ -1242,6 +1395,14 @@ relatedTo: hop }, include
                     (row as Record<string, unknown>)[key] = relatedList;
                 }
             } catch (e) {
+                // A relation that failed to load is not a relation that is absent.
+                // Without this the request answers 200 with the field quietly
+                // missing — and because a Postgres error poisons the surrounding
+                // transaction, every later relation in the same request is
+                // swallowed too, so one failure becomes a response missing
+                // several fields. Same guard the four other catches in this file
+                // already use.
+                if (reachedDatabase(e)) throw e;
                 logger.warn(`[include] Failed to batch load many '${key}'`, { error: e });
             }
         }
@@ -1308,8 +1469,9 @@ relatedTo: hop }, include
         }
 
         // Fallback: db.select + N+1 relation loading
+        const visibleOne = visibleColumnProjection(getTableColumns(table), collection);
         const result = await this.db
-            .select()
+            .select(visibleOne as never)
             .from(table)
             .where(eq(idField, parsedId))
             .limit(1);
@@ -1349,6 +1511,14 @@ relatedTo: hop }, include
                     }));
                 }
             } catch (e) {
+                // A relation that failed to load is not a relation that is absent.
+                // Without this the request answers 200 with the field quietly
+                // missing — and because a Postgres error poisons the surrounding
+                // transaction, every later relation in the same request is
+                // swallowed too, so one failure becomes a response missing
+                // several fields. Same guard the four other catches in this file
+                // already use.
+                if (reachedDatabase(e)) throw e;
                 logger.warn(`[include] Failed to load relation '${key}'`, { error: e });
             }
         }
@@ -1363,12 +1533,26 @@ relatedTo: hop }, include
         collectionPath: string,
         options: {
             filter?: FilterValues<Extract<keyof M, string>>;
+            /**
+             * An `or(...)`/`and(...)` group, applied alongside `filter`.
+             *
+             * Declared *and applied*, because this is the path every REST search
+             * and vector read takes: `fetchCollectionForRest` skips `db.query`
+             * whenever a `searchString` or a `vectorSearch` is present. The
+             * group arrived here on `options` from the very beginning and was
+             * simply never read, so `?searchString=x&or=(...)` served every row
+             * matching `x` that RLS allowed — while `count` (which does apply
+             * it) reported the narrowed total, so `meta.total` and `data`
+             * described different sets of rows.
+             */
+            logical?: LogicalCondition;
             orderBy?: string;
             order?: "desc" | "asc";
             limit?: number;
             offset?: number;
             startAfter?: Record<string, unknown>;
             searchString?: string;
+            searchExplain?: boolean;
             vectorSearch?: VectorSearchParams;
             relatedTo?: NestedPathHop;
         } = {}
@@ -1384,17 +1568,41 @@ relatedTo: hop }, include
             vectorMeta = DrizzleConditionBuilder.buildVectorSearchConditions(table, options.vectorSearch);
         }
 
+        // A generated search column is an index in column form; `SELECT *`
+        // would ship it to every caller. The projection is undefined — and the
+        // SQL therefore unchanged — for any table without one.
+        const visible = visibleColumnProjection(getTableColumns(table), collection);
+
+        // Relevance, alongside the row, exactly as `_distance` rides along with
+        // a vector search. Present only when the collection opted in and the
+        // request carried a search string, so a caller can order by it, show
+        // it, or blend it with a score of their own.
+        const rankSelect = options.searchString
+            ? DrizzleConditionBuilder.buildSearchRankExpression(options.searchString, table, collection)
+            : undefined;
+
+        // Only when asked: a `ts_headline` per declared field per row.
+        const matchesSelect = options.searchString && options.searchExplain
+            ? DrizzleConditionBuilder.buildSearchMatchesExpression(options.searchString, table, collection)
+            : undefined;
+
         let query = vectorMeta
-            ? this.db.select({ table_row: table,
+            ? this.db.select({ table_row: (visible ?? table) as never,
 _distance: vectorMeta.distanceSelect }).from(table).$dynamic()
-            : this.db.select().from(table).$dynamic();
+            : rankSelect
+                ? this.db.select({
+                    table_row: (visible ?? table) as never,
+                    _score: rankSelect,
+                    ...(matchesSelect ? { _matches: matchesSelect } : {})
+                }).from(table).$dynamic()
+                : (visible ? this.db.select(visible as never).from(table).$dynamic() : this.db.select().from(table).$dynamic());
         const allConditions: SQL[] = [];
 
         if (options.relatedTo) allConditions.push(this.buildRelationScope(options.relatedTo));
 
         if (options.searchString) {
             const searchConditions = DrizzleConditionBuilder.buildSearchConditions(
-                options.searchString, collection.properties, table
+                options.searchString, collection.properties, table, collection
             );
             if (searchConditions.length === 0) return [];
             allConditions.push(DrizzleConditionBuilder.combineConditionsWithOr(searchConditions)!);
@@ -1405,6 +1613,11 @@ _distance: vectorMeta.distanceSelect }).from(table).$dynamic()
             if (filterConditions.length > 0) allConditions.push(...filterConditions);
         }
 
+        if (options.logical) {
+            const logicalCondition = DrizzleConditionBuilder.buildLogicalConditions(options.logical, table, collectionPath, this.filterContext(collectionPath, table));
+            if (logicalCondition) allConditions.push(logicalCondition);
+        }
+
         if (vectorMeta?.filter) {
             allConditions.push(vectorMeta.filter);
         }
@@ -1418,7 +1631,7 @@ _distance: vectorMeta.distanceSelect }).from(table).$dynamic()
         if (vectorMeta) {
             orderExpressions.push(asc(vectorMeta.orderBy));
         } else if (options.orderBy) {
-            const orderByField = this.resolveOrderByField(table, options.orderBy, collection);
+            const orderByField = this.resolveOrderTarget(table, options.orderBy, collection, options.searchString);
             if (orderByField) {
                 orderExpressions.push(options.order === "asc" ? asc(orderByField) : desc(orderByField));
             }
@@ -1443,6 +1656,14 @@ _distance: vectorMeta.distanceSelect }).from(table).$dynamic()
             }));
         }
 
+        if (rankSelect) {
+            return (rawResults as { table_row: Record<string, unknown>; _score: unknown; _matches?: unknown }[]).map(r => ({
+                ...r.table_row,
+                _score: typeof r._score === "number" ? r._score : parseFloat(String(r._score)),
+                ...(matchesSelect ? { _matches: r._matches ?? [] } : {})
+            }));
+        }
+
         return rawResults as Record<string, unknown>[];
     }