@rebasepro/server-postgres 0.13.0 → 0.13.1-canary.g249daa1

package/src/services/channel-presence.ts

@@ -28,6 +28,7 @@
 
 import { sql } from "drizzle-orm";
 import { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { revokeInternalTableSql } from "@rebasepro/common";
 
 /** A tracked client, as any instance sees it. */
 export interface PresenceRow {
@@ -71,6 +72,11 @@ export class ChannelPresenceStore {
             ON rebase.channel_presence (last_seen)
         `);
 
+        // The roster of every client on every channel, with no RLS — presence
+        // authorization is a channel rule, not a row policy. Revoke the
+        // schema-wide grant the driver handed out before this table existed.
+        await this.db.execute(sql.raw(revokeInternalTableSql("rebase", "channel_presence")));
+
         this.tablesReady = true;
     }
 

package/src/services/dataService.ts

@@ -60,6 +60,8 @@ export class DataService implements DataRepository {
         collectionPath: string,
         options: {
             filter?: FilterValues<Extract<keyof M, string>>;
+            /** An `or(...)`/`and(...)` group, applied alongside `filter`. */
+            logical?: LogicalCondition;
             orderBy?: string;
             order?: "desc" | "asc";
             limit?: number;
@@ -85,6 +87,7 @@ export class DataService implements DataRepository {
             order?: "desc" | "asc";
             limit?: number;
             databaseId?: string;
+            searchExplain?: boolean;
         } = {}
     ): Promise<Record<string, unknown>[]> {
         return this.fetchService.searchRows<M>(collectionPath, searchString, options);

package/src/services/realtimeService.ts

@@ -4,7 +4,7 @@ import { Client as PgClient } from "pg";
 import { randomUUID } from "crypto";
 import { DataService } from "./dataService";
 
-import { ANONYMOUS_USER_ID, FetchCollectionProps, ListenCollectionProps, ListenOneProps, DataDriver, CollectionUpdateMessage, SingleUpdateMessage, CollectionPatchMessage, WebSocketMessage, FilterValues, CollectionConfig, RebaseCallContext, resolveClientListLimit } from "@rebasepro/types";
+import { ANONYMOUS_USER_ID, FetchCollectionProps, ListenCollectionProps, ListenOneProps, DataDriver, CollectionUpdateMessage, SingleUpdateMessage, CollectionPatchMessage, WebSocketMessage, FilterValues, LogicalCondition, CollectionConfig, RebaseCallContext, resolveClientListLimit } from "@rebasepro/types";
 import { NodePgDatabase } from "drizzle-orm/node-postgres";
 import { sql as drizzleSql } from "drizzle-orm";
 import { RealtimeProvider, CollectionSubscriptionConfig, SingleSubscriptionConfig } from "../interfaces";
@@ -41,6 +41,30 @@ type RealTimeListenCollectionProps = ListenCollectionProps & {
     subscriptionId: string
 };
 
+/**
+ * The narrowing a collection subscription was created with, kept so that every
+ * refetch answers the same query the initial fetch did.
+ *
+ * Named once because it used to be written out inline in five places, and a
+ * field missing from one of them is accepted over the wire and then silently
+ * ignored: `offset` was declared on the incoming props and never stored, so a
+ * live list on page three served page one, and `logical` was never stored
+ * either, so an `or(...)` subscription was pushed every row in the table.
+ */
+type StoredCollectionRequest = {
+    filter?: Record<string, unknown>;
+    logical?: LogicalCondition;
+    orderBy?: string;
+    order?: "desc" | "asc";
+    limit?: number;
+    offset?: number;
+    startAfter?: Record<string, unknown>;
+    databaseId?: string;
+    searchString?: string;
+    /** Ask each row which declared search field matched — populates `_matches`. */
+    searchExplain?: boolean;
+};
+
 type RealTimeListenEntityProps = ListenOneProps & { subscriptionId: string };
 
 /**
@@ -119,16 +143,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
         path: string;
         id?: string | number;
         // Store full collection request parameters for proper refetching
-        collectionRequest?: {
-            filter?: Record<string, unknown>;
-            orderBy?: string;
-            order?: "desc" | "asc";
-            limit?: number;
-            offset?: number;
-            startAfter?: Record<string, unknown>;
-            databaseId?: string;
-            searchString?: string;
-        };
+        collectionRequest?: StoredCollectionRequest;
         // Auth context for RLS — when set, refetches run in a transaction
         // with set_config('app.uid', ...) / set_config('app.user_roles', ...)
         authContext?: SubscriptionAuthContext;
@@ -212,16 +227,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
         type: "collection" | "single";
         path: string;
         id?: string | number;
-        collectionRequest?: {
-            filter?: Record<string, unknown>;
-            orderBy?: string;
-            order?: "desc" | "asc";
-            limit?: number;
-            offset?: number;
-            startAfter?: Record<string, unknown>;
-            databaseId?: string;
-            searchString?: string;
-        };
+        collectionRequest?: StoredCollectionRequest;
         authContext?: SubscriptionAuthContext;
     }) {
         this.debugLog("📋 [RealtimeService] Registering DataDriver subscription:", subscriptionId, subscription.authContext ? "(with auth)" : "(no auth)");
@@ -262,7 +268,8 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                 limit: config.limit,
                 startAfter: config.startAfter as Record<string, unknown> | undefined,
                 databaseId: config.databaseId,
-                searchString: config.searchString
+                searchString: config.searchString,
+                searchExplain: config.searchExplain
             }
         });
 
@@ -448,27 +455,25 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                 path: request.path,
                 collectionRequest: {
                     filter: request.filter,
+                    logical: request.logical,
                     orderBy: request.orderBy,
                     order: request.order,
                     limit: boundedLimit,
+                    offset: request.offset,
                     startAfter: request.startAfter as Record<string, unknown> | undefined,
                     databaseId: request.collection?.databaseId,
-                    searchString: request.searchString
+                    searchString: request.searchString,
+                    searchExplain: request.searchExplain
                 },
                 authContext
             });
 
-            // Send initial data
+            // Send initial data. Built from the request the subscription just
+            // stored, so the first answer and every refetch after it cannot
+            // describe different queries.
             const rows = await this.fetchCollectionWithAuth(
                 request.path,
-                {
-                    filter: request.filter,
-                    orderBy: request.orderBy,
-                    order: request.order,
-                    limit: boundedLimit,
-                    startAfter: request.startAfter as Record<string, unknown> | undefined,
-                    searchString: request.searchString
-                },
+                this._subscriptions.get(subscriptionId)!.collectionRequest!,
                 authContext
             );
 
@@ -679,7 +684,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
     private debouncedCollectionRefetch(
         subscriptionId: string,
         notifyPath: string,
-        subscription: { clientId: string; collectionRequest?: { filter?: Record<string, unknown>; orderBy?: string; order?: "desc" | "asc"; limit?: number; offset?: number; startAfter?: Record<string, unknown>; databaseId?: string; searchString?: string }; authContext?: SubscriptionAuthContext }
+        subscription: { clientId: string; collectionRequest?: StoredCollectionRequest; authContext?: SubscriptionAuthContext }
     ) {
         const timerKey = `ws_${subscriptionId}`;
         const existing = this.refetchTimers.get(timerKey);
@@ -705,7 +710,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
     private debouncedDriverRefetch(
         subscriptionId: string,
         notifyPath: string,
-        subscription: { collectionRequest?: { filter?: Record<string, unknown>; orderBy?: string; order?: "desc" | "asc"; limit?: number; offset?: number; startAfter?: Record<string, unknown>; databaseId?: string; searchString?: string }; authContext?: SubscriptionAuthContext },
+        subscription: { collectionRequest?: StoredCollectionRequest; authContext?: SubscriptionAuthContext },
         callback: (data: Record<string, unknown>[] | Record<string, unknown> | null) => void
     ) {
         const timerKey = `drv_${subscriptionId}`;
@@ -731,7 +736,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
      */
     private async fetchCollectionWithAuth(
         notifyPath: string,
-        collectionRequest: { filter?: Record<string, unknown>; orderBy?: string; order?: "desc" | "asc"; limit?: number; offset?: number; startAfter?: Record<string, unknown>; databaseId?: string; searchString?: string },
+        collectionRequest: StoredCollectionRequest,
         authContext?: SubscriptionAuthContext
     ): Promise<Record<string, unknown>[]> {
         if (this.driver) {
@@ -740,12 +745,14 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
                 path: notifyPath,
                 collection: collection,
                 filter: collectionRequest.filter as FetchCollectionProps["filter"],
+                logical: collectionRequest.logical,
                 orderBy: collectionRequest.orderBy,
                 order: collectionRequest.order,
                 limit: collectionRequest.limit,
                 offset: collectionRequest.offset,
                 startAfter: collectionRequest.startAfter,
-                searchString: collectionRequest.searchString
+                searchString: collectionRequest.searchString,
+                searchExplain: collectionRequest.searchExplain
             });
 
             // Always wrap in a transaction with session vars, defaulting to anonymous context if missing.
@@ -766,12 +773,14 @@ roles: ["anon"] };
                             orderBy: collectionRequest.orderBy,
                             order: collectionRequest.order,
                             limit: collectionRequest.limit,
-                            databaseId: collectionRequest.databaseId
+                            databaseId: collectionRequest.databaseId,
+                            searchExplain: collectionRequest.searchExplain
                         }
                     );
                 } else {
                     fetchedEntities = await txEntityService.fetchCollection(notifyPath, {
                         filter: collectionRequest.filter as FilterValues<string>,
+                        logical: collectionRequest.logical,
                         orderBy: collectionRequest.orderBy,
                         order: collectionRequest.order,
                         limit: collectionRequest.limit,

package/src/utils/drizzle-conditions.ts

@@ -9,6 +9,13 @@ import {
     getColumnName, getTableName, normalizeToEntityRelation, resolveCollectionRelations
 } from "@rebasepro/common";
 import { generateForeignKeyName } from "@rebasepro/utils";
+/**
+ * Postgres's own default for `pg_trgm.word_similarity_threshold`. Named here
+ * because the fuzzy predicate has to know when the index-backed operator agrees
+ * with the collection's declared threshold and when it would narrow too far.
+ */
+const PG_TRGM_WORD_SIMILARITY_DEFAULT = 0.6;
+import { buildSearchColumnSpec, SEARCH_UNACCENT_FN, type SearchColumnSpec } from "../schema/search-column";
 import { PostgresCollectionRegistry } from "../collections/PostgresCollectionRegistry";
 import { ConditionBuilderStatic } from "../interfaces";
 import { ApiError, logger } from "@rebasepro/server";
@@ -1275,15 +1282,37 @@ whereConditions };
     }
 
     /**
-     * Build search conditions for text fields
+     * Build search conditions for text fields.
+     *
+     * Two shapes, chosen by whether the collection declared a `search` block:
+     *
+     * - **Declared** — one `@@ websearch_to_tsquery` against the generated
+     *   `tsvector` column. Stems, drops stopwords, AND-es the terms, reaches
+     *   inside JSONB and arrays, and uses the GIN index.
+     * - **Not declared** — the original `ILIKE '%term%'` OR-ed across top-level
+     *   string properties, unchanged.
+     *
+     * The second is the default and stays the default. A collection that has
+     * not opted in compiles to exactly the SQL it compiled to before this
+     * branch existed, which is the only reason it is safe to have added it.
+     *
+     * `collection` is optional so that the callers which genuinely have no
+     * collection in hand — nested paths, derived views — keep working; without
+     * one there is no `search` block to read and the ILIKE path is correct.
      */
     static buildSearchConditions(
         searchString: string,
         properties: Record<string, unknown>,
-        table: PgTable<any>
+        table: PgTable<any>,
+        collection?: CollectionConfig
     ): SQL[] {
         const searchConditions: SQL[] = [];
 
+        const ftsCondition = collection
+            ? DrizzleConditionBuilder.buildFullTextCondition(searchString, table, collection)
+            : undefined;
+        if (ftsCondition) return [ftsCondition];
+
         for (const [key, prop] of Object.entries(properties)) {
             const p = prop as Record<string, unknown>;
             // Only include string properties that don't have enum defined
@@ -1307,6 +1336,198 @@ whereConditions };
         return searchConditions;
     }
 
+    /**
+     * The `@@` predicate for a collection that declared a `search` block, or
+     * undefined for one that did not.
+     *
+     * The query is normalized exactly as the indexed content was — same text
+     * search configuration, same accent folding. Skipping that on the query
+     * side is the subtle way to get a search that matches nothing: the column
+     * would hold `gestion` while the query asked for `gestión`.
+     *
+     * `websearch_to_tsquery` rather than `plainto_tsquery` because it is the
+     * one that behaves the way a search box looks like it should — quoted
+     * phrases, `or`, and a leading `-` to exclude — and because it never throws
+     * on user input, which `to_tsquery` does on so much as a stray parenthesis.
+     */
+    static buildFullTextCondition(
+        searchString: string,
+        table: PgTable<any>,
+        collection: CollectionConfig
+    ): SQL | undefined {
+        let spec: SearchColumnSpec | undefined;
+        try {
+            spec = buildSearchColumnSpec(collection);
+        } catch {
+            // Reported at boot. Falling back to ILIKE here keeps reads serving.
+            return undefined;
+        }
+        if (!spec) return undefined;
+
+        const column = table[spec.column as keyof typeof table] as AnyPgColumn | undefined;
+        if (!column) {
+            // The block is declared but the column is not on the table yet —
+            // a database that has not been migrated. ILIKE still answers.
+            return undefined;
+        }
+
+        const query = DrizzleConditionBuilder.normalizedTsQuery(searchString, spec);
+        const exact = sql`${column} @@ ${query}`;
+
+        if (!spec.fuzzy) return exact;
+
+        const fuzzyColumn = table[spec.fuzzy.column as keyof typeof table] as AnyPgColumn | undefined;
+        if (!fuzzyColumn) return exact;
+
+        const needle = spec.unaccent
+            ? sql`${sql.raw(SEARCH_UNACCENT_FN)}(${searchString})`
+            : sql`${searchString}`;
+
+        // `word_similarity(query, document)`, not `similarity`. `similarity`
+        // scores two strings as wholes, so a short query against a whole row's
+        // text scores near zero however well it matches part of it — measured:
+        // "iso 14001 auditor" against one candidate's concatenated fields
+        // scores 0.228 by `similarity` and 0.783 by `word_similarity`. The
+        // first is below any usable threshold, which would have made `fuzzy`
+        // a setting that quietly did nothing.
+        //
+        // Argument order matters: the first operand is the needle, and the
+        // score is its similarity to the best-matching extent of the second.
+        //
+        // Both the function and the operator are schema-qualified: pg_trgm is
+        // installed into `public`, and an unqualified reference resolves
+        // through `search_path`, which does not necessarily reach it.
+        const similar = sql`public.word_similarity(${needle}, ${fuzzyColumn}) >= ${spec.fuzzy.threshold}`;
+        // `<%` is the index-backed form, but it tests against the session's
+        // `pg_trgm.word_similarity_threshold` (0.6), not ours. Above that
+        // default the operator narrows using the trigram index and the explicit
+        // score refines; at or below it, the operator would exclude rows the
+        // declared threshold admits, so the score stands alone and the planner
+        // scans — correct either way, and only the faster path is conditional.
+        const fuzzy = spec.fuzzy.threshold > PG_TRGM_WORD_SIMILARITY_DEFAULT
+            ? sql`(${needle} OPERATOR(public.<%) ${fuzzyColumn} AND ${similar})`
+            : similar;
+
+        return sql`(${exact} OR ${fuzzy})`;
+    }
+
+    /**
+     * `websearch_to_tsquery(<config>, <normalized search string>)`.
+     *
+     * Split out because the ranking expression needs the identical query — a
+     * row ranked against a different tsquery than it was matched against is a
+     * ranking of something else.
+     */
+    static normalizedTsQuery(searchString: string, spec: SearchColumnSpec): SQL {
+        const normalized = spec.unaccent
+            ? sql`${sql.raw(SEARCH_UNACCENT_FN)}(${searchString})`
+            : sql`${searchString}`;
+        return sql`websearch_to_tsquery(${spec.language}, ${normalized})`;
+    }
+
+    /**
+     * A JSONB array of `{ field, snippet }` naming which declared fields matched
+     * and showing the text around each hit — what backs `_matches`.
+     *
+     * A ranked list answers "which rows", never "why this row". For a talent
+     * pool that difference is the product: a candidate surfacing for
+     * "iso 14001" on a *certification* is a different candidate from one whose
+     * bio happens to mention the standard, and the score cannot tell them apart.
+     *
+     * Built as a correlated subquery over a `VALUES` list of the declared
+     * fields, rather than one `CASE` per field, so the shape does not change
+     * with the number of fields and the empty result is a plain `[]`.
+     *
+     * `ts_headline` runs over the same normalized text that was indexed. Over
+     * the *original* text it would find nothing to mark whenever `unaccent` is
+     * on — the query's lexemes are folded and the document's are not — and
+     * would return the text silently unhighlighted. Folded-but-marked beats
+     * pretty-but-inert.
+     *
+     * Undefined when the collection has not opted in, when the column is not on
+     * the table yet, or when the caller did not ask: this costs a `ts_headline`
+     * per field per row and `ts_headline` re-parses the document.
+     */
+    static buildSearchMatchesExpression(
+        searchString: string,
+        table: PgTable<any>,
+        collection: CollectionConfig
+    ): SQL | undefined {
+        let spec: SearchColumnSpec | undefined;
+        try {
+            spec = buildSearchColumnSpec(collection);
+        } catch {
+            return undefined;
+        }
+        if (!spec || spec.fields.length === 0) return undefined;
+        if (!table[spec.column as keyof typeof table]) return undefined;
+
+        const query = DrizzleConditionBuilder.normalizedTsQuery(searchString, spec);
+        const config = sql`${spec.language}`;
+
+        // `ord` keeps the author's declared field order in the output, so the
+        // most important field they named reads first rather than whichever
+        // Postgres aggregated first.
+        const rows = spec.fields.map((f, i) =>
+            sql`(${i}, ${f.path}, ${sql.raw(f.textSql)})`
+        );
+
+        return sql`(
+            SELECT coalesce(jsonb_agg(s.m ORDER BY f.ord), '[]'::jsonb)
+            FROM (VALUES ${sql.join(rows, sql`, `)}) AS f(ord, path, txt)
+            CROSS JOIN LATERAL (
+                SELECT jsonb_build_object(
+                    'field', f.path,
+                    'snippet', ts_headline(${config}::regconfig, f.txt, ${query},
+                        'StartSel=<mark>,StopSel=</mark>,MaxWords=14,MinWords=1,MaxFragments=1,FragmentDelimiter= … ')
+                ) AS m
+                WHERE to_tsvector(${config}::regconfig, f.txt) @@ ${query}
+            ) s
+        )`;
+    }
+
+    /**
+     * `ts_rank(<column>, <query>)` for the collection, or undefined when it has
+     * not opted in. This is what backs `orderBy: ["_score", "desc"]`.
+     */
+    static buildSearchRankExpression(
+        searchString: string,
+        table: PgTable<any>,
+        collection: CollectionConfig
+    ): SQL | undefined {
+        let spec: SearchColumnSpec | undefined;
+        try {
+            spec = buildSearchColumnSpec(collection);
+        } catch {
+            return undefined;
+        }
+        if (!spec) return undefined;
+        const column = table[spec.column as keyof typeof table] as AnyPgColumn | undefined;
+        if (!column) return undefined;
+
+        const rank = sql`ts_rank(${column}, ${DrizzleConditionBuilder.normalizedTsQuery(searchString, spec)})`;
+        if (!spec.fuzzy) return rank;
+
+        const fuzzyColumn = table[spec.fuzzy.column as keyof typeof table] as AnyPgColumn | undefined;
+        if (!fuzzyColumn) return rank;
+
+        // With `fuzzy` on, `ts_rank` alone is not a ranking — it is zero for
+        // every row the trigram path matched and the exact path did not, which
+        // is the whole population of a typo'd query. Measured on the real
+        // sustentalent pool: "auditor de iso14000" matches four candidates,
+        // every one of them at ts_rank 0, so ordering by rank alone returned
+        // the best match in whatever order the table felt like.
+        //
+        // Summed rather than blended with tuned constants: a row that matched
+        // exactly contributes both terms, so it outranks a fuzzy-only row of
+        // equal similarity without needing a coefficient to say so. Both terms
+        // are non-negative and monotonic, which is all the ordering needs.
+        const needle = spec.unaccent
+            ? sql`${sql.raw(SEARCH_UNACCENT_FN)}(${searchString})`
+            : sql`${searchString}`;
+        return sql`(${rank} + public.word_similarity(${needle}, ${fuzzyColumn}))`;
+    }
+
     /**
      * Build a unique field check condition
      */