@rebasepro/server-postgres 0.13.0 → 0.13.1-canary.g06dbe5b

package/src/utils/drizzle-conditions.ts

@@ -1,14 +1,21 @@
 import { and, eq, or, sql, SQL, ilike, inArray, getTableColumns } from "drizzle-orm";
-import { AnyPgColumn, PgTable, PgVarchar, PgText, PgChar } from "drizzle-orm/pg-core";
+import { AnyPgColumn, PgTable } from "drizzle-orm/pg-core";
 import {
     CollectionConfig, FilterValues, WhereFilterOp, JoinStep, LogicalCondition, FilterCondition,
     ResolvedRelation, ResolvedBelongsTo, ResolvedHasOne, ResolvedHasMany,
     ResolvedForeignKeyOnTarget, ResolvedManyToMany, hasForeignKeyOnTarget, isManyToMany
 } from "@rebasepro/types";
 import {
-    getColumnName, getTableName, normalizeToEntityRelation, resolveCollectionRelations
+    getColumnName, getTableName, normalizeToEntityRelation, resolveCollectionRelations, toFilterTuples
 } from "@rebasepro/common";
 import { generateForeignKeyName } from "@rebasepro/utils";
+/**
+ * Postgres's own default for `pg_trgm.word_similarity_threshold`. Named here
+ * because the fuzzy predicate has to know when the index-backed operator agrees
+ * with the collection's declared threshold and when it would narrow too far.
+ */
+const PG_TRGM_WORD_SIMILARITY_DEFAULT = 0.6;
+import { buildSearchColumnSpec, SEARCH_UNACCENT_FN, type SearchColumnSpec } from "../schema/search-column";
 import { PostgresCollectionRegistry } from "../collections/PostgresCollectionRegistry";
 import { ConditionBuilderStatic } from "../interfaces";
 import { ApiError, logger } from "@rebasepro/server";
@@ -28,6 +35,69 @@ import { getColumnMeta } from "../services/collection-helpers";
  */
 export type UnknownFilterFieldsMode = "error" | "warn";
 
+/**
+ * A user's search term, made safe to drop inside a `%…%` LIKE pattern.
+ *
+ * The term is already a bind parameter, so this is not about injection. It is
+ * about the two things a LIKE metacharacter does when it arrives from a search
+ * box:
+ *
+ *  1. **It changes the query.** `%` and `_` are wildcards, so searching for
+ *     `50%` returned every row and `a_c` matched `abc`. Nothing the caller
+ *     could type would find a literal `%`.
+ *  2. **It is a cost the caller chooses.** Postgres matches LIKE by
+ *     backtracking: each `%` re-tries every remaining offset, so
+ *     `?searchString=a%a%a%a%a%a%a%b` is polynomial with an attacker-chosen
+ *     exponent — evaluated per row, OR-ed across every string property of the
+ *     collection, on a sequential scan (a leading `%` cannot use an index), and
+ *     the page limit does not bound it because the scan happens first.
+ *
+ * This is the server-side half of the pattern `like-pattern-redos.test.ts`
+ * hardened the offline evaluator against; that test's own note ("the same
+ * translation in the Mongo driver hands the expression to the database, where
+ * it occupies a server thread instead") describes this call site.
+ *
+ * Backslash is the default `ESCAPE` character for LIKE, and the pattern is
+ * bound rather than interpolated, so a single backslash here reaches the
+ * matcher as one. Escaping the escape character first is what keeps a term
+ * ending in `\` from swallowing the closing `%`.
+ *
+ * Note this is a *substring search*, not the `like` filter operator: a caller
+ * who wants wildcards has `?title=like.foo%` for that, where the pattern is the
+ * documented input.
+ */
+export const escapeLikePattern = (value: string): string =>
+    value.replace(/[\\%_]/g, ch => `\\${ch}`);
+
+/** Column types `ILIKE '%…%'` is defined on. */
+const ILIKE_SQL_TYPES = /^(text|varchar|character varying|char|character|bpchar|citext)\b/;
+
+/**
+ * Can this column be matched with `ILIKE`?
+ *
+ * Asked of the column's *declared SQL type*, never with `instanceof`. The
+ * previous version tested `column instanceof PgVarchar || … PgText || … PgChar`,
+ * and `instanceof` compares class identity: it is only true when the column was
+ * constructed by the very same copy of `drizzle-orm` that this module imported.
+ *
+ * An application's generated schema builds its tables with the app's own
+ * `drizzle-orm`, and this driver declares its own dependency on one. When the
+ * two ranges do not overlap — an app scaffolded against `^0.44` with a driver
+ * asking for `^0.45` — a strict installer gives the driver a second copy, every
+ * check returns false, no condition is produced, and the caller compiles that
+ * into an impossible `WHERE`. The result is a 200 with an empty page for every
+ * search on every collection without a `search` block: the failure looks
+ * exactly like "nothing matched". Observed in production, not theorised.
+ *
+ * `getSQLType()` is a value the column reports about itself, so it crosses
+ * module instances the way a class identity cannot. It also happens to fix
+ * `citext`, which the `instanceof` list never covered.
+ */
+const supportsILike = (column: AnyPgColumn): boolean => {
+    const sqlType = typeof column?.getSQLType === "function" ? column.getSQLType().toLowerCase() : "";
+    return ILIKE_SQL_TYPES.test(sqlType);
+};
+
 /**
  * Process-wide default, set once when the driver is constructed.
  *
@@ -465,11 +535,9 @@ export class DrizzleConditionBuilder {
             const target = this.resolveFilterTarget(table, field, collectionPath, mode, options);
             if (!target) continue;
 
-            const paramsList = Array.isArray(filterParam) && filterParam.length > 0 && Array.isArray(filterParam[0])
-                ? (filterParam as [WhereFilterOp, any][])
-                : [filterParam as [WhereFilterOp, any]];
-
-            for (const [op, value] of paramsList) {
+            // One tuple or an array of them — the grammar, read the same way by
+            // every compiler. See `toFilterTuples`.
+            for (const [op, value] of toFilterTuples(filterParam)) {
                 const condition = this.compileFilterTarget(target, op, value, field, collectionPath);
                 if (condition) {
                     conditions.push(condition);
@@ -1275,38 +1343,261 @@ whereConditions };
     }
 
     /**
-     * Build search conditions for text fields
+     * Build search conditions for text fields.
+     *
+     * Two shapes, chosen by whether the collection declared a `search` block:
+     *
+     * - **Declared** — one `@@ websearch_to_tsquery` against the generated
+     *   `tsvector` column. Stems, drops stopwords, AND-es the terms, reaches
+     *   inside JSONB and arrays, and uses the GIN index.
+     * - **Not declared** — the original `ILIKE '%term%'` OR-ed across top-level
+     *   string properties, with the term escaped (see {@link escapeLikePattern})
+     *   so it is matched as the literal text the user typed.
+     *
+     * The second is the default and stays the default. A collection that has
+     * not opted in compiles to exactly the SQL it compiled to before this
+     * branch existed, which is the only reason it is safe to have added it.
+     *
+     * `collection` is optional so that the callers which genuinely have no
+     * collection in hand — nested paths, derived views — keep working; without
+     * one there is no `search` block to read and the ILIKE path is correct.
      */
     static buildSearchConditions(
         searchString: string,
         properties: Record<string, unknown>,
-        table: PgTable<any>
+        table: PgTable<any>,
+        collection?: CollectionConfig
     ): SQL[] {
         const searchConditions: SQL[] = [];
 
+        const ftsCondition = collection
+            ? DrizzleConditionBuilder.buildFullTextCondition(searchString, table, collection)
+            : undefined;
+        if (ftsCondition) return [ftsCondition];
+
+        let declaredStringProperties = 0;
+
         for (const [key, prop] of Object.entries(properties)) {
             const p = prop as Record<string, unknown>;
             // Only include string properties that don't have enum defined
             // PostgreSQL enum and uuid columns don't support ILIKE, so we skip them
             if (p.type === "string" && !p.enum && p.isId !== "uuid") {
+                declaredStringProperties++;
                 const fieldColumn = table[key as keyof typeof table] as AnyPgColumn;
-                if (fieldColumn) {
-                    // Verify that the underlying database column supports string pattern-matching
-                    const supportsILike =
-                        fieldColumn instanceof PgVarchar ||
-                        fieldColumn instanceof PgText ||
-                        fieldColumn instanceof PgChar ||
-                        (fieldColumn && typeof fieldColumn === "object" && !("columnType" in fieldColumn));
-                    if (supportsILike) {
-                        searchConditions.push(ilike(fieldColumn, `%${searchString}%`));
-                    }
+                if (fieldColumn && supportsILike(fieldColumn)) {
+                    searchConditions.push(ilike(fieldColumn, `%${escapeLikePattern(searchString)}%`));
                 }
             }
         }
 
+        // Every string property was rejected, so the caller is about to turn an
+        // empty condition list into "match nothing" — a 200 with an empty page,
+        // which reads as "no such row" rather than as the breakage it is. Say so
+        // once per query: this is how the `instanceof` version of
+        // {@link supportsILike} failed silently in the field for months.
+        if (declaredStringProperties > 0 && searchConditions.length === 0) {
+            logger.warn(
+                `[search] "${collection?.slug ?? "collection"}" declares ${declaredStringProperties} string ` +
+                "property(ies) but none compiled to a searchable column, so this search can only return nothing. " +
+                "Check that the generated schema's column types are text/varchar/char."
+            );
+        }
+
         return searchConditions;
     }
 
+    /**
+     * The `@@` predicate for a collection that declared a `search` block, or
+     * undefined for one that did not.
+     *
+     * The query is normalized exactly as the indexed content was — same text
+     * search configuration, same accent folding. Skipping that on the query
+     * side is the subtle way to get a search that matches nothing: the column
+     * would hold `gestion` while the query asked for `gestión`.
+     *
+     * `websearch_to_tsquery` rather than `plainto_tsquery` because it is the
+     * one that behaves the way a search box looks like it should — quoted
+     * phrases, `or`, and a leading `-` to exclude — and because it never throws
+     * on user input, which `to_tsquery` does on so much as a stray parenthesis.
+     */
+    static buildFullTextCondition(
+        searchString: string,
+        table: PgTable<any>,
+        collection: CollectionConfig
+    ): SQL | undefined {
+        let spec: SearchColumnSpec | undefined;
+        try {
+            spec = buildSearchColumnSpec(collection);
+        } catch {
+            // Reported at boot. Falling back to ILIKE here keeps reads serving.
+            return undefined;
+        }
+        if (!spec) return undefined;
+
+        const column = table[spec.column as keyof typeof table] as AnyPgColumn | undefined;
+        if (!column) {
+            // The block is declared but the column is not on the table yet —
+            // a database that has not been migrated. ILIKE still answers.
+            return undefined;
+        }
+
+        const query = DrizzleConditionBuilder.normalizedTsQuery(searchString, spec);
+        const exact = sql`${column} @@ ${query}`;
+
+        if (!spec.fuzzy) return exact;
+
+        const fuzzyColumn = table[spec.fuzzy.column as keyof typeof table] as AnyPgColumn | undefined;
+        if (!fuzzyColumn) return exact;
+
+        const needle = spec.unaccent
+            ? sql`${sql.raw(SEARCH_UNACCENT_FN)}(${searchString})`
+            : sql`${searchString}`;
+
+        // `word_similarity(query, document)`, not `similarity`. `similarity`
+        // scores two strings as wholes, so a short query against a whole row's
+        // text scores near zero however well it matches part of it — measured:
+        // "iso 14001 auditor" against one candidate's concatenated fields
+        // scores 0.228 by `similarity` and 0.783 by `word_similarity`. The
+        // first is below any usable threshold, which would have made `fuzzy`
+        // a setting that quietly did nothing.
+        //
+        // Argument order matters: the first operand is the needle, and the
+        // score is its similarity to the best-matching extent of the second.
+        //
+        // Both the function and the operator are schema-qualified: pg_trgm is
+        // installed into `public`, and an unqualified reference resolves
+        // through `search_path`, which does not necessarily reach it.
+        const similar = sql`public.word_similarity(${needle}, ${fuzzyColumn}) >= ${spec.fuzzy.threshold}`;
+        // `<%` is the index-backed form, but it tests against the session's
+        // `pg_trgm.word_similarity_threshold` (0.6), not ours. Above that
+        // default the operator narrows using the trigram index and the explicit
+        // score refines; at or below it, the operator would exclude rows the
+        // declared threshold admits, so the score stands alone and the planner
+        // scans — correct either way, and only the faster path is conditional.
+        const fuzzy = spec.fuzzy.threshold > PG_TRGM_WORD_SIMILARITY_DEFAULT
+            ? sql`(${needle} OPERATOR(public.<%) ${fuzzyColumn} AND ${similar})`
+            : similar;
+
+        return sql`(${exact} OR ${fuzzy})`;
+    }
+
+    /**
+     * `websearch_to_tsquery(<config>, <normalized search string>)`.
+     *
+     * Split out because the ranking expression needs the identical query — a
+     * row ranked against a different tsquery than it was matched against is a
+     * ranking of something else.
+     */
+    static normalizedTsQuery(searchString: string, spec: SearchColumnSpec): SQL {
+        const normalized = spec.unaccent
+            ? sql`${sql.raw(SEARCH_UNACCENT_FN)}(${searchString})`
+            : sql`${searchString}`;
+        return sql`websearch_to_tsquery(${spec.language}, ${normalized})`;
+    }
+
+    /**
+     * A JSONB array of `{ field, snippet }` naming which declared fields matched
+     * and showing the text around each hit — what backs `_matches`.
+     *
+     * A ranked list answers "which rows", never "why this row". For a talent
+     * pool that difference is the product: a candidate surfacing for
+     * "iso 14001" on a *certification* is a different candidate from one whose
+     * bio happens to mention the standard, and the score cannot tell them apart.
+     *
+     * Built as a correlated subquery over a `VALUES` list of the declared
+     * fields, rather than one `CASE` per field, so the shape does not change
+     * with the number of fields and the empty result is a plain `[]`.
+     *
+     * `ts_headline` runs over the same normalized text that was indexed. Over
+     * the *original* text it would find nothing to mark whenever `unaccent` is
+     * on — the query's lexemes are folded and the document's are not — and
+     * would return the text silently unhighlighted. Folded-but-marked beats
+     * pretty-but-inert.
+     *
+     * Undefined when the collection has not opted in, when the column is not on
+     * the table yet, or when the caller did not ask: this costs a `ts_headline`
+     * per field per row and `ts_headline` re-parses the document.
+     */
+    static buildSearchMatchesExpression(
+        searchString: string,
+        table: PgTable<any>,
+        collection: CollectionConfig
+    ): SQL | undefined {
+        let spec: SearchColumnSpec | undefined;
+        try {
+            spec = buildSearchColumnSpec(collection);
+        } catch {
+            return undefined;
+        }
+        if (!spec || spec.fields.length === 0) return undefined;
+        if (!table[spec.column as keyof typeof table]) return undefined;
+
+        const query = DrizzleConditionBuilder.normalizedTsQuery(searchString, spec);
+        const config = sql`${spec.language}`;
+
+        // `ord` keeps the author's declared field order in the output, so the
+        // most important field they named reads first rather than whichever
+        // Postgres aggregated first.
+        const rows = spec.fields.map((f, i) =>
+            sql`(${i}, ${f.path}, ${sql.raw(f.textSql)})`
+        );
+
+        return sql`(
+            SELECT coalesce(jsonb_agg(s.m ORDER BY f.ord), '[]'::jsonb)
+            FROM (VALUES ${sql.join(rows, sql`, `)}) AS f(ord, path, txt)
+            CROSS JOIN LATERAL (
+                SELECT jsonb_build_object(
+                    'field', f.path,
+                    'snippet', ts_headline(${config}::regconfig, f.txt, ${query},
+                        'StartSel=<mark>,StopSel=</mark>,MaxWords=14,MinWords=1,MaxFragments=1,FragmentDelimiter= … ')
+                ) AS m
+                WHERE to_tsvector(${config}::regconfig, f.txt) @@ ${query}
+            ) s
+        )`;
+    }
+
+    /**
+     * `ts_rank(<column>, <query>)` for the collection, or undefined when it has
+     * not opted in. This is what backs `orderBy: ["_score", "desc"]`.
+     */
+    static buildSearchRankExpression(
+        searchString: string,
+        table: PgTable<any>,
+        collection: CollectionConfig
+    ): SQL | undefined {
+        let spec: SearchColumnSpec | undefined;
+        try {
+            spec = buildSearchColumnSpec(collection);
+        } catch {
+            return undefined;
+        }
+        if (!spec) return undefined;
+        const column = table[spec.column as keyof typeof table] as AnyPgColumn | undefined;
+        if (!column) return undefined;
+
+        const rank = sql`ts_rank(${column}, ${DrizzleConditionBuilder.normalizedTsQuery(searchString, spec)})`;
+        if (!spec.fuzzy) return rank;
+
+        const fuzzyColumn = table[spec.fuzzy.column as keyof typeof table] as AnyPgColumn | undefined;
+        if (!fuzzyColumn) return rank;
+
+        // With `fuzzy` on, `ts_rank` alone is not a ranking — it is zero for
+        // every row the trigram path matched and the exact path did not, which
+        // is the whole population of a typo'd query. Measured on the real
+        // sustentalent pool: "auditor de iso14000" matches four candidates,
+        // every one of them at ts_rank 0, so ordering by rank alone returned
+        // the best match in whatever order the table felt like.
+        //
+        // Summed rather than blended with tuned constants: a row that matched
+        // exactly contributes both terms, so it outranks a fuzzy-only row of
+        // equal similarity without needing a coefficient to say so. Both terms
+        // are non-negative and monotonic, which is all the ordering needs.
+        const needle = spec.unaccent
+            ? sql`${sql.raw(SEARCH_UNACCENT_FN)}(${searchString})`
+            : sql`${searchString}`;
+        return sql`(${rank} + public.word_similarity(${needle}, ${fuzzyColumn}))`;
+    }
+
     /**
      * Build a unique field check condition
      */
@@ -1538,6 +1829,16 @@ whereConditions };
      * - `orderBy`: SQL expression to ORDER BY distance (ascending = closest first)
      * - `filter`: optional WHERE clause for distance threshold
      * - `distanceSelect`: SQL expression for selecting the distance as `_distance`
+     *
+     * `property` is `?vector_search=` off the querystring, so it is an untrusted
+     * *name*, and it used to be looked up straight in the drizzle table object.
+     * Two ways that went wrong, both answering 500 to a malformed request:
+     * `?vector_search=title` built `"title" <=> '[1,2]'::vector`, which the
+     * database rejects with "operator does not exist"; and a table object also
+     * carries non-column keys (`_`, methods), which passed the `if (!column)`
+     * guard and compiled to nonsense. The name is resolved against the table's
+     * actual columns and required to be a `vector` — anything else is the
+     * caller's mistake and gets a 400 that says so.
      */
     static buildVectorSearchConditions(
         table: PgTable<any>,
@@ -1548,10 +1849,7 @@ whereConditions };
             threshold?: number;
         }
     ): { orderBy: SQL; filter?: SQL; distanceSelect: SQL } {
-        const column = table[vectorSearch.property as keyof typeof table] as AnyPgColumn;
-        if (!column) {
-            throw new Error(`Vector column '${vectorSearch.property}' not found in table`);
-        }
+        const column = DrizzleConditionBuilder.resolveVectorColumn(table, vectorSearch.property);
 
         // The vector is interpolated as a raw SQL literal below (pgvector has no
         // bind form for the `::vector` cast), so every element must be a finite
@@ -1592,8 +1890,52 @@ whereConditions };
             distanceSelect: sql`(${column} ${sql.raw(operator)} ${sql.raw(vectorLiteral)})`
         };
     }
+
+    /**
+     * The `vector` column a request named, or a 400 explaining what it named.
+     *
+     * `getTableColumns` rather than a key lookup: it returns only the columns,
+     * so `_`, `getSQL` and every other property of a drizzle table stop looking
+     * like candidates. The type check is on the *physical* column
+     * (`vector(1536)`) rather than on the declared property, so it holds for an
+     * introspected collection too, where the property carries no Rebase type.
+     */
+    private static resolveVectorColumn(table: PgTable<any>, property: string): AnyPgColumn {
+        const columns = getTableColumns(table) as Record<string, AnyPgColumn> | undefined;
+        const column = columns?.[property];
+        if (!column) {
+            const known = Object.entries(columns ?? {})
+                .filter(([, c]) => isVectorColumn(c))
+                .map(([name]) => name);
+            throw ApiError.badRequest(
+                `Unknown vector property "${property}". ` +
+                (known.length > 0
+                    ? `This collection's vector properties are: ${known.join(", ")}.`
+                    : "This collection declares no `vector` property to search."),
+                "UNKNOWN_VECTOR_PROPERTY"
+            );
+        }
+        if (!isVectorColumn(column)) {
+            throw ApiError.badRequest(
+                `Property "${property}" is not a vector column (it is \`${columnSqlType(column) || "unknown"}\`), ` +
+                "so it has no distance operator. Name the property declared as `{ type: \"vector\" }`.",
+                "UNKNOWN_VECTOR_PROPERTY"
+            );
+        }
+        return column;
+    }
 }
 
+/** The column's SQL type, for a value that may not be a drizzle column at all. */
+const columnSqlType = (column: unknown): string => {
+    const getSQLType = (column as { getSQLType?: () => string })?.getSQLType;
+    return typeof getSQLType === "function" ? getSQLType.call(column).toLowerCase() : "";
+};
+
+/** True for `vector(1536)` and its pgvector siblings, whatever the width. */
+const isVectorColumn = (column: unknown): boolean =>
+    /^(vector|halfvec|sparsevec)\b/.test(columnSqlType(column));
+
 /**
  * Alias for DrizzleConditionBuilder for consistent naming with other database implementations.
  * This allows code to use PostgresConditionBuilder alongside future MongoConditionBuilder, etc.

package/src/utils/pg-error-utils.ts

@@ -314,9 +314,14 @@ export function sanitizeErrorForClient(error: unknown, context: string): { messa
             column: pgError.column,
             table: pgError.table,
             constraint: pgError.constraint,
-            dataType: pgError.dataType,
-            // Also log the outer Drizzle wrapper message for full context
-            drizzleMessage: error instanceof Error ? error.message : String(error)
+            dataType: pgError.dataType
+            // The outer Drizzle wrapper message used to be logged here "for
+            // full context": it is `Failed query: <sql>\nparams: <values>`, so
+            // it published the statement and every bound value (an email, a
+            // password hash) on every realtime data failure. The SQLSTATE,
+            // detail, table, column and constraint above are the diagnostic
+            // value; the wrapper added only the leak. `logger` strips the
+            // wrapper as well, but the field itself carried nothing else.
         });
         return pgErrorToFriendlyMessage(pgError, context);
     }