@rebasepro/server-postgres 0.13.1-canary.gef9608c → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/PostgresBackendDriver.d.ts +48 -1
- package/dist/PostgresBootstrapper.d.ts +26 -0
- package/dist/auth/services.d.ts +21 -0
- package/dist/{src-CU6WZGYV.js → auth-users-columns-BfQHf9JE.js} +1111 -92
- package/dist/auth-users-columns-BfQHf9JE.js.map +1 -0
- package/dist/{backup-service-CD8o_1Sl.js → backup-service-BH0Dzo_h.js} +2 -3
- package/dist/{backup-service-CD8o_1Sl.js.map → backup-service-BH0Dzo_h.js.map} +1 -1
- package/dist/cli-helpers.d.ts +56 -0
- package/dist/cli-output.d.ts +34 -0
- package/dist/data-transformer.d.ts +7 -2
- package/dist/data_driver-ULAyJEi9.js +193 -0
- package/dist/data_driver-ULAyJEi9.js.map +1 -0
- package/dist/ensure-collection-policies-8vuu-n4r.js +124 -0
- package/dist/ensure-collection-policies-8vuu-n4r.js.map +1 -0
- package/dist/{ensure-collection-tables-BLIIACla.js → ensure-collection-tables-CbvaGuVn.js} +162 -16
- package/dist/ensure-collection-tables-CbvaGuVn.js.map +1 -0
- package/dist/index.es.js +1720 -946
- package/dist/index.es.js.map +1 -1
- package/dist/rls-bootstrap-sql-69hYT8nr.js +244 -0
- package/dist/rls-bootstrap-sql-69hYT8nr.js.map +1 -0
- package/dist/rls-enforcement-BJ_3wxwg.js +425 -0
- package/dist/rls-enforcement-BJ_3wxwg.js.map +1 -0
- package/dist/schema/auth-schema.d.ts +102 -0
- package/dist/schema/auth-users-columns.d.ts +97 -0
- package/dist/schema/doctor-policy-checks.d.ts +28 -0
- package/dist/schema/doctor.d.ts +41 -25
- package/dist/schema/ensure-collection-policies.d.ts +33 -9
- package/dist/schema/ensure-collection-tables.d.ts +60 -6
- package/dist/schema/generate-drizzle-schema-logic.d.ts +9 -1
- package/dist/schema/generate-postgres-ddl-logic.d.ts +48 -0
- package/dist/schema/introspect-db-inference.d.ts +8 -1
- package/dist/schema/introspect-db-logic.d.ts +49 -0
- package/dist/schema/introspect-db-project.d.ts +21 -0
- package/dist/schema/rls-bootstrap-sql.d.ts +135 -0
- package/dist/schema/search-column.d.ts +248 -0
- package/dist/security/policy-drift.d.ts +34 -0
- package/dist/security/rls-enforcement.d.ts +61 -5
- package/dist/services/FetchService.d.ts +24 -0
- package/dist/services/PersistService.d.ts +21 -17
- package/dist/services/RelationService.d.ts +9 -57
- package/dist/services/RelationWriteService.d.ts +82 -0
- package/dist/services/collection-helpers.d.ts +42 -0
- package/dist/services/dataService.d.ts +3 -0
- package/dist/services/junction-writes.d.ts +82 -0
- package/dist/services/realtimeService.d.ts +139 -2
- package/dist/services/write-denial.d.ts +36 -0
- package/dist/{src-DoU9yPqq.js → src-DCdn3Val.js} +124 -3
- package/dist/src-DCdn3Val.js.map +1 -0
- package/dist/utils/drizzle-conditions.d.ts +124 -2
- package/dist/{websocket-B2LsrINK.js → websocket-C8ZqVBiV.js} +75 -18
- package/dist/websocket-C8ZqVBiV.js.map +1 -0
- package/package.json +8 -7
- package/src/PostgresBackendDriver.ts +172 -6
- package/src/PostgresBootstrapper.ts +136 -11
- package/src/auth/ensure-tables.ts +212 -91
- package/src/auth/services.ts +82 -5
- package/src/backup/backup-cli.ts +59 -57
- package/src/cli-errors.ts +6 -6
- package/src/cli-helpers.ts +124 -11
- package/src/cli-output.ts +43 -0
- package/src/cli.ts +299 -168
- package/src/collections/buildRegistry.ts +3 -1
- package/src/data-transformer.ts +129 -25
- package/src/history/ensure-history-table.ts +9 -2
- package/src/schema/auth-schema.ts +17 -1
- package/src/schema/auth-users-columns.ts +131 -0
- package/src/schema/doctor-cli.ts +14 -65
- package/src/schema/doctor-policy-checks.ts +105 -0
- package/src/schema/doctor.ts +149 -72
- package/src/schema/ensure-collection-policies.ts +99 -6
- package/src/schema/ensure-collection-tables.ts +366 -30
- package/src/schema/generate-drizzle-schema-logic.ts +146 -66
- package/src/schema/generate-drizzle-schema.ts +11 -10
- package/src/schema/generate-postgres-ddl-logic.ts +277 -10
- package/src/schema/generate-postgres-ddl.ts +38 -14
- package/src/schema/generated-schema-staleness.ts +14 -7
- package/src/schema/introspect-db-inference.ts +9 -2
- package/src/schema/introspect-db-logic.ts +251 -75
- package/src/schema/introspect-db-project.ts +78 -0
- package/src/schema/introspect-db.ts +42 -25
- package/src/schema/introspect-runtime.ts +14 -2
- package/src/schema/rls-bootstrap-sql.ts +288 -0
- package/src/schema/search-column.ts +643 -0
- package/src/security/anonymous-grants.test.ts +4 -2
- package/src/security/policy-drift.test.ts +104 -3
- package/src/security/policy-drift.ts +129 -7
- package/src/security/rls-enforcement.ts +150 -7
- package/src/services/BranchService.ts +5 -0
- package/src/services/FetchService.ts +243 -22
- package/src/services/PersistService.ts +68 -42
- package/src/services/RelationService.ts +37 -696
- package/src/services/RelationWriteService.ts +653 -0
- package/src/services/cdc/trigger-cdc.ts +5 -1
- package/src/services/channel-history.ts +14 -0
- package/src/services/channel-presence.ts +13 -0
- package/src/services/collection-helpers.ts +89 -4
- package/src/services/dataService.ts +3 -0
- package/src/services/junction-writes.ts +295 -0
- package/src/services/pg-notify-listener.ts +1 -1
- package/src/services/realtimeService.ts +347 -86
- package/src/services/write-denial.ts +55 -0
- package/src/utils/drizzle-conditions.ts +433 -35
- package/src/utils/pg-error-utils.ts +8 -3
- package/src/websocket.ts +113 -16
- package/dist/ensure-collection-policies-Bck0ky4u.js +0 -57
- package/dist/ensure-collection-policies-Bck0ky4u.js.map +0 -1
- package/dist/ensure-collection-tables-BLIIACla.js.map +0 -1
- package/dist/policy-CeA1JcxP.js +0 -105
- package/dist/policy-CeA1JcxP.js.map +0 -1
- package/dist/schema/auth-bootstrap-sql.d.ts +0 -24
- package/dist/src-CU6WZGYV.js.map +0 -1
- package/dist/src-DoU9yPqq.js.map +0 -1
- package/dist/websocket-B2LsrINK.js.map +0 -1
- package/src/schema/auth-bootstrap-sql.ts +0 -47
|
@@ -1,14 +1,22 @@
|
|
|
1
1
|
import { and, eq, or, sql, SQL, ilike, inArray, getTableColumns } from "drizzle-orm";
|
|
2
|
-
import { AnyPgColumn, PgTable
|
|
2
|
+
import { AnyPgColumn, PgTable } from "drizzle-orm/pg-core";
|
|
3
3
|
import {
|
|
4
|
+
ALL_WHERE_FILTER_OPS,
|
|
4
5
|
CollectionConfig, FilterValues, WhereFilterOp, JoinStep, LogicalCondition, FilterCondition,
|
|
5
6
|
ResolvedRelation, ResolvedBelongsTo, ResolvedHasOne, ResolvedHasMany,
|
|
6
7
|
ResolvedForeignKeyOnTarget, ResolvedManyToMany, hasForeignKeyOnTarget, isManyToMany
|
|
7
8
|
} from "@rebasepro/types";
|
|
8
9
|
import {
|
|
9
|
-
getColumnName, getTableName, normalizeToEntityRelation, resolveCollectionRelations
|
|
10
|
+
fieldKeyForColumn, getColumnName, getTableName, normalizeToEntityRelation, resolveCollectionRelations, toFilterTuples
|
|
10
11
|
} from "@rebasepro/common";
|
|
11
|
-
import { generateForeignKeyName } from "@rebasepro/utils";
|
|
12
|
+
import { generateForeignKeyName, toWireKey } from "@rebasepro/utils";
|
|
13
|
+
/**
|
|
14
|
+
* Postgres's own default for `pg_trgm.word_similarity_threshold`. Named here
|
|
15
|
+
* because the fuzzy predicate has to know when the index-backed operator agrees
|
|
16
|
+
* with the collection's declared threshold and when it would narrow too far.
|
|
17
|
+
*/
|
|
18
|
+
const PG_TRGM_WORD_SIMILARITY_DEFAULT = 0.6;
|
|
19
|
+
import { buildSearchColumnSpec, SEARCH_UNACCENT_FN, type SearchColumnSpec } from "../schema/search-column";
|
|
12
20
|
import { PostgresCollectionRegistry } from "../collections/PostgresCollectionRegistry";
|
|
13
21
|
import { ConditionBuilderStatic } from "../interfaces";
|
|
14
22
|
import { ApiError, logger } from "@rebasepro/server";
|
|
@@ -28,6 +36,101 @@ import { getColumnMeta } from "../services/collection-helpers";
|
|
|
28
36
|
*/
|
|
29
37
|
export type UnknownFilterFieldsMode = "error" | "warn";
|
|
30
38
|
|
|
39
|
+
/**
|
|
40
|
+
* A user's search term, made safe to drop inside a `%…%` LIKE pattern.
|
|
41
|
+
*
|
|
42
|
+
* The term is already a bind parameter, so this is not about injection. It is
|
|
43
|
+
* about the two things a LIKE metacharacter does when it arrives from a search
|
|
44
|
+
* box:
|
|
45
|
+
*
|
|
46
|
+
* 1. **It changes the query.** `%` and `_` are wildcards, so searching for
|
|
47
|
+
* `50%` returned every row and `a_c` matched `abc`. Nothing the caller
|
|
48
|
+
* could type would find a literal `%`.
|
|
49
|
+
* 2. **It is a cost the caller chooses.** Postgres matches LIKE by
|
|
50
|
+
* backtracking: each `%` re-tries every remaining offset, so
|
|
51
|
+
* `?searchString=a%a%a%a%a%a%a%b` is polynomial with an attacker-chosen
|
|
52
|
+
* exponent — evaluated per row, OR-ed across every string property of the
|
|
53
|
+
* collection, on a sequential scan (a leading `%` cannot use an index), and
|
|
54
|
+
* the page limit does not bound it because the scan happens first.
|
|
55
|
+
*
|
|
56
|
+
* This is the server-side half of the pattern `like-pattern-redos.test.ts`
|
|
57
|
+
* hardened the offline evaluator against; that test's own note ("the same
|
|
58
|
+
* translation in the Mongo driver hands the expression to the database, where
|
|
59
|
+
* it occupies a server thread instead") describes this call site.
|
|
60
|
+
*
|
|
61
|
+
* Backslash is the default `ESCAPE` character for LIKE, and the pattern is
|
|
62
|
+
* bound rather than interpolated, so a single backslash here reaches the
|
|
63
|
+
* matcher as one. Escaping the escape character first is what keeps a term
|
|
64
|
+
* ending in `\` from swallowing the closing `%`.
|
|
65
|
+
*
|
|
66
|
+
* Note this is a *substring search*, not the `like` filter operator: a caller
|
|
67
|
+
* who wants wildcards has `?title=like.foo%` for that, where the pattern is the
|
|
68
|
+
* documented input.
|
|
69
|
+
*/
|
|
70
|
+
export const escapeLikePattern = (value: string): string =>
|
|
71
|
+
value.replace(/[\\%_]/g, ch => `\\${ch}`);
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* The Drizzle column a relation's column name addresses on a table.
|
|
75
|
+
*
|
|
76
|
+
* A relation names its link in *column* terms — `localKey: "author_id"`,
|
|
77
|
+
* `foreignKeyOnTarget: "author_id"` — because that is what the database and
|
|
78
|
+
* every FK constraint call it. A Drizzle table is keyed by the *wire* name,
|
|
79
|
+
* `authorId`. Indexing the table with the column, which is what every one of
|
|
80
|
+
* these call sites used to do, therefore finds nothing the moment the two
|
|
81
|
+
* differ: for a `columnName`-carrying property that was already true, and it is
|
|
82
|
+
* now true of every derived foreign key.
|
|
83
|
+
*
|
|
84
|
+
* `undefined` rather than a throw: each caller already has a message naming the
|
|
85
|
+
* relation it was resolving, which is worth more than a generic one here.
|
|
86
|
+
*/
|
|
87
|
+
const relationColumn = (
|
|
88
|
+
table: PgTable<any>,
|
|
89
|
+
collection: CollectionConfig | undefined,
|
|
90
|
+
column: string
|
|
91
|
+
): AnyPgColumn | undefined => {
|
|
92
|
+
const key = fieldKeyForColumn(collection, column);
|
|
93
|
+
return (key in table ? table[key as keyof typeof table] as AnyPgColumn : undefined) || undefined;
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
/** The target collection of a relation, or `undefined` if its thunk cannot resolve. */
|
|
97
|
+
const targetOf = (relation: ResolvedRelation): CollectionConfig | undefined => {
|
|
98
|
+
try {
|
|
99
|
+
return relation.target();
|
|
100
|
+
} catch {
|
|
101
|
+
return undefined;
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/** Column types `ILIKE '%…%'` is defined on. */
|
|
106
|
+
const ILIKE_SQL_TYPES = /^(text|varchar|character varying|char|character|bpchar|citext)\b/;
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Can this column be matched with `ILIKE`?
|
|
110
|
+
*
|
|
111
|
+
* Asked of the column's *declared SQL type*, never with `instanceof`. The
|
|
112
|
+
* previous version tested `column instanceof PgVarchar || … PgText || … PgChar`,
|
|
113
|
+
* and `instanceof` compares class identity: it is only true when the column was
|
|
114
|
+
* constructed by the very same copy of `drizzle-orm` that this module imported.
|
|
115
|
+
*
|
|
116
|
+
* An application's generated schema builds its tables with the app's own
|
|
117
|
+
* `drizzle-orm`, and this driver declares its own dependency on one. When the
|
|
118
|
+
* two ranges do not overlap — an app scaffolded against `^0.44` with a driver
|
|
119
|
+
* asking for `^0.45` — a strict installer gives the driver a second copy, every
|
|
120
|
+
* check returns false, no condition is produced, and the caller compiles that
|
|
121
|
+
* into an impossible `WHERE`. The result is a 200 with an empty page for every
|
|
122
|
+
* search on every collection without a `search` block: the failure looks
|
|
123
|
+
* exactly like "nothing matched". Observed in production, not theorised.
|
|
124
|
+
*
|
|
125
|
+
* `getSQLType()` is a value the column reports about itself, so it crosses
|
|
126
|
+
* module instances the way a class identity cannot. It also happens to fix
|
|
127
|
+
* `citext`, which the `instanceof` list never covered.
|
|
128
|
+
*/
|
|
129
|
+
const supportsILike = (column: AnyPgColumn): boolean => {
|
|
130
|
+
const sqlType = typeof column?.getSQLType === "function" ? column.getSQLType().toLowerCase() : "";
|
|
131
|
+
return ILIKE_SQL_TYPES.test(sqlType);
|
|
132
|
+
};
|
|
133
|
+
|
|
31
134
|
/**
|
|
32
135
|
* Process-wide default, set once when the driver is constructed.
|
|
33
136
|
*
|
|
@@ -233,7 +336,7 @@ export class DrizzleConditionBuilder {
|
|
|
233
336
|
|
|
234
337
|
case "hasOne":
|
|
235
338
|
case "hasMany": {
|
|
236
|
-
const fkColumn = targetTable
|
|
339
|
+
const fkColumn = relationColumn(targetTable, targetOf(relation), relation.foreignKeyOnTarget);
|
|
237
340
|
if (!fkColumn) {
|
|
238
341
|
throw new Error(
|
|
239
342
|
`Foreign key column '${relation.foreignKeyOnTarget}' not found in the target table of ` +
|
|
@@ -386,7 +489,7 @@ export class DrizzleConditionBuilder {
|
|
|
386
489
|
|
|
387
490
|
// Owning relation, resolved: the relation names its own local key.
|
|
388
491
|
if (relation?.kind === "belongsTo") {
|
|
389
|
-
const foreignKey =
|
|
492
|
+
const foreignKey = relationColumn(table, collection, relation.localKey);
|
|
390
493
|
if (foreignKey) return { kind: "column", column: foreignKey };
|
|
391
494
|
}
|
|
392
495
|
|
|
@@ -401,7 +504,7 @@ export class DrizzleConditionBuilder {
|
|
|
401
504
|
// correlating on the id anyway silently matches nothing —
|
|
402
505
|
// "filter by this relation" would quietly return zero rows.
|
|
403
506
|
const correlationColumn = hasForeignKeyOnTarget(relation) && relation.sourceKey
|
|
404
|
-
?
|
|
507
|
+
? relationColumn(table, collection, relation.sourceKey)
|
|
405
508
|
: sourceIdColumn;
|
|
406
509
|
if (!correlationColumn) {
|
|
407
510
|
throw new Error(
|
|
@@ -414,10 +517,17 @@ export class DrizzleConditionBuilder {
|
|
|
414
517
|
}
|
|
415
518
|
}
|
|
416
519
|
|
|
417
|
-
// No collection in hand — the
|
|
418
|
-
//
|
|
419
|
-
// `
|
|
420
|
-
|
|
520
|
+
// No collection in hand — the shapes an owning relation's key takes by
|
|
521
|
+
// default (e.g. `project` → `projectId`, `userProfile` →
|
|
522
|
+
// `userProfileId`). The snake forms stay in the list because a project
|
|
523
|
+
// may have authored the property under its column name, which is still
|
|
524
|
+
// its wire name.
|
|
525
|
+
for (const guess of [
|
|
526
|
+
`${field}Id`,
|
|
527
|
+
toWireKey(generateForeignKeyName(field)),
|
|
528
|
+
`${field}_id`,
|
|
529
|
+
generateForeignKeyName(field)
|
|
530
|
+
]) {
|
|
421
531
|
const foreignKey = columnAt(guess);
|
|
422
532
|
if (foreignKey) return { kind: "column", column: foreignKey };
|
|
423
533
|
}
|
|
@@ -465,11 +575,9 @@ export class DrizzleConditionBuilder {
|
|
|
465
575
|
const target = this.resolveFilterTarget(table, field, collectionPath, mode, options);
|
|
466
576
|
if (!target) continue;
|
|
467
577
|
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
for (const [op, value] of paramsList) {
|
|
578
|
+
// One tuple or an array of them — the grammar, read the same way by
|
|
579
|
+
// every compiler. See `toFilterTuples`.
|
|
580
|
+
for (const [op, value] of toFilterTuples(filterParam)) {
|
|
473
581
|
const condition = this.compileFilterTarget(target, op, value, field, collectionPath);
|
|
474
582
|
if (condition) {
|
|
475
583
|
conditions.push(condition);
|
|
@@ -603,7 +711,7 @@ export class DrizzleConditionBuilder {
|
|
|
603
711
|
`(collection '${targetCollection.slug}')`
|
|
604
712
|
);
|
|
605
713
|
}
|
|
606
|
-
const fkColumn = targetTable
|
|
714
|
+
const fkColumn = relationColumn(targetTable, targetCollection, relation.foreignKeyOnTarget);
|
|
607
715
|
if (!fkColumn) {
|
|
608
716
|
throw new Error(
|
|
609
717
|
`Foreign key column '${relation.foreignKeyOnTarget}' not found in the target table of ` +
|
|
@@ -844,8 +952,24 @@ export class DrizzleConditionBuilder {
|
|
|
844
952
|
case "is-not-null":
|
|
845
953
|
return sql`${column} IS NOT NULL`;
|
|
846
954
|
default:
|
|
847
|
-
|
|
848
|
-
|
|
955
|
+
// The relation path five hundred lines up already refuses this,
|
|
956
|
+
// and says why: "returning `null` for an operator this cannot
|
|
957
|
+
// express would drop the condition", and a dropped condition
|
|
958
|
+
// widens the result. The column path is its twin and kept the
|
|
959
|
+
// warning — so `{ status: ["contains", "x"] }` filtered on
|
|
960
|
+
// nothing and answered 200 with every row, which reads as data
|
|
961
|
+
// that matched.
|
|
962
|
+
//
|
|
963
|
+
// The wire layer rejects operator-shaped unknowns before they
|
|
964
|
+
// arrive (`UnknownFilterOperatorError`, 400). What reaches here
|
|
965
|
+
// came from in-process `rebase.data`, a stored filter preset or
|
|
966
|
+
// a config — none of them typechecked at the call site, all of
|
|
967
|
+
// them able to name an operator that no longer exists.
|
|
968
|
+
throw ApiError.badRequest(
|
|
969
|
+
`Unknown filter operator '${op}'. Valid operators: ${ALL_WHERE_FILTER_OPS.join(", ")}.`,
|
|
970
|
+
"UNKNOWN_FILTER_OPERATOR",
|
|
971
|
+
{ operator: op, validOperators: ALL_WHERE_FILTER_OPS }
|
|
972
|
+
);
|
|
849
973
|
}
|
|
850
974
|
}
|
|
851
975
|
|
|
@@ -1246,7 +1370,7 @@ whereConditions };
|
|
|
1246
1370
|
return match(targetIdCol);
|
|
1247
1371
|
}
|
|
1248
1372
|
|
|
1249
|
-
const foreignKeyCol = targetTable
|
|
1373
|
+
const foreignKeyCol = relationColumn(targetTable, targetOf(relation), relation.foreignKeyOnTarget);
|
|
1250
1374
|
if (!foreignKeyCol) {
|
|
1251
1375
|
throw new Error(
|
|
1252
1376
|
`Foreign key column '${relation.foreignKeyOnTarget}' not found in the target table of relation ` +
|
|
@@ -1275,38 +1399,261 @@ whereConditions };
|
|
|
1275
1399
|
}
|
|
1276
1400
|
|
|
1277
1401
|
/**
|
|
1278
|
-
* Build search conditions for text fields
|
|
1402
|
+
* Build search conditions for text fields.
|
|
1403
|
+
*
|
|
1404
|
+
* Two shapes, chosen by whether the collection declared a `search` block:
|
|
1405
|
+
*
|
|
1406
|
+
* - **Declared** — one `@@ websearch_to_tsquery` against the generated
|
|
1407
|
+
* `tsvector` column. Stems, drops stopwords, AND-es the terms, reaches
|
|
1408
|
+
* inside JSONB and arrays, and uses the GIN index.
|
|
1409
|
+
* - **Not declared** — the original `ILIKE '%term%'` OR-ed across top-level
|
|
1410
|
+
* string properties, with the term escaped (see {@link escapeLikePattern})
|
|
1411
|
+
* so it is matched as the literal text the user typed.
|
|
1412
|
+
*
|
|
1413
|
+
* The second is the default and stays the default. A collection that has
|
|
1414
|
+
* not opted in compiles to exactly the SQL it compiled to before this
|
|
1415
|
+
* branch existed, which is the only reason it is safe to have added it.
|
|
1416
|
+
*
|
|
1417
|
+
* `collection` is optional so that the callers which genuinely have no
|
|
1418
|
+
* collection in hand — nested paths, derived views — keep working; without
|
|
1419
|
+
* one there is no `search` block to read and the ILIKE path is correct.
|
|
1279
1420
|
*/
|
|
1280
1421
|
static buildSearchConditions(
|
|
1281
1422
|
searchString: string,
|
|
1282
1423
|
properties: Record<string, unknown>,
|
|
1283
|
-
table: PgTable<any
|
|
1424
|
+
table: PgTable<any>,
|
|
1425
|
+
collection?: CollectionConfig
|
|
1284
1426
|
): SQL[] {
|
|
1285
1427
|
const searchConditions: SQL[] = [];
|
|
1286
1428
|
|
|
1429
|
+
const ftsCondition = collection
|
|
1430
|
+
? DrizzleConditionBuilder.buildFullTextCondition(searchString, table, collection)
|
|
1431
|
+
: undefined;
|
|
1432
|
+
if (ftsCondition) return [ftsCondition];
|
|
1433
|
+
|
|
1434
|
+
let declaredStringProperties = 0;
|
|
1435
|
+
|
|
1287
1436
|
for (const [key, prop] of Object.entries(properties)) {
|
|
1288
1437
|
const p = prop as Record<string, unknown>;
|
|
1289
1438
|
// Only include string properties that don't have enum defined
|
|
1290
1439
|
// PostgreSQL enum and uuid columns don't support ILIKE, so we skip them
|
|
1291
1440
|
if (p.type === "string" && !p.enum && p.isId !== "uuid") {
|
|
1441
|
+
declaredStringProperties++;
|
|
1292
1442
|
const fieldColumn = table[key as keyof typeof table] as AnyPgColumn;
|
|
1293
|
-
if (fieldColumn) {
|
|
1294
|
-
|
|
1295
|
-
const supportsILike =
|
|
1296
|
-
fieldColumn instanceof PgVarchar ||
|
|
1297
|
-
fieldColumn instanceof PgText ||
|
|
1298
|
-
fieldColumn instanceof PgChar ||
|
|
1299
|
-
(fieldColumn && typeof fieldColumn === "object" && !("columnType" in fieldColumn));
|
|
1300
|
-
if (supportsILike) {
|
|
1301
|
-
searchConditions.push(ilike(fieldColumn, `%${searchString}%`));
|
|
1302
|
-
}
|
|
1443
|
+
if (fieldColumn && supportsILike(fieldColumn)) {
|
|
1444
|
+
searchConditions.push(ilike(fieldColumn, `%${escapeLikePattern(searchString)}%`));
|
|
1303
1445
|
}
|
|
1304
1446
|
}
|
|
1305
1447
|
}
|
|
1306
1448
|
|
|
1449
|
+
// Every string property was rejected, so the caller is about to turn an
|
|
1450
|
+
// empty condition list into "match nothing" — a 200 with an empty page,
|
|
1451
|
+
// which reads as "no such row" rather than as the breakage it is. Say so
|
|
1452
|
+
// once per query: this is how the `instanceof` version of
|
|
1453
|
+
// {@link supportsILike} failed silently in the field for months.
|
|
1454
|
+
if (declaredStringProperties > 0 && searchConditions.length === 0) {
|
|
1455
|
+
logger.warn(
|
|
1456
|
+
`[search] "${collection?.slug ?? "collection"}" declares ${declaredStringProperties} string ` +
|
|
1457
|
+
"property(ies) but none compiled to a searchable column, so this search can only return nothing. " +
|
|
1458
|
+
"Check that the generated schema's column types are text/varchar/char."
|
|
1459
|
+
);
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1307
1462
|
return searchConditions;
|
|
1308
1463
|
}
|
|
1309
1464
|
|
|
1465
|
+
/**
|
|
1466
|
+
* The `@@` predicate for a collection that declared a `search` block, or
|
|
1467
|
+
* undefined for one that did not.
|
|
1468
|
+
*
|
|
1469
|
+
* The query is normalized exactly as the indexed content was — same text
|
|
1470
|
+
* search configuration, same accent folding. Skipping that on the query
|
|
1471
|
+
* side is the subtle way to get a search that matches nothing: the column
|
|
1472
|
+
* would hold `gestion` while the query asked for `gestión`.
|
|
1473
|
+
*
|
|
1474
|
+
* `websearch_to_tsquery` rather than `plainto_tsquery` because it is the
|
|
1475
|
+
* one that behaves the way a search box looks like it should — quoted
|
|
1476
|
+
* phrases, `or`, and a leading `-` to exclude — and because it never throws
|
|
1477
|
+
* on user input, which `to_tsquery` does on so much as a stray parenthesis.
|
|
1478
|
+
*/
|
|
1479
|
+
static buildFullTextCondition(
|
|
1480
|
+
searchString: string,
|
|
1481
|
+
table: PgTable<any>,
|
|
1482
|
+
collection: CollectionConfig
|
|
1483
|
+
): SQL | undefined {
|
|
1484
|
+
let spec: SearchColumnSpec | undefined;
|
|
1485
|
+
try {
|
|
1486
|
+
spec = buildSearchColumnSpec(collection);
|
|
1487
|
+
} catch {
|
|
1488
|
+
// Reported at boot. Falling back to ILIKE here keeps reads serving.
|
|
1489
|
+
return undefined;
|
|
1490
|
+
}
|
|
1491
|
+
if (!spec) return undefined;
|
|
1492
|
+
|
|
1493
|
+
const column = table[spec.column as keyof typeof table] as AnyPgColumn | undefined;
|
|
1494
|
+
if (!column) {
|
|
1495
|
+
// The block is declared but the column is not on the table yet —
|
|
1496
|
+
// a database that has not been migrated. ILIKE still answers.
|
|
1497
|
+
return undefined;
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
const query = DrizzleConditionBuilder.normalizedTsQuery(searchString, spec);
|
|
1501
|
+
const exact = sql`${column} @@ ${query}`;
|
|
1502
|
+
|
|
1503
|
+
if (!spec.fuzzy) return exact;
|
|
1504
|
+
|
|
1505
|
+
const fuzzyColumn = table[spec.fuzzy.column as keyof typeof table] as AnyPgColumn | undefined;
|
|
1506
|
+
if (!fuzzyColumn) return exact;
|
|
1507
|
+
|
|
1508
|
+
const needle = spec.unaccent
|
|
1509
|
+
? sql`${sql.raw(SEARCH_UNACCENT_FN)}(${searchString})`
|
|
1510
|
+
: sql`${searchString}`;
|
|
1511
|
+
|
|
1512
|
+
// `word_similarity(query, document)`, not `similarity`. `similarity`
|
|
1513
|
+
// scores two strings as wholes, so a short query against a whole row's
|
|
1514
|
+
// text scores near zero however well it matches part of it — measured:
|
|
1515
|
+
// "iso 14001 auditor" against one candidate's concatenated fields
|
|
1516
|
+
// scores 0.228 by `similarity` and 0.783 by `word_similarity`. The
|
|
1517
|
+
// first is below any usable threshold, which would have made `fuzzy`
|
|
1518
|
+
// a setting that quietly did nothing.
|
|
1519
|
+
//
|
|
1520
|
+
// Argument order matters: the first operand is the needle, and the
|
|
1521
|
+
// score is its similarity to the best-matching extent of the second.
|
|
1522
|
+
//
|
|
1523
|
+
// Both the function and the operator are schema-qualified: pg_trgm is
|
|
1524
|
+
// installed into `public`, and an unqualified reference resolves
|
|
1525
|
+
// through `search_path`, which does not necessarily reach it.
|
|
1526
|
+
const similar = sql`public.word_similarity(${needle}, ${fuzzyColumn}) >= ${spec.fuzzy.threshold}`;
|
|
1527
|
+
// `<%` is the index-backed form, but it tests against the session's
|
|
1528
|
+
// `pg_trgm.word_similarity_threshold` (0.6), not ours. Above that
|
|
1529
|
+
// default the operator narrows using the trigram index and the explicit
|
|
1530
|
+
// score refines; at or below it, the operator would exclude rows the
|
|
1531
|
+
// declared threshold admits, so the score stands alone and the planner
|
|
1532
|
+
// scans — correct either way, and only the faster path is conditional.
|
|
1533
|
+
const fuzzy = spec.fuzzy.threshold > PG_TRGM_WORD_SIMILARITY_DEFAULT
|
|
1534
|
+
? sql`(${needle} OPERATOR(public.<%) ${fuzzyColumn} AND ${similar})`
|
|
1535
|
+
: similar;
|
|
1536
|
+
|
|
1537
|
+
return sql`(${exact} OR ${fuzzy})`;
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
/**
|
|
1541
|
+
* `websearch_to_tsquery(<config>, <normalized search string>)`.
|
|
1542
|
+
*
|
|
1543
|
+
* Split out because the ranking expression needs the identical query — a
|
|
1544
|
+
* row ranked against a different tsquery than it was matched against is a
|
|
1545
|
+
* ranking of something else.
|
|
1546
|
+
*/
|
|
1547
|
+
static normalizedTsQuery(searchString: string, spec: SearchColumnSpec): SQL {
|
|
1548
|
+
const normalized = spec.unaccent
|
|
1549
|
+
? sql`${sql.raw(SEARCH_UNACCENT_FN)}(${searchString})`
|
|
1550
|
+
: sql`${searchString}`;
|
|
1551
|
+
return sql`websearch_to_tsquery(${spec.language}, ${normalized})`;
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
/**
|
|
1555
|
+
* A JSONB array of `{ field, snippet }` naming which declared fields matched
|
|
1556
|
+
* and showing the text around each hit — what backs `_matches`.
|
|
1557
|
+
*
|
|
1558
|
+
* A ranked list answers "which rows", never "why this row". For a talent
|
|
1559
|
+
* pool that difference is the product: a candidate surfacing for
|
|
1560
|
+
* "iso 14001" on a *certification* is a different candidate from one whose
|
|
1561
|
+
* bio happens to mention the standard, and the score cannot tell them apart.
|
|
1562
|
+
*
|
|
1563
|
+
* Built as a correlated subquery over a `VALUES` list of the declared
|
|
1564
|
+
* fields, rather than one `CASE` per field, so the shape does not change
|
|
1565
|
+
* with the number of fields and the empty result is a plain `[]`.
|
|
1566
|
+
*
|
|
1567
|
+
* `ts_headline` runs over the same normalized text that was indexed. Over
|
|
1568
|
+
* the *original* text it would find nothing to mark whenever `unaccent` is
|
|
1569
|
+
* on — the query's lexemes are folded and the document's are not — and
|
|
1570
|
+
* would return the text silently unhighlighted. Folded-but-marked beats
|
|
1571
|
+
* pretty-but-inert.
|
|
1572
|
+
*
|
|
1573
|
+
* Undefined when the collection has not opted in, when the column is not on
|
|
1574
|
+
* the table yet, or when the caller did not ask: this costs a `ts_headline`
|
|
1575
|
+
* per field per row and `ts_headline` re-parses the document.
|
|
1576
|
+
*/
|
|
1577
|
+
static buildSearchMatchesExpression(
|
|
1578
|
+
searchString: string,
|
|
1579
|
+
table: PgTable<any>,
|
|
1580
|
+
collection: CollectionConfig
|
|
1581
|
+
): SQL | undefined {
|
|
1582
|
+
let spec: SearchColumnSpec | undefined;
|
|
1583
|
+
try {
|
|
1584
|
+
spec = buildSearchColumnSpec(collection);
|
|
1585
|
+
} catch {
|
|
1586
|
+
return undefined;
|
|
1587
|
+
}
|
|
1588
|
+
if (!spec || spec.fields.length === 0) return undefined;
|
|
1589
|
+
if (!table[spec.column as keyof typeof table]) return undefined;
|
|
1590
|
+
|
|
1591
|
+
const query = DrizzleConditionBuilder.normalizedTsQuery(searchString, spec);
|
|
1592
|
+
const config = sql`${spec.language}`;
|
|
1593
|
+
|
|
1594
|
+
// `ord` keeps the author's declared field order in the output, so the
|
|
1595
|
+
// most important field they named reads first rather than whichever
|
|
1596
|
+
// Postgres aggregated first.
|
|
1597
|
+
const rows = spec.fields.map((f, i) =>
|
|
1598
|
+
sql`(${i}, ${f.path}, ${sql.raw(f.textSql)})`
|
|
1599
|
+
);
|
|
1600
|
+
|
|
1601
|
+
return sql`(
|
|
1602
|
+
SELECT coalesce(jsonb_agg(s.m ORDER BY f.ord), '[]'::jsonb)
|
|
1603
|
+
FROM (VALUES ${sql.join(rows, sql`, `)}) AS f(ord, path, txt)
|
|
1604
|
+
CROSS JOIN LATERAL (
|
|
1605
|
+
SELECT jsonb_build_object(
|
|
1606
|
+
'field', f.path,
|
|
1607
|
+
'snippet', ts_headline(${config}::regconfig, f.txt, ${query},
|
|
1608
|
+
'StartSel=<mark>,StopSel=</mark>,MaxWords=14,MinWords=1,MaxFragments=1,FragmentDelimiter= … ')
|
|
1609
|
+
) AS m
|
|
1610
|
+
WHERE to_tsvector(${config}::regconfig, f.txt) @@ ${query}
|
|
1611
|
+
) s
|
|
1612
|
+
)`;
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
/**
|
|
1616
|
+
* `ts_rank(<column>, <query>)` for the collection, or undefined when it has
|
|
1617
|
+
* not opted in. This is what backs `orderBy: ["_score", "desc"]`.
|
|
1618
|
+
*/
|
|
1619
|
+
static buildSearchRankExpression(
|
|
1620
|
+
searchString: string,
|
|
1621
|
+
table: PgTable<any>,
|
|
1622
|
+
collection: CollectionConfig
|
|
1623
|
+
): SQL | undefined {
|
|
1624
|
+
let spec: SearchColumnSpec | undefined;
|
|
1625
|
+
try {
|
|
1626
|
+
spec = buildSearchColumnSpec(collection);
|
|
1627
|
+
} catch {
|
|
1628
|
+
return undefined;
|
|
1629
|
+
}
|
|
1630
|
+
if (!spec) return undefined;
|
|
1631
|
+
const column = table[spec.column as keyof typeof table] as AnyPgColumn | undefined;
|
|
1632
|
+
if (!column) return undefined;
|
|
1633
|
+
|
|
1634
|
+
const rank = sql`ts_rank(${column}, ${DrizzleConditionBuilder.normalizedTsQuery(searchString, spec)})`;
|
|
1635
|
+
if (!spec.fuzzy) return rank;
|
|
1636
|
+
|
|
1637
|
+
const fuzzyColumn = table[spec.fuzzy.column as keyof typeof table] as AnyPgColumn | undefined;
|
|
1638
|
+
if (!fuzzyColumn) return rank;
|
|
1639
|
+
|
|
1640
|
+
// With `fuzzy` on, `ts_rank` alone is not a ranking — it is zero for
|
|
1641
|
+
// every row the trigram path matched and the exact path did not, which
|
|
1642
|
+
// is the whole population of a typo'd query. Measured on the real
|
|
1643
|
+
// sustentalent pool: "auditor de iso14000" matches four candidates,
|
|
1644
|
+
// every one of them at ts_rank 0, so ordering by rank alone returned
|
|
1645
|
+
// the best match in whatever order the table felt like.
|
|
1646
|
+
//
|
|
1647
|
+
// Summed rather than blended with tuned constants: a row that matched
|
|
1648
|
+
// exactly contributes both terms, so it outranks a fuzzy-only row of
|
|
1649
|
+
// equal similarity without needing a coefficient to say so. Both terms
|
|
1650
|
+
// are non-negative and monotonic, which is all the ordering needs.
|
|
1651
|
+
const needle = spec.unaccent
|
|
1652
|
+
? sql`${sql.raw(SEARCH_UNACCENT_FN)}(${searchString})`
|
|
1653
|
+
: sql`${searchString}`;
|
|
1654
|
+
return sql`(${rank} + public.word_similarity(${needle}, ${fuzzyColumn}))`;
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1310
1657
|
/**
|
|
1311
1658
|
* Build a unique field check condition
|
|
1312
1659
|
*/
|
|
@@ -1538,6 +1885,16 @@ whereConditions };
|
|
|
1538
1885
|
* - `orderBy`: SQL expression to ORDER BY distance (ascending = closest first)
|
|
1539
1886
|
* - `filter`: optional WHERE clause for distance threshold
|
|
1540
1887
|
* - `distanceSelect`: SQL expression for selecting the distance as `_distance`
|
|
1888
|
+
*
|
|
1889
|
+
* `property` is `?vector_search=` off the querystring, so it is an untrusted
|
|
1890
|
+
* *name*, and it used to be looked up straight in the drizzle table object.
|
|
1891
|
+
* Two ways that went wrong, both answering 500 to a malformed request:
|
|
1892
|
+
* `?vector_search=title` built `"title" <=> '[1,2]'::vector`, which the
|
|
1893
|
+
* database rejects with "operator does not exist"; and a table object also
|
|
1894
|
+
* carries non-column keys (`_`, methods), which passed the `if (!column)`
|
|
1895
|
+
* guard and compiled to nonsense. The name is resolved against the table's
|
|
1896
|
+
* actual columns and required to be a `vector` — anything else is the
|
|
1897
|
+
* caller's mistake and gets a 400 that says so.
|
|
1541
1898
|
*/
|
|
1542
1899
|
static buildVectorSearchConditions(
|
|
1543
1900
|
table: PgTable<any>,
|
|
@@ -1548,10 +1905,7 @@ whereConditions };
|
|
|
1548
1905
|
threshold?: number;
|
|
1549
1906
|
}
|
|
1550
1907
|
): { orderBy: SQL; filter?: SQL; distanceSelect: SQL } {
|
|
1551
|
-
const column = table
|
|
1552
|
-
if (!column) {
|
|
1553
|
-
throw new Error(`Vector column '${vectorSearch.property}' not found in table`);
|
|
1554
|
-
}
|
|
1908
|
+
const column = DrizzleConditionBuilder.resolveVectorColumn(table, vectorSearch.property);
|
|
1555
1909
|
|
|
1556
1910
|
// The vector is interpolated as a raw SQL literal below (pgvector has no
|
|
1557
1911
|
// bind form for the `::vector` cast), so every element must be a finite
|
|
@@ -1592,8 +1946,52 @@ whereConditions };
|
|
|
1592
1946
|
distanceSelect: sql`(${column} ${sql.raw(operator)} ${sql.raw(vectorLiteral)})`
|
|
1593
1947
|
};
|
|
1594
1948
|
}
|
|
1949
|
+
|
|
1950
|
+
/**
|
|
1951
|
+
* The `vector` column a request named, or a 400 explaining what it named.
|
|
1952
|
+
*
|
|
1953
|
+
* `getTableColumns` rather than a key lookup: it returns only the columns,
|
|
1954
|
+
* so `_`, `getSQL` and every other property of a drizzle table stop looking
|
|
1955
|
+
* like candidates. The type check is on the *physical* column
|
|
1956
|
+
* (`vector(1536)`) rather than on the declared property, so it holds for an
|
|
1957
|
+
* introspected collection too, where the property carries no Rebase type.
|
|
1958
|
+
*/
|
|
1959
|
+
private static resolveVectorColumn(table: PgTable<any>, property: string): AnyPgColumn {
|
|
1960
|
+
const columns = getTableColumns(table) as Record<string, AnyPgColumn> | undefined;
|
|
1961
|
+
const column = columns?.[property];
|
|
1962
|
+
if (!column) {
|
|
1963
|
+
const known = Object.entries(columns ?? {})
|
|
1964
|
+
.filter(([, c]) => isVectorColumn(c))
|
|
1965
|
+
.map(([name]) => name);
|
|
1966
|
+
throw ApiError.badRequest(
|
|
1967
|
+
`Unknown vector property "${property}". ` +
|
|
1968
|
+
(known.length > 0
|
|
1969
|
+
? `This collection's vector properties are: ${known.join(", ")}.`
|
|
1970
|
+
: "This collection declares no `vector` property to search."),
|
|
1971
|
+
"UNKNOWN_VECTOR_PROPERTY"
|
|
1972
|
+
);
|
|
1973
|
+
}
|
|
1974
|
+
if (!isVectorColumn(column)) {
|
|
1975
|
+
throw ApiError.badRequest(
|
|
1976
|
+
`Property "${property}" is not a vector column (it is \`${columnSqlType(column) || "unknown"}\`), ` +
|
|
1977
|
+
"so it has no distance operator. Name the property declared as `{ type: \"vector\" }`.",
|
|
1978
|
+
"UNKNOWN_VECTOR_PROPERTY"
|
|
1979
|
+
);
|
|
1980
|
+
}
|
|
1981
|
+
return column;
|
|
1982
|
+
}
|
|
1595
1983
|
}
|
|
1596
1984
|
|
|
1985
|
+
/** The column's SQL type, for a value that may not be a drizzle column at all. */
|
|
1986
|
+
const columnSqlType = (column: unknown): string => {
|
|
1987
|
+
const getSQLType = (column as { getSQLType?: () => string })?.getSQLType;
|
|
1988
|
+
return typeof getSQLType === "function" ? getSQLType.call(column).toLowerCase() : "";
|
|
1989
|
+
};
|
|
1990
|
+
|
|
1991
|
+
/** True for `vector(1536)` and its pgvector siblings, whatever the width. */
|
|
1992
|
+
const isVectorColumn = (column: unknown): boolean =>
|
|
1993
|
+
/^(vector|halfvec|sparsevec)\b/.test(columnSqlType(column));
|
|
1994
|
+
|
|
1597
1995
|
/**
|
|
1598
1996
|
* Alias for DrizzleConditionBuilder for consistent naming with other database implementations.
|
|
1599
1997
|
* This allows code to use PostgresConditionBuilder alongside future MongoConditionBuilder, etc.
|
|
@@ -314,9 +314,14 @@ export function sanitizeErrorForClient(error: unknown, context: string): { messa
|
|
|
314
314
|
column: pgError.column,
|
|
315
315
|
table: pgError.table,
|
|
316
316
|
constraint: pgError.constraint,
|
|
317
|
-
dataType: pgError.dataType
|
|
318
|
-
//
|
|
319
|
-
|
|
317
|
+
dataType: pgError.dataType
|
|
318
|
+
// The outer Drizzle wrapper message used to be logged here "for
|
|
319
|
+
// full context": it is `Failed query: <sql>\nparams: <values>`, so
|
|
320
|
+
// it published the statement and every bound value (an email, a
|
|
321
|
+
// password hash) on every realtime data failure. The SQLSTATE,
|
|
322
|
+
// detail, table, column and constraint above are the diagnostic
|
|
323
|
+
// value; the wrapper added only the leak. `logger` strips the
|
|
324
|
+
// wrapper as well, but the field itself carried nothing else.
|
|
320
325
|
});
|
|
321
326
|
return pgErrorToFriendlyMessage(pgError, context);
|
|
322
327
|
}
|