@rebasepro/server-postgres 0.13.1-canary.gf57a27e → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/PostgresBootstrapper.d.ts +26 -0
- package/dist/auth/services.d.ts +21 -0
- package/dist/{auth-users-columns-Dt9g712t.js → auth-users-columns-BfQHf9JE.js} +525 -63
- package/dist/auth-users-columns-BfQHf9JE.js.map +1 -0
- package/dist/{backup-service-Bww-Lg0s.js → backup-service-BH0Dzo_h.js} +2 -3
- package/dist/{backup-service-Bww-Lg0s.js.map → backup-service-BH0Dzo_h.js.map} +1 -1
- package/dist/cli-output.d.ts +34 -0
- package/dist/data-transformer.d.ts +7 -2
- package/dist/data_driver-ULAyJEi9.js +193 -0
- package/dist/data_driver-ULAyJEi9.js.map +1 -0
- package/dist/ensure-collection-policies-8vuu-n4r.js +124 -0
- package/dist/ensure-collection-policies-8vuu-n4r.js.map +1 -0
- package/dist/{ensure-collection-tables-DRxaUG96.js → ensure-collection-tables-CbvaGuVn.js} +89 -10
- package/dist/ensure-collection-tables-CbvaGuVn.js.map +1 -0
- package/dist/index.es.js +1310 -1060
- package/dist/index.es.js.map +1 -1
- package/dist/{rls-bootstrap-sql-Bpv3nUZo.js → rls-bootstrap-sql-69hYT8nr.js} +2 -2
- package/dist/{rls-bootstrap-sql-Bpv3nUZo.js.map → rls-bootstrap-sql-69hYT8nr.js.map} +1 -1
- package/dist/rls-enforcement-BJ_3wxwg.js +425 -0
- package/dist/rls-enforcement-BJ_3wxwg.js.map +1 -0
- package/dist/schema/auth-schema.d.ts +102 -0
- package/dist/schema/doctor-policy-checks.d.ts +28 -0
- package/dist/schema/doctor.d.ts +41 -25
- package/dist/schema/ensure-collection-policies.d.ts +33 -9
- package/dist/schema/ensure-collection-tables.d.ts +60 -6
- package/dist/schema/generate-drizzle-schema-logic.d.ts +9 -1
- package/dist/schema/introspect-db-inference.d.ts +8 -1
- package/dist/schema/introspect-db-logic.d.ts +49 -0
- package/dist/schema/introspect-db-project.d.ts +21 -0
- package/dist/schema/search-column.d.ts +49 -0
- package/dist/security/policy-drift.d.ts +34 -0
- package/dist/security/rls-enforcement.d.ts +8 -3
- package/dist/services/FetchService.d.ts +9 -0
- package/dist/services/PersistService.d.ts +21 -17
- package/dist/services/RelationService.d.ts +9 -57
- package/dist/services/RelationWriteService.d.ts +82 -0
- package/dist/services/collection-helpers.d.ts +42 -0
- package/dist/services/dataService.d.ts +2 -0
- package/dist/services/junction-writes.d.ts +82 -0
- package/dist/services/realtimeService.d.ts +137 -2
- package/dist/services/write-denial.d.ts +36 -0
- package/dist/{src-C_wvdMnl.js → src-DCdn3Val.js} +35 -3
- package/dist/src-DCdn3Val.js.map +1 -0
- package/dist/utils/drizzle-conditions.d.ts +54 -1
- package/dist/{websocket-D0TBU3ia.js → websocket-C8ZqVBiV.js} +75 -18
- package/dist/websocket-C8ZqVBiV.js.map +1 -0
- package/package.json +6 -6
- package/src/PostgresBackendDriver.ts +7 -3
- package/src/PostgresBootstrapper.ts +95 -9
- package/src/auth/ensure-tables.ts +27 -5
- package/src/auth/services.ts +82 -5
- package/src/backup/backup-cli.ts +59 -57
- package/src/cli-errors.ts +6 -6
- package/src/cli-helpers.ts +4 -4
- package/src/cli-output.ts +43 -0
- package/src/cli.ts +155 -147
- package/src/collections/buildRegistry.ts +3 -1
- package/src/data-transformer.ts +111 -25
- package/src/history/ensure-history-table.ts +2 -2
- package/src/schema/auth-schema.ts +17 -1
- package/src/schema/doctor-cli.ts +14 -65
- package/src/schema/doctor-policy-checks.ts +105 -0
- package/src/schema/doctor.ts +149 -72
- package/src/schema/ensure-collection-policies.ts +99 -6
- package/src/schema/ensure-collection-tables.ts +214 -17
- package/src/schema/generate-drizzle-schema-logic.ts +121 -65
- package/src/schema/generate-drizzle-schema.ts +11 -10
- package/src/schema/generate-postgres-ddl-logic.ts +28 -1
- package/src/schema/generate-postgres-ddl.ts +14 -13
- package/src/schema/generated-schema-staleness.ts +7 -5
- package/src/schema/introspect-db-inference.ts +9 -2
- package/src/schema/introspect-db-logic.ts +251 -75
- package/src/schema/introspect-db-project.ts +78 -0
- package/src/schema/introspect-db.ts +42 -25
- package/src/schema/introspect-runtime.ts +14 -2
- package/src/schema/search-column.ts +85 -0
- package/src/security/policy-drift.test.ts +104 -3
- package/src/security/policy-drift.ts +129 -7
- package/src/security/rls-enforcement.ts +9 -4
- package/src/services/FetchService.ts +105 -7
- package/src/services/PersistService.ts +68 -42
- package/src/services/RelationService.ts +35 -695
- package/src/services/RelationWriteService.ts +653 -0
- package/src/services/cdc/trigger-cdc.ts +5 -1
- package/src/services/channel-history.ts +9 -3
- package/src/services/channel-presence.ts +10 -3
- package/src/services/collection-helpers.ts +89 -4
- package/src/services/dataService.ts +2 -0
- package/src/services/junction-writes.ts +295 -0
- package/src/services/pg-notify-listener.ts +1 -1
- package/src/services/realtimeService.ts +337 -82
- package/src/services/write-denial.ts +55 -0
- package/src/utils/drizzle-conditions.ts +211 -34
- package/src/utils/pg-error-utils.ts +8 -3
- package/src/websocket.ts +113 -16
- package/dist/auth-users-columns-Dt9g712t.js.map +0 -1
- package/dist/ensure-collection-policies-CwYUliAa.js +0 -57
- package/dist/ensure-collection-policies-CwYUliAa.js.map +0 -1
- package/dist/ensure-collection-tables-DRxaUG96.js.map +0 -1
- package/dist/policy-CPkCqVTz.js +0 -105
- package/dist/policy-CPkCqVTz.js.map +0 -1
- package/dist/src-C_wvdMnl.js.map +0 -1
- package/dist/websocket-D0TBU3ia.js.map +0 -1
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { and, eq, or, sql, SQL, ilike, inArray, getTableColumns } from "drizzle-orm";
|
|
2
|
-
import { AnyPgColumn, PgTable
|
|
2
|
+
import { AnyPgColumn, PgTable } from "drizzle-orm/pg-core";
|
|
3
3
|
import {
|
|
4
|
+
ALL_WHERE_FILTER_OPS,
|
|
4
5
|
CollectionConfig, FilterValues, WhereFilterOp, JoinStep, LogicalCondition, FilterCondition,
|
|
5
6
|
ResolvedRelation, ResolvedBelongsTo, ResolvedHasOne, ResolvedHasMany,
|
|
6
7
|
ResolvedForeignKeyOnTarget, ResolvedManyToMany, hasForeignKeyOnTarget, isManyToMany
|
|
7
8
|
} from "@rebasepro/types";
|
|
8
9
|
import {
|
|
9
|
-
getColumnName, getTableName, normalizeToEntityRelation, resolveCollectionRelations
|
|
10
|
+
fieldKeyForColumn, getColumnName, getTableName, normalizeToEntityRelation, resolveCollectionRelations, toFilterTuples
|
|
10
11
|
} from "@rebasepro/common";
|
|
11
|
-
import { generateForeignKeyName } from "@rebasepro/utils";
|
|
12
|
+
import { generateForeignKeyName, toWireKey } from "@rebasepro/utils";
|
|
12
13
|
/**
|
|
13
14
|
* Postgres's own default for `pg_trgm.word_similarity_threshold`. Named here
|
|
14
15
|
* because the fuzzy predicate has to know when the index-backed operator agrees
|
|
@@ -35,6 +36,101 @@ import { getColumnMeta } from "../services/collection-helpers";
|
|
|
35
36
|
*/
|
|
36
37
|
export type UnknownFilterFieldsMode = "error" | "warn";
|
|
37
38
|
|
|
39
|
+
/**
|
|
40
|
+
* A user's search term, made safe to drop inside a `%…%` LIKE pattern.
|
|
41
|
+
*
|
|
42
|
+
* The term is already a bind parameter, so this is not about injection. It is
|
|
43
|
+
* about the two things a LIKE metacharacter does when it arrives from a search
|
|
44
|
+
* box:
|
|
45
|
+
*
|
|
46
|
+
* 1. **It changes the query.** `%` and `_` are wildcards, so searching for
|
|
47
|
+
* `50%` returned every row and `a_c` matched `abc`. Nothing the caller
|
|
48
|
+
* could type would find a literal `%`.
|
|
49
|
+
* 2. **It is a cost the caller chooses.** Postgres matches LIKE by
|
|
50
|
+
* backtracking: each `%` re-tries every remaining offset, so
|
|
51
|
+
* `?searchString=a%a%a%a%a%a%a%b` is polynomial with an attacker-chosen
|
|
52
|
+
* exponent — evaluated per row, OR-ed across every string property of the
|
|
53
|
+
* collection, on a sequential scan (a leading `%` cannot use an index), and
|
|
54
|
+
* the page limit does not bound it because the scan happens first.
|
|
55
|
+
*
|
|
56
|
+
* This is the server-side half of the pattern `like-pattern-redos.test.ts`
|
|
57
|
+
* hardened the offline evaluator against; that test's own note ("the same
|
|
58
|
+
* translation in the Mongo driver hands the expression to the database, where
|
|
59
|
+
* it occupies a server thread instead") describes this call site.
|
|
60
|
+
*
|
|
61
|
+
* Backslash is the default `ESCAPE` character for LIKE, and the pattern is
|
|
62
|
+
* bound rather than interpolated, so a single backslash here reaches the
|
|
63
|
+
* matcher as one. Escaping the escape character first is what keeps a term
|
|
64
|
+
* ending in `\` from swallowing the closing `%`.
|
|
65
|
+
*
|
|
66
|
+
* Note this is a *substring search*, not the `like` filter operator: a caller
|
|
67
|
+
* who wants wildcards has `?title=like.foo%` for that, where the pattern is the
|
|
68
|
+
* documented input.
|
|
69
|
+
*/
|
|
70
|
+
export const escapeLikePattern = (value: string): string =>
|
|
71
|
+
value.replace(/[\\%_]/g, ch => `\\${ch}`);
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* The Drizzle column a relation's column name addresses on a table.
|
|
75
|
+
*
|
|
76
|
+
* A relation names its link in *column* terms — `localKey: "author_id"`,
|
|
77
|
+
* `foreignKeyOnTarget: "author_id"` — because that is what the database and
|
|
78
|
+
* every FK constraint call it. A Drizzle table is keyed by the *wire* name,
|
|
79
|
+
* `authorId`. Indexing the table with the column, which is what every one of
|
|
80
|
+
* these call sites used to do, therefore finds nothing the moment the two
|
|
81
|
+
* differ: for a `columnName`-carrying property that was already true, and it is
|
|
82
|
+
* now true of every derived foreign key.
|
|
83
|
+
*
|
|
84
|
+
* `undefined` rather than a throw: each caller already has a message naming the
|
|
85
|
+
* relation it was resolving, which is worth more than a generic one here.
|
|
86
|
+
*/
|
|
87
|
+
const relationColumn = (
|
|
88
|
+
table: PgTable<any>,
|
|
89
|
+
collection: CollectionConfig | undefined,
|
|
90
|
+
column: string
|
|
91
|
+
): AnyPgColumn | undefined => {
|
|
92
|
+
const key = fieldKeyForColumn(collection, column);
|
|
93
|
+
return (key in table ? table[key as keyof typeof table] as AnyPgColumn : undefined) || undefined;
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
/** The target collection of a relation, or `undefined` if its thunk cannot resolve. */
|
|
97
|
+
const targetOf = (relation: ResolvedRelation): CollectionConfig | undefined => {
|
|
98
|
+
try {
|
|
99
|
+
return relation.target();
|
|
100
|
+
} catch {
|
|
101
|
+
return undefined;
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/** Column types `ILIKE '%…%'` is defined on. */
|
|
106
|
+
const ILIKE_SQL_TYPES = /^(text|varchar|character varying|char|character|bpchar|citext)\b/;
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Can this column be matched with `ILIKE`?
|
|
110
|
+
*
|
|
111
|
+
* Asked of the column's *declared SQL type*, never with `instanceof`. The
|
|
112
|
+
* previous version tested `column instanceof PgVarchar || … PgText || … PgChar`,
|
|
113
|
+
* and `instanceof` compares class identity: it is only true when the column was
|
|
114
|
+
* constructed by the very same copy of `drizzle-orm` that this module imported.
|
|
115
|
+
*
|
|
116
|
+
* An application's generated schema builds its tables with the app's own
|
|
117
|
+
* `drizzle-orm`, and this driver declares its own dependency on one. When the
|
|
118
|
+
* two ranges do not overlap — an app scaffolded against `^0.44` with a driver
|
|
119
|
+
* asking for `^0.45` — a strict installer gives the driver a second copy, every
|
|
120
|
+
* check returns false, no condition is produced, and the caller compiles that
|
|
121
|
+
* into an impossible `WHERE`. The result is a 200 with an empty page for every
|
|
122
|
+
* search on every collection without a `search` block: the failure looks
|
|
123
|
+
* exactly like "nothing matched". Observed in production, not theorised.
|
|
124
|
+
*
|
|
125
|
+
* `getSQLType()` is a value the column reports about itself, so it crosses
|
|
126
|
+
* module instances the way a class identity cannot. It also happens to fix
|
|
127
|
+
* `citext`, which the `instanceof` list never covered.
|
|
128
|
+
*/
|
|
129
|
+
const supportsILike = (column: AnyPgColumn): boolean => {
|
|
130
|
+
const sqlType = typeof column?.getSQLType === "function" ? column.getSQLType().toLowerCase() : "";
|
|
131
|
+
return ILIKE_SQL_TYPES.test(sqlType);
|
|
132
|
+
};
|
|
133
|
+
|
|
38
134
|
/**
|
|
39
135
|
* Process-wide default, set once when the driver is constructed.
|
|
40
136
|
*
|
|
@@ -240,7 +336,7 @@ export class DrizzleConditionBuilder {
|
|
|
240
336
|
|
|
241
337
|
case "hasOne":
|
|
242
338
|
case "hasMany": {
|
|
243
|
-
const fkColumn = targetTable
|
|
339
|
+
const fkColumn = relationColumn(targetTable, targetOf(relation), relation.foreignKeyOnTarget);
|
|
244
340
|
if (!fkColumn) {
|
|
245
341
|
throw new Error(
|
|
246
342
|
`Foreign key column '${relation.foreignKeyOnTarget}' not found in the target table of ` +
|
|
@@ -393,7 +489,7 @@ export class DrizzleConditionBuilder {
|
|
|
393
489
|
|
|
394
490
|
// Owning relation, resolved: the relation names its own local key.
|
|
395
491
|
if (relation?.kind === "belongsTo") {
|
|
396
|
-
const foreignKey =
|
|
492
|
+
const foreignKey = relationColumn(table, collection, relation.localKey);
|
|
397
493
|
if (foreignKey) return { kind: "column", column: foreignKey };
|
|
398
494
|
}
|
|
399
495
|
|
|
@@ -408,7 +504,7 @@ export class DrizzleConditionBuilder {
|
|
|
408
504
|
// correlating on the id anyway silently matches nothing —
|
|
409
505
|
// "filter by this relation" would quietly return zero rows.
|
|
410
506
|
const correlationColumn = hasForeignKeyOnTarget(relation) && relation.sourceKey
|
|
411
|
-
?
|
|
507
|
+
? relationColumn(table, collection, relation.sourceKey)
|
|
412
508
|
: sourceIdColumn;
|
|
413
509
|
if (!correlationColumn) {
|
|
414
510
|
throw new Error(
|
|
@@ -421,10 +517,17 @@ export class DrizzleConditionBuilder {
|
|
|
421
517
|
}
|
|
422
518
|
}
|
|
423
519
|
|
|
424
|
-
// No collection in hand — the
|
|
425
|
-
//
|
|
426
|
-
// `
|
|
427
|
-
|
|
520
|
+
// No collection in hand — the shapes an owning relation's key takes by
|
|
521
|
+
// default (e.g. `project` → `projectId`, `userProfile` →
|
|
522
|
+
// `userProfileId`). The snake forms stay in the list because a project
|
|
523
|
+
// may have authored the property under its column name, which is still
|
|
524
|
+
// its wire name.
|
|
525
|
+
for (const guess of [
|
|
526
|
+
`${field}Id`,
|
|
527
|
+
toWireKey(generateForeignKeyName(field)),
|
|
528
|
+
`${field}_id`,
|
|
529
|
+
generateForeignKeyName(field)
|
|
530
|
+
]) {
|
|
428
531
|
const foreignKey = columnAt(guess);
|
|
429
532
|
if (foreignKey) return { kind: "column", column: foreignKey };
|
|
430
533
|
}
|
|
@@ -472,11 +575,9 @@ export class DrizzleConditionBuilder {
|
|
|
472
575
|
const target = this.resolveFilterTarget(table, field, collectionPath, mode, options);
|
|
473
576
|
if (!target) continue;
|
|
474
577
|
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
for (const [op, value] of paramsList) {
|
|
578
|
+
// One tuple or an array of them — the grammar, read the same way by
|
|
579
|
+
// every compiler. See `toFilterTuples`.
|
|
580
|
+
for (const [op, value] of toFilterTuples(filterParam)) {
|
|
480
581
|
const condition = this.compileFilterTarget(target, op, value, field, collectionPath);
|
|
481
582
|
if (condition) {
|
|
482
583
|
conditions.push(condition);
|
|
@@ -610,7 +711,7 @@ export class DrizzleConditionBuilder {
|
|
|
610
711
|
`(collection '${targetCollection.slug}')`
|
|
611
712
|
);
|
|
612
713
|
}
|
|
613
|
-
const fkColumn = targetTable
|
|
714
|
+
const fkColumn = relationColumn(targetTable, targetCollection, relation.foreignKeyOnTarget);
|
|
614
715
|
if (!fkColumn) {
|
|
615
716
|
throw new Error(
|
|
616
717
|
`Foreign key column '${relation.foreignKeyOnTarget}' not found in the target table of ` +
|
|
@@ -851,8 +952,24 @@ export class DrizzleConditionBuilder {
|
|
|
851
952
|
case "is-not-null":
|
|
852
953
|
return sql`${column} IS NOT NULL`;
|
|
853
954
|
default:
|
|
854
|
-
|
|
855
|
-
|
|
955
|
+
// The relation path five hundred lines up already refuses this,
|
|
956
|
+
// and says why: "returning `null` for an operator this cannot
|
|
957
|
+
// express would drop the condition", and a dropped condition
|
|
958
|
+
// widens the result. The column path is its twin and kept the
|
|
959
|
+
// warning — so `{ status: ["contains", "x"] }` filtered on
|
|
960
|
+
// nothing and answered 200 with every row, which reads as data
|
|
961
|
+
// that matched.
|
|
962
|
+
//
|
|
963
|
+
// The wire layer rejects operator-shaped unknowns before they
|
|
964
|
+
// arrive (`UnknownFilterOperatorError`, 400). What reaches here
|
|
965
|
+
// came from in-process `rebase.data`, a stored filter preset or
|
|
966
|
+
// a config — none of them typechecked at the call site, all of
|
|
967
|
+
// them able to name an operator that no longer exists.
|
|
968
|
+
throw ApiError.badRequest(
|
|
969
|
+
`Unknown filter operator '${op}'. Valid operators: ${ALL_WHERE_FILTER_OPS.join(", ")}.`,
|
|
970
|
+
"UNKNOWN_FILTER_OPERATOR",
|
|
971
|
+
{ operator: op, validOperators: ALL_WHERE_FILTER_OPS }
|
|
972
|
+
);
|
|
856
973
|
}
|
|
857
974
|
}
|
|
858
975
|
|
|
@@ -1253,7 +1370,7 @@ whereConditions };
|
|
|
1253
1370
|
return match(targetIdCol);
|
|
1254
1371
|
}
|
|
1255
1372
|
|
|
1256
|
-
const foreignKeyCol = targetTable
|
|
1373
|
+
const foreignKeyCol = relationColumn(targetTable, targetOf(relation), relation.foreignKeyOnTarget);
|
|
1257
1374
|
if (!foreignKeyCol) {
|
|
1258
1375
|
throw new Error(
|
|
1259
1376
|
`Foreign key column '${relation.foreignKeyOnTarget}' not found in the target table of relation ` +
|
|
@@ -1290,7 +1407,8 @@ whereConditions };
|
|
|
1290
1407
|
* `tsvector` column. Stems, drops stopwords, AND-es the terms, reaches
|
|
1291
1408
|
* inside JSONB and arrays, and uses the GIN index.
|
|
1292
1409
|
* - **Not declared** — the original `ILIKE '%term%'` OR-ed across top-level
|
|
1293
|
-
* string properties,
|
|
1410
|
+
* string properties, with the term escaped (see {@link escapeLikePattern})
|
|
1411
|
+
* so it is matched as the literal text the user typed.
|
|
1294
1412
|
*
|
|
1295
1413
|
* The second is the default and stays the default. A collection that has
|
|
1296
1414
|
* not opted in compiles to exactly the SQL it compiled to before this
|
|
@@ -1313,26 +1431,34 @@ whereConditions };
|
|
|
1313
1431
|
: undefined;
|
|
1314
1432
|
if (ftsCondition) return [ftsCondition];
|
|
1315
1433
|
|
|
1434
|
+
let declaredStringProperties = 0;
|
|
1435
|
+
|
|
1316
1436
|
for (const [key, prop] of Object.entries(properties)) {
|
|
1317
1437
|
const p = prop as Record<string, unknown>;
|
|
1318
1438
|
// Only include string properties that don't have enum defined
|
|
1319
1439
|
// PostgreSQL enum and uuid columns don't support ILIKE, so we skip them
|
|
1320
1440
|
if (p.type === "string" && !p.enum && p.isId !== "uuid") {
|
|
1441
|
+
declaredStringProperties++;
|
|
1321
1442
|
const fieldColumn = table[key as keyof typeof table] as AnyPgColumn;
|
|
1322
|
-
if (fieldColumn) {
|
|
1323
|
-
|
|
1324
|
-
const supportsILike =
|
|
1325
|
-
fieldColumn instanceof PgVarchar ||
|
|
1326
|
-
fieldColumn instanceof PgText ||
|
|
1327
|
-
fieldColumn instanceof PgChar ||
|
|
1328
|
-
(fieldColumn && typeof fieldColumn === "object" && !("columnType" in fieldColumn));
|
|
1329
|
-
if (supportsILike) {
|
|
1330
|
-
searchConditions.push(ilike(fieldColumn, `%${searchString}%`));
|
|
1331
|
-
}
|
|
1443
|
+
if (fieldColumn && supportsILike(fieldColumn)) {
|
|
1444
|
+
searchConditions.push(ilike(fieldColumn, `%${escapeLikePattern(searchString)}%`));
|
|
1332
1445
|
}
|
|
1333
1446
|
}
|
|
1334
1447
|
}
|
|
1335
1448
|
|
|
1449
|
+
// Every string property was rejected, so the caller is about to turn an
|
|
1450
|
+
// empty condition list into "match nothing" — a 200 with an empty page,
|
|
1451
|
+
// which reads as "no such row" rather than as the breakage it is. Say so
|
|
1452
|
+
// once per query: this is how the `instanceof` version of
|
|
1453
|
+
// {@link supportsILike} failed silently in the field for months.
|
|
1454
|
+
if (declaredStringProperties > 0 && searchConditions.length === 0) {
|
|
1455
|
+
logger.warn(
|
|
1456
|
+
`[search] "${collection?.slug ?? "collection"}" declares ${declaredStringProperties} string ` +
|
|
1457
|
+
"property(ies) but none compiled to a searchable column, so this search can only return nothing. " +
|
|
1458
|
+
"Check that the generated schema's column types are text/varchar/char."
|
|
1459
|
+
);
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1336
1462
|
return searchConditions;
|
|
1337
1463
|
}
|
|
1338
1464
|
|
|
@@ -1759,6 +1885,16 @@ whereConditions };
|
|
|
1759
1885
|
* - `orderBy`: SQL expression to ORDER BY distance (ascending = closest first)
|
|
1760
1886
|
* - `filter`: optional WHERE clause for distance threshold
|
|
1761
1887
|
* - `distanceSelect`: SQL expression for selecting the distance as `_distance`
|
|
1888
|
+
*
|
|
1889
|
+
* `property` is `?vector_search=` off the querystring, so it is an untrusted
|
|
1890
|
+
* *name*, and it used to be looked up straight in the drizzle table object.
|
|
1891
|
+
* Two ways that went wrong, both answering 500 to a malformed request:
|
|
1892
|
+
* `?vector_search=title` built `"title" <=> '[1,2]'::vector`, which the
|
|
1893
|
+
* database rejects with "operator does not exist"; and a table object also
|
|
1894
|
+
* carries non-column keys (`_`, methods), which passed the `if (!column)`
|
|
1895
|
+
* guard and compiled to nonsense. The name is resolved against the table's
|
|
1896
|
+
* actual columns and required to be a `vector` — anything else is the
|
|
1897
|
+
* caller's mistake and gets a 400 that says so.
|
|
1762
1898
|
*/
|
|
1763
1899
|
static buildVectorSearchConditions(
|
|
1764
1900
|
table: PgTable<any>,
|
|
@@ -1769,10 +1905,7 @@ whereConditions };
|
|
|
1769
1905
|
threshold?: number;
|
|
1770
1906
|
}
|
|
1771
1907
|
): { orderBy: SQL; filter?: SQL; distanceSelect: SQL } {
|
|
1772
|
-
const column = table
|
|
1773
|
-
if (!column) {
|
|
1774
|
-
throw new Error(`Vector column '${vectorSearch.property}' not found in table`);
|
|
1775
|
-
}
|
|
1908
|
+
const column = DrizzleConditionBuilder.resolveVectorColumn(table, vectorSearch.property);
|
|
1776
1909
|
|
|
1777
1910
|
// The vector is interpolated as a raw SQL literal below (pgvector has no
|
|
1778
1911
|
// bind form for the `::vector` cast), so every element must be a finite
|
|
@@ -1813,8 +1946,52 @@ whereConditions };
|
|
|
1813
1946
|
distanceSelect: sql`(${column} ${sql.raw(operator)} ${sql.raw(vectorLiteral)})`
|
|
1814
1947
|
};
|
|
1815
1948
|
}
|
|
1949
|
+
|
|
1950
|
+
/**
|
|
1951
|
+
* The `vector` column a request named, or a 400 explaining what it named.
|
|
1952
|
+
*
|
|
1953
|
+
* `getTableColumns` rather than a key lookup: it returns only the columns,
|
|
1954
|
+
* so `_`, `getSQL` and every other property of a drizzle table stop looking
|
|
1955
|
+
* like candidates. The type check is on the *physical* column
|
|
1956
|
+
* (`vector(1536)`) rather than on the declared property, so it holds for an
|
|
1957
|
+
* introspected collection too, where the property carries no Rebase type.
|
|
1958
|
+
*/
|
|
1959
|
+
private static resolveVectorColumn(table: PgTable<any>, property: string): AnyPgColumn {
|
|
1960
|
+
const columns = getTableColumns(table) as Record<string, AnyPgColumn> | undefined;
|
|
1961
|
+
const column = columns?.[property];
|
|
1962
|
+
if (!column) {
|
|
1963
|
+
const known = Object.entries(columns ?? {})
|
|
1964
|
+
.filter(([, c]) => isVectorColumn(c))
|
|
1965
|
+
.map(([name]) => name);
|
|
1966
|
+
throw ApiError.badRequest(
|
|
1967
|
+
`Unknown vector property "${property}". ` +
|
|
1968
|
+
(known.length > 0
|
|
1969
|
+
? `This collection's vector properties are: ${known.join(", ")}.`
|
|
1970
|
+
: "This collection declares no `vector` property to search."),
|
|
1971
|
+
"UNKNOWN_VECTOR_PROPERTY"
|
|
1972
|
+
);
|
|
1973
|
+
}
|
|
1974
|
+
if (!isVectorColumn(column)) {
|
|
1975
|
+
throw ApiError.badRequest(
|
|
1976
|
+
`Property "${property}" is not a vector column (it is \`${columnSqlType(column) || "unknown"}\`), ` +
|
|
1977
|
+
"so it has no distance operator. Name the property declared as `{ type: \"vector\" }`.",
|
|
1978
|
+
"UNKNOWN_VECTOR_PROPERTY"
|
|
1979
|
+
);
|
|
1980
|
+
}
|
|
1981
|
+
return column;
|
|
1982
|
+
}
|
|
1816
1983
|
}
|
|
1817
1984
|
|
|
1985
|
+
/** The column's SQL type, for a value that may not be a drizzle column at all. */
|
|
1986
|
+
const columnSqlType = (column: unknown): string => {
|
|
1987
|
+
const getSQLType = (column as { getSQLType?: () => string })?.getSQLType;
|
|
1988
|
+
return typeof getSQLType === "function" ? getSQLType.call(column).toLowerCase() : "";
|
|
1989
|
+
};
|
|
1990
|
+
|
|
1991
|
+
/** True for `vector(1536)` and its pgvector siblings, whatever the width. */
|
|
1992
|
+
const isVectorColumn = (column: unknown): boolean =>
|
|
1993
|
+
/^(vector|halfvec|sparsevec)\b/.test(columnSqlType(column));
|
|
1994
|
+
|
|
1818
1995
|
/**
|
|
1819
1996
|
* Alias for DrizzleConditionBuilder for consistent naming with other database implementations.
|
|
1820
1997
|
* This allows code to use PostgresConditionBuilder alongside future MongoConditionBuilder, etc.
|
|
@@ -314,9 +314,14 @@ export function sanitizeErrorForClient(error: unknown, context: string): { messa
|
|
|
314
314
|
column: pgError.column,
|
|
315
315
|
table: pgError.table,
|
|
316
316
|
constraint: pgError.constraint,
|
|
317
|
-
dataType: pgError.dataType
|
|
318
|
-
//
|
|
319
|
-
|
|
317
|
+
dataType: pgError.dataType
|
|
318
|
+
// The outer Drizzle wrapper message used to be logged here "for
|
|
319
|
+
// full context": it is `Failed query: <sql>\nparams: <values>`, so
|
|
320
|
+
// it published the statement and every bound value (an email, a
|
|
321
|
+
// password hash) on every realtime data failure. The SQLSTATE,
|
|
322
|
+
// detail, table, column and constraint above are the diagnostic
|
|
323
|
+
// value; the wrapper added only the leak. `logger` strips the
|
|
324
|
+
// wrapper as well, but the field itself carried nothing else.
|
|
320
325
|
});
|
|
321
326
|
return pgErrorToFriendlyMessage(pgError, context);
|
|
322
327
|
}
|
package/src/websocket.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { RealtimeService } from "./services/realtimeService";
|
|
2
2
|
import { PostgresBackendDriver } from "./PostgresBackendDriver";
|
|
3
3
|
import type { DataDriver, DeleteProps, FetchCollectionProps, FetchOneProps, SaveProps, TableMetadata, BranchInfo, AuthAdapter } from "@rebasepro/types";
|
|
4
|
-
import { ANONYMOUS_USER_ID, isSQLAdmin, isSchemaAdmin } from "@rebasepro/types";
|
|
4
|
+
import { ANONYMOUS_USER_ID, isSQLAdmin, isSchemaAdmin, resolveClientListLimit, ListLimitError } from "@rebasepro/types";
|
|
5
5
|
import type { User } from "@rebasepro/types";
|
|
6
6
|
|
|
7
7
|
import { WebSocketServer, WebSocket } from "ws";
|
|
@@ -39,6 +39,9 @@ interface ClientSession {
|
|
|
39
39
|
/** Sliding window message counter for rate limiting */
|
|
40
40
|
messageCount: number;
|
|
41
41
|
messageWindowStart: number;
|
|
42
|
+
/** The same window, counted separately for channel frames. */
|
|
43
|
+
channelMessageCount: number;
|
|
44
|
+
channelWindowStart: number;
|
|
42
45
|
}
|
|
43
46
|
|
|
44
47
|
|
|
@@ -47,6 +50,33 @@ const WS_RATE_LIMIT = 2000;
|
|
|
47
50
|
/** Rate limit window in milliseconds (60 seconds) */
|
|
48
51
|
const WS_RATE_WINDOW_MS = 60_000;
|
|
49
52
|
|
|
53
|
+
/**
|
|
54
|
+
* Channel frames get their own budget, because they are a different workload.
|
|
55
|
+
*
|
|
56
|
+
* 2000/minute is 33/second, which is generous for queries and subscriptions and
|
|
57
|
+
* an order of magnitude below what the documented channel idiom asks for: the
|
|
58
|
+
* capacity note in `docs/backend/realtime.md` uses 60 fps cursor movement as
|
|
59
|
+
* its worked example, and the presence idiom re-`track()`s on every move, so
|
|
60
|
+
* one client sustaining that sends ~120 frames/second — 7200 a minute. Sharing
|
|
61
|
+
* one counter meant the cursor stream ate the query budget and then froze for
|
|
62
|
+
* the rest of the window.
|
|
63
|
+
*
|
|
64
|
+
* The number is sized to that documented workload and nothing more; it is not
|
|
65
|
+
* a considered product limit (see `docs/channel-authorization.md`).
|
|
66
|
+
*/
|
|
67
|
+
const WS_CHANNEL_RATE_LIMIT = 7200;
|
|
68
|
+
|
|
69
|
+
/** Frames counted against the channel budget rather than the general one. */
|
|
70
|
+
const CHANNEL_MESSAGE_TYPES = new Set([
|
|
71
|
+
"join_channel",
|
|
72
|
+
"leave_channel",
|
|
73
|
+
"broadcast",
|
|
74
|
+
"presence_track",
|
|
75
|
+
"presence_untrack",
|
|
76
|
+
"presence_state",
|
|
77
|
+
"channel_history"
|
|
78
|
+
]);
|
|
79
|
+
|
|
50
80
|
/** Admin-only WebSocket message types */
|
|
51
81
|
const ADMIN_ONLY_TYPES = new Set([
|
|
52
82
|
"EXECUTE_SQL",
|
|
@@ -139,7 +169,9 @@ export function createPostgresWebSocket(
|
|
|
139
169
|
clientSessions.set(clientId, { ws,
|
|
140
170
|
authenticated: !requireAuth,
|
|
141
171
|
messageCount: 0,
|
|
142
|
-
messageWindowStart: Date.now()
|
|
172
|
+
messageWindowStart: Date.now(),
|
|
173
|
+
channelMessageCount: 0,
|
|
174
|
+
channelWindowStart: Date.now() });
|
|
143
175
|
realtimeService.addClient(clientId, ws);
|
|
144
176
|
|
|
145
177
|
ws.on("close", () => {
|
|
@@ -247,19 +279,34 @@ roles: verifiedUser.roles }
|
|
|
247
279
|
}
|
|
248
280
|
}
|
|
249
281
|
|
|
250
|
-
// Rate limiting: reject if client exceeds message limit
|
|
282
|
+
// Rate limiting: reject if client exceeds message limit.
|
|
283
|
+
// Channel frames are counted against their own budget — see
|
|
284
|
+
// WS_CHANNEL_RATE_LIMIT for why one shared counter starved them.
|
|
251
285
|
{
|
|
252
286
|
const session = clientSessions.get(clientId);
|
|
253
287
|
if (session) {
|
|
254
288
|
const now = Date.now();
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
session.
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
289
|
+
const isChannelFrame = CHANNEL_MESSAGE_TYPES.has(type);
|
|
290
|
+
if (isChannelFrame) {
|
|
291
|
+
if (now - session.channelWindowStart > WS_RATE_WINDOW_MS) {
|
|
292
|
+
session.channelMessageCount = 0;
|
|
293
|
+
session.channelWindowStart = now;
|
|
294
|
+
}
|
|
295
|
+
session.channelMessageCount++;
|
|
296
|
+
if (session.channelMessageCount > WS_CHANNEL_RATE_LIMIT) {
|
|
297
|
+
sendError("ERROR", "RATE_LIMITED", "Too many channel messages. Please slow down.");
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
} else {
|
|
301
|
+
if (now - session.messageWindowStart > WS_RATE_WINDOW_MS) {
|
|
302
|
+
session.messageCount = 0;
|
|
303
|
+
session.messageWindowStart = now;
|
|
304
|
+
}
|
|
305
|
+
session.messageCount++;
|
|
306
|
+
if (session.messageCount > WS_RATE_LIMIT) {
|
|
307
|
+
sendError("ERROR", "RATE_LIMITED", "Too many requests. Please slow down.");
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
263
310
|
}
|
|
264
311
|
}
|
|
265
312
|
}
|
|
@@ -312,7 +359,19 @@ roles: verifiedUser.roles }
|
|
|
312
359
|
wsDebug("📋 [WebSocket Server] Processing FETCH_COLLECTION request");
|
|
313
360
|
const request: FetchCollectionProps = payload;
|
|
314
361
|
const delegate = await getScopedDelegate();
|
|
315
|
-
|
|
362
|
+
// Bound the client-supplied limit with the SAME guarantee
|
|
363
|
+
// the REST ingress and `subscribe_collection` apply
|
|
364
|
+
// (`resolveClientListLimit`). Without it an absent limit
|
|
365
|
+
// reached the driver as `undefined`, which emits no LIMIT
|
|
366
|
+
// clause — one socket frame streamed the whole table, on
|
|
367
|
+
// the one transport that skipped the ceiling every other
|
|
368
|
+
// read path enforces.
|
|
369
|
+
const rows = await delegate.fetchCollection({
|
|
370
|
+
...request,
|
|
371
|
+
limit: resolveClientListLimit(request.limit, {
|
|
372
|
+
vectorSearch: !!request.vectorSearch
|
|
373
|
+
})
|
|
374
|
+
});
|
|
316
375
|
wsDebug("📋 [WebSocket Server] FETCH_COLLECTION result - rows count:", rows.length);
|
|
317
376
|
const response = {
|
|
318
377
|
type: "FETCH_COLLECTION_SUCCESS",
|
|
@@ -400,6 +459,12 @@ colors: true }));
|
|
|
400
459
|
|
|
401
460
|
|
|
402
461
|
case "COUNT": {
|
|
462
|
+
// Deliberately NOT routed through `resolveClientListLimit`:
|
|
463
|
+
// this answers with a scalar, and the driver drops `limit`
|
|
464
|
+
// on the way to `SELECT count(*)`. Clamping here could only
|
|
465
|
+
// ever make `total` describe fewer rows than the collection
|
|
466
|
+
// holds — the page size is the caller's business, the total
|
|
467
|
+
// is not.
|
|
403
468
|
const request: FetchCollectionProps = payload;
|
|
404
469
|
const delegate = await getScopedDelegate();
|
|
405
470
|
const count = await delegate.count!(request);
|
|
@@ -426,14 +491,29 @@ colors: true }));
|
|
|
426
491
|
wsDebug(`⚡ [WebSocket Server] SQL executed. Returned ${Array.isArray(result) ? result.length : "non-array"} rows.`);
|
|
427
492
|
}
|
|
428
493
|
const auditSession = clientSessions.get(clientId);
|
|
429
|
-
console.log
|
|
430
|
-
|
|
431
|
-
|
|
494
|
+
// Through `logger`, not `console.log`: this line is
|
|
495
|
+
// emitted in production, and a bare console call
|
|
496
|
+
// has no severity, no timestamp, no JSON envelope
|
|
497
|
+
// and no LOG_LEVEL gate, so it lands in Cloud
|
|
498
|
+
// Logging as unstructured text the queries written
|
|
499
|
+
// for every other line cannot match.
|
|
500
|
+
//
|
|
501
|
+
// The bound values are counted, never written — the
|
|
502
|
+
// statement is the audit signal, the parameters are
|
|
503
|
+
// whatever row the operator was touching. (stdout is
|
|
504
|
+
// not an audit sink either; a real trail belongs in
|
|
505
|
+
// a table with an actor and a retention policy.)
|
|
506
|
+
logger.info("[SQL Audit] WebSocket SQL execution", {
|
|
507
|
+
sql: typeof sql === "string" ? sql.substring(0, 500) : String(sql),
|
|
508
|
+
database: options?.database,
|
|
509
|
+
role: options?.role,
|
|
510
|
+
paramCount: Array.isArray(options?.params) ? options.params.length : 0,
|
|
432
511
|
resultRows: Array.isArray(result) ? result.length : "unknown",
|
|
433
512
|
uid: auditSession?.user?.uid ?? "unknown",
|
|
434
513
|
roles: auditSession?.user?.roles ?? [],
|
|
435
514
|
isAdmin: auditSession?.user?.isAdmin ?? false,
|
|
436
|
-
|
|
515
|
+
requestId
|
|
516
|
+
});
|
|
437
517
|
const response = {
|
|
438
518
|
type: "EXECUTE_SQL_SUCCESS",
|
|
439
519
|
payload: { result },
|
|
@@ -644,6 +724,23 @@ roles: ["anon"] };
|
|
|
644
724
|
logger.error("❌ [WebSocket Server] Unknown message type", { detail: type });
|
|
645
725
|
}
|
|
646
726
|
} catch (error: unknown) {
|
|
727
|
+
// A refused `limit` is the caller's mistake, not a server fault.
|
|
728
|
+
// Left to the generic branch below it answers INTERNAL_ERROR
|
|
729
|
+
// with the message suppressed in production — so the one thing
|
|
730
|
+
// that would tell the caller what to send instead is exactly
|
|
731
|
+
// what gets dropped. Answered here the way
|
|
732
|
+
// `subscribe_collection` already answers it: INVALID_LIMIT,
|
|
733
|
+
// message intact. The text names the ceiling and nothing else.
|
|
734
|
+
if (error instanceof ListLimitError) {
|
|
735
|
+
logger.warn(`[WebSocket Server] Refused a list read: ${error.message}`);
|
|
736
|
+
ws.send(JSON.stringify({
|
|
737
|
+
type: "ERROR",
|
|
738
|
+
requestId,
|
|
739
|
+
payload: { error: { message: error.message,
|
|
740
|
+
code: "INVALID_LIMIT" } }
|
|
741
|
+
}));
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
647
744
|
logger.error("💥 [WebSocket Server] Error handling message", { error: error });
|
|
648
745
|
if (error instanceof Error) {
|
|
649
746
|
logger.error("Stack trace", { detail: error.stack });
|