@rebasepro/server-postgres 0.10.1-canary.a54c057 → 0.10.1-canary.af2af91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/PostgresBootstrapper.d.ts +7 -3
- package/dist/auth/services.d.ts +43 -4
- package/dist/backup/backup-logic.d.ts +23 -0
- package/dist/backup/backup-service.d.ts +44 -2
- package/dist/backup/pg-tools.d.ts +41 -1
- package/dist/chunk-DSJWtz9O.js +40 -0
- package/dist/cli-helpers.d.ts +33 -1
- package/dist/ensure-collection-tables-BVvtkkRm.js +304 -0
- package/dist/ensure-collection-tables-BVvtkkRm.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.es.js +1854 -4745
- package/dist/index.es.js.map +1 -1
- package/dist/schema/auth-schema.d.ts +170 -0
- package/dist/schema/destructive-sql.d.ts +49 -0
- package/dist/schema/ensure-collection-tables.d.ts +79 -0
- package/dist/schema/generate-postgres-ddl-logic.d.ts +4 -1
- package/dist/services/FetchService.d.ts +21 -8
- package/dist/services/PersistService.d.ts +12 -0
- package/dist/services/RelationService.d.ts +30 -0
- package/dist/services/cdc/CdcListener.d.ts +7 -14
- package/dist/services/cdc/junction-tables.d.ts +38 -0
- package/dist/services/channel-bus/ChannelBus.d.ts +29 -0
- package/dist/services/channel-bus/PostgresChannelBus.d.ts +111 -0
- package/dist/services/channel-bus/index.d.ts +55 -0
- package/dist/services/channel-history.d.ts +11 -0
- package/dist/services/channel-presence.d.ts +66 -0
- package/dist/services/nested-path.d.ts +59 -0
- package/dist/services/pg-notify-listener.d.ts +47 -0
- package/dist/services/realtimeService.d.ts +133 -6
- package/dist/src-CBgtrPhJ.js +336 -0
- package/dist/src-CBgtrPhJ.js.map +1 -0
- package/dist/src-lcfUP4xg.js +4106 -0
- package/dist/src-lcfUP4xg.js.map +1 -0
- package/dist/utils/drizzle-conditions.d.ts +41 -0
- package/package.json +8 -9
- package/src/PostgresBootstrapper.ts +82 -4
- package/src/auth/ensure-tables.ts +70 -56
- package/src/auth/services.ts +186 -48
- package/src/backup/backup-cli.ts +60 -1
- package/src/backup/backup-cron.ts +24 -1
- package/src/backup/backup-logic.ts +62 -0
- package/src/backup/backup-service.ts +132 -13
- package/src/backup/pg-tools.ts +70 -2
- package/src/cli-helpers.ts +82 -27
- package/src/cli.ts +152 -6
- package/src/index.ts +4 -0
- package/src/schema/auth-schema.ts +41 -3
- package/src/schema/destructive-sql.ts +94 -0
- package/src/schema/doctor.ts +6 -6
- package/src/schema/ensure-collection-tables.test.ts +156 -0
- package/src/schema/ensure-collection-tables.ts +297 -0
- package/src/schema/generate-drizzle-schema-logic.ts +13 -9
- package/src/schema/generate-postgres-ddl-logic.ts +22 -15
- package/src/schema/introspect-db-inference.ts +13 -13
- package/src/schema/introspect-db-logic.ts +6 -6
- package/src/services/FetchService.ts +104 -114
- package/src/services/PersistService.ts +127 -85
- package/src/services/RelationService.ts +95 -6
- package/src/services/cdc/CdcListener.ts +27 -91
- package/src/services/cdc/junction-tables.ts +91 -0
- package/src/services/channel-bus/ChannelBus.ts +44 -0
- package/src/services/channel-bus/PostgresChannelBus.ts +299 -0
- package/src/services/channel-bus/index.ts +123 -0
- package/src/services/channel-history.ts +35 -0
- package/src/services/channel-presence.ts +148 -0
- package/src/services/nested-path.ts +130 -0
- package/src/services/pg-notify-listener.ts +137 -0
- package/src/services/realtimeService.ts +443 -14
- package/src/utils/drizzle-conditions.ts +154 -0
|
@@ -4,18 +4,21 @@ import { Client as PgClient } from "pg";
|
|
|
4
4
|
import { randomUUID } from "crypto";
|
|
5
5
|
import { DataService } from "./dataService";
|
|
6
6
|
|
|
7
|
-
import { FetchCollectionProps, ListenCollectionProps, ListenOneProps, DataDriver, CollectionUpdateMessage, SingleUpdateMessage, CollectionPatchMessage, WebSocketMessage, FilterValues, CollectionConfig, RebaseCallContext } from "@rebasepro/types";
|
|
7
|
+
import { FetchCollectionProps, ListenCollectionProps, ListenOneProps, DataDriver, CollectionUpdateMessage, SingleUpdateMessage, CollectionPatchMessage, WebSocketMessage, FilterValues, CollectionConfig, RebaseCallContext, resolveClientListLimit } from "@rebasepro/types";
|
|
8
8
|
import { NodePgDatabase } from "drizzle-orm/node-postgres";
|
|
9
9
|
import { sql as drizzleSql } from "drizzle-orm";
|
|
10
10
|
import { RealtimeProvider, CollectionSubscriptionConfig, SingleSubscriptionConfig } from "../interfaces";
|
|
11
11
|
import { PostgresCollectionRegistry } from "../collections/PostgresCollectionRegistry";
|
|
12
12
|
import { buildPropertyCallbacks, getTableName } from "@rebasepro/common";
|
|
13
13
|
import { applyAuthContext } from "../security/rls-enforcement";
|
|
14
|
+
import { buildJunctionLinkMap, type JunctionLink } from "./cdc/junction-tables";
|
|
14
15
|
import { logger } from "@rebasepro/server";
|
|
15
16
|
import { sanitizeErrorForClient } from "../utils/pg-error-utils";
|
|
16
17
|
import { CdcListener, type CdcChangeEvent } from "./cdc/CdcListener";
|
|
17
18
|
import { deriveRowAddress, getPrimaryKeys, type PrimaryKeyInfo } from "./collection-helpers";
|
|
18
19
|
import { ChannelHistoryStore, type ResolvedRetention } from "./channel-history";
|
|
20
|
+
import { ChannelPresenceStore } from "./channel-presence";
|
|
21
|
+
import { ChannelBus, ChannelBusFrame, MemoryChannelBus, frameByteLength } from "./channel-bus";
|
|
19
22
|
import type { ChannelHistoryEntry, ChannelRetentionRule } from "@rebasepro/types";
|
|
20
23
|
|
|
21
24
|
/** Channel name used for Postgres LISTEN/NOTIFY cross-instance realtime. */
|
|
@@ -76,8 +79,38 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
|
|
|
76
79
|
* wait on each other.
|
|
77
80
|
*/
|
|
78
81
|
private channelSendQueues = new Map<string, Promise<void>>();
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Cross-instance transport for channel frames and presence.
|
|
85
|
+
*
|
|
86
|
+
* Defaults to the memory bus, which publishes nowhere — so a single-instance
|
|
87
|
+
* deployment runs the same fan-out it always did, with one resolved promise
|
|
88
|
+
* per broadcast for company. See `channel-bus/ChannelBus.ts`.
|
|
89
|
+
*/
|
|
90
|
+
private bus: ChannelBus = new MemoryChannelBus();
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* The shared presence roster, present only when a real bus is active.
|
|
94
|
+
*
|
|
95
|
+
* Fan-out alone is not enough for presence: `presence_state` has to answer
|
|
96
|
+
* with everyone in the channel, and per-process maps can only answer for
|
|
97
|
+
* this replica's clients. See `channel-presence.ts`.
|
|
98
|
+
*/
|
|
99
|
+
private presenceStore?: ChannelPresenceStore;
|
|
100
|
+
|
|
101
|
+
/** Sweeps roster rows left behind by instances that stopped heartbeating. */
|
|
102
|
+
private presenceSweepInterval?: ReturnType<typeof setInterval>;
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Channels whose oversized ephemeral broadcasts have already been reported,
|
|
106
|
+
* so a hot channel logs the problem once rather than once per message.
|
|
107
|
+
*/
|
|
108
|
+
private oversizedBroadcastWarned = new Set<string>();
|
|
109
|
+
|
|
79
110
|
private presenceInterval?: ReturnType<typeof setInterval>;
|
|
80
111
|
private static readonly PRESENCE_TIMEOUT_MS = 30000; // 30s
|
|
112
|
+
/** How often stale roster rows from other instances are reaped. */
|
|
113
|
+
private static readonly PRESENCE_SWEEP_INTERVAL_MS = 10000; // 10s
|
|
81
114
|
private dataService: DataService;
|
|
82
115
|
// Enhanced subscriptions storage with full request parameters
|
|
83
116
|
private _subscriptions = new Map<string, {
|
|
@@ -127,6 +160,9 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
|
|
|
127
160
|
private cdcListener?: CdcListener;
|
|
128
161
|
/** Whether database-level CDC is the active cross-instance change source. */
|
|
129
162
|
private cdcActive = false;
|
|
163
|
+
/** Junction table → the child lists its rows belong to, built when CDC starts. */
|
|
164
|
+
private junctionLinkMap?: Map<string, JunctionLink[]>;
|
|
165
|
+
|
|
130
166
|
/** Reverse lookup: `schema.table` (and bare `table`) → collection, built when CDC starts. */
|
|
131
167
|
private cdcTableMap?: Map<string, CollectionConfig>;
|
|
132
168
|
/**
|
|
@@ -307,15 +343,19 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
|
|
|
307
343
|
for (const [channel, members] of this.channels.entries()) {
|
|
308
344
|
if (members.has(clientId)) {
|
|
309
345
|
members.delete(clientId);
|
|
310
|
-
this.removePresence(clientId, channel);
|
|
346
|
+
this.removePresence(clientId, channel, { skipStore: true });
|
|
311
347
|
if (members.size === 0) this.channels.delete(channel);
|
|
312
348
|
}
|
|
313
349
|
}
|
|
314
350
|
|
|
315
351
|
// Remove from all presence channels
|
|
316
352
|
for (const [channel] of this.presence) {
|
|
317
|
-
this.removePresence(clientId, channel);
|
|
353
|
+
this.removePresence(clientId, channel, { skipStore: true });
|
|
318
354
|
}
|
|
355
|
+
|
|
356
|
+
// One statement for every channel the client was in, rather than one
|
|
357
|
+
// per channel above — a disconnect is the common case, not a rare one.
|
|
358
|
+
void this.presenceStoreOp(() => this.presenceStore!.removeClient(clientId), "client removal");
|
|
319
359
|
}
|
|
320
360
|
|
|
321
361
|
private async handleMessage(clientId: string, message: WebSocketMessage, authContext?: SubscriptionAuthContext) {
|
|
@@ -391,6 +431,16 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
|
|
|
391
431
|
return;
|
|
392
432
|
}
|
|
393
433
|
|
|
434
|
+
// Bound the client-supplied limit with the SAME guarantee the REST
|
|
435
|
+
// ingress applies (`resolveClientListLimit`): clamp to the hard max
|
|
436
|
+
// and default an absent limit by mode. A subscription is re-fetched
|
|
437
|
+
// on every matching write, so an unbounded one is a DoS amplified
|
|
438
|
+
// per write — resolve it once and reuse for the stored request and
|
|
439
|
+
// the initial fetch.
|
|
440
|
+
const boundedLimit = resolveClientListLimit(request.limit, {
|
|
441
|
+
vectorSearch: !!request.vectorSearch
|
|
442
|
+
});
|
|
443
|
+
|
|
394
444
|
// Store subscription with full request parameters and auth context for RLS
|
|
395
445
|
this._subscriptions.set(subscriptionId, {
|
|
396
446
|
clientId,
|
|
@@ -400,7 +450,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
|
|
|
400
450
|
filter: request.filter,
|
|
401
451
|
orderBy: request.orderBy,
|
|
402
452
|
order: request.order,
|
|
403
|
-
limit:
|
|
453
|
+
limit: boundedLimit,
|
|
404
454
|
startAfter: request.startAfter as Record<string, unknown> | undefined,
|
|
405
455
|
databaseId: request.collection?.databaseId,
|
|
406
456
|
searchString: request.searchString
|
|
@@ -415,7 +465,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
|
|
|
415
465
|
filter: request.filter,
|
|
416
466
|
orderBy: request.orderBy,
|
|
417
467
|
order: request.order,
|
|
418
|
-
limit:
|
|
468
|
+
limit: boundedLimit,
|
|
419
469
|
startAfter: request.startAfter as Record<string, unknown> | undefined,
|
|
420
470
|
searchString: request.searchString
|
|
421
471
|
},
|
|
@@ -1088,6 +1138,11 @@ roles: activeAuth.roles },
|
|
|
1088
1138
|
const retention = this.channelHistory?.retentionFor(channel);
|
|
1089
1139
|
if (!retention) {
|
|
1090
1140
|
this.fanOutBroadcast(clientId, channel, event, payload);
|
|
1141
|
+
// Other instances get the same frame, but never before the clients
|
|
1142
|
+
// on this one: the local fan-out above is synchronous and the
|
|
1143
|
+
// publish is not, which is also what keeps the ephemeral path free
|
|
1144
|
+
// of any await for a single-instance deployment.
|
|
1145
|
+
this.publishBroadcast(clientId, channel, event, payload);
|
|
1091
1146
|
return;
|
|
1092
1147
|
}
|
|
1093
1148
|
|
|
@@ -1137,6 +1192,7 @@ roles: activeAuth.roles },
|
|
|
1137
1192
|
}
|
|
1138
1193
|
|
|
1139
1194
|
this.fanOutBroadcast(clientId, channel, event, payload, seq);
|
|
1195
|
+
this.publishBroadcast(clientId, channel, event, payload, seq);
|
|
1140
1196
|
|
|
1141
1197
|
try {
|
|
1142
1198
|
await this.channelHistory!.prune(channel, retention);
|
|
@@ -1170,6 +1226,186 @@ roles: activeAuth.roles },
|
|
|
1170
1226
|
}
|
|
1171
1227
|
}
|
|
1172
1228
|
|
|
1229
|
+
// =============================================================================
|
|
1230
|
+
// Cross-Instance Channel Bus
|
|
1231
|
+
// =============================================================================
|
|
1232
|
+
|
|
1233
|
+
/**
|
|
1234
|
+
* Install the transport that carries channel frames between instances.
|
|
1235
|
+
*
|
|
1236
|
+
* Called once at boot. A bus that cannot start is reported and replaced with
|
|
1237
|
+
* the memory bus: losing cross-instance fan-out degrades collaboration to
|
|
1238
|
+
* what it was before this existed, whereas refusing to boot takes the whole
|
|
1239
|
+
* backend down for it.
|
|
1240
|
+
*/
|
|
1241
|
+
async configureChannelBus(bus: ChannelBus): Promise<void> {
|
|
1242
|
+
if (bus.kind === "memory") {
|
|
1243
|
+
this.bus = bus;
|
|
1244
|
+
return;
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
try {
|
|
1248
|
+
await bus.start((frame) => this.handleBusFrame(frame));
|
|
1249
|
+
} catch (error) {
|
|
1250
|
+
logger.warn(
|
|
1251
|
+
`⚠️ [ChannelBus] Could not start the "${bus.kind}" channel bus — channel broadcast and presence ` +
|
|
1252
|
+
"stay per-instance. Clients served by different replicas will not see each other.",
|
|
1253
|
+
{ error }
|
|
1254
|
+
);
|
|
1255
|
+
await bus.stop().catch(() => { /* best effort */ });
|
|
1256
|
+
this.bus = new MemoryChannelBus();
|
|
1257
|
+
return;
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
this.bus = bus;
|
|
1261
|
+
|
|
1262
|
+
// Presence needs shared *state*, not just shared fan-out — see
|
|
1263
|
+
// `channel-presence.ts`. It comes up with the bus and only with it.
|
|
1264
|
+
try {
|
|
1265
|
+
const store = new ChannelPresenceStore(this.db, this.instanceId);
|
|
1266
|
+
await store.ensureTables();
|
|
1267
|
+
this.presenceStore = store;
|
|
1268
|
+
this.ensurePresenceSweep();
|
|
1269
|
+
} catch (error) {
|
|
1270
|
+
logger.warn(
|
|
1271
|
+
"⚠️ [ChannelBus] Could not create the shared presence table — presence rosters will only list " +
|
|
1272
|
+
"clients connected to this instance (broadcast is unaffected).",
|
|
1273
|
+
{ error }
|
|
1274
|
+
);
|
|
1275
|
+
this.presenceStore = undefined;
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
logger.info(
|
|
1279
|
+
`📡 [ChannelBus] Cross-instance channels active via ${bus.kind} (instanceId: ${this.instanceId}).`
|
|
1280
|
+
);
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
/** Which transport is in use — `"memory"` means per-instance only. */
|
|
1284
|
+
public getChannelBusKind(): ChannelBus["kind"] {
|
|
1285
|
+
return this.bus.kind;
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
/**
|
|
1289
|
+
* Send a broadcast to the other instances.
|
|
1290
|
+
*
|
|
1291
|
+
* Fire-and-forget by design: the clients on this instance have already been
|
|
1292
|
+
* served, and a bus that is briefly unreachable must not turn a broadcast
|
|
1293
|
+
* into an error for the sender.
|
|
1294
|
+
*/
|
|
1295
|
+
private publishBroadcast(clientId: string, channel: string, event: string, payload: unknown, seq?: number): void {
|
|
1296
|
+
if (this.bus.kind === "memory") return;
|
|
1297
|
+
|
|
1298
|
+
const frame: ChannelBusFrame = {
|
|
1299
|
+
kind: "broadcast",
|
|
1300
|
+
sid: this.instanceId,
|
|
1301
|
+
channel,
|
|
1302
|
+
event,
|
|
1303
|
+
from: clientId,
|
|
1304
|
+
...(seq !== undefined ? { seq } : {}),
|
|
1305
|
+
payload
|
|
1306
|
+
};
|
|
1307
|
+
|
|
1308
|
+
// Postgres caps a NOTIFY payload at 8 KB. A retained message is already
|
|
1309
|
+
// durable and addressable, so it travels as a pointer and each receiver
|
|
1310
|
+
// reads the body back — the same shape as the entity path, which
|
|
1311
|
+
// notifies an address and refetches the row.
|
|
1312
|
+
if (frameByteLength(frame) > this.bus.maxFrameBytes) {
|
|
1313
|
+
if (seq === undefined) {
|
|
1314
|
+
this.reportOversizedBroadcast(clientId, channel);
|
|
1315
|
+
return;
|
|
1316
|
+
}
|
|
1317
|
+
void this.publishFrame({
|
|
1318
|
+
kind: "broadcast_ref",
|
|
1319
|
+
sid: this.instanceId,
|
|
1320
|
+
channel,
|
|
1321
|
+
from: clientId,
|
|
1322
|
+
seq
|
|
1323
|
+
});
|
|
1324
|
+
return;
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
void this.publishFrame(frame);
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1330
|
+
private async publishFrame(frame: ChannelBusFrame): Promise<void> {
|
|
1331
|
+
try {
|
|
1332
|
+
await this.bus.publish(frame);
|
|
1333
|
+
} catch (error) {
|
|
1334
|
+
logger.error("❌ [ChannelBus] Failed to publish frame — other instances did not receive it", {
|
|
1335
|
+
detail: `${frame.kind} on "${frame.channel}"`,
|
|
1336
|
+
error
|
|
1337
|
+
});
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
/**
|
|
1342
|
+
* Tell the sender that a message was delivered locally but nowhere else.
|
|
1343
|
+
*
|
|
1344
|
+
* Staying quiet here would be the worst option available: on one instance
|
|
1345
|
+
* the app works, on two it works for half the users, and nothing in the
|
|
1346
|
+
* logs connects the two. The fix is a one-liner in config — give the
|
|
1347
|
+
* channel a retention rule and the message travels as a pointer instead —
|
|
1348
|
+
* so the message says exactly that.
|
|
1349
|
+
*/
|
|
1350
|
+
private reportOversizedBroadcast(clientId: string, channel: string): void {
|
|
1351
|
+
const remedy =
|
|
1352
|
+
`Add a retention rule for "${channel}" (realtime.channels) — retained messages travel by reference ` +
|
|
1353
|
+
"and have no size limit.";
|
|
1354
|
+
|
|
1355
|
+
if (!this.oversizedBroadcastWarned.has(channel)) {
|
|
1356
|
+
this.oversizedBroadcastWarned.add(channel);
|
|
1357
|
+
logger.warn(
|
|
1358
|
+
`⚠️ [ChannelBus] A broadcast on ephemeral channel "${channel}" exceeds the ` +
|
|
1359
|
+
`${this.bus.maxFrameBytes}-byte limit of the ${this.bus.kind} bus and reached only this instance. ` +
|
|
1360
|
+
remedy
|
|
1361
|
+
);
|
|
1362
|
+
}
|
|
1363
|
+
this.sendError(
|
|
1364
|
+
clientId,
|
|
1365
|
+
`Broadcast on "${channel}" was too large to reach other instances. ${remedy}`,
|
|
1366
|
+
undefined,
|
|
1367
|
+
"CHANNEL_BUS_PAYLOAD_TOO_LARGE"
|
|
1368
|
+
);
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
/**
|
|
1372
|
+
* Deliver a frame published by another instance to this one's clients.
|
|
1373
|
+
*
|
|
1374
|
+
* Frames we published ourselves are dropped on arrival — the local fan-out
|
|
1375
|
+
* happened before the publish — exactly as the entity-change handler skips
|
|
1376
|
+
* its own `sid`.
|
|
1377
|
+
*/
|
|
1378
|
+
private async handleBusFrame(frame: ChannelBusFrame): Promise<void> {
|
|
1379
|
+
if (frame.sid === this.instanceId) return;
|
|
1380
|
+
|
|
1381
|
+
switch (frame.kind) {
|
|
1382
|
+
case "broadcast":
|
|
1383
|
+
this.fanOutBroadcast(frame.from ?? "", frame.channel, frame.event, frame.payload, frame.seq);
|
|
1384
|
+
return;
|
|
1385
|
+
|
|
1386
|
+
case "broadcast_ref": {
|
|
1387
|
+
// Nothing to read back for: skip the query rather than pay for
|
|
1388
|
+
// a message no client here is waiting for.
|
|
1389
|
+
if (!this.channels.get(frame.channel)?.size) return;
|
|
1390
|
+
|
|
1391
|
+
const entry = await this.channelHistory?.getBySeq(frame.channel, frame.seq);
|
|
1392
|
+
if (!entry) {
|
|
1393
|
+
logger.warn(
|
|
1394
|
+
`⚠️ [ChannelBus] Message ${frame.seq} on "${frame.channel}" is no longer retained — ` +
|
|
1395
|
+
"clients on this instance will need to replay (channel_history) to catch up."
|
|
1396
|
+
);
|
|
1397
|
+
return;
|
|
1398
|
+
}
|
|
1399
|
+
this.fanOutBroadcast(frame.from ?? "", frame.channel, entry.event, entry.payload, entry.seq);
|
|
1400
|
+
return;
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
case "presence_diff":
|
|
1404
|
+
this.deliverPresenceDiff(frame.channel, frame.joins, frame.leaves);
|
|
1405
|
+
return;
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1173
1409
|
// =============================================================================
|
|
1174
1410
|
// Channel History
|
|
1175
1411
|
// =============================================================================
|
|
@@ -1247,32 +1483,59 @@ roles: activeAuth.roles },
|
|
|
1247
1483
|
// Presence
|
|
1248
1484
|
// =============================================================================
|
|
1249
1485
|
|
|
1250
|
-
/**
|
|
1486
|
+
/**
|
|
1487
|
+
* Track presence in a channel.
|
|
1488
|
+
*
|
|
1489
|
+
* The client re-sends this every ~20s as a heartbeat against the 30s
|
|
1490
|
+
* timeout, so most calls carry the state that is already recorded. Those
|
|
1491
|
+
* refresh `last_seen` and stop there: re-announcing an unchanged state to
|
|
1492
|
+
* every instance would put a bus message per client per heartbeat on the
|
|
1493
|
+
* wire to tell everyone nothing happened.
|
|
1494
|
+
*/
|
|
1251
1495
|
trackPresence(clientId: string, channel: string, state: Record<string, unknown>): void {
|
|
1252
1496
|
if (!this.presence.has(channel)) {
|
|
1253
1497
|
this.presence.set(channel, new Map());
|
|
1254
1498
|
}
|
|
1255
1499
|
|
|
1256
1500
|
const channelPresence = this.presence.get(channel)!;
|
|
1501
|
+
const previous = channelPresence.get(clientId);
|
|
1502
|
+
const changed = !previous || JSON.stringify(previous.state) !== JSON.stringify(state);
|
|
1257
1503
|
channelPresence.set(clientId, { state,
|
|
1258
1504
|
lastSeen: Date.now() });
|
|
1259
1505
|
|
|
1506
|
+
// Refresh the shared roster on every heartbeat — that timestamp is what
|
|
1507
|
+
// tells other instances this client is still here.
|
|
1508
|
+
void this.presenceStoreOp(() => this.presenceStore!.track(channel, clientId, state), "track");
|
|
1509
|
+
|
|
1260
1510
|
// Broadcast join / state update to channel
|
|
1261
|
-
this.
|
|
1511
|
+
this.deliverPresenceDiff(channel, { [clientId]: state }, {});
|
|
1512
|
+
if (changed) {
|
|
1513
|
+
this.publishPresenceDiff(channel, { [clientId]: state }, {});
|
|
1514
|
+
}
|
|
1262
1515
|
|
|
1263
1516
|
// Start cleanup interval if not running
|
|
1264
1517
|
this.ensurePresenceCleanup();
|
|
1265
1518
|
}
|
|
1266
1519
|
|
|
1267
|
-
/**
|
|
1268
|
-
|
|
1520
|
+
/**
|
|
1521
|
+
* Remove presence from a channel.
|
|
1522
|
+
*
|
|
1523
|
+
* `skipStore` is for the socket-close path, which clears every channel at
|
|
1524
|
+
* once and then deletes the client's rows in a single statement instead of
|
|
1525
|
+
* one per channel.
|
|
1526
|
+
*/
|
|
1527
|
+
removePresence(clientId: string, channel: string, options?: { skipStore?: boolean }): void {
|
|
1269
1528
|
const channelPresence = this.presence.get(channel);
|
|
1270
1529
|
if (!channelPresence) return;
|
|
1271
1530
|
|
|
1272
1531
|
const entry = channelPresence.get(clientId);
|
|
1273
1532
|
if (entry) {
|
|
1274
1533
|
channelPresence.delete(clientId);
|
|
1275
|
-
this.
|
|
1534
|
+
this.deliverPresenceDiff(channel, {}, { [clientId]: entry.state });
|
|
1535
|
+
this.publishPresenceDiff(channel, {}, { [clientId]: entry.state });
|
|
1536
|
+
if (!options?.skipStore) {
|
|
1537
|
+
void this.presenceStoreOp(() => this.presenceStore!.remove(channel, clientId), "remove");
|
|
1538
|
+
}
|
|
1276
1539
|
}
|
|
1277
1540
|
|
|
1278
1541
|
if (channelPresence.size === 0) {
|
|
@@ -1280,17 +1543,50 @@ lastSeen: Date.now() });
|
|
|
1280
1543
|
}
|
|
1281
1544
|
}
|
|
1282
1545
|
|
|
1283
|
-
/**
|
|
1546
|
+
/**
|
|
1547
|
+
* Send the full roster for a channel to one client.
|
|
1548
|
+
*
|
|
1549
|
+
* Answered from the shared table when there is one, because "who is in this
|
|
1550
|
+
* document?" has a single answer that must not depend on which replica the
|
|
1551
|
+
* asker happens to be connected to. Without a bus there is nothing to share
|
|
1552
|
+
* and the local map *is* the roster — that path stays synchronous, which is
|
|
1553
|
+
* what it always was.
|
|
1554
|
+
*/
|
|
1284
1555
|
sendPresenceState(clientId: string, channel: string): void {
|
|
1556
|
+
if (!this.presenceStore) {
|
|
1557
|
+
this.sendPresenceStateMessage(clientId, channel, this.localPresences(channel));
|
|
1558
|
+
return;
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
void this.presenceStore.roster(channel)
|
|
1562
|
+
.then((presences) => {
|
|
1563
|
+
this.sendPresenceStateMessage(clientId, channel, presences);
|
|
1564
|
+
})
|
|
1565
|
+
.catch((error) => {
|
|
1566
|
+
// A roster the asker can act on beats none: fall back to the
|
|
1567
|
+
// clients we can see rather than leaving the request unanswered.
|
|
1568
|
+
logger.warn(`⚠️ [Presence] Could not read the shared roster for "${channel}" — answering with this instance's clients only.`, { error });
|
|
1569
|
+
this.sendPresenceStateMessage(clientId, channel, this.localPresences(channel));
|
|
1570
|
+
});
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
/** Presence of the clients connected to this instance. */
|
|
1574
|
+
private localPresences(channel: string): Record<string, Record<string, unknown>> {
|
|
1285
1575
|
const channelPresence = this.presence.get(channel);
|
|
1286
1576
|
const presences: Record<string, Record<string, unknown>> = {};
|
|
1287
|
-
|
|
1288
1577
|
if (channelPresence) {
|
|
1289
1578
|
for (const [id, { state }] of channelPresence) {
|
|
1290
1579
|
presences[id] = state;
|
|
1291
1580
|
}
|
|
1292
1581
|
}
|
|
1582
|
+
return presences;
|
|
1583
|
+
}
|
|
1293
1584
|
|
|
1585
|
+
private sendPresenceStateMessage(
|
|
1586
|
+
clientId: string,
|
|
1587
|
+
channel: string,
|
|
1588
|
+
presences: Record<string, Record<string, unknown>>
|
|
1589
|
+
): void {
|
|
1294
1590
|
const ws = this.clients.get(clientId);
|
|
1295
1591
|
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
1296
1592
|
ws.send(JSON.stringify({
|
|
@@ -1301,8 +1597,8 @@ lastSeen: Date.now() });
|
|
|
1301
1597
|
}
|
|
1302
1598
|
}
|
|
1303
1599
|
|
|
1304
|
-
/**
|
|
1305
|
-
private
|
|
1600
|
+
/** Deliver a presence diff to this instance's members of the channel. */
|
|
1601
|
+
private deliverPresenceDiff(
|
|
1306
1602
|
channel: string,
|
|
1307
1603
|
joins: Record<string, Record<string, unknown>>,
|
|
1308
1604
|
leaves: Record<string, Record<string, unknown>>
|
|
@@ -1325,6 +1621,26 @@ lastSeen: Date.now() });
|
|
|
1325
1621
|
}
|
|
1326
1622
|
}
|
|
1327
1623
|
|
|
1624
|
+
/** Tell the other instances about a presence change. */
|
|
1625
|
+
private publishPresenceDiff(
|
|
1626
|
+
channel: string,
|
|
1627
|
+
joins: Record<string, Record<string, unknown>>,
|
|
1628
|
+
leaves: Record<string, Record<string, unknown>>
|
|
1629
|
+
): void {
|
|
1630
|
+
if (this.bus.kind === "memory") return;
|
|
1631
|
+
void this.publishFrame({ kind: "presence_diff", sid: this.instanceId, channel, joins, leaves });
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
/** Run a roster write when there is a roster, and never let it throw. */
|
|
1635
|
+
private async presenceStoreOp(op: () => Promise<void>, label: string): Promise<void> {
|
|
1636
|
+
if (!this.presenceStore) return;
|
|
1637
|
+
try {
|
|
1638
|
+
await op();
|
|
1639
|
+
} catch (error) {
|
|
1640
|
+
logger.warn(`⚠️ [Presence] Shared roster ${label} failed`, { error });
|
|
1641
|
+
}
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1328
1644
|
/** Periodic cleanup for stale presences */
|
|
1329
1645
|
private ensurePresenceCleanup(): void {
|
|
1330
1646
|
if (this.presenceInterval) return;
|
|
@@ -1345,6 +1661,43 @@ lastSeen: Date.now() });
|
|
|
1345
1661
|
}, 10000); // Check every 10s
|
|
1346
1662
|
}
|
|
1347
1663
|
|
|
1664
|
+
/**
|
|
1665
|
+
* Reap roster rows whose owning instance stopped heartbeating.
|
|
1666
|
+
*
|
|
1667
|
+
* This is the cross-instance half of the sweep above, and it doubles as
|
|
1668
|
+
* crash recovery: a pod that dies takes its clients with it but leaves
|
|
1669
|
+
* their rows behind, and after one TTL window they look exactly like any
|
|
1670
|
+
* other client that went quiet. The delete returns what it removed, so
|
|
1671
|
+
* whichever instance wins the race is the one that announces the
|
|
1672
|
+
* departures — once for the cluster, not once per replica.
|
|
1673
|
+
*/
|
|
1674
|
+
private ensurePresenceSweep(): void {
|
|
1675
|
+
if (this.presenceSweepInterval || !this.presenceStore) return;
|
|
1676
|
+
|
|
1677
|
+
this.presenceSweepInterval = setInterval(
|
|
1678
|
+
() => void this.sweepStalePresence(),
|
|
1679
|
+
RealtimeService.PRESENCE_SWEEP_INTERVAL_MS
|
|
1680
|
+
);
|
|
1681
|
+
|
|
1682
|
+
// Never hold the process open for housekeeping.
|
|
1683
|
+
(this.presenceSweepInterval as unknown as { unref?: () => void }).unref?.();
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1686
|
+
/** One pass of the stale-roster sweep. See {@link ensurePresenceSweep}. */
|
|
1687
|
+
private async sweepStalePresence(): Promise<void> {
|
|
1688
|
+
if (!this.presenceStore) return;
|
|
1689
|
+
try {
|
|
1690
|
+
const removed = await this.presenceStore.sweepStale(RealtimeService.PRESENCE_TIMEOUT_MS);
|
|
1691
|
+
for (const row of removed) {
|
|
1692
|
+
this.debugLog(`👻 [Presence] Reaped stale presence ${row.clientId} on "${row.channel}"`);
|
|
1693
|
+
this.deliverPresenceDiff(row.channel, {}, { [row.clientId]: row.state });
|
|
1694
|
+
this.publishPresenceDiff(row.channel, {}, { [row.clientId]: row.state });
|
|
1695
|
+
}
|
|
1696
|
+
} catch (error) {
|
|
1697
|
+
logger.warn("⚠️ [Presence] Stale-roster sweep failed", { error });
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1348
1701
|
// =============================================================================
|
|
1349
1702
|
// Lifecycle / Cleanup
|
|
1350
1703
|
// =============================================================================
|
|
@@ -1383,10 +1736,30 @@ lastSeen: Date.now() });
|
|
|
1383
1736
|
clearInterval(this.presenceInterval);
|
|
1384
1737
|
this.presenceInterval = undefined;
|
|
1385
1738
|
}
|
|
1739
|
+
if (this.presenceSweepInterval) {
|
|
1740
|
+
clearInterval(this.presenceSweepInterval);
|
|
1741
|
+
this.presenceSweepInterval = undefined;
|
|
1742
|
+
}
|
|
1743
|
+
this.oversizedBroadcastWarned.clear();
|
|
1744
|
+
|
|
1745
|
+
// Drop this instance's roster rows now rather than leaving every other
|
|
1746
|
+
// replica to wait out a TTL window on ghosts — a rolling deploy would
|
|
1747
|
+
// otherwise show 30s of departed users on every restart.
|
|
1748
|
+
if (this.presenceStore) {
|
|
1749
|
+
try {
|
|
1750
|
+
await this.presenceStore.removeInstance();
|
|
1751
|
+
} catch (error) {
|
|
1752
|
+
logger.warn("⚠️ [Presence] Could not clear this instance's roster rows on shutdown", { error });
|
|
1753
|
+
}
|
|
1754
|
+
this.presenceStore = undefined;
|
|
1755
|
+
}
|
|
1386
1756
|
|
|
1387
1757
|
// 4. Disconnect the dedicated LISTEN client(s)
|
|
1388
1758
|
await this.stopListening();
|
|
1389
1759
|
await this.stopCdc();
|
|
1760
|
+
await this.bus.stop().catch((error) =>
|
|
1761
|
+
logger.warn("⚠️ [ChannelBus] Error while stopping the channel bus", { error }));
|
|
1762
|
+
this.bus = new MemoryChannelBus();
|
|
1390
1763
|
|
|
1391
1764
|
// 5. Drop client references (don't close — server.close drains them)
|
|
1392
1765
|
this.clients.clear();
|
|
@@ -1426,6 +1799,7 @@ lastSeen: Date.now() });
|
|
|
1426
1799
|
return;
|
|
1427
1800
|
}
|
|
1428
1801
|
this.cdcTableMap = this.buildCdcTableMap();
|
|
1802
|
+
this.junctionLinkMap = buildJunctionLinkMap(this.registry);
|
|
1429
1803
|
this.cdcListener = new CdcListener(connectionString, (event) => this.handleCdcEvent(event));
|
|
1430
1804
|
try {
|
|
1431
1805
|
// start() validates the initial connection; if it can't be established
|
|
@@ -1436,6 +1810,7 @@ lastSeen: Date.now() });
|
|
|
1436
1810
|
await this.cdcListener.stop().catch(() => { /* best effort */ });
|
|
1437
1811
|
this.cdcListener = undefined;
|
|
1438
1812
|
this.cdcTableMap = undefined;
|
|
1813
|
+
this.junctionLinkMap = undefined;
|
|
1439
1814
|
throw err;
|
|
1440
1815
|
}
|
|
1441
1816
|
this.cdcActive = true;
|
|
@@ -1453,6 +1828,7 @@ lastSeen: Date.now() });
|
|
|
1453
1828
|
this.cdcListener = undefined;
|
|
1454
1829
|
}
|
|
1455
1830
|
this.cdcTableMap = undefined;
|
|
1831
|
+
this.junctionLinkMap = undefined;
|
|
1456
1832
|
this.recentAppEmits.clear();
|
|
1457
1833
|
}
|
|
1458
1834
|
|
|
@@ -1493,6 +1869,10 @@ lastSeen: Date.now() });
|
|
|
1493
1869
|
private async handleCdcEvent(event: CdcChangeEvent): Promise<void> {
|
|
1494
1870
|
const collection = this.resolveCollectionForTable(event.schema, event.table);
|
|
1495
1871
|
if (!collection) {
|
|
1872
|
+
// A junction table backs no collection, but its rows *are* a child
|
|
1873
|
+
// list. Route the change to the lists it changes before giving up.
|
|
1874
|
+
if (await this.handleJunctionCdcEvent(event)) return;
|
|
1875
|
+
|
|
1496
1876
|
// Unmapped table (not backed by a collection) — nothing to deliver.
|
|
1497
1877
|
this.debugLog(`📡 [CDC] Ignoring change on unmapped table ${event.schema}.${event.table}`);
|
|
1498
1878
|
return;
|
|
@@ -1509,6 +1889,55 @@ lastSeen: Date.now() });
|
|
|
1509
1889
|
await this.notifyUpdate(path, id, row, databaseId, /* broadcast */ false, /* origin */ "cdc");
|
|
1510
1890
|
}
|
|
1511
1891
|
|
|
1892
|
+
/**
|
|
1893
|
+
* Deliver a change on a many-to-many junction table as a change to the child
|
|
1894
|
+
* lists it belongs to.
|
|
1895
|
+
*
|
|
1896
|
+
* Linking a tag to a post writes only `posts_tags`. That table backs no
|
|
1897
|
+
* collection, so change capture dropped the event as unmapped and the
|
|
1898
|
+
* subscribers of `posts/1/tags` never heard about it — every other write in
|
|
1899
|
+
* the system was realtime, and this one silently was not. The junction row
|
|
1900
|
+
* carries both ids, so it names its own paths exactly.
|
|
1901
|
+
*
|
|
1902
|
+
* Notifies the nested path rather than either endpoint collection, because
|
|
1903
|
+
* invalidation walks *parent* paths and never child ones: telling `tags` it
|
|
1904
|
+
* changed would not reach a subscription on `posts/1/tags`.
|
|
1905
|
+
*
|
|
1906
|
+
* Returns whether the table was recognised as a junction.
|
|
1907
|
+
*/
|
|
1908
|
+
private async handleJunctionCdcEvent(event: CdcChangeEvent): Promise<boolean> {
|
|
1909
|
+
const links = this.junctionLinkMap?.get(`${event.schema}.${event.table}`)
|
|
1910
|
+
?? this.junctionLinkMap?.get(event.table);
|
|
1911
|
+
if (!links?.length) return false;
|
|
1912
|
+
|
|
1913
|
+
for (const link of links) {
|
|
1914
|
+
const sourceId = event.row?.[link.sourceColumn];
|
|
1915
|
+
const targetId = event.row?.[link.targetColumn];
|
|
1916
|
+
if (sourceId === undefined || sourceId === null || targetId === undefined || targetId === null) {
|
|
1917
|
+
this.debugLog(
|
|
1918
|
+
`📡 [CDC] Junction row on ${event.table} is missing '${link.sourceColumn}'/'${link.targetColumn}' — skipping.`
|
|
1919
|
+
);
|
|
1920
|
+
continue;
|
|
1921
|
+
}
|
|
1922
|
+
|
|
1923
|
+
const path = `${link.parentCollection.slug}/${String(sourceId)}/${link.relationKey}`;
|
|
1924
|
+
// An unlink removes the target from this list; a link invalidates it
|
|
1925
|
+
// so each subscriber refetches under its own RLS context.
|
|
1926
|
+
const row = event.op === "DELETE" ? null : { _rebase_invalidated: true };
|
|
1927
|
+
|
|
1928
|
+
await this.notifyUpdate(
|
|
1929
|
+
path,
|
|
1930
|
+
String(targetId),
|
|
1931
|
+
row,
|
|
1932
|
+
(link.parentCollection as { databaseId?: string }).databaseId,
|
|
1933
|
+
/* broadcast */ false,
|
|
1934
|
+
/* origin */ "cdc"
|
|
1935
|
+
);
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1938
|
+
return true;
|
|
1939
|
+
}
|
|
1940
|
+
|
|
1512
1941
|
/** Compute the canonical (possibly composite) id string from a captured row. */
|
|
1513
1942
|
private extractIdFromCdcRow(collection: CollectionConfig, row: Record<string, unknown>): string {
|
|
1514
1943
|
// Unaddressable falls back to a collection-level invalidation: single-row
|