@rebasepro/server-postgres 0.9.1-canary.16c42e9 → 0.9.1-canary.26fe4b2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/dist/PostgresBackendDriver.d.ts +18 -0
- package/dist/PostgresBootstrapper.d.ts +7 -1
- package/dist/auth/services.d.ts +53 -53
- package/dist/index.es.js +730 -194
- package/dist/index.es.js.map +1 -1
- package/dist/schema/auth-bootstrap-sql.d.ts +1 -1
- package/dist/schema/auth-schema.d.ts +24 -24
- package/dist/schema/doctor.d.ts +1 -1
- package/dist/schema/introspect-db-logic.d.ts +0 -5
- package/dist/schema/introspect-db-naming.d.ts +10 -0
- package/dist/security/policy-drift.d.ts +30 -0
- package/dist/security/rls-enforcement.d.ts +2 -2
- package/dist/services/channel-history.d.ts +118 -0
- package/dist/services/realtimeService.d.ts +69 -2
- package/package.json +8 -31
- package/src/PostgresBackendDriver.ts +56 -5
- package/src/PostgresBootstrapper.ts +18 -1
- package/src/auth/ensure-tables.ts +97 -17
- package/src/auth/services.ts +134 -133
- package/src/cli.ts +60 -0
- package/src/schema/auth-bootstrap-sql.ts +7 -1
- package/src/schema/auth-schema.ts +13 -13
- package/src/schema/doctor.ts +45 -20
- package/src/schema/introspect-db-inference.ts +1 -1
- package/src/schema/introspect-db-logic.ts +1 -10
- package/src/schema/introspect-db-naming.ts +15 -0
- package/src/schema/introspect-runtime.ts +1 -1
- package/src/security/policy-drift.test.ts +106 -1
- package/src/security/policy-drift.ts +56 -0
- package/src/security/rls-enforcement.ts +11 -5
- package/src/services/channel-history.ts +343 -0
- package/src/services/realtimeService.ts +198 -10
- package/src/services/row-pipeline.ts +27 -3
- package/src/websocket.ts +30 -11
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ordered, replayable per-channel message history.
|
|
3
|
+
*
|
|
4
|
+
* Broadcast on its own is fire-and-forget to whoever is connected at the
|
|
5
|
+
* instant it is sent: fine for presence and for "someone saved" notifications,
|
|
6
|
+
* not enough for op-based collaborative editing, where a client that blinks
|
|
7
|
+
* out for two seconds has to resync a whole document rather than catch up on
|
|
8
|
+
* the four operations it missed. This adds the missing half — every retained
|
|
9
|
+
* broadcast gets a per-channel sequence number, and a client can ask for
|
|
10
|
+
* everything after the last one it saw.
|
|
11
|
+
*
|
|
12
|
+
* Three decisions worth stating, because each rules out a simpler-looking one:
|
|
13
|
+
*
|
|
14
|
+
* - **Retention is server-side and opt-in.** A channel is created by whoever
|
|
15
|
+
* names it, so a client-supplied history depth would let any visitor commit
|
|
16
|
+
* the backend to unbounded storage. And presence channels — the common case
|
|
17
|
+
* — must not pay for this: with no rules configured nothing is written, no
|
|
18
|
+
* table is created, and `broadcast` runs exactly the code it ran before.
|
|
19
|
+
*
|
|
20
|
+
* - **Sequence numbers come from the database, not from a counter in this
|
|
21
|
+
* process.** They have to survive a restart and be shared across instances;
|
|
22
|
+
* an in-memory counter would restart at 1 after a deploy and hand a
|
|
23
|
+
* reconnecting client a replay from the wrong era, silently.
|
|
24
|
+
*
|
|
25
|
+
* - **The cursor row outlives the messages it numbered.** Pruning is what
|
|
26
|
+
* makes retention affordable, but pruning the cursor along with the messages
|
|
27
|
+
* would restart the sequence and make `sinceSeq` mean something different
|
|
28
|
+
* before and after — the worst kind of bug, because replay would still
|
|
29
|
+
* return rows and they would look plausible. Cursors are tiny and are kept
|
|
30
|
+
* forever; see {@link prune}, which touches only `channel_messages`.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import { sql } from "drizzle-orm";
|
|
34
|
+
import { NodePgDatabase } from "drizzle-orm/node-postgres";
|
|
35
|
+
import type { ChannelHistoryEntry, ChannelRetentionRule } from "@rebasepro/types";
|
|
36
|
+
import { logger } from "@rebasepro/server";
|
|
37
|
+
|
|
38
|
+
/** How many messages a replay returns when the caller does not say. */
|
|
39
|
+
const DEFAULT_REPLAY_LIMIT = 200;
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Hard ceiling on one replay, whatever the caller asks for.
|
|
43
|
+
*
|
|
44
|
+
* A reconnecting client names its own `limit`, so this is the only thing
|
|
45
|
+
* standing between a stale `sinceSeq` and a single frame carrying a channel's
|
|
46
|
+
* entire retained history. A client that is further behind than this is told so
|
|
47
|
+
* via `latestSeq` and can decide to resync wholesale instead of paging.
|
|
48
|
+
*/
|
|
49
|
+
const MAX_REPLAY_LIMIT = 1000;
|
|
50
|
+
|
|
51
|
+
/** Minimum gap between two prunes of the same channel. */
|
|
52
|
+
const PRUNE_THROTTLE_MS = 30_000;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Parse a retention TTL into milliseconds.
|
|
56
|
+
*
|
|
57
|
+
* Accepts a raw millisecond count or a short duration string (`"30s"`, `"15m"`,
|
|
58
|
+
* `"24h"`, `"7d"`). Returns undefined for anything unparseable, which the
|
|
59
|
+
* caller treats as "no TTL" — a misspelt duration must not silently become an
|
|
60
|
+
* aggressive one.
|
|
61
|
+
*/
|
|
62
|
+
export function parseTtlMs(ttl: number | string | undefined): number | undefined {
|
|
63
|
+
if (ttl === undefined || ttl === null) return undefined;
|
|
64
|
+
if (typeof ttl === "number") return Number.isFinite(ttl) && ttl > 0 ? ttl : undefined;
|
|
65
|
+
|
|
66
|
+
const match = /^\s*(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)\s*$/i.exec(ttl);
|
|
67
|
+
if (!match) {
|
|
68
|
+
logger.warn(`⚠️ [ChannelHistory] Ignoring unparseable retention ttl "${ttl}" — expected e.g. "30s", "15m", "24h", "7d".`);
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
const value = parseFloat(match[1]);
|
|
72
|
+
const unit = match[2].toLowerCase();
|
|
73
|
+
const multiplier = unit === "ms" ? 1
|
|
74
|
+
: unit === "s" ? 1_000
|
|
75
|
+
: unit === "m" ? 60_000
|
|
76
|
+
: unit === "h" ? 3_600_000
|
|
77
|
+
: 86_400_000;
|
|
78
|
+
const ms = value * multiplier;
|
|
79
|
+
return ms > 0 ? ms : undefined;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Whether `channel` is covered by `rule`.
|
|
84
|
+
*
|
|
85
|
+
* Exact match, or a trailing `*` acting as a prefix. Not a general glob: this
|
|
86
|
+
* decides what reaches disk, and a pattern language whose reach is not obvious
|
|
87
|
+
* at a glance is the wrong tool for that job.
|
|
88
|
+
*/
|
|
89
|
+
export function channelMatchesRule(channel: string, rule: ChannelRetentionRule): boolean {
|
|
90
|
+
const pattern = rule.match;
|
|
91
|
+
if (pattern === "*") return true;
|
|
92
|
+
if (pattern.endsWith("*")) return channel.startsWith(pattern.slice(0, -1));
|
|
93
|
+
return channel === pattern;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** A rule with its TTL already resolved to milliseconds. */
|
|
97
|
+
export interface ResolvedRetention {
|
|
98
|
+
limit?: number;
|
|
99
|
+
ttlMs?: number;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Persistence and replay for retained channels.
|
|
104
|
+
*
|
|
105
|
+
* Inert unless constructed with at least one rule: {@link enabled} is false,
|
|
106
|
+
* {@link ensureTables} does nothing, and {@link retentionFor} answers undefined
|
|
107
|
+
* for every channel, so the realtime service never reaches the SQL below.
|
|
108
|
+
*/
|
|
109
|
+
export class ChannelHistoryStore {
|
|
110
|
+
private rules: ChannelRetentionRule[];
|
|
111
|
+
/** Resolved rule per channel name, so the match runs once per channel. */
|
|
112
|
+
private resolved = new Map<string, ResolvedRetention | null>();
|
|
113
|
+
/** Channel → timestamp of its last prune, for {@link PRUNE_THROTTLE_MS}. */
|
|
114
|
+
private lastPruned = new Map<string, number>();
|
|
115
|
+
private tablesReady = false;
|
|
116
|
+
|
|
117
|
+
constructor(private db: NodePgDatabase<Record<string, unknown>>, rules: ChannelRetentionRule[] = []) {
|
|
118
|
+
this.rules = rules.filter(rule => {
|
|
119
|
+
if (!rule?.match) {
|
|
120
|
+
logger.warn("⚠️ [ChannelHistory] Ignoring a retention rule with no `match`.");
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
const hasBound = rule.limit !== undefined || rule.ttl !== undefined;
|
|
124
|
+
if (!hasBound) {
|
|
125
|
+
// Unbounded retention is almost never intended and cannot be
|
|
126
|
+
// walked back once the table has grown, so it is refused rather
|
|
127
|
+
// than honoured.
|
|
128
|
+
logger.warn(`⚠️ [ChannelHistory] Retention rule "${rule.match}" sets neither \`limit\` nor \`ttl\` — ignoring it, as it would retain forever.`);
|
|
129
|
+
return false;
|
|
130
|
+
}
|
|
131
|
+
return true;
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Whether any channel retains anything at all. */
|
|
136
|
+
get enabled(): boolean {
|
|
137
|
+
return this.rules.length > 0;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* The retention that applies to `channel`, or undefined when none does.
|
|
142
|
+
*
|
|
143
|
+
* First matching rule wins, so callers order them most-specific first.
|
|
144
|
+
*/
|
|
145
|
+
retentionFor(channel: string): ResolvedRetention | undefined {
|
|
146
|
+
if (!this.enabled || !channel) return undefined;
|
|
147
|
+
|
|
148
|
+
const cached = this.resolved.get(channel);
|
|
149
|
+
if (cached !== undefined) return cached ?? undefined;
|
|
150
|
+
|
|
151
|
+
const rule = this.rules.find(r => channelMatchesRule(channel, r));
|
|
152
|
+
const resolved: ResolvedRetention | null = rule
|
|
153
|
+
? { limit: rule.limit, ttlMs: parseTtlMs(rule.ttl) }
|
|
154
|
+
: null;
|
|
155
|
+
|
|
156
|
+
// Bounded by the number of distinct channel names seen, which is the
|
|
157
|
+
// same thing the in-memory channel and presence maps are bounded by.
|
|
158
|
+
this.resolved.set(channel, resolved);
|
|
159
|
+
return resolved ?? undefined;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Create the history tables. Idempotent, and a no-op when no rule is set —
|
|
164
|
+
* a deployment that never retains anything gets no schema for it.
|
|
165
|
+
*/
|
|
166
|
+
async ensureTables(): Promise<void> {
|
|
167
|
+
if (!this.enabled || this.tablesReady) return;
|
|
168
|
+
|
|
169
|
+
await this.db.execute(sql`CREATE SCHEMA IF NOT EXISTS rebase`);
|
|
170
|
+
|
|
171
|
+
// The primary key is exactly the replay query's access path
|
|
172
|
+
// (`channel = $1 AND seq > $2 ORDER BY seq`), so it needs no further
|
|
173
|
+
// index of its own.
|
|
174
|
+
await this.db.execute(sql`
|
|
175
|
+
CREATE TABLE IF NOT EXISTS rebase.channel_messages (
|
|
176
|
+
channel TEXT NOT NULL,
|
|
177
|
+
seq BIGINT NOT NULL,
|
|
178
|
+
event TEXT NOT NULL,
|
|
179
|
+
payload JSONB,
|
|
180
|
+
sender_id TEXT,
|
|
181
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
182
|
+
PRIMARY KEY (channel, seq)
|
|
183
|
+
)
|
|
184
|
+
`);
|
|
185
|
+
|
|
186
|
+
// Only for the TTL arm of pruning; the limit arm rides the primary key.
|
|
187
|
+
await this.db.execute(sql`
|
|
188
|
+
CREATE INDEX IF NOT EXISTS idx_channel_messages_created
|
|
189
|
+
ON rebase.channel_messages (created_at)
|
|
190
|
+
`);
|
|
191
|
+
|
|
192
|
+
// Never pruned — see the note at the top of this file. One row per
|
|
193
|
+
// channel that has ever retained a message.
|
|
194
|
+
await this.db.execute(sql`
|
|
195
|
+
CREATE TABLE IF NOT EXISTS rebase.channel_cursors (
|
|
196
|
+
channel TEXT PRIMARY KEY,
|
|
197
|
+
last_seq BIGINT NOT NULL
|
|
198
|
+
)
|
|
199
|
+
`);
|
|
200
|
+
|
|
201
|
+
this.tablesReady = true;
|
|
202
|
+
logger.info(`✅ [ChannelHistory] Retained channels ready (${this.rules.length} rule(s)).`);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Append a broadcast and return the sequence number it was given.
|
|
207
|
+
*
|
|
208
|
+
* The sequence is allocated by the same statement that stores the message,
|
|
209
|
+
* so a crash between the two is not a possibility. `ON CONFLICT DO UPDATE`
|
|
210
|
+
* takes a row lock on the channel's cursor, which is what makes concurrent
|
|
211
|
+
* broadcasts to one channel line up in a single order — and what keeps
|
|
212
|
+
* different channels from contending with each other at all.
|
|
213
|
+
*/
|
|
214
|
+
async append(
|
|
215
|
+
channel: string,
|
|
216
|
+
event: string,
|
|
217
|
+
payload: unknown,
|
|
218
|
+
senderId?: string
|
|
219
|
+
): Promise<{ seq: number; at: string }> {
|
|
220
|
+
const result = await this.db.execute(sql`
|
|
221
|
+
WITH next AS (
|
|
222
|
+
INSERT INTO rebase.channel_cursors (channel, last_seq)
|
|
223
|
+
VALUES (${channel}, 1)
|
|
224
|
+
ON CONFLICT (channel)
|
|
225
|
+
DO UPDATE SET last_seq = rebase.channel_cursors.last_seq + 1
|
|
226
|
+
RETURNING last_seq
|
|
227
|
+
)
|
|
228
|
+
INSERT INTO rebase.channel_messages (channel, seq, event, payload, sender_id)
|
|
229
|
+
SELECT ${channel}, next.last_seq, ${event}, ${JSON.stringify(payload ?? null)}::jsonb, ${senderId ?? null}
|
|
230
|
+
FROM next
|
|
231
|
+
RETURNING seq, created_at
|
|
232
|
+
`);
|
|
233
|
+
|
|
234
|
+
const row = result.rows[0] as { seq: string | number; created_at: Date | string } | undefined;
|
|
235
|
+
if (!row) throw new Error(`Failed to append to channel history for "${channel}"`);
|
|
236
|
+
|
|
237
|
+
return {
|
|
238
|
+
// BIGINT comes back as a string from node-postgres; the wire type is
|
|
239
|
+
// a number, and a channel would need 2^53 messages to notice.
|
|
240
|
+
seq: Number(row.seq),
|
|
241
|
+
at: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at)
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Everything retained for `channel` after `sinceSeq`, oldest first.
|
|
247
|
+
*
|
|
248
|
+
* `latestSeq` is reported whether or not the messages were capped, so a
|
|
249
|
+
* client that is further behind than one page can tell.
|
|
250
|
+
*/
|
|
251
|
+
async replay(
|
|
252
|
+
channel: string,
|
|
253
|
+
sinceSeq = 0,
|
|
254
|
+
limit = DEFAULT_REPLAY_LIMIT
|
|
255
|
+
): Promise<{ messages: ChannelHistoryEntry[]; latestSeq: number }> {
|
|
256
|
+
const capped = Math.max(1, Math.min(Math.floor(limit) || DEFAULT_REPLAY_LIMIT, MAX_REPLAY_LIMIT));
|
|
257
|
+
const after = Number.isFinite(sinceSeq) && sinceSeq > 0 ? Math.floor(sinceSeq) : 0;
|
|
258
|
+
|
|
259
|
+
const result = await this.db.execute(sql`
|
|
260
|
+
SELECT seq, event, payload, sender_id, created_at
|
|
261
|
+
FROM rebase.channel_messages
|
|
262
|
+
WHERE channel = ${channel} AND seq > ${after}
|
|
263
|
+
ORDER BY seq ASC
|
|
264
|
+
LIMIT ${capped}
|
|
265
|
+
`);
|
|
266
|
+
|
|
267
|
+
const messages = (result.rows as Array<{
|
|
268
|
+
seq: string | number;
|
|
269
|
+
event: string;
|
|
270
|
+
payload: unknown;
|
|
271
|
+
sender_id: string | null;
|
|
272
|
+
created_at: Date | string;
|
|
273
|
+
}>).map(row => ({
|
|
274
|
+
seq: Number(row.seq),
|
|
275
|
+
event: row.event,
|
|
276
|
+
payload: row.payload,
|
|
277
|
+
senderId: row.sender_id ?? undefined,
|
|
278
|
+
at: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at)
|
|
279
|
+
}));
|
|
280
|
+
|
|
281
|
+
// Read from the cursor rather than from the messages: the cursor is the
|
|
282
|
+
// authority on how far the channel has got, and still says so after
|
|
283
|
+
// pruning has removed the messages it counted.
|
|
284
|
+
const cursor = await this.db.execute(sql`
|
|
285
|
+
SELECT last_seq FROM rebase.channel_cursors WHERE channel = ${channel}
|
|
286
|
+
`);
|
|
287
|
+
const cursorRow = cursor.rows[0] as { last_seq: string | number } | undefined;
|
|
288
|
+
const latestSeq = cursorRow ? Number(cursorRow.last_seq) : 0;
|
|
289
|
+
|
|
290
|
+
return { messages, latestSeq };
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Enforce a channel's retention bounds.
|
|
295
|
+
*
|
|
296
|
+
* Throttled per channel, so a burst of operations prunes once rather than
|
|
297
|
+
* once per message — the cost then tracks elapsed time instead of write
|
|
298
|
+
* volume, which is what makes retention affordable on a hot channel.
|
|
299
|
+
*/
|
|
300
|
+
async prune(channel: string, retention: ResolvedRetention): Promise<number> {
|
|
301
|
+
const now = Date.now();
|
|
302
|
+
const last = this.lastPruned.get(channel) ?? 0;
|
|
303
|
+
if (now - last < PRUNE_THROTTLE_MS) return 0;
|
|
304
|
+
this.lastPruned.set(channel, now);
|
|
305
|
+
|
|
306
|
+
let deleted = 0;
|
|
307
|
+
|
|
308
|
+
if (retention.ttlMs !== undefined) {
|
|
309
|
+
const result = await this.db.execute(sql`
|
|
310
|
+
DELETE FROM rebase.channel_messages
|
|
311
|
+
WHERE channel = ${channel}
|
|
312
|
+
AND created_at < NOW() - MAKE_INTERVAL(secs => ${retention.ttlMs / 1000})
|
|
313
|
+
`);
|
|
314
|
+
deleted += result.rowCount ?? 0;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
if (retention.limit !== undefined && retention.limit > 0) {
|
|
318
|
+
// OFFSET past the newest `limit` rows to find the highest seq that
|
|
319
|
+
// is no longer wanted, then delete everything at or below it. Fewer
|
|
320
|
+
// rows than the limit leaves the subquery empty, and the comparison
|
|
321
|
+
// with NULL deletes nothing.
|
|
322
|
+
const result = await this.db.execute(sql`
|
|
323
|
+
DELETE FROM rebase.channel_messages
|
|
324
|
+
WHERE channel = ${channel}
|
|
325
|
+
AND seq <= (
|
|
326
|
+
SELECT seq FROM rebase.channel_messages
|
|
327
|
+
WHERE channel = ${channel}
|
|
328
|
+
ORDER BY seq DESC
|
|
329
|
+
OFFSET ${Math.floor(retention.limit)} LIMIT 1
|
|
330
|
+
)
|
|
331
|
+
`);
|
|
332
|
+
deleted += result.rowCount ?? 0;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
return deleted;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** Forget throttle and match caches. Called on shutdown. */
|
|
339
|
+
clear(): void {
|
|
340
|
+
this.resolved.clear();
|
|
341
|
+
this.lastPruned.clear();
|
|
342
|
+
}
|
|
343
|
+
}
|
|
@@ -15,6 +15,8 @@ import { logger } from "@rebasepro/server";
|
|
|
15
15
|
import { sanitizeErrorForClient } from "../utils/pg-error-utils";
|
|
16
16
|
import { CdcListener, type CdcChangeEvent } from "./cdc/CdcListener";
|
|
17
17
|
import { deriveRowAddress, getPrimaryKeys, type PrimaryKeyInfo } from "./collection-helpers";
|
|
18
|
+
import { ChannelHistoryStore, type ResolvedRetention } from "./channel-history";
|
|
19
|
+
import type { ChannelHistoryEntry, ChannelRetentionRule } from "@rebasepro/types";
|
|
18
20
|
|
|
19
21
|
/** Channel name used for Postgres LISTEN/NOTIFY cross-instance realtime. */
|
|
20
22
|
const PG_NOTIFY_CHANNEL = "rebase_entity_changes";
|
|
@@ -24,7 +26,7 @@ const PG_NOTIFY_CHANNEL = "rebase_entity_changes";
|
|
|
24
26
|
* Mirrors the session variables set by PostgresBackendDriver.withAuth().
|
|
25
27
|
*/
|
|
26
28
|
export interface SubscriptionAuthContext {
|
|
27
|
-
|
|
29
|
+
uid: string;
|
|
28
30
|
roles: string[];
|
|
29
31
|
}
|
|
30
32
|
|
|
@@ -52,6 +54,28 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
|
|
|
52
54
|
|
|
53
55
|
// Presence: channel → Map<clientId, { state, lastSeen }>
|
|
54
56
|
private presence = new Map<string, Map<string, { state: Record<string, unknown>; lastSeen: number }>>();
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Ordered, replayable history for channels that opt into it.
|
|
60
|
+
*
|
|
61
|
+
* Undefined until {@link configureChannelHistory} is called, and inert even
|
|
62
|
+
* then unless retention rules were supplied — so presence and ephemeral
|
|
63
|
+
* notification channels never touch it. See `channel-history.ts`.
|
|
64
|
+
*/
|
|
65
|
+
private channelHistory?: ChannelHistoryStore;
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* One promise chain per retained channel, so that assigning a sequence
|
|
69
|
+
* number and fanning the message out happen in the same order for every
|
|
70
|
+
* message on that channel.
|
|
71
|
+
*
|
|
72
|
+
* Without it, two concurrent broadcasts can be numbered 4 and 5 by the
|
|
73
|
+
* database and still reach subscribers as 5 then 4 — live order and replay
|
|
74
|
+
* order would disagree, which is exactly the divergence sequence numbers
|
|
75
|
+
* are supposed to rule out. Keyed by channel, so unrelated channels never
|
|
76
|
+
* wait on each other.
|
|
77
|
+
*/
|
|
78
|
+
private channelSendQueues = new Map<string, Promise<void>>();
|
|
55
79
|
private presenceInterval?: ReturnType<typeof setInterval>;
|
|
56
80
|
private static readonly PRESENCE_TIMEOUT_MS = 30000; // 30s
|
|
57
81
|
private dataService: DataService;
|
|
@@ -73,7 +97,7 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
|
|
|
73
97
|
searchString?: string;
|
|
74
98
|
};
|
|
75
99
|
// Auth context for RLS — when set, refetches run in a transaction
|
|
76
|
-
// with set_config('app.
|
|
100
|
+
// with set_config('app.uid', ...) / set_config('app.user_roles', ...)
|
|
77
101
|
authContext?: SubscriptionAuthContext;
|
|
78
102
|
}>();
|
|
79
103
|
|
|
@@ -322,6 +346,14 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
|
|
|
322
346
|
payload?.payload
|
|
323
347
|
);
|
|
324
348
|
break;
|
|
349
|
+
case "channel_history":
|
|
350
|
+
await this.handleChannelHistoryRequest(
|
|
351
|
+
clientId,
|
|
352
|
+
payload?.channel as string,
|
|
353
|
+
payload?.sinceSeq as number | undefined,
|
|
354
|
+
payload?.limit as number | undefined
|
|
355
|
+
);
|
|
356
|
+
break;
|
|
325
357
|
|
|
326
358
|
// ── Presence ──
|
|
327
359
|
case "presence_track":
|
|
@@ -669,10 +701,10 @@ export class RealtimeService extends EventEmitter implements RealtimeProvider {
|
|
|
669
701
|
// Always wrap in a transaction with session vars, defaulting to anonymous context if missing.
|
|
670
702
|
// Refetches are reads: apply the same GUCs + reader-role downgrade as the
|
|
671
703
|
// driver's read path, so realtime cannot leak rows the initial fetch hid.
|
|
672
|
-
const activeAuth = authContext || {
|
|
704
|
+
const activeAuth = authContext || { uid: "anon",
|
|
673
705
|
roles: ["anon"] };
|
|
674
706
|
return await this.db.transaction(async (tx) => {
|
|
675
|
-
await applyAuthContext(tx, {
|
|
707
|
+
await applyAuthContext(tx, { uid: activeAuth.uid, roles: activeAuth.roles }, this.rlsUserRole);
|
|
676
708
|
const txEntityService = new DataService(tx, this.registry);
|
|
677
709
|
let fetchedEntities;
|
|
678
710
|
if (collectionRequest.searchString) {
|
|
@@ -711,7 +743,7 @@ roles: ["anon"] };
|
|
|
711
743
|
|
|
712
744
|
if (globalCallbacks?.afterRead || callbacks?.afterRead || propertyCallbacks?.afterRead) {
|
|
713
745
|
const contextForCallback = {
|
|
714
|
-
user: { uid: activeAuth.
|
|
746
|
+
user: { uid: activeAuth.uid,
|
|
715
747
|
roles: activeAuth.roles },
|
|
716
748
|
driver: this.driver,
|
|
717
749
|
data: (this.driver && "data" in this.driver) ? (this.driver as DataDriverWithData).data : undefined
|
|
@@ -849,10 +881,10 @@ roles: activeAuth.roles },
|
|
|
849
881
|
|
|
850
882
|
// Always wrap in a transaction with session vars, defaulting to anonymous context if missing.
|
|
851
883
|
// Same read isolation as collection refetches: GUCs + reader-role downgrade.
|
|
852
|
-
const activeAuth = authContext || {
|
|
884
|
+
const activeAuth = authContext || { uid: "anon",
|
|
853
885
|
roles: ["anon"] };
|
|
854
886
|
return await this.db.transaction(async (tx) => {
|
|
855
|
-
await applyAuthContext(tx, {
|
|
887
|
+
await applyAuthContext(tx, { uid: activeAuth.uid, roles: activeAuth.roles }, this.rlsUserRole);
|
|
856
888
|
const txEntityService = new DataService(tx, this.registry);
|
|
857
889
|
let processedEntity = await txEntityService.fetchOne(notifyPath, id, collection?.databaseId);
|
|
858
890
|
|
|
@@ -867,7 +899,7 @@ roles: ["anon"] };
|
|
|
867
899
|
|
|
868
900
|
if (globalCallbacks?.afterRead || callbacks?.afterRead || propertyCallbacks?.afterRead) {
|
|
869
901
|
const contextForCallback = {
|
|
870
|
-
user: { uid: activeAuth.
|
|
902
|
+
user: { uid: activeAuth.uid,
|
|
871
903
|
roles: activeAuth.roles },
|
|
872
904
|
driver: this.driver,
|
|
873
905
|
data: (this.driver && "data" in this.driver) ? (this.driver as DataDriverWithData).data : undefined
|
|
@@ -1039,8 +1071,85 @@ roles: activeAuth.roles },
|
|
|
1039
1071
|
this.removePresence(clientId, channel);
|
|
1040
1072
|
}
|
|
1041
1073
|
|
|
1042
|
-
/**
|
|
1074
|
+
/**
|
|
1075
|
+
* Broadcast a message to all clients in a channel except the sender.
|
|
1076
|
+
*
|
|
1077
|
+
* On a channel with no retention rule this is what it always was: a
|
|
1078
|
+
* synchronous fan-out to whoever is connected, with no sequence number, no
|
|
1079
|
+
* SQL and no await — the body below runs to completion before returning.
|
|
1080
|
+
*
|
|
1081
|
+
* On a retained channel the message is durably numbered first and only then
|
|
1082
|
+
* delivered, through a per-channel queue so that delivery order matches
|
|
1083
|
+
* sequence order. That ordering is the whole point: a client that catches up
|
|
1084
|
+
* with `sinceSeq` has to arrive at the same state as one that never
|
|
1085
|
+
* disconnected.
|
|
1086
|
+
*/
|
|
1043
1087
|
broadcastToChannel(clientId: string, channel: string, event: string, payload: unknown): void {
|
|
1088
|
+
const retention = this.channelHistory?.retentionFor(channel);
|
|
1089
|
+
if (!retention) {
|
|
1090
|
+
this.fanOutBroadcast(clientId, channel, event, payload);
|
|
1091
|
+
return;
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
const previous = this.channelSendQueues.get(channel) ?? Promise.resolve();
|
|
1095
|
+
const next = previous
|
|
1096
|
+
// A failed predecessor must not poison the chain — the next message
|
|
1097
|
+
// on this channel is independent and still deserves to be sent.
|
|
1098
|
+
.catch(() => { /* already reported below */ })
|
|
1099
|
+
.then(() => this.persistAndFanOut(clientId, channel, event, payload, retention));
|
|
1100
|
+
|
|
1101
|
+
this.channelSendQueues.set(channel, next);
|
|
1102
|
+
void next.finally(() => {
|
|
1103
|
+
// Only clear if nothing has queued behind us in the meantime.
|
|
1104
|
+
if (this.channelSendQueues.get(channel) === next) this.channelSendQueues.delete(channel);
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
/**
|
|
1109
|
+
* Number a broadcast, store it, then deliver it.
|
|
1110
|
+
*
|
|
1111
|
+
* A message that cannot be stored is **not** delivered. Delivering it would
|
|
1112
|
+
* put it in front of live subscribers while leaving it absent from every
|
|
1113
|
+
* future replay — the two views of the channel would disagree permanently,
|
|
1114
|
+
* and no later message could repair the gap. Failing loudly to the sender
|
|
1115
|
+
* instead lets it retry, which for an operation stream is the only outcome
|
|
1116
|
+
* that keeps clients convergent.
|
|
1117
|
+
*/
|
|
1118
|
+
private async persistAndFanOut(
|
|
1119
|
+
clientId: string,
|
|
1120
|
+
channel: string,
|
|
1121
|
+
event: string,
|
|
1122
|
+
payload: unknown,
|
|
1123
|
+
retention: ResolvedRetention
|
|
1124
|
+
): Promise<void> {
|
|
1125
|
+
let seq: number;
|
|
1126
|
+
try {
|
|
1127
|
+
({ seq } = await this.channelHistory!.append(channel, event, payload, clientId));
|
|
1128
|
+
} catch (error) {
|
|
1129
|
+
logger.error(`❌ [ChannelHistory] Could not persist broadcast on "${channel}" — message dropped`, { error });
|
|
1130
|
+
this.sendError(
|
|
1131
|
+
clientId,
|
|
1132
|
+
`Could not persist broadcast on retained channel "${channel}"`,
|
|
1133
|
+
undefined,
|
|
1134
|
+
"CHANNEL_HISTORY_WRITE_FAILED"
|
|
1135
|
+
);
|
|
1136
|
+
return;
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
this.fanOutBroadcast(clientId, channel, event, payload, seq);
|
|
1140
|
+
|
|
1141
|
+
try {
|
|
1142
|
+
await this.channelHistory!.prune(channel, retention);
|
|
1143
|
+
} catch (error) {
|
|
1144
|
+
// Retention is a housekeeping concern; the message is already
|
|
1145
|
+
// delivered and durable, so a failed prune must not surface as a
|
|
1146
|
+
// broadcast failure. It will be retried on the next message.
|
|
1147
|
+
logger.warn(`⚠️ [ChannelHistory] Prune failed for "${channel}"`, { error });
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
/** Deliver a broadcast frame to every member of a channel but the sender. */
|
|
1152
|
+
private fanOutBroadcast(clientId: string, channel: string, event: string, payload: unknown, seq?: number): void {
|
|
1044
1153
|
const members = this.channels.get(channel);
|
|
1045
1154
|
if (!members) return;
|
|
1046
1155
|
|
|
@@ -1048,7 +1157,8 @@ roles: activeAuth.roles },
|
|
|
1048
1157
|
type: "broadcast",
|
|
1049
1158
|
channel,
|
|
1050
1159
|
event,
|
|
1051
|
-
payload
|
|
1160
|
+
payload,
|
|
1161
|
+
...(seq !== undefined ? { seq } : {})
|
|
1052
1162
|
});
|
|
1053
1163
|
|
|
1054
1164
|
for (const memberId of members) {
|
|
@@ -1060,6 +1170,79 @@ roles: activeAuth.roles },
|
|
|
1060
1170
|
}
|
|
1061
1171
|
}
|
|
1062
1172
|
|
|
1173
|
+
// =============================================================================
|
|
1174
|
+
// Channel History
|
|
1175
|
+
// =============================================================================
|
|
1176
|
+
|
|
1177
|
+
/**
|
|
1178
|
+
* Install retention rules and create the tables they need.
|
|
1179
|
+
*
|
|
1180
|
+
* Safe to call with no rules (and safe not to call at all): the store stays
|
|
1181
|
+
* inert, no schema is created, and broadcast keeps its original
|
|
1182
|
+
* fire-and-forget path.
|
|
1183
|
+
*/
|
|
1184
|
+
async configureChannelHistory(rules: ChannelRetentionRule[] | undefined): Promise<void> {
|
|
1185
|
+
this.channelHistory = new ChannelHistoryStore(this.db, rules ?? []);
|
|
1186
|
+
if (!this.channelHistory.enabled) return;
|
|
1187
|
+
await this.channelHistory.ensureTables();
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
/** Whether any channel is configured to retain messages. */
|
|
1191
|
+
public isChannelHistoryEnabled(): boolean {
|
|
1192
|
+
return this.channelHistory?.enabled ?? false;
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
/**
|
|
1196
|
+
* Answer a client's catch-up request.
|
|
1197
|
+
*
|
|
1198
|
+
* A channel with no retention rule is answered with `retained: false`
|
|
1199
|
+
* rather than an empty list, so the client can tell "you missed nothing"
|
|
1200
|
+
* apart from "this channel never keeps anything" — the second means its
|
|
1201
|
+
* reconnect strategy has to be a full resync, and silence would leave it
|
|
1202
|
+
* guessing.
|
|
1203
|
+
*/
|
|
1204
|
+
private async handleChannelHistoryRequest(
|
|
1205
|
+
clientId: string,
|
|
1206
|
+
channel: string,
|
|
1207
|
+
sinceSeq?: number,
|
|
1208
|
+
limit?: number
|
|
1209
|
+
): Promise<void> {
|
|
1210
|
+
if (!channel) return;
|
|
1211
|
+
|
|
1212
|
+
const retention = this.channelHistory?.retentionFor(channel);
|
|
1213
|
+
if (!retention) {
|
|
1214
|
+
this.sendChannelHistory(clientId, channel, [], false);
|
|
1215
|
+
return;
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
try {
|
|
1219
|
+
const { messages, latestSeq } = await this.channelHistory!.replay(channel, sinceSeq, limit);
|
|
1220
|
+
this.sendChannelHistory(clientId, channel, messages, true, latestSeq);
|
|
1221
|
+
} catch (error) {
|
|
1222
|
+
logger.error(`❌ [ChannelHistory] Replay failed for "${channel}"`, { error });
|
|
1223
|
+
this.sendError(clientId, `Could not replay history for channel "${channel}"`, undefined, "CHANNEL_HISTORY_READ_FAILED");
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
private sendChannelHistory(
|
|
1228
|
+
clientId: string,
|
|
1229
|
+
channel: string,
|
|
1230
|
+
messages: ChannelHistoryEntry[],
|
|
1231
|
+
retained: boolean,
|
|
1232
|
+
latestSeq?: number
|
|
1233
|
+
): void {
|
|
1234
|
+
const ws = this.clients.get(clientId);
|
|
1235
|
+
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
1236
|
+
ws.send(JSON.stringify({
|
|
1237
|
+
type: "channel_history",
|
|
1238
|
+
channel,
|
|
1239
|
+
messages,
|
|
1240
|
+
retained,
|
|
1241
|
+
...(latestSeq !== undefined ? { latestSeq } : {})
|
|
1242
|
+
}));
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1063
1246
|
// =============================================================================
|
|
1064
1247
|
// Presence
|
|
1065
1248
|
// =============================================================================
|
|
@@ -1191,6 +1374,11 @@ lastSeen: Date.now() });
|
|
|
1191
1374
|
// 3. Clear broadcast channels and presence
|
|
1192
1375
|
this.channels.clear();
|
|
1193
1376
|
this.presence.clear();
|
|
1377
|
+
// Pending history writes hold the pool open; let them settle before the
|
|
1378
|
+
// caller closes it, but never let a rejected one break shutdown.
|
|
1379
|
+
await Promise.allSettled([...this.channelSendQueues.values()]);
|
|
1380
|
+
this.channelSendQueues.clear();
|
|
1381
|
+
this.channelHistory?.clear();
|
|
1194
1382
|
if (this.presenceInterval) {
|
|
1195
1383
|
clearInterval(this.presenceInterval);
|
|
1196
1384
|
this.presenceInterval = undefined;
|