@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/src/realtime.ts
CHANGED
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* queues the ack's cursor write without awaiting it, so `drain()` is the
|
|
8
8
|
* awaitable boundary a host uses before closing its storage or ending a
|
|
9
9
|
* hibernatable event (§8.2). Initial
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* registration comes from the live units of the client record (§8.1); a sync
|
|
11
|
+
* round completed on the
|
|
12
12
|
* connection replaces it at round end (§8.7). Deltas are complete SSP2
|
|
13
13
|
* response messages pushed as `0x00`-tagged binary (§8.2/§8.7); sync
|
|
14
14
|
* rounds ride the same socket as `0x01`-tagged byte-stream chunks driven
|
|
@@ -17,25 +17,28 @@
|
|
|
17
17
|
* the `sync` wake-up (§8.3).
|
|
18
18
|
*/
|
|
19
19
|
import {
|
|
20
|
-
type CommitChange,
|
|
21
20
|
DecodeError,
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
decodeRequest,
|
|
22
|
+
encodeResponse,
|
|
24
23
|
encodePresenceError,
|
|
25
24
|
encodePresenceFanout,
|
|
26
25
|
isSupportedProtocolWireVersion,
|
|
27
26
|
MessageStreamScanner,
|
|
28
27
|
PROTOCOL_WIRE_VERSION,
|
|
29
28
|
type PresenceKind,
|
|
30
|
-
parseRealtimePresencePublish,
|
|
31
29
|
REALTIME_TAG_DELTA,
|
|
32
30
|
REALTIME_TAG_ROUND,
|
|
33
|
-
type ResponseFrame,
|
|
34
31
|
type ScopeMap,
|
|
32
|
+
unitId,
|
|
35
33
|
type WakeReason,
|
|
36
34
|
} from '@syncular/core';
|
|
37
|
-
import type {
|
|
35
|
+
import type {
|
|
36
|
+
ServerLimits,
|
|
37
|
+
SyncRequestContext,
|
|
38
|
+
SyncServerConfig,
|
|
39
|
+
} from './context';
|
|
38
40
|
import {
|
|
41
|
+
limitsOf,
|
|
39
42
|
REMOTE_COMMAND_CLIENT_ID_PREFIX,
|
|
40
43
|
RESOLVER_OUTAGE,
|
|
41
44
|
touchAuthenticatedPartition,
|
|
@@ -51,18 +54,15 @@ import {
|
|
|
51
54
|
import { emitEvent, type SyncularServerEvents } from './events';
|
|
52
55
|
import { composeEvents } from './events-ring';
|
|
53
56
|
import { createSyncResponseStream } from './handler';
|
|
57
|
+
import { commitFrame, coveredTables, planUnit } from './pull';
|
|
54
58
|
import {
|
|
55
59
|
type CompiledSchema,
|
|
56
60
|
type CompiledTable,
|
|
57
61
|
compileSchema,
|
|
58
62
|
} from './schema';
|
|
59
|
-
import { schemaWindowOf
|
|
60
|
-
import {
|
|
61
|
-
|
|
62
|
-
matchesEffective,
|
|
63
|
-
type ResolvedScopes,
|
|
64
|
-
} from './scopes';
|
|
65
|
-
import type { ServerStorage, StoredCommit } from './storage';
|
|
63
|
+
import { schemaWindowOf } from './schema-window';
|
|
64
|
+
import { matchesEffective, type ResolvedScopes } from './scopes';
|
|
65
|
+
import type { ClientRecord, ServerStorage, StoredCommit } from './storage';
|
|
66
66
|
|
|
67
67
|
/**
|
|
68
68
|
* Realtime adds fanout/presence tuning to the canonical sync-server config;
|
|
@@ -72,9 +72,19 @@ export interface RealtimeHubConfig extends Omit<SyncServerConfig, 'realtime'> {
|
|
|
72
72
|
/** Deltas larger than this become `delta-too-large` wake-ups (§8.2). */
|
|
73
73
|
readonly maxDeltaBytes?: number;
|
|
74
74
|
/**
|
|
75
|
-
* §8.
|
|
76
|
-
*
|
|
77
|
-
*
|
|
75
|
+
* §8.2 flow control: sent deltas a session may leave unacknowledged before
|
|
76
|
+
* the server stops sending it deltas and answers with a `catchup-required`
|
|
77
|
+
* wake-up. Default 256.
|
|
78
|
+
*/
|
|
79
|
+
readonly maxUnackedDeltas?: number;
|
|
80
|
+
/** §8.2 flow control: bytes of unacknowledged deltas per session. Default
|
|
81
|
+
* 4 MiB. */
|
|
82
|
+
readonly maxUnackedDeltaBytes?: number;
|
|
83
|
+
/**
|
|
84
|
+
* §8.6.2 presence cap per connection: the UTF-8 bytes of each held scope
|
|
85
|
+
* key plus its serialized document, summed over the connection's keys. A
|
|
86
|
+
* publish that would exceed it is answered with `presence.too_large` and
|
|
87
|
+
* changes nothing. Default 8,192.
|
|
78
88
|
*/
|
|
79
89
|
readonly maxPresenceBytes?: number;
|
|
80
90
|
/**
|
|
@@ -104,10 +114,20 @@ export interface RealtimeConnectOptions {
|
|
|
104
114
|
* `session.close()` from its socket-close handler as usual.
|
|
105
115
|
*/
|
|
106
116
|
readonly closeSocket?: () => void;
|
|
117
|
+
/**
|
|
118
|
+
* §8.6.1: called after every change to this connection's own presence
|
|
119
|
+
* documents with all of them, scope key → serialized document, so a host
|
|
120
|
+
* whose memory can be evicted keeps them with the connection and hands
|
|
121
|
+
* them back to `restore`. Not called once the connection closed.
|
|
122
|
+
*/
|
|
123
|
+
readonly onPresence?: (presence: ReadonlyMap<string, string>) => void;
|
|
107
124
|
}
|
|
108
125
|
|
|
126
|
+
/** One active (unit, table) pair of a live unit (§8.1). */
|
|
109
127
|
interface Registration {
|
|
110
|
-
|
|
128
|
+
/** The schema version of the client's latest round: the delta codec (§9). */
|
|
129
|
+
readonly schema: CompiledSchema;
|
|
130
|
+
/** The unit id (§4.1). */
|
|
111
131
|
readonly id: string;
|
|
112
132
|
readonly table: string;
|
|
113
133
|
readonly schemaVersion: number;
|
|
@@ -115,28 +135,30 @@ interface Registration {
|
|
|
115
135
|
}
|
|
116
136
|
|
|
117
137
|
const DEFAULT_MAX_DELTA_BYTES = 1024 * 1024;
|
|
118
|
-
const DEFAULT_MAX_PRESENCE_BYTES =
|
|
138
|
+
const DEFAULT_MAX_PRESENCE_BYTES = 8192;
|
|
139
|
+
const DEFAULT_MAX_UNACKED_DELTAS = 256;
|
|
140
|
+
const DEFAULT_MAX_UNACKED_DELTA_BYTES = 4 * 1024 * 1024;
|
|
119
141
|
|
|
120
142
|
type PresenceDoc = Record<string, unknown>;
|
|
121
143
|
|
|
144
|
+
/** A held presence document, its JSON text, and its charge against the
|
|
145
|
+
* per-connection cap (§8.6.2). */
|
|
146
|
+
interface PresenceEntry {
|
|
147
|
+
readonly doc: PresenceDoc;
|
|
148
|
+
readonly json: string;
|
|
149
|
+
readonly bytes: number;
|
|
150
|
+
}
|
|
151
|
+
|
|
122
152
|
/**
|
|
123
|
-
* §8.6 presence
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
* presence (§8.6.1). Fanout is scoped to registered peers only — the
|
|
127
|
-
* privacy floor (§8.6.3).
|
|
153
|
+
* §8.6 presence index: per `(partition, scopeKey)`, the sessions holding a
|
|
154
|
+
* document there. The documents live with their session (§8.6.1). Pure
|
|
155
|
+
* in-memory state that never touches `ServerStorage`.
|
|
128
156
|
*/
|
|
129
157
|
class PresenceRegistry {
|
|
130
|
-
/** partition → scopeKey →
|
|
131
|
-
readonly #byKey = new Map<
|
|
132
|
-
string,
|
|
133
|
-
Map<string, Map<RealtimeSession, PresenceDoc>>
|
|
134
|
-
>();
|
|
158
|
+
/** partition → scopeKey → sessions with a document on the key. */
|
|
159
|
+
readonly #byKey = new Map<string, Map<string, Set<RealtimeSession>>>();
|
|
135
160
|
|
|
136
|
-
|
|
137
|
-
partition: string,
|
|
138
|
-
scopeKey: string,
|
|
139
|
-
): Map<RealtimeSession, PresenceDoc> {
|
|
161
|
+
add(partition: string, scopeKey: string, session: RealtimeSession): void {
|
|
140
162
|
let byScope = this.#byKey.get(partition);
|
|
141
163
|
if (byScope === undefined) {
|
|
142
164
|
byScope = new Map();
|
|
@@ -144,72 +166,22 @@ class PresenceRegistry {
|
|
|
144
166
|
}
|
|
145
167
|
let sessions = byScope.get(scopeKey);
|
|
146
168
|
if (sessions === undefined) {
|
|
147
|
-
sessions = new
|
|
169
|
+
sessions = new Set();
|
|
148
170
|
byScope.set(scopeKey, sessions);
|
|
149
171
|
}
|
|
150
|
-
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
/** Current documents on a key, excluding one session (the snapshot,
|
|
154
|
-
* §8.6.4). */
|
|
155
|
-
snapshot(
|
|
156
|
-
partition: string,
|
|
157
|
-
scopeKey: string,
|
|
158
|
-
exclude: RealtimeSession,
|
|
159
|
-
): Array<{ session: RealtimeSession; doc: PresenceDoc }> {
|
|
160
|
-
const sessions = this.#byKey.get(partition)?.get(scopeKey);
|
|
161
|
-
if (sessions === undefined) return [];
|
|
162
|
-
const out: Array<{ session: RealtimeSession; doc: PresenceDoc }> = [];
|
|
163
|
-
for (const [session, doc] of sessions) {
|
|
164
|
-
if (session !== exclude) out.push({ session, doc });
|
|
165
|
-
}
|
|
166
|
-
return out;
|
|
172
|
+
sessions.add(session);
|
|
167
173
|
}
|
|
168
174
|
|
|
169
|
-
|
|
170
|
-
* the session's FIRST document on the key (join) or a replacement
|
|
171
|
-
* (update). */
|
|
172
|
-
set(
|
|
173
|
-
partition: string,
|
|
174
|
-
scopeKey: string,
|
|
175
|
-
session: RealtimeSession,
|
|
176
|
-
doc: PresenceDoc,
|
|
177
|
-
): 'join' | 'update' {
|
|
178
|
-
const sessions = this.#keyMap(partition, scopeKey);
|
|
179
|
-
const kind = sessions.has(session) ? 'update' : 'join';
|
|
180
|
-
sessions.set(session, doc);
|
|
181
|
-
return kind;
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
/** Remove a session's document for a key. Returns true if one existed. */
|
|
185
|
-
clear(
|
|
186
|
-
partition: string,
|
|
187
|
-
scopeKey: string,
|
|
188
|
-
session: RealtimeSession,
|
|
189
|
-
): boolean {
|
|
175
|
+
delete(partition: string, scopeKey: string, session: RealtimeSession): void {
|
|
190
176
|
const sessions = this.#byKey.get(partition)?.get(scopeKey);
|
|
191
|
-
if (sessions === undefined) return
|
|
192
|
-
|
|
177
|
+
if (sessions === undefined) return;
|
|
178
|
+
sessions.delete(session);
|
|
193
179
|
if (sessions.size === 0) this.#byKey.get(partition)?.delete(scopeKey);
|
|
194
|
-
return existed;
|
|
195
180
|
}
|
|
196
181
|
|
|
197
|
-
/**
|
|
198
|
-
* and registration-change re-derivation). */
|
|
199
|
-
keysOf(partition: string, session: RealtimeSession): string[] {
|
|
200
|
-
const byScope = this.#byKey.get(partition);
|
|
201
|
-
if (byScope === undefined) return [];
|
|
202
|
-
const keys: string[] = [];
|
|
203
|
-
for (const [scopeKey, sessions] of byScope) {
|
|
204
|
-
if (sessions.has(session)) keys.push(scopeKey);
|
|
205
|
-
}
|
|
206
|
-
return keys;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
/** Sessions currently registered as peers on a key (for fanout). */
|
|
182
|
+
/** Sessions holding a document on a key. */
|
|
210
183
|
peers(partition: string, scopeKey: string): RealtimeSession[] {
|
|
211
|
-
|
|
212
|
-
return sessions === undefined ? [] : [...sessions.keys()];
|
|
184
|
+
return [...(this.#byKey.get(partition)?.get(scopeKey) ?? [])];
|
|
213
185
|
}
|
|
214
186
|
}
|
|
215
187
|
|
|
@@ -248,6 +220,11 @@ export class RealtimeSession {
|
|
|
248
220
|
lastKnownSeq: number;
|
|
249
221
|
/** Replaced at socket-round completion (§8.7); initial set from §8.1. */
|
|
250
222
|
registrations: readonly Registration[];
|
|
223
|
+
/** The scope keys `registrations` cover: the presence grant (§8.6.3). */
|
|
224
|
+
scopeKeys: ReadonlySet<string>;
|
|
225
|
+
/** Sent deltas the client has not acknowledged, in commit order (§8.2). */
|
|
226
|
+
#unacked: { commitSeq: number; bytes: number }[] = [];
|
|
227
|
+
#unackedBytes = 0;
|
|
251
228
|
/** Epoch-ms (hub clock) at registration, for `realtime.closed`. */
|
|
252
229
|
readonly openedAtMs: number;
|
|
253
230
|
#send: (data: string | Uint8Array) => void | Promise<void>;
|
|
@@ -261,6 +238,8 @@ export class RealtimeSession {
|
|
|
261
238
|
* token identifies the owning round so a finished round's cleanup
|
|
262
239
|
* never clobbers a follow-up round's in-flight marker. */
|
|
263
240
|
#requestScanner: MessageStreamScanner | undefined;
|
|
241
|
+
/** Bytes the scanner holds, against `maxRequestBytes` (§1.7). */
|
|
242
|
+
#requestBytes = 0;
|
|
264
243
|
#activeRound: symbol | undefined;
|
|
265
244
|
/** Deltas wait for every round response and registration refresh to finish. */
|
|
266
245
|
#roundsRefreshing = 0;
|
|
@@ -269,6 +248,11 @@ export class RealtimeSession {
|
|
|
269
248
|
#deferredBytes = 0;
|
|
270
249
|
#deferredRegistrations: string | undefined;
|
|
271
250
|
#closed = false;
|
|
251
|
+
/** §8.6.1: this connection's presence documents by scope key, and their
|
|
252
|
+
* summed charge against `maxPresenceBytes` (§8.6.2). */
|
|
253
|
+
readonly #presence = new Map<string, PresenceEntry>();
|
|
254
|
+
#presenceBytes = 0;
|
|
255
|
+
#onPresence: ((presence: ReadonlyMap<string, string>) => void) | undefined;
|
|
272
256
|
/** §8.6.4 rate cap: per-scope-key last-fanout time + a pending latest
|
|
273
257
|
* document coalesced while over the cap. */
|
|
274
258
|
#presenceRate = new Map<
|
|
@@ -314,9 +298,11 @@ export class RealtimeSession {
|
|
|
314
298
|
this.wakePending = cursor < latestSeq;
|
|
315
299
|
this.#wakeRequiredThroughSeq = cursor < latestSeq ? latestSeq : 0;
|
|
316
300
|
this.registrations = registrations;
|
|
301
|
+
this.scopeKeys = hub.scopeKeysOf(registrations);
|
|
317
302
|
this.openedAtMs = clock();
|
|
318
303
|
this.#send = options.send;
|
|
319
304
|
this.#closeSocket = options.closeSocket;
|
|
305
|
+
this.#onPresence = options.onPresence;
|
|
320
306
|
this.#hub = hub;
|
|
321
307
|
this.#clock = clock;
|
|
322
308
|
this.#maxDeltaBytes = maxDeltaBytes;
|
|
@@ -337,12 +323,15 @@ export class RealtimeSession {
|
|
|
337
323
|
// §8.6.2: client→server presence carries `event: 'presence'` (server
|
|
338
324
|
// control messages carry `event`; the ack carries `type`, §8.1).
|
|
339
325
|
if ((parsed as { event?: unknown }).event === 'presence') {
|
|
340
|
-
this.#handlePresence(
|
|
326
|
+
this.#handlePresence((parsed as { data?: unknown }).data);
|
|
341
327
|
return;
|
|
342
328
|
}
|
|
343
329
|
if ((parsed as { type?: unknown }).type !== 'ack') return;
|
|
344
330
|
const cursor = (parsed as { cursor?: unknown }).cursor;
|
|
345
331
|
if (typeof cursor !== 'number' || !Number.isSafeInteger(cursor)) return;
|
|
332
|
+
while ((this.#unacked[0]?.commitSeq ?? Infinity) <= cursor) {
|
|
333
|
+
this.#unackedBytes -= this.#unacked.shift()!.bytes;
|
|
334
|
+
}
|
|
346
335
|
this.cursor = Math.max(this.cursor, cursor);
|
|
347
336
|
// The client may have caught up through another binding (§8.4). Its ack
|
|
348
337
|
// proves those commits no longer need socket notifications, so the next
|
|
@@ -367,21 +356,31 @@ export class RealtimeSession {
|
|
|
367
356
|
});
|
|
368
357
|
}
|
|
369
358
|
|
|
370
|
-
/**
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
359
|
+
/**
|
|
360
|
+
* §8.6.2 inbound presence publish/leave from the client: the `data` of the
|
|
361
|
+
* control message `handleMessage` parsed, a non-empty `scopeKey` and a
|
|
362
|
+
* `doc` that is an object or `null`.
|
|
363
|
+
*/
|
|
364
|
+
#handlePresence(data: unknown): void {
|
|
365
|
+
const publish =
|
|
366
|
+
typeof data === 'object' && data !== null && !Array.isArray(data)
|
|
367
|
+
? (data as { scopeKey?: unknown; doc?: unknown })
|
|
368
|
+
: undefined;
|
|
369
|
+
const scopeKey = publish?.scopeKey;
|
|
370
|
+
const doc = publish?.doc;
|
|
371
|
+
if (
|
|
372
|
+
typeof scopeKey !== 'string' ||
|
|
373
|
+
scopeKey.length === 0 ||
|
|
374
|
+
(doc !== null && (typeof doc !== 'object' || Array.isArray(doc)))
|
|
375
|
+
) {
|
|
376
376
|
// A malformed known event is a parse error (§8.1) — a broken client;
|
|
377
377
|
// drop the connection loudly rather than silently ignore.
|
|
378
378
|
this.#violation('malformed presence control message (§8.6.2)');
|
|
379
379
|
return;
|
|
380
380
|
}
|
|
381
|
-
const { scopeKey, doc } = publish.data;
|
|
382
381
|
// §8.6.3 authorization: the publish key must be in the connection's
|
|
383
382
|
// registered effective scopes.
|
|
384
|
-
if (!this
|
|
383
|
+
if (!this.scopeKeys.has(scopeKey)) {
|
|
385
384
|
this.#sendSafe(
|
|
386
385
|
encodePresenceError(scopeKey, 'presence.forbidden', this.#clock()),
|
|
387
386
|
);
|
|
@@ -391,20 +390,40 @@ export class RealtimeSession {
|
|
|
391
390
|
this.#leavePresence(scopeKey);
|
|
392
391
|
return;
|
|
393
392
|
}
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
393
|
+
const entry = this.#presenceEntry(scopeKey, doc as PresenceDoc);
|
|
394
|
+
if (entry !== undefined) this.#publishPresence(scopeKey, entry);
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* §8.6.2 size cap per connection: the key's and document's UTF-8 bytes,
|
|
399
|
+
* added to the connection's documents on its other keys, stay within
|
|
400
|
+
* `maxPresenceBytes`. Over the cap the publisher alone hears
|
|
401
|
+
* `presence.too_large` and nothing changes.
|
|
402
|
+
*/
|
|
403
|
+
#presenceEntry(
|
|
404
|
+
scopeKey: string,
|
|
405
|
+
doc: PresenceDoc,
|
|
406
|
+
): PresenceEntry | undefined {
|
|
407
|
+
const json = JSON.stringify(doc);
|
|
408
|
+
const budget =
|
|
409
|
+
this.#hub.maxPresenceBytes -
|
|
410
|
+
this.#presenceBytes +
|
|
411
|
+
(this.#presence.get(scopeKey)?.bytes ?? 0);
|
|
412
|
+
// A string never has fewer UTF-8 bytes than UTF-16 code units, so the
|
|
413
|
+
// unit count rejects first.
|
|
414
|
+
if (scopeKey.length + json.length <= budget) {
|
|
415
|
+
const bytes = new TextEncoder().encode(scopeKey + json).byteLength;
|
|
416
|
+
if (bytes <= budget) return { doc, json, bytes };
|
|
401
417
|
}
|
|
402
|
-
this.#
|
|
418
|
+
this.#sendSafe(
|
|
419
|
+
encodePresenceError(scopeKey, 'presence.too_large', this.#clock()),
|
|
420
|
+
);
|
|
421
|
+
return undefined;
|
|
403
422
|
}
|
|
404
423
|
|
|
405
424
|
/** §8.6.4: store + fan out a publish, honoring the rate cap (latest-wins
|
|
406
425
|
* coalesce). */
|
|
407
|
-
#publishPresence(scopeKey: string,
|
|
426
|
+
#publishPresence(scopeKey: string, entry: PresenceEntry): void {
|
|
408
427
|
const minInterval = this.#hub.presenceMinIntervalMs;
|
|
409
428
|
if (minInterval > 0) {
|
|
410
429
|
const now = this.#clock();
|
|
@@ -412,24 +431,48 @@ export class RealtimeSession {
|
|
|
412
431
|
if (rate !== undefined && now - rate.lastFanoutMs < minInterval) {
|
|
413
432
|
// Over the cap: keep only the latest document, fan it out at the
|
|
414
433
|
// window edge (never an error, never a stale/lost latest, §8.6.4).
|
|
415
|
-
rate.pending = doc;
|
|
434
|
+
rate.pending = entry.doc;
|
|
416
435
|
if (rate.timer === undefined) {
|
|
417
436
|
const delay = minInterval - (now - rate.lastFanoutMs);
|
|
418
437
|
rate.timer = setTimeout(() => {
|
|
419
438
|
delete rate.timer;
|
|
420
439
|
const pending = rate.pending;
|
|
421
440
|
delete rate.pending;
|
|
422
|
-
|
|
441
|
+
// The connection's other keys may have grown since: the cap
|
|
442
|
+
// holds when the document is stored.
|
|
443
|
+
const latest =
|
|
444
|
+
pending === undefined
|
|
445
|
+
? undefined
|
|
446
|
+
: this.#presenceEntry(scopeKey, pending);
|
|
447
|
+
if (latest !== undefined) this.#fanoutPublish(scopeKey, latest);
|
|
423
448
|
}, delay);
|
|
424
449
|
}
|
|
425
450
|
return;
|
|
426
451
|
}
|
|
427
452
|
}
|
|
428
|
-
this.#fanoutPublish(scopeKey,
|
|
453
|
+
this.#fanoutPublish(scopeKey, entry);
|
|
429
454
|
}
|
|
430
455
|
|
|
431
|
-
#fanoutPublish(scopeKey: string,
|
|
432
|
-
|
|
456
|
+
#fanoutPublish(scopeKey: string, entry: PresenceEntry): void {
|
|
457
|
+
// §8.6.1: the peer identity is (actorId, clientId). Another session of
|
|
458
|
+
// this identity (a stale socket of a reconnected client) loses its
|
|
459
|
+
// document silently, so its later close announces nothing.
|
|
460
|
+
let replaced = false;
|
|
461
|
+
for (const peer of this.#hub.presence.peers(this.partition, scopeKey)) {
|
|
462
|
+
if (
|
|
463
|
+
peer !== this &&
|
|
464
|
+
peer.actorId === this.actorId &&
|
|
465
|
+
peer.clientId === this.clientId
|
|
466
|
+
) {
|
|
467
|
+
replaced = peer.forgetPresence(scopeKey) || replaced;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
const held = this.#presence.get(scopeKey);
|
|
471
|
+
const kind = replaced || held !== undefined ? 'update' : 'join';
|
|
472
|
+
this.#presenceBytes += entry.bytes - (held?.bytes ?? 0);
|
|
473
|
+
this.#presence.set(scopeKey, entry);
|
|
474
|
+
this.#hub.presence.add(this.partition, scopeKey, this);
|
|
475
|
+
this.#reportPresence();
|
|
433
476
|
let rate = this.#presenceRate.get(scopeKey);
|
|
434
477
|
if (rate === undefined) {
|
|
435
478
|
rate = { lastFanoutMs: this.#clock() };
|
|
@@ -437,17 +480,52 @@ export class RealtimeSession {
|
|
|
437
480
|
} else {
|
|
438
481
|
rate.lastFanoutMs = this.#clock();
|
|
439
482
|
}
|
|
440
|
-
this.#hub.fanoutPresence(this, scopeKey, kind, doc);
|
|
483
|
+
this.#hub.fanoutPresence(this, scopeKey, kind, entry.doc);
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/** §8.6.1: hand the host this connection's documents after a change. */
|
|
487
|
+
#reportPresence(): void {
|
|
488
|
+
if (this.#closed || this.#onPresence === undefined) return;
|
|
489
|
+
this.#onPresence(
|
|
490
|
+
new Map([...this.#presence].map(([key, entry]) => [key, entry.json])),
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/** §8.6.1: hold the documents a host kept for this connection across an
|
|
495
|
+
* eviction, without fanout: peers already saw them. */
|
|
496
|
+
seedPresence(presence: ReadonlyMap<string, string>): void {
|
|
497
|
+
for (const [scopeKey, json] of presence) {
|
|
498
|
+
const bytes = new TextEncoder().encode(scopeKey + json).byteLength;
|
|
499
|
+
this.#presence.set(scopeKey, {
|
|
500
|
+
doc: JSON.parse(json) as PresenceDoc,
|
|
501
|
+
json,
|
|
502
|
+
bytes,
|
|
503
|
+
});
|
|
504
|
+
this.#presenceBytes += bytes;
|
|
505
|
+
this.#hub.presence.add(this.partition, scopeKey, this);
|
|
506
|
+
}
|
|
441
507
|
}
|
|
442
508
|
|
|
443
509
|
/** §8.6.1/§8.6.3: clear this session's document for a key and fan a
|
|
444
510
|
* leave to the remaining peers. */
|
|
445
511
|
#leavePresence(scopeKey: string): void {
|
|
446
|
-
|
|
512
|
+
if (this.forgetPresence(scopeKey))
|
|
513
|
+
this.#hub.fanoutPresence(this, scopeKey, 'leave', null);
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/** Clear this session's document for a key without fanout, including a
|
|
517
|
+
* coalesced publish still pending. Returns whether a document existed. */
|
|
518
|
+
forgetPresence(scopeKey: string): boolean {
|
|
447
519
|
const rate = this.#presenceRate.get(scopeKey);
|
|
448
520
|
if (rate?.timer !== undefined) clearTimeout(rate.timer);
|
|
449
521
|
this.#presenceRate.delete(scopeKey);
|
|
450
|
-
|
|
522
|
+
const held = this.#presence.get(scopeKey);
|
|
523
|
+
if (held === undefined) return false;
|
|
524
|
+
this.#presence.delete(scopeKey);
|
|
525
|
+
this.#presenceBytes -= held.bytes;
|
|
526
|
+
this.#hub.presence.delete(this.partition, scopeKey, this);
|
|
527
|
+
this.#reportPresence();
|
|
528
|
+
return true;
|
|
451
529
|
}
|
|
452
530
|
|
|
453
531
|
/** Deliver a fanout event to this session (called by the hub for peers,
|
|
@@ -471,17 +549,12 @@ export class RealtimeSession {
|
|
|
471
549
|
);
|
|
472
550
|
}
|
|
473
551
|
|
|
474
|
-
/** The scope keys this connection currently holds (§8.6.3). */
|
|
475
|
-
#currentScopeKeys(): Set<string> {
|
|
476
|
-
return this.#hub.scopeKeysOf(this.registrations);
|
|
477
|
-
}
|
|
478
|
-
|
|
479
552
|
/** §8.6.3: drop presence on keys the connection no longer holds and
|
|
480
553
|
* deliver the snapshot on keys it newly holds. Called after a
|
|
481
554
|
* registration change (§8.7 round end, §8.1 reconnect handled by
|
|
482
555
|
* connect's snapshot). */
|
|
483
556
|
reconcilePresence(previousKeys: ReadonlySet<string>): void {
|
|
484
|
-
const currentKeys = this
|
|
557
|
+
const currentKeys = this.scopeKeys;
|
|
485
558
|
// Keys lost: leave my own published docs; presence delivery stops
|
|
486
559
|
// naturally (peers no longer fan to me — I am not registered).
|
|
487
560
|
for (const key of previousKeys) {
|
|
@@ -496,17 +569,15 @@ export class RealtimeSession {
|
|
|
496
569
|
/** §8.6.4 snapshot: a burst of `join`s for peers already present on a
|
|
497
570
|
* key, delivered to this session only. */
|
|
498
571
|
#deliverSnapshot(scopeKey: string): void {
|
|
499
|
-
for (const peer of this.#hub.presence.
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
this,
|
|
503
|
-
)) {
|
|
572
|
+
for (const peer of this.#hub.presence.peers(this.partition, scopeKey)) {
|
|
573
|
+
const held = peer.#presence.get(scopeKey);
|
|
574
|
+
if (peer === this || held === undefined) continue;
|
|
504
575
|
this.receivePresence(
|
|
505
576
|
scopeKey,
|
|
506
577
|
'join',
|
|
507
|
-
peer.
|
|
508
|
-
peer.
|
|
509
|
-
|
|
578
|
+
peer.actorId,
|
|
579
|
+
peer.clientId,
|
|
580
|
+
held.doc,
|
|
510
581
|
);
|
|
511
582
|
}
|
|
512
583
|
}
|
|
@@ -517,9 +588,7 @@ export class RealtimeSession {
|
|
|
517
588
|
this.#deferredDeltas = [];
|
|
518
589
|
this.#deferredBytes = 0;
|
|
519
590
|
this.#deferredRegistrations = undefined;
|
|
520
|
-
for (const key of this.#
|
|
521
|
-
this.#leavePresence(key);
|
|
522
|
-
}
|
|
591
|
+
for (const key of [...this.#presence.keys()]) this.#leavePresence(key);
|
|
523
592
|
for (const rate of this.#presenceRate.values()) {
|
|
524
593
|
if (rate.timer !== undefined) clearTimeout(rate.timer);
|
|
525
594
|
}
|
|
@@ -529,7 +598,7 @@ export class RealtimeSession {
|
|
|
529
598
|
/** Snapshot delivery on connect (§8.6.4) — the newly-registered
|
|
530
599
|
* connection sees who is already present on each of its keys. */
|
|
531
600
|
deliverInitialPresence(): void {
|
|
532
|
-
for (const key of this
|
|
601
|
+
for (const key of this.scopeKeys) this.#deliverSnapshot(key);
|
|
533
602
|
}
|
|
534
603
|
|
|
535
604
|
/**
|
|
@@ -554,6 +623,17 @@ export class RealtimeSession {
|
|
|
554
623
|
return;
|
|
555
624
|
}
|
|
556
625
|
const chunk = bytes.subarray(1);
|
|
626
|
+
// §1.7: the assembled request counts against `maxRequestBytes`; past it
|
|
627
|
+
// the stream has no acceptable end, so the connection drops (§8.7).
|
|
628
|
+
this.#requestBytes =
|
|
629
|
+
this.#requestScanner === undefined
|
|
630
|
+
? chunk.length
|
|
631
|
+
: this.#requestBytes + chunk.length;
|
|
632
|
+
if (this.#requestBytes > this.#hub.limits.maxRequestBytes) {
|
|
633
|
+
this.#requestScanner = undefined;
|
|
634
|
+
this.#violation('request exceeds maxRequestBytes (§1.7)');
|
|
635
|
+
return;
|
|
636
|
+
}
|
|
557
637
|
this.#requestScanner ??= new MessageStreamScanner();
|
|
558
638
|
let done: ReturnType<MessageStreamScanner['push']>;
|
|
559
639
|
try {
|
|
@@ -595,9 +675,8 @@ export class RealtimeSession {
|
|
|
595
675
|
try {
|
|
596
676
|
// §8.7: the round's clientId must match the connection's —
|
|
597
677
|
// registration identity would otherwise be ambiguous.
|
|
598
|
-
const
|
|
599
|
-
|
|
600
|
-
if (decoded.msgKind !== 'request' || header?.type !== 'REQ_HEADER') {
|
|
678
|
+
const header = decodeRequest(requestBytes).frames[0];
|
|
679
|
+
if (header?.type !== 'REQ_HEADER') {
|
|
601
680
|
throw syncError('sync.invalid_request', 'expected a request message');
|
|
602
681
|
}
|
|
603
682
|
if (header.clientId !== this.clientId) {
|
|
@@ -732,8 +811,9 @@ export class RealtimeSession {
|
|
|
732
811
|
* failed, because that round applied nothing.
|
|
733
812
|
*/
|
|
734
813
|
/**
|
|
735
|
-
* A pull outside this connection changed the client's
|
|
736
|
-
* round in flight on the connection reloads
|
|
814
|
+
* A pull outside this connection changed the client's registration list
|
|
815
|
+
* or live set. A round in flight on the connection reloads it at its end
|
|
816
|
+
* anyway.
|
|
737
817
|
*/
|
|
738
818
|
async refreshAfterPull(): Promise<void> {
|
|
739
819
|
if (this.#activeRound !== undefined) return;
|
|
@@ -748,6 +828,7 @@ export class RealtimeSession {
|
|
|
748
828
|
this.partition,
|
|
749
829
|
this.actorId,
|
|
750
830
|
this.clientId,
|
|
831
|
+
await this.#storage.getClientRecord(this.partition, this.clientId),
|
|
751
832
|
);
|
|
752
833
|
} catch {
|
|
753
834
|
if (!failClosed) {
|
|
@@ -764,8 +845,9 @@ export class RealtimeSession {
|
|
|
764
845
|
// §8.6.3: a registration change re-derives the presence grant — capture
|
|
765
846
|
// the keys before, reconcile after (leaves on lost keys, snapshots on
|
|
766
847
|
// gained keys).
|
|
767
|
-
const previousKeys = this
|
|
848
|
+
const previousKeys = this.scopeKeys;
|
|
768
849
|
this.registrations = registrations;
|
|
850
|
+
this.scopeKeys = this.#hub.scopeKeysOf(registrations);
|
|
769
851
|
this.reconcilePresence(previousKeys);
|
|
770
852
|
}
|
|
771
853
|
|
|
@@ -813,12 +895,7 @@ export class RealtimeSession {
|
|
|
813
895
|
}
|
|
814
896
|
|
|
815
897
|
sendHeartbeat(): void {
|
|
816
|
-
this.#sendSafe(
|
|
817
|
-
JSON.stringify({
|
|
818
|
-
event: 'heartbeat',
|
|
819
|
-
data: { timestamp: this.#clock() },
|
|
820
|
-
}),
|
|
821
|
-
);
|
|
898
|
+
this.#sendSafe(encodeHeartbeat(this.#clock()));
|
|
822
899
|
}
|
|
823
900
|
|
|
824
901
|
sendWake(reason: WakeReason): void {
|
|
@@ -859,33 +936,15 @@ export class RealtimeSession {
|
|
|
859
936
|
deliverCommit(commit: StoredCommit): void {
|
|
860
937
|
const contiguous = commit.commitSeq === this.lastKnownSeq + 1;
|
|
861
938
|
this.lastKnownSeq = Math.max(this.lastKnownSeq, commit.commitSeq);
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
matchesEffective(change.scopes, registration.effective),
|
|
872
|
-
)
|
|
873
|
-
.map(
|
|
874
|
-
(change): CommitChange => ({
|
|
875
|
-
tableIndex: 0,
|
|
876
|
-
rowId: change.rowId,
|
|
877
|
-
op: change.op,
|
|
878
|
-
...(change.rowVersion !== undefined
|
|
879
|
-
? { rowVersion: change.rowVersion }
|
|
880
|
-
: {}),
|
|
881
|
-
scopes: change.scopes,
|
|
882
|
-
...(change.payload !== undefined
|
|
883
|
-
? { row: this.#hub.projectPayload(registration, change.payload) }
|
|
884
|
-
: {}),
|
|
885
|
-
}),
|
|
886
|
-
);
|
|
887
|
-
if (changes.length > 0) sections.push({ registration, changes });
|
|
888
|
-
}
|
|
939
|
+
// §8.2: one COMMIT holding every change that matches a registration of
|
|
940
|
+
// the connection, once each, in change order.
|
|
941
|
+
const changes = commit.changes.filter((change) =>
|
|
942
|
+
this.registrations.some(
|
|
943
|
+
(registration) =>
|
|
944
|
+
change.table === registration.table &&
|
|
945
|
+
matchesEffective(change.scopes, registration.effective),
|
|
946
|
+
),
|
|
947
|
+
);
|
|
889
948
|
if (!contiguous) {
|
|
890
949
|
// Pushes allocate commitSeq under the partition lock but notify after
|
|
891
950
|
// commit, so racing notifications may arrive out of order. A gap,
|
|
@@ -913,44 +972,30 @@ export class RealtimeSession {
|
|
|
913
972
|
}
|
|
914
973
|
this.#deferredRegistrations = registrations;
|
|
915
974
|
}
|
|
916
|
-
if (
|
|
975
|
+
if (changes.length === 0) return;
|
|
917
976
|
if (this.wakePending) {
|
|
918
977
|
// §8.2: deltas must be cursor-contiguous; while the client is behind
|
|
919
978
|
// we send (coalescible) wake-ups instead of a delta past its cursor.
|
|
920
979
|
this.sendWake('catchup-required');
|
|
921
980
|
return;
|
|
922
981
|
}
|
|
923
|
-
const
|
|
924
|
-
{
|
|
925
|
-
type: 'RESP_HEADER',
|
|
926
|
-
...(this.wireVersion >= 2
|
|
927
|
-
? { logEpoch: this.logEpoch, resetRequired: false }
|
|
928
|
-
: {}),
|
|
929
|
-
},
|
|
930
|
-
];
|
|
931
|
-
for (const section of sections) {
|
|
932
|
-
frames.push({
|
|
933
|
-
type: 'SUB_START',
|
|
934
|
-
id: section.registration.id,
|
|
935
|
-
status: 'active',
|
|
936
|
-
reasonCode: '',
|
|
937
|
-
effectiveScopes: section.registration.effective,
|
|
938
|
-
bootstrap: false,
|
|
939
|
-
});
|
|
940
|
-
frames.push({
|
|
941
|
-
type: 'COMMIT',
|
|
942
|
-
commitSeq: commit.commitSeq,
|
|
943
|
-
createdAtMs: commit.createdAtMs,
|
|
944
|
-
actorId: commit.actorId,
|
|
945
|
-
tables: [section.registration.table],
|
|
946
|
-
changes: section.changes,
|
|
947
|
-
});
|
|
948
|
-
frames.push({ type: 'SUB_END', nextCursor: commit.commitSeq });
|
|
949
|
-
}
|
|
950
|
-
const bytes = encodeMessage({
|
|
982
|
+
const bytes = encodeResponse({
|
|
951
983
|
wireVersion: this.wireVersion,
|
|
952
984
|
msgKind: 'response',
|
|
953
|
-
frames
|
|
985
|
+
frames: [
|
|
986
|
+
{ type: 'RESP_HEADER', logEpoch: this.logEpoch, resetRequired: false },
|
|
987
|
+
{ type: 'LIVE_START', status: 'active', reasonCode: '' },
|
|
988
|
+
commitFrame(
|
|
989
|
+
{ ...commit, changes },
|
|
990
|
+
this.#hub.schema,
|
|
991
|
+
this.registrations[0]!.schema,
|
|
992
|
+
),
|
|
993
|
+
{
|
|
994
|
+
type: 'LIVE_END',
|
|
995
|
+
throughSeq: commit.commitSeq,
|
|
996
|
+
echoDigest: new Uint8Array(),
|
|
997
|
+
},
|
|
998
|
+
],
|
|
954
999
|
});
|
|
955
1000
|
if (bytes.length > this.#maxDeltaBytes) {
|
|
956
1001
|
this.sendWake('delta-too-large');
|
|
@@ -960,7 +1005,7 @@ export class RealtimeSession {
|
|
|
960
1005
|
const tagged = new Uint8Array(bytes.length + 1);
|
|
961
1006
|
tagged[0] = REALTIME_TAG_DELTA;
|
|
962
1007
|
tagged.set(bytes, 1);
|
|
963
|
-
const changeCount =
|
|
1008
|
+
const changeCount = changes.length;
|
|
964
1009
|
if (this.#roundsRefreshing > 0) {
|
|
965
1010
|
if (this.#deferredBytes + tagged.length > this.#maxDeltaBytes) {
|
|
966
1011
|
this.sendWake('catchup-required');
|
|
@@ -978,6 +1023,21 @@ export class RealtimeSession {
|
|
|
978
1023
|
}
|
|
979
1024
|
|
|
980
1025
|
#sendDelta(tagged: Uint8Array, commitSeq: number, changes: number): void {
|
|
1026
|
+
// A wake earlier in a deferred flush suppresses the rest of it.
|
|
1027
|
+
if (this.wakePending) return;
|
|
1028
|
+
// §8.2 flow control: a client this far behind on acks catches up by pull;
|
|
1029
|
+
// its acknowledgement after that round lifts the suppression. One delta
|
|
1030
|
+
// always fits, so `maxDeltaBytes` alone bounds a single message.
|
|
1031
|
+
if (
|
|
1032
|
+
this.#unacked.length >= this.#hub.maxUnackedDeltas ||
|
|
1033
|
+
(this.#unacked.length > 0 &&
|
|
1034
|
+
this.#unackedBytes + tagged.length > this.#hub.maxUnackedDeltaBytes)
|
|
1035
|
+
) {
|
|
1036
|
+
this.sendWake('catchup-required');
|
|
1037
|
+
return;
|
|
1038
|
+
}
|
|
1039
|
+
this.#unacked.push({ commitSeq, bytes: tagged.length });
|
|
1040
|
+
this.#unackedBytes += tagged.length;
|
|
981
1041
|
this.#sendSafe(tagged);
|
|
982
1042
|
this.cursor = Math.max(this.cursor, commitSeq);
|
|
983
1043
|
const events = this.#events;
|
|
@@ -1012,6 +1072,8 @@ export interface RealtimeConnection {
|
|
|
1012
1072
|
export class RealtimeHub {
|
|
1013
1073
|
#config: RealtimeHubConfig;
|
|
1014
1074
|
readonly #schema: CompiledSchema;
|
|
1075
|
+
/** The validated configured limits (§1.7); a bad value throws here. */
|
|
1076
|
+
readonly limits: ServerLimits;
|
|
1015
1077
|
readonly #sessions = new Set<RealtimeSession>();
|
|
1016
1078
|
/** §8.6 presence registry — ephemeral, in-memory, never persisted. */
|
|
1017
1079
|
readonly presence = new PresenceRegistry();
|
|
@@ -1020,14 +1082,12 @@ export class RealtimeHub {
|
|
|
1020
1082
|
this.#config = config;
|
|
1021
1083
|
this.#schema = compileSchema(config.schema);
|
|
1022
1084
|
schemaWindowOf(config.schema, config.schemaWindow);
|
|
1085
|
+
this.limits = limitsOf(config);
|
|
1023
1086
|
}
|
|
1024
1087
|
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
registration.codec,
|
|
1029
|
-
payload,
|
|
1030
|
-
);
|
|
1088
|
+
/** The current compiled schema: the storage codec of fanned-out changes. */
|
|
1089
|
+
get schema(): CompiledSchema {
|
|
1090
|
+
return this.#schema;
|
|
1031
1091
|
}
|
|
1032
1092
|
|
|
1033
1093
|
/**
|
|
@@ -1064,6 +1124,16 @@ export class RealtimeHub {
|
|
|
1064
1124
|
return this.#config.maxPresenceBytes ?? DEFAULT_MAX_PRESENCE_BYTES;
|
|
1065
1125
|
}
|
|
1066
1126
|
|
|
1127
|
+
/** §8.2 flow control: unacknowledged deltas per session. */
|
|
1128
|
+
get maxUnackedDeltas(): number {
|
|
1129
|
+
return this.#config.maxUnackedDeltas ?? DEFAULT_MAX_UNACKED_DELTAS;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
/** §8.2 flow control: unacknowledged delta bytes per session. */
|
|
1133
|
+
get maxUnackedDeltaBytes(): number {
|
|
1134
|
+
return this.#config.maxUnackedDeltaBytes ?? DEFAULT_MAX_UNACKED_DELTA_BYTES;
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1067
1137
|
/** §8.6.4 presence rate cap (ms); 0 = off (reference default). */
|
|
1068
1138
|
get presenceMinIntervalMs(): number {
|
|
1069
1139
|
return this.#config.presenceMinIntervalMs ?? 0;
|
|
@@ -1080,7 +1150,7 @@ export class RealtimeHub {
|
|
|
1080
1150
|
* The publisher does not receive its own fanout.
|
|
1081
1151
|
*/
|
|
1082
1152
|
fanoutPresence(
|
|
1083
|
-
origin: RealtimeSession,
|
|
1153
|
+
origin: Pick<RealtimeSession, 'partition' | 'actorId' | 'clientId'>,
|
|
1084
1154
|
scopeKey: string,
|
|
1085
1155
|
kind: PresenceKind,
|
|
1086
1156
|
doc: PresenceDoc | null,
|
|
@@ -1089,7 +1159,7 @@ export class RealtimeHub {
|
|
|
1089
1159
|
if (session === origin) continue;
|
|
1090
1160
|
if (session.partition !== origin.partition) continue;
|
|
1091
1161
|
// Receive authorization (§8.6.3): the receiver must itself hold the key.
|
|
1092
|
-
if (!
|
|
1162
|
+
if (!session.scopeKeys.has(scopeKey)) continue;
|
|
1093
1163
|
session.receivePresence(
|
|
1094
1164
|
scopeKey,
|
|
1095
1165
|
kind,
|
|
@@ -1101,18 +1171,15 @@ export class RealtimeHub {
|
|
|
1101
1171
|
}
|
|
1102
1172
|
|
|
1103
1173
|
/**
|
|
1104
|
-
* Resolve the client record's
|
|
1174
|
+
* Resolve the client record's live units into per-connection
|
|
1105
1175
|
* registrations (§8.1 at upgrade; §8.7 at socket-round end).
|
|
1106
1176
|
*/
|
|
1107
1177
|
async loadRegistrations(
|
|
1108
1178
|
partition: string,
|
|
1109
1179
|
actorId: string,
|
|
1110
1180
|
clientId: string,
|
|
1181
|
+
record: ClientRecord | undefined,
|
|
1111
1182
|
): Promise<Registration[]> {
|
|
1112
|
-
const record = await this.#config.storage.getClientRecord(
|
|
1113
|
-
partition,
|
|
1114
|
-
clientId,
|
|
1115
|
-
);
|
|
1116
1183
|
let resolved: ResolvedScopes;
|
|
1117
1184
|
try {
|
|
1118
1185
|
const allowed = await this.#config.resolveScopes({
|
|
@@ -1140,35 +1207,35 @@ export class RealtimeHub {
|
|
|
1140
1207
|
}
|
|
1141
1208
|
resolved = { ok: false };
|
|
1142
1209
|
}
|
|
1210
|
+
// §8.1: the stored list minus the catching-up units, each live unit's
|
|
1211
|
+
// active pairs under the schema version of the client's latest round.
|
|
1212
|
+
const schema = schemaWindowOf(
|
|
1213
|
+
this.#config.schema,
|
|
1214
|
+
this.#config.schemaWindow,
|
|
1215
|
+
).find((candidate) => candidate.version === record?.schemaVersion);
|
|
1143
1216
|
const registrations: Registration[] = [];
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
const
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
codec: table,
|
|
1167
|
-
table: subscription.table,
|
|
1168
|
-
schemaVersion:
|
|
1169
|
-
subscription.schemaVersion ?? this.#config.schema.version,
|
|
1170
|
-
effective: outcome.effective,
|
|
1171
|
-
});
|
|
1217
|
+
if (schema === undefined) return registrations;
|
|
1218
|
+
for (const scopes of record?.registration?.units ?? []) {
|
|
1219
|
+
const id = unitId(scopes);
|
|
1220
|
+
if (record?.catchingUp.some((unit) => unit.id === id)) continue;
|
|
1221
|
+
let tables: CompiledTable[];
|
|
1222
|
+
try {
|
|
1223
|
+
tables = coveredTables(schema, scopes);
|
|
1224
|
+
} catch {
|
|
1225
|
+
// A unit the client's schema version no longer serves registers
|
|
1226
|
+
// nothing; the client's next pull fails loud on it (§4.1).
|
|
1227
|
+
continue;
|
|
1228
|
+
}
|
|
1229
|
+
const unit = planUnit(scopes, tables, resolved);
|
|
1230
|
+
for (const [table, effective] of unit.pairs) {
|
|
1231
|
+
registrations.push({
|
|
1232
|
+
schema,
|
|
1233
|
+
id: unit.id,
|
|
1234
|
+
table,
|
|
1235
|
+
schemaVersion: schema.version,
|
|
1236
|
+
effective,
|
|
1237
|
+
});
|
|
1238
|
+
}
|
|
1172
1239
|
}
|
|
1173
1240
|
return registrations;
|
|
1174
1241
|
}
|
|
@@ -1217,17 +1284,90 @@ export class RealtimeHub {
|
|
|
1217
1284
|
}
|
|
1218
1285
|
|
|
1219
1286
|
/**
|
|
1220
|
-
* Register a connected socket (§8.1): load the client's
|
|
1221
|
-
*
|
|
1287
|
+
* Register a connected socket (§8.1): load the client record's live
|
|
1288
|
+
* units, resolve their effective scopes, send `hello`.
|
|
1222
1289
|
*/
|
|
1223
|
-
|
|
1290
|
+
connect(options: RealtimeConnectOptions): Promise<RealtimeSession> {
|
|
1291
|
+
return this.#open(options, undefined);
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
/**
|
|
1295
|
+
* Rebuild the sessions of sockets that outlived their host's memory, such as
|
|
1296
|
+
* a hibernated Durable Object (§8.1). Each client saw `hello` and the
|
|
1297
|
+
* presence snapshot at its real upgrade, so neither is sent again. The
|
|
1298
|
+
* partition is touched and its latest sequence read once per partition, and
|
|
1299
|
+
* each client record once. The caller holds back commits until this
|
|
1300
|
+
* resolves, so the shared latest sequence stays current.
|
|
1301
|
+
*
|
|
1302
|
+
* `presence` is the connection's last `onPresence` report; the session
|
|
1303
|
+
* holds it again without fanout (§8.6.1). Once every session is back, a
|
|
1304
|
+
* key the reloaded registrations no longer cover leaves, and so does every
|
|
1305
|
+
* key of a socket whose session could not be rebuilt.
|
|
1306
|
+
*/
|
|
1307
|
+
async restore(
|
|
1308
|
+
sockets: readonly (RealtimeConnectOptions & {
|
|
1309
|
+
readonly presence?: ReadonlyMap<string, string>;
|
|
1310
|
+
})[],
|
|
1311
|
+
): Promise<PromiseSettledResult<RealtimeSession>[]> {
|
|
1312
|
+
const partitions = new Map<
|
|
1313
|
+
string,
|
|
1314
|
+
{ logEpoch: string; latestSeq: number }
|
|
1315
|
+
>();
|
|
1316
|
+
const results: PromiseSettledResult<RealtimeSession>[] = [];
|
|
1317
|
+
for (const options of sockets) {
|
|
1318
|
+
try {
|
|
1319
|
+
let shared = partitions.get(options.partition);
|
|
1320
|
+
if (shared === undefined) {
|
|
1321
|
+
const { logEpoch } = await touchAuthenticatedPartition({
|
|
1322
|
+
...this.#config,
|
|
1323
|
+
partition: options.partition,
|
|
1324
|
+
actorId: options.actorId,
|
|
1325
|
+
});
|
|
1326
|
+
shared = {
|
|
1327
|
+
logEpoch,
|
|
1328
|
+
latestSeq: await this.#config.storage.getMaxCommitSeq(
|
|
1329
|
+
options.partition,
|
|
1330
|
+
),
|
|
1331
|
+
};
|
|
1332
|
+
partitions.set(options.partition, shared);
|
|
1333
|
+
}
|
|
1334
|
+
const value = await this.#open(options, shared);
|
|
1335
|
+
if (options.presence !== undefined)
|
|
1336
|
+
value.seedPresence(options.presence);
|
|
1337
|
+
results.push({ status: 'fulfilled', value });
|
|
1338
|
+
} catch (reason) {
|
|
1339
|
+
results.push({ status: 'rejected', reason });
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
for (const [i, result] of results.entries()) {
|
|
1343
|
+
const held = [...(sockets[i]!.presence?.keys() ?? [])];
|
|
1344
|
+
if (result.status === 'fulfilled') {
|
|
1345
|
+
const session = result.value;
|
|
1346
|
+
session.reconcilePresence(new Set([...session.scopeKeys, ...held]));
|
|
1347
|
+
} else {
|
|
1348
|
+
for (const key of held) {
|
|
1349
|
+
this.fanoutPresence(sockets[i]!, key, 'leave', null);
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
return results;
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
async #open(
|
|
1357
|
+
options: RealtimeConnectOptions,
|
|
1358
|
+
restored: { logEpoch: string; latestSeq: number } | undefined,
|
|
1359
|
+
): Promise<RealtimeSession> {
|
|
1224
1360
|
const { storage } = this.#config;
|
|
1225
1361
|
const clock = this.#config.clock ?? Date.now;
|
|
1226
|
-
const
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1362
|
+
const logEpoch =
|
|
1363
|
+
restored?.logEpoch ??
|
|
1364
|
+
(
|
|
1365
|
+
await touchAuthenticatedPartition({
|
|
1366
|
+
...this.#config,
|
|
1367
|
+
partition: options.partition,
|
|
1368
|
+
actorId: options.actorId,
|
|
1369
|
+
})
|
|
1370
|
+
).logEpoch;
|
|
1231
1371
|
if (options.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
|
|
1232
1372
|
throw syncError(
|
|
1233
1373
|
'sync.invalid_client_id',
|
|
@@ -1248,8 +1388,10 @@ export class RealtimeHub {
|
|
|
1248
1388
|
options.partition,
|
|
1249
1389
|
options.actorId,
|
|
1250
1390
|
options.clientId,
|
|
1391
|
+
record,
|
|
1251
1392
|
);
|
|
1252
|
-
const latestSeq =
|
|
1393
|
+
const latestSeq =
|
|
1394
|
+
restored?.latestSeq ?? (await storage.getMaxCommitSeq(options.partition));
|
|
1253
1395
|
const cursor = record?.cursor ?? -1;
|
|
1254
1396
|
const session = new RealtimeSession(
|
|
1255
1397
|
this,
|
|
@@ -1260,44 +1402,40 @@ export class RealtimeHub {
|
|
|
1260
1402
|
clock,
|
|
1261
1403
|
this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES,
|
|
1262
1404
|
storage,
|
|
1263
|
-
|
|
1405
|
+
logEpoch,
|
|
1264
1406
|
record !== undefined && isSupportedProtocolWireVersion(record.wireVersion)
|
|
1265
1407
|
? record.wireVersion
|
|
1266
1408
|
: PROTOCOL_WIRE_VERSION,
|
|
1267
1409
|
this.#config.events,
|
|
1268
1410
|
);
|
|
1269
1411
|
this.#sessions.add(session);
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
if (helloResult instanceof Promise) helloResult.catch(() => {});
|
|
1298
|
-
// §8.6.4: the newly-registered connection sees who is already present
|
|
1299
|
-
// on each key it holds (a burst of joins, after hello).
|
|
1300
|
-
session.deliverInitialPresence();
|
|
1412
|
+
if (restored === undefined) {
|
|
1413
|
+
const helloResult = options.send(
|
|
1414
|
+
JSON.stringify({
|
|
1415
|
+
event: 'hello',
|
|
1416
|
+
data: {
|
|
1417
|
+
protocolVersion: 1,
|
|
1418
|
+
sessionId: session.sessionId,
|
|
1419
|
+
actorId: options.actorId,
|
|
1420
|
+
clientId: options.clientId,
|
|
1421
|
+
cursor,
|
|
1422
|
+
latestCursor: latestSeq,
|
|
1423
|
+
requiresSync:
|
|
1424
|
+
record?.registration === undefined ||
|
|
1425
|
+
cursor < latestSeq ||
|
|
1426
|
+
!schemaWindowOf(
|
|
1427
|
+
this.#config.schema,
|
|
1428
|
+
this.#config.schemaWindow,
|
|
1429
|
+
).some((schema) => schema.version === record.schemaVersion),
|
|
1430
|
+
timestamp: clock(),
|
|
1431
|
+
},
|
|
1432
|
+
}),
|
|
1433
|
+
);
|
|
1434
|
+
if (helloResult instanceof Promise) helloResult.catch(() => {});
|
|
1435
|
+
// §8.6.4: the newly-registered connection sees who is already present
|
|
1436
|
+
// on each key it holds (a burst of joins, after hello).
|
|
1437
|
+
session.deliverInitialPresence();
|
|
1438
|
+
}
|
|
1301
1439
|
const events = this.#config.events;
|
|
1302
1440
|
if (events !== undefined) {
|
|
1303
1441
|
emitEvent(events, {
|
|
@@ -1345,7 +1483,8 @@ export class RealtimeHub {
|
|
|
1345
1483
|
}
|
|
1346
1484
|
|
|
1347
1485
|
/**
|
|
1348
|
-
* RealtimeNotifier: a pull changed a client's
|
|
1486
|
+
* RealtimeNotifier: a pull changed a client's registration list or live
|
|
1487
|
+
* set (§8.1).
|
|
1349
1488
|
* Its sockets reload their registrations with the round-end semantics, so
|
|
1350
1489
|
* an unreadable record keeps the previous registrations.
|
|
1351
1490
|
*/
|
|
@@ -1396,6 +1535,23 @@ export class RealtimeHub {
|
|
|
1396
1535
|
wakeAll(reason: WakeReason): void {
|
|
1397
1536
|
for (const session of this.#sessions) session.sendWake(reason);
|
|
1398
1537
|
}
|
|
1538
|
+
|
|
1539
|
+
/**
|
|
1540
|
+
* §8.5: send `heartbeat` to every session. The host calls this on its
|
|
1541
|
+
* heartbeat interval (suggested 30 s); a client that hears nothing for twice
|
|
1542
|
+
* the interval treats the socket as lost.
|
|
1543
|
+
*/
|
|
1544
|
+
heartbeat(): void {
|
|
1545
|
+
for (const session of this.#sessions) session.sendHeartbeat();
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
/** The §8.5 `heartbeat` control message. */
|
|
1550
|
+
export function encodeHeartbeat(timestampMs: number): string {
|
|
1551
|
+
return JSON.stringify({
|
|
1552
|
+
event: 'heartbeat',
|
|
1553
|
+
data: { timestamp: timestampMs },
|
|
1554
|
+
});
|
|
1399
1555
|
}
|
|
1400
1556
|
|
|
1401
1557
|
export function createRealtimeHub(config: RealtimeHubConfig): RealtimeHub {
|
|
@@ -1410,13 +1566,14 @@ function errorResponseBytes(
|
|
|
1410
1566
|
wireVersion: number,
|
|
1411
1567
|
logEpoch: string,
|
|
1412
1568
|
): Uint8Array {
|
|
1413
|
-
return
|
|
1569
|
+
return encodeResponse({
|
|
1414
1570
|
wireVersion,
|
|
1415
1571
|
msgKind: 'response',
|
|
1416
1572
|
frames: [
|
|
1417
1573
|
{
|
|
1418
1574
|
type: 'RESP_HEADER',
|
|
1419
|
-
|
|
1575
|
+
logEpoch,
|
|
1576
|
+
resetRequired: false,
|
|
1420
1577
|
},
|
|
1421
1578
|
{
|
|
1422
1579
|
type: 'ERROR',
|