@syncular/server 0.57.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
package/dist/realtime.js
CHANGED
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
* one handler, two framings); the only JSON data-plane server event is
|
|
17
17
|
* the `sync` wake-up (§8.3).
|
|
18
18
|
*/
|
|
19
|
-
import { DecodeError, decodeMessage, encodeMessage, encodePresenceError, encodePresenceFanout, isSupportedProtocolWireVersion, MessageStreamScanner, PROTOCOL_WIRE_VERSION,
|
|
20
|
-
import { REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
|
|
19
|
+
import { DecodeError, decodeMessage, encodeMessage, encodePresenceError, encodePresenceFanout, isSupportedProtocolWireVersion, MessageStreamScanner, PROTOCOL_WIRE_VERSION, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
|
|
20
|
+
import { limitsOf, REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
|
|
21
21
|
import { adapterSyncError, reportError, SyncError, syncError, } from './errors.js';
|
|
22
22
|
import { emitEvent } from './events.js';
|
|
23
23
|
import { composeEvents } from './events-ring.js';
|
|
@@ -26,18 +26,18 @@ import { compileSchema, } from './schema.js';
|
|
|
26
26
|
import { schemaWindowOf, projectRowPayload } from './schema-window.js';
|
|
27
27
|
import { computeEffective, matchesEffective, } from './scopes.js';
|
|
28
28
|
const DEFAULT_MAX_DELTA_BYTES = 1024 * 1024;
|
|
29
|
-
const DEFAULT_MAX_PRESENCE_BYTES =
|
|
29
|
+
const DEFAULT_MAX_PRESENCE_BYTES = 8192;
|
|
30
|
+
const DEFAULT_MAX_UNACKED_DELTAS = 256;
|
|
31
|
+
const DEFAULT_MAX_UNACKED_DELTA_BYTES = 4 * 1024 * 1024;
|
|
30
32
|
/**
|
|
31
|
-
* §8.6 presence
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
* presence (§8.6.1). Fanout is scoped to registered peers only — the
|
|
35
|
-
* privacy floor (§8.6.3).
|
|
33
|
+
* §8.6 presence index: per `(partition, scopeKey)`, the sessions holding a
|
|
34
|
+
* document there. The documents live with their session (§8.6.1). Pure
|
|
35
|
+
* in-memory state that never touches `ServerStorage`.
|
|
36
36
|
*/
|
|
37
37
|
class PresenceRegistry {
|
|
38
|
-
/** partition → scopeKey →
|
|
38
|
+
/** partition → scopeKey → sessions with a document on the key. */
|
|
39
39
|
#byKey = new Map();
|
|
40
|
-
|
|
40
|
+
add(partition, scopeKey, session) {
|
|
41
41
|
let byScope = this.#byKey.get(partition);
|
|
42
42
|
if (byScope === undefined) {
|
|
43
43
|
byScope = new Map();
|
|
@@ -45,60 +45,22 @@ class PresenceRegistry {
|
|
|
45
45
|
}
|
|
46
46
|
let sessions = byScope.get(scopeKey);
|
|
47
47
|
if (sessions === undefined) {
|
|
48
|
-
sessions = new
|
|
48
|
+
sessions = new Set();
|
|
49
49
|
byScope.set(scopeKey, sessions);
|
|
50
50
|
}
|
|
51
|
-
|
|
51
|
+
sessions.add(session);
|
|
52
52
|
}
|
|
53
|
-
|
|
54
|
-
* §8.6.4). */
|
|
55
|
-
snapshot(partition, scopeKey, exclude) {
|
|
53
|
+
delete(partition, scopeKey, session) {
|
|
56
54
|
const sessions = this.#byKey.get(partition)?.get(scopeKey);
|
|
57
55
|
if (sessions === undefined)
|
|
58
|
-
return
|
|
59
|
-
|
|
60
|
-
for (const [session, doc] of sessions) {
|
|
61
|
-
if (session !== exclude)
|
|
62
|
-
out.push({ session, doc });
|
|
63
|
-
}
|
|
64
|
-
return out;
|
|
65
|
-
}
|
|
66
|
-
/** Store/replace a session's document for a key. Returns whether this is
|
|
67
|
-
* the session's FIRST document on the key (join) or a replacement
|
|
68
|
-
* (update). */
|
|
69
|
-
set(partition, scopeKey, session, doc) {
|
|
70
|
-
const sessions = this.#keyMap(partition, scopeKey);
|
|
71
|
-
const kind = sessions.has(session) ? 'update' : 'join';
|
|
72
|
-
sessions.set(session, doc);
|
|
73
|
-
return kind;
|
|
74
|
-
}
|
|
75
|
-
/** Remove a session's document for a key. Returns true if one existed. */
|
|
76
|
-
clear(partition, scopeKey, session) {
|
|
77
|
-
const sessions = this.#byKey.get(partition)?.get(scopeKey);
|
|
78
|
-
if (sessions === undefined)
|
|
79
|
-
return false;
|
|
80
|
-
const existed = sessions.delete(session);
|
|
56
|
+
return;
|
|
57
|
+
sessions.delete(session);
|
|
81
58
|
if (sessions.size === 0)
|
|
82
59
|
this.#byKey.get(partition)?.delete(scopeKey);
|
|
83
|
-
return existed;
|
|
84
60
|
}
|
|
85
|
-
/**
|
|
86
|
-
* and registration-change re-derivation). */
|
|
87
|
-
keysOf(partition, session) {
|
|
88
|
-
const byScope = this.#byKey.get(partition);
|
|
89
|
-
if (byScope === undefined)
|
|
90
|
-
return [];
|
|
91
|
-
const keys = [];
|
|
92
|
-
for (const [scopeKey, sessions] of byScope) {
|
|
93
|
-
if (sessions.has(session))
|
|
94
|
-
keys.push(scopeKey);
|
|
95
|
-
}
|
|
96
|
-
return keys;
|
|
97
|
-
}
|
|
98
|
-
/** Sessions currently registered as peers on a key (for fanout). */
|
|
61
|
+
/** Sessions holding a document on a key. */
|
|
99
62
|
peers(partition, scopeKey) {
|
|
100
|
-
|
|
101
|
-
return sessions === undefined ? [] : [...sessions.keys()];
|
|
63
|
+
return [...(this.#byKey.get(partition)?.get(scopeKey) ?? [])];
|
|
102
64
|
}
|
|
103
65
|
}
|
|
104
66
|
/** The scope keys a set of registrations covers (§3.1: `prefix:value`). A
|
|
@@ -135,6 +97,11 @@ export class RealtimeSession {
|
|
|
135
97
|
lastKnownSeq;
|
|
136
98
|
/** Replaced at socket-round completion (§8.7); initial set from §8.1. */
|
|
137
99
|
registrations;
|
|
100
|
+
/** The scope keys `registrations` cover: the presence grant (§8.6.3). */
|
|
101
|
+
scopeKeys;
|
|
102
|
+
/** Sent deltas the client has not acknowledged, in commit order (§8.2). */
|
|
103
|
+
#unacked = [];
|
|
104
|
+
#unackedBytes = 0;
|
|
138
105
|
/** Epoch-ms (hub clock) at registration, for `realtime.closed`. */
|
|
139
106
|
openedAtMs;
|
|
140
107
|
#send;
|
|
@@ -148,6 +115,8 @@ export class RealtimeSession {
|
|
|
148
115
|
* token identifies the owning round so a finished round's cleanup
|
|
149
116
|
* never clobbers a follow-up round's in-flight marker. */
|
|
150
117
|
#requestScanner;
|
|
118
|
+
/** Bytes the scanner holds, against `maxRequestBytes` (§1.7). */
|
|
119
|
+
#requestBytes = 0;
|
|
151
120
|
#activeRound;
|
|
152
121
|
/** Deltas wait for every round response and registration refresh to finish. */
|
|
153
122
|
#roundsRefreshing = 0;
|
|
@@ -155,6 +124,11 @@ export class RealtimeSession {
|
|
|
155
124
|
#deferredBytes = 0;
|
|
156
125
|
#deferredRegistrations;
|
|
157
126
|
#closed = false;
|
|
127
|
+
/** §8.6.1: this connection's presence documents by scope key, and their
|
|
128
|
+
* summed charge against `maxPresenceBytes` (§8.6.2). */
|
|
129
|
+
#presence = new Map();
|
|
130
|
+
#presenceBytes = 0;
|
|
131
|
+
#onPresence;
|
|
158
132
|
/** §8.6.4 rate cap: per-scope-key last-fanout time + a pending latest
|
|
159
133
|
* document coalesced while over the cap. */
|
|
160
134
|
#presenceRate = new Map();
|
|
@@ -180,9 +154,11 @@ export class RealtimeSession {
|
|
|
180
154
|
this.wakePending = cursor < latestSeq;
|
|
181
155
|
this.#wakeRequiredThroughSeq = cursor < latestSeq ? latestSeq : 0;
|
|
182
156
|
this.registrations = registrations;
|
|
157
|
+
this.scopeKeys = hub.scopeKeysOf(registrations);
|
|
183
158
|
this.openedAtMs = clock();
|
|
184
159
|
this.#send = options.send;
|
|
185
160
|
this.#closeSocket = options.closeSocket;
|
|
161
|
+
this.#onPresence = options.onPresence;
|
|
186
162
|
this.#hub = hub;
|
|
187
163
|
this.#clock = clock;
|
|
188
164
|
this.#maxDeltaBytes = maxDeltaBytes;
|
|
@@ -204,7 +180,7 @@ export class RealtimeSession {
|
|
|
204
180
|
// §8.6.2: client→server presence carries `event: 'presence'` (server
|
|
205
181
|
// control messages carry `event`; the ack carries `type`, §8.1).
|
|
206
182
|
if (parsed.event === 'presence') {
|
|
207
|
-
this.#handlePresence(
|
|
183
|
+
this.#handlePresence(parsed.data);
|
|
208
184
|
return;
|
|
209
185
|
}
|
|
210
186
|
if (parsed.type !== 'ack')
|
|
@@ -212,6 +188,9 @@ export class RealtimeSession {
|
|
|
212
188
|
const cursor = parsed.cursor;
|
|
213
189
|
if (typeof cursor !== 'number' || !Number.isSafeInteger(cursor))
|
|
214
190
|
return;
|
|
191
|
+
while ((this.#unacked[0]?.commitSeq ?? Infinity) <= cursor) {
|
|
192
|
+
this.#unackedBytes -= this.#unacked.shift().bytes;
|
|
193
|
+
}
|
|
215
194
|
this.cursor = Math.max(this.cursor, cursor);
|
|
216
195
|
// The client may have caught up through another binding (§8.4). Its ack
|
|
217
196
|
// proves those commits no longer need socket notifications, so the next
|
|
@@ -236,22 +215,28 @@ export class RealtimeSession {
|
|
|
236
215
|
throw failure;
|
|
237
216
|
});
|
|
238
217
|
}
|
|
239
|
-
/**
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
218
|
+
/**
|
|
219
|
+
* §8.6.2 inbound presence publish/leave from the client: the `data` of the
|
|
220
|
+
* control message `handleMessage` parsed, a non-empty `scopeKey` and a
|
|
221
|
+
* `doc` that is an object or `null`.
|
|
222
|
+
*/
|
|
223
|
+
#handlePresence(data) {
|
|
224
|
+
const publish = typeof data === 'object' && data !== null && !Array.isArray(data)
|
|
225
|
+
? data
|
|
226
|
+
: undefined;
|
|
227
|
+
const scopeKey = publish?.scopeKey;
|
|
228
|
+
const doc = publish?.doc;
|
|
229
|
+
if (typeof scopeKey !== 'string' ||
|
|
230
|
+
scopeKey.length === 0 ||
|
|
231
|
+
(doc !== null && (typeof doc !== 'object' || Array.isArray(doc)))) {
|
|
246
232
|
// A malformed known event is a parse error (§8.1) — a broken client;
|
|
247
233
|
// drop the connection loudly rather than silently ignore.
|
|
248
234
|
this.#violation('malformed presence control message (§8.6.2)');
|
|
249
235
|
return;
|
|
250
236
|
}
|
|
251
|
-
const { scopeKey, doc } = publish.data;
|
|
252
237
|
// §8.6.3 authorization: the publish key must be in the connection's
|
|
253
238
|
// registered effective scopes.
|
|
254
|
-
if (!this
|
|
239
|
+
if (!this.scopeKeys.has(scopeKey)) {
|
|
255
240
|
this.#sendSafe(encodePresenceError(scopeKey, 'presence.forbidden', this.#clock()));
|
|
256
241
|
return;
|
|
257
242
|
}
|
|
@@ -259,17 +244,34 @@ export class RealtimeSession {
|
|
|
259
244
|
this.#leavePresence(scopeKey);
|
|
260
245
|
return;
|
|
261
246
|
}
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
247
|
+
const entry = this.#presenceEntry(scopeKey, doc);
|
|
248
|
+
if (entry !== undefined)
|
|
249
|
+
this.#publishPresence(scopeKey, entry);
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* §8.6.2 size cap per connection: the key's and document's UTF-8 bytes,
|
|
253
|
+
* added to the connection's documents on its other keys, stay within
|
|
254
|
+
* `maxPresenceBytes`. Over the cap the publisher alone hears
|
|
255
|
+
* `presence.too_large` and nothing changes.
|
|
256
|
+
*/
|
|
257
|
+
#presenceEntry(scopeKey, doc) {
|
|
258
|
+
const json = JSON.stringify(doc);
|
|
259
|
+
const budget = this.#hub.maxPresenceBytes -
|
|
260
|
+
this.#presenceBytes +
|
|
261
|
+
(this.#presence.get(scopeKey)?.bytes ?? 0);
|
|
262
|
+
// A string never has fewer UTF-8 bytes than UTF-16 code units, so the
|
|
263
|
+
// unit count rejects first.
|
|
264
|
+
if (scopeKey.length + json.length <= budget) {
|
|
265
|
+
const bytes = new TextEncoder().encode(scopeKey + json).byteLength;
|
|
266
|
+
if (bytes <= budget)
|
|
267
|
+
return { doc, json, bytes };
|
|
267
268
|
}
|
|
268
|
-
this.#
|
|
269
|
+
this.#sendSafe(encodePresenceError(scopeKey, 'presence.too_large', this.#clock()));
|
|
270
|
+
return undefined;
|
|
269
271
|
}
|
|
270
272
|
/** §8.6.4: store + fan out a publish, honoring the rate cap (latest-wins
|
|
271
273
|
* coalesce). */
|
|
272
|
-
#publishPresence(scopeKey,
|
|
274
|
+
#publishPresence(scopeKey, entry) {
|
|
273
275
|
const minInterval = this.#hub.presenceMinIntervalMs;
|
|
274
276
|
if (minInterval > 0) {
|
|
275
277
|
const now = this.#clock();
|
|
@@ -277,24 +279,45 @@ export class RealtimeSession {
|
|
|
277
279
|
if (rate !== undefined && now - rate.lastFanoutMs < minInterval) {
|
|
278
280
|
// Over the cap: keep only the latest document, fan it out at the
|
|
279
281
|
// window edge (never an error, never a stale/lost latest, §8.6.4).
|
|
280
|
-
rate.pending = doc;
|
|
282
|
+
rate.pending = entry.doc;
|
|
281
283
|
if (rate.timer === undefined) {
|
|
282
284
|
const delay = minInterval - (now - rate.lastFanoutMs);
|
|
283
285
|
rate.timer = setTimeout(() => {
|
|
284
286
|
delete rate.timer;
|
|
285
287
|
const pending = rate.pending;
|
|
286
288
|
delete rate.pending;
|
|
287
|
-
|
|
288
|
-
|
|
289
|
+
// The connection's other keys may have grown since: the cap
|
|
290
|
+
// holds when the document is stored.
|
|
291
|
+
const latest = pending === undefined
|
|
292
|
+
? undefined
|
|
293
|
+
: this.#presenceEntry(scopeKey, pending);
|
|
294
|
+
if (latest !== undefined)
|
|
295
|
+
this.#fanoutPublish(scopeKey, latest);
|
|
289
296
|
}, delay);
|
|
290
297
|
}
|
|
291
298
|
return;
|
|
292
299
|
}
|
|
293
300
|
}
|
|
294
|
-
this.#fanoutPublish(scopeKey,
|
|
301
|
+
this.#fanoutPublish(scopeKey, entry);
|
|
295
302
|
}
|
|
296
|
-
#fanoutPublish(scopeKey,
|
|
297
|
-
|
|
303
|
+
#fanoutPublish(scopeKey, entry) {
|
|
304
|
+
// §8.6.1: the peer identity is (actorId, clientId). Another session of
|
|
305
|
+
// this identity (a stale socket of a reconnected client) loses its
|
|
306
|
+
// document silently, so its later close announces nothing.
|
|
307
|
+
let replaced = false;
|
|
308
|
+
for (const peer of this.#hub.presence.peers(this.partition, scopeKey)) {
|
|
309
|
+
if (peer !== this &&
|
|
310
|
+
peer.actorId === this.actorId &&
|
|
311
|
+
peer.clientId === this.clientId) {
|
|
312
|
+
replaced = peer.forgetPresence(scopeKey) || replaced;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
const held = this.#presence.get(scopeKey);
|
|
316
|
+
const kind = replaced || held !== undefined ? 'update' : 'join';
|
|
317
|
+
this.#presenceBytes += entry.bytes - (held?.bytes ?? 0);
|
|
318
|
+
this.#presence.set(scopeKey, entry);
|
|
319
|
+
this.#hub.presence.add(this.partition, scopeKey, this);
|
|
320
|
+
this.#reportPresence();
|
|
298
321
|
let rate = this.#presenceRate.get(scopeKey);
|
|
299
322
|
if (rate === undefined) {
|
|
300
323
|
rate = { lastFanoutMs: this.#clock() };
|
|
@@ -303,34 +326,61 @@ export class RealtimeSession {
|
|
|
303
326
|
else {
|
|
304
327
|
rate.lastFanoutMs = this.#clock();
|
|
305
328
|
}
|
|
306
|
-
this.#hub.fanoutPresence(this, scopeKey, kind, doc);
|
|
329
|
+
this.#hub.fanoutPresence(this, scopeKey, kind, entry.doc);
|
|
330
|
+
}
|
|
331
|
+
/** §8.6.1: hand the host this connection's documents after a change. */
|
|
332
|
+
#reportPresence() {
|
|
333
|
+
if (this.#closed || this.#onPresence === undefined)
|
|
334
|
+
return;
|
|
335
|
+
this.#onPresence(new Map([...this.#presence].map(([key, entry]) => [key, entry.json])));
|
|
336
|
+
}
|
|
337
|
+
/** §8.6.1: hold the documents a host kept for this connection across an
|
|
338
|
+
* eviction, without fanout: peers already saw them. */
|
|
339
|
+
seedPresence(presence) {
|
|
340
|
+
for (const [scopeKey, json] of presence) {
|
|
341
|
+
const bytes = new TextEncoder().encode(scopeKey + json).byteLength;
|
|
342
|
+
this.#presence.set(scopeKey, {
|
|
343
|
+
doc: JSON.parse(json),
|
|
344
|
+
json,
|
|
345
|
+
bytes,
|
|
346
|
+
});
|
|
347
|
+
this.#presenceBytes += bytes;
|
|
348
|
+
this.#hub.presence.add(this.partition, scopeKey, this);
|
|
349
|
+
}
|
|
307
350
|
}
|
|
308
351
|
/** §8.6.1/§8.6.3: clear this session's document for a key and fan a
|
|
309
352
|
* leave to the remaining peers. */
|
|
310
353
|
#leavePresence(scopeKey) {
|
|
311
|
-
|
|
354
|
+
if (this.forgetPresence(scopeKey))
|
|
355
|
+
this.#hub.fanoutPresence(this, scopeKey, 'leave', null);
|
|
356
|
+
}
|
|
357
|
+
/** Clear this session's document for a key without fanout, including a
|
|
358
|
+
* coalesced publish still pending. Returns whether a document existed. */
|
|
359
|
+
forgetPresence(scopeKey) {
|
|
312
360
|
const rate = this.#presenceRate.get(scopeKey);
|
|
313
361
|
if (rate?.timer !== undefined)
|
|
314
362
|
clearTimeout(rate.timer);
|
|
315
363
|
this.#presenceRate.delete(scopeKey);
|
|
316
|
-
|
|
317
|
-
|
|
364
|
+
const held = this.#presence.get(scopeKey);
|
|
365
|
+
if (held === undefined)
|
|
366
|
+
return false;
|
|
367
|
+
this.#presence.delete(scopeKey);
|
|
368
|
+
this.#presenceBytes -= held.bytes;
|
|
369
|
+
this.#hub.presence.delete(this.partition, scopeKey, this);
|
|
370
|
+
this.#reportPresence();
|
|
371
|
+
return true;
|
|
318
372
|
}
|
|
319
373
|
/** Deliver a fanout event to this session (called by the hub for peers,
|
|
320
374
|
* §8.6.3 receive authorization checked by the caller). */
|
|
321
375
|
receivePresence(scopeKey, kind, actorId, clientId, doc) {
|
|
322
376
|
this.#sendSafe(encodePresenceFanout(scopeKey, kind, actorId, clientId, doc, this.#clock()));
|
|
323
377
|
}
|
|
324
|
-
/** The scope keys this connection currently holds (§8.6.3). */
|
|
325
|
-
#currentScopeKeys() {
|
|
326
|
-
return this.#hub.scopeKeysOf(this.registrations);
|
|
327
|
-
}
|
|
328
378
|
/** §8.6.3: drop presence on keys the connection no longer holds and
|
|
329
379
|
* deliver the snapshot on keys it newly holds. Called after a
|
|
330
380
|
* registration change (§8.7 round end, §8.1 reconnect handled by
|
|
331
381
|
* connect's snapshot). */
|
|
332
382
|
reconcilePresence(previousKeys) {
|
|
333
|
-
const currentKeys = this
|
|
383
|
+
const currentKeys = this.scopeKeys;
|
|
334
384
|
// Keys lost: leave my own published docs; presence delivery stops
|
|
335
385
|
// naturally (peers no longer fan to me — I am not registered).
|
|
336
386
|
for (const key of previousKeys) {
|
|
@@ -346,8 +396,11 @@ export class RealtimeSession {
|
|
|
346
396
|
/** §8.6.4 snapshot: a burst of `join`s for peers already present on a
|
|
347
397
|
* key, delivered to this session only. */
|
|
348
398
|
#deliverSnapshot(scopeKey) {
|
|
349
|
-
for (const peer of this.#hub.presence.
|
|
350
|
-
|
|
399
|
+
for (const peer of this.#hub.presence.peers(this.partition, scopeKey)) {
|
|
400
|
+
const held = peer.#presence.get(scopeKey);
|
|
401
|
+
if (peer === this || held === undefined)
|
|
402
|
+
continue;
|
|
403
|
+
this.receivePresence(scopeKey, 'join', peer.actorId, peer.clientId, held.doc);
|
|
351
404
|
}
|
|
352
405
|
}
|
|
353
406
|
/** §8.6.1: on disconnect, leave every key this session was present on. */
|
|
@@ -356,9 +409,8 @@ export class RealtimeSession {
|
|
|
356
409
|
this.#deferredDeltas = [];
|
|
357
410
|
this.#deferredBytes = 0;
|
|
358
411
|
this.#deferredRegistrations = undefined;
|
|
359
|
-
for (const key of this.#
|
|
412
|
+
for (const key of [...this.#presence.keys()])
|
|
360
413
|
this.#leavePresence(key);
|
|
361
|
-
}
|
|
362
414
|
for (const rate of this.#presenceRate.values()) {
|
|
363
415
|
if (rate.timer !== undefined)
|
|
364
416
|
clearTimeout(rate.timer);
|
|
@@ -368,7 +420,7 @@ export class RealtimeSession {
|
|
|
368
420
|
/** Snapshot delivery on connect (§8.6.4) — the newly-registered
|
|
369
421
|
* connection sees who is already present on each of its keys. */
|
|
370
422
|
deliverInitialPresence() {
|
|
371
|
-
for (const key of this
|
|
423
|
+
for (const key of this.scopeKeys)
|
|
372
424
|
this.#deliverSnapshot(key);
|
|
373
425
|
}
|
|
374
426
|
/**
|
|
@@ -394,6 +446,17 @@ export class RealtimeSession {
|
|
|
394
446
|
return;
|
|
395
447
|
}
|
|
396
448
|
const chunk = bytes.subarray(1);
|
|
449
|
+
// §1.7: the assembled request counts against `maxRequestBytes`; past it
|
|
450
|
+
// the stream has no acceptable end, so the connection drops (§8.7).
|
|
451
|
+
this.#requestBytes =
|
|
452
|
+
this.#requestScanner === undefined
|
|
453
|
+
? chunk.length
|
|
454
|
+
: this.#requestBytes + chunk.length;
|
|
455
|
+
if (this.#requestBytes > this.#hub.limits.maxRequestBytes) {
|
|
456
|
+
this.#requestScanner = undefined;
|
|
457
|
+
this.#violation('request exceeds maxRequestBytes (§1.7)');
|
|
458
|
+
return;
|
|
459
|
+
}
|
|
397
460
|
this.#requestScanner ??= new MessageStreamScanner();
|
|
398
461
|
let done;
|
|
399
462
|
try {
|
|
@@ -559,7 +622,7 @@ export class RealtimeSession {
|
|
|
559
622
|
const generation = ++this.#registrationGeneration;
|
|
560
623
|
let registrations;
|
|
561
624
|
try {
|
|
562
|
-
registrations = await this.#hub.loadRegistrations(this.partition, this.actorId, this.clientId);
|
|
625
|
+
registrations = await this.#hub.loadRegistrations(this.partition, this.actorId, this.clientId, await this.#storage.getClientRecord(this.partition, this.clientId));
|
|
563
626
|
}
|
|
564
627
|
catch {
|
|
565
628
|
if (!failClosed) {
|
|
@@ -577,8 +640,9 @@ export class RealtimeSession {
|
|
|
577
640
|
// §8.6.3: a registration change re-derives the presence grant — capture
|
|
578
641
|
// the keys before, reconcile after (leaves on lost keys, snapshots on
|
|
579
642
|
// gained keys).
|
|
580
|
-
const previousKeys = this
|
|
643
|
+
const previousKeys = this.scopeKeys;
|
|
581
644
|
this.registrations = registrations;
|
|
645
|
+
this.scopeKeys = this.#hub.scopeKeysOf(registrations);
|
|
582
646
|
this.reconcilePresence(previousKeys);
|
|
583
647
|
}
|
|
584
648
|
/** §8.7 protocol violation: drop the connection, fail loud. The
|
|
@@ -615,10 +679,7 @@ export class RealtimeSession {
|
|
|
615
679
|
}
|
|
616
680
|
}
|
|
617
681
|
sendHeartbeat() {
|
|
618
|
-
this.#sendSafe(
|
|
619
|
-
event: 'heartbeat',
|
|
620
|
-
data: { timestamp: this.#clock() },
|
|
621
|
-
}));
|
|
682
|
+
this.#sendSafe(encodeHeartbeat(this.#clock()));
|
|
622
683
|
}
|
|
623
684
|
sendWake(reason) {
|
|
624
685
|
this.#deferredDeltas = [];
|
|
@@ -708,9 +769,8 @@ export class RealtimeSession {
|
|
|
708
769
|
const frames = [
|
|
709
770
|
{
|
|
710
771
|
type: 'RESP_HEADER',
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
: {}),
|
|
772
|
+
logEpoch: this.logEpoch,
|
|
773
|
+
resetRequired: false,
|
|
714
774
|
},
|
|
715
775
|
];
|
|
716
776
|
for (const section of sections) {
|
|
@@ -762,6 +822,20 @@ export class RealtimeSession {
|
|
|
762
822
|
this.#sendDelta(tagged, commit.commitSeq, changeCount);
|
|
763
823
|
}
|
|
764
824
|
#sendDelta(tagged, commitSeq, changes) {
|
|
825
|
+
// A wake earlier in a deferred flush suppresses the rest of it.
|
|
826
|
+
if (this.wakePending)
|
|
827
|
+
return;
|
|
828
|
+
// §8.2 flow control: a client this far behind on acks catches up by pull;
|
|
829
|
+
// its acknowledgement after that round lifts the suppression. One delta
|
|
830
|
+
// always fits, so `maxDeltaBytes` alone bounds a single message.
|
|
831
|
+
if (this.#unacked.length >= this.#hub.maxUnackedDeltas ||
|
|
832
|
+
(this.#unacked.length > 0 &&
|
|
833
|
+
this.#unackedBytes + tagged.length > this.#hub.maxUnackedDeltaBytes)) {
|
|
834
|
+
this.sendWake('catchup-required');
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
837
|
+
this.#unacked.push({ commitSeq, bytes: tagged.length });
|
|
838
|
+
this.#unackedBytes += tagged.length;
|
|
765
839
|
this.#sendSafe(tagged);
|
|
766
840
|
this.cursor = Math.max(this.cursor, commitSeq);
|
|
767
841
|
const events = this.#events;
|
|
@@ -786,6 +860,8 @@ export class RealtimeSession {
|
|
|
786
860
|
export class RealtimeHub {
|
|
787
861
|
#config;
|
|
788
862
|
#schema;
|
|
863
|
+
/** The validated configured limits (§1.7); a bad value throws here. */
|
|
864
|
+
limits;
|
|
789
865
|
#sessions = new Set();
|
|
790
866
|
/** §8.6 presence registry — ephemeral, in-memory, never persisted. */
|
|
791
867
|
presence = new PresenceRegistry();
|
|
@@ -793,6 +869,7 @@ export class RealtimeHub {
|
|
|
793
869
|
this.#config = config;
|
|
794
870
|
this.#schema = compileSchema(config.schema);
|
|
795
871
|
schemaWindowOf(config.schema, config.schemaWindow);
|
|
872
|
+
this.limits = limitsOf(config);
|
|
796
873
|
}
|
|
797
874
|
projectPayload(registration, payload) {
|
|
798
875
|
return projectRowPayload(this.#schema.tables.get(registration.table), registration.codec, payload);
|
|
@@ -827,6 +904,14 @@ export class RealtimeHub {
|
|
|
827
904
|
get maxPresenceBytes() {
|
|
828
905
|
return this.#config.maxPresenceBytes ?? DEFAULT_MAX_PRESENCE_BYTES;
|
|
829
906
|
}
|
|
907
|
+
/** §8.2 flow control: unacknowledged deltas per session. */
|
|
908
|
+
get maxUnackedDeltas() {
|
|
909
|
+
return this.#config.maxUnackedDeltas ?? DEFAULT_MAX_UNACKED_DELTAS;
|
|
910
|
+
}
|
|
911
|
+
/** §8.2 flow control: unacknowledged delta bytes per session. */
|
|
912
|
+
get maxUnackedDeltaBytes() {
|
|
913
|
+
return this.#config.maxUnackedDeltaBytes ?? DEFAULT_MAX_UNACKED_DELTA_BYTES;
|
|
914
|
+
}
|
|
830
915
|
/** §8.6.4 presence rate cap (ms); 0 = off (reference default). */
|
|
831
916
|
get presenceMinIntervalMs() {
|
|
832
917
|
return this.#config.presenceMinIntervalMs ?? 0;
|
|
@@ -847,7 +932,7 @@ export class RealtimeHub {
|
|
|
847
932
|
if (session.partition !== origin.partition)
|
|
848
933
|
continue;
|
|
849
934
|
// Receive authorization (§8.6.3): the receiver must itself hold the key.
|
|
850
|
-
if (!
|
|
935
|
+
if (!session.scopeKeys.has(scopeKey))
|
|
851
936
|
continue;
|
|
852
937
|
session.receivePresence(scopeKey, kind, origin.actorId, origin.clientId, doc);
|
|
853
938
|
}
|
|
@@ -856,8 +941,7 @@ export class RealtimeHub {
|
|
|
856
941
|
* Resolve the client record's subscription list into per-connection
|
|
857
942
|
* registrations (§8.1 at upgrade; §8.7 at socket-round end).
|
|
858
943
|
*/
|
|
859
|
-
async loadRegistrations(partition, actorId, clientId) {
|
|
860
|
-
const record = await this.#config.storage.getClientRecord(partition, clientId);
|
|
944
|
+
async loadRegistrations(partition, actorId, clientId, record) {
|
|
861
945
|
let resolved;
|
|
862
946
|
try {
|
|
863
947
|
const allowed = await this.#config.resolveScopes({
|
|
@@ -899,7 +983,7 @@ export class RealtimeHub {
|
|
|
899
983
|
const keysValid = Object.keys(subscription.scopes).every((key) => table.declaredVariables.has(key));
|
|
900
984
|
if (!keysValid)
|
|
901
985
|
continue;
|
|
902
|
-
const outcome = computeEffective(subscription.scopes, resolved);
|
|
986
|
+
const outcome = computeEffective(table.declaredVariables, subscription.scopes, resolved);
|
|
903
987
|
if (outcome.status !== 'active')
|
|
904
988
|
continue;
|
|
905
989
|
registrations.push({
|
|
@@ -949,14 +1033,72 @@ export class RealtimeHub {
|
|
|
949
1033
|
* Register a connected socket (§8.1): load the client's last pull's
|
|
950
1034
|
* subscription list, resolve + intersect scopes, send `hello`.
|
|
951
1035
|
*/
|
|
952
|
-
|
|
1036
|
+
connect(options) {
|
|
1037
|
+
return this.#open(options, undefined);
|
|
1038
|
+
}
|
|
1039
|
+
/**
|
|
1040
|
+
* Rebuild the sessions of sockets that outlived their host's memory, such as
|
|
1041
|
+
* a hibernated Durable Object (§8.1). Each client saw `hello` and the
|
|
1042
|
+
* presence snapshot at its real upgrade, so neither is sent again. The
|
|
1043
|
+
* partition is touched and its latest sequence read once per partition, and
|
|
1044
|
+
* each client record once. The caller holds back commits until this
|
|
1045
|
+
* resolves, so the shared latest sequence stays current.
|
|
1046
|
+
*
|
|
1047
|
+
* `presence` is the connection's last `onPresence` report; the session
|
|
1048
|
+
* holds it again without fanout (§8.6.1). Once every session is back, a
|
|
1049
|
+
* key the reloaded registrations no longer cover leaves, and so does every
|
|
1050
|
+
* key of a socket whose session could not be rebuilt.
|
|
1051
|
+
*/
|
|
1052
|
+
async restore(sockets) {
|
|
1053
|
+
const partitions = new Map();
|
|
1054
|
+
const results = [];
|
|
1055
|
+
for (const options of sockets) {
|
|
1056
|
+
try {
|
|
1057
|
+
let shared = partitions.get(options.partition);
|
|
1058
|
+
if (shared === undefined) {
|
|
1059
|
+
const { logEpoch } = await touchAuthenticatedPartition({
|
|
1060
|
+
...this.#config,
|
|
1061
|
+
partition: options.partition,
|
|
1062
|
+
actorId: options.actorId,
|
|
1063
|
+
});
|
|
1064
|
+
shared = {
|
|
1065
|
+
logEpoch,
|
|
1066
|
+
latestSeq: await this.#config.storage.getMaxCommitSeq(options.partition),
|
|
1067
|
+
};
|
|
1068
|
+
partitions.set(options.partition, shared);
|
|
1069
|
+
}
|
|
1070
|
+
const value = await this.#open(options, shared);
|
|
1071
|
+
if (options.presence !== undefined)
|
|
1072
|
+
value.seedPresence(options.presence);
|
|
1073
|
+
results.push({ status: 'fulfilled', value });
|
|
1074
|
+
}
|
|
1075
|
+
catch (reason) {
|
|
1076
|
+
results.push({ status: 'rejected', reason });
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
for (const [i, result] of results.entries()) {
|
|
1080
|
+
const held = [...(sockets[i].presence?.keys() ?? [])];
|
|
1081
|
+
if (result.status === 'fulfilled') {
|
|
1082
|
+
const session = result.value;
|
|
1083
|
+
session.reconcilePresence(new Set([...session.scopeKeys, ...held]));
|
|
1084
|
+
}
|
|
1085
|
+
else {
|
|
1086
|
+
for (const key of held) {
|
|
1087
|
+
this.fanoutPresence(sockets[i], key, 'leave', null);
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
return results;
|
|
1092
|
+
}
|
|
1093
|
+
async #open(options, restored) {
|
|
953
1094
|
const { storage } = this.#config;
|
|
954
1095
|
const clock = this.#config.clock ?? Date.now;
|
|
955
|
-
const
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
1096
|
+
const logEpoch = restored?.logEpoch ??
|
|
1097
|
+
(await touchAuthenticatedPartition({
|
|
1098
|
+
...this.#config,
|
|
1099
|
+
partition: options.partition,
|
|
1100
|
+
actorId: options.actorId,
|
|
1101
|
+
})).logEpoch;
|
|
960
1102
|
if (options.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
|
|
961
1103
|
throw syncError('sync.invalid_client_id', 'clientId uses a reserved server-command namespace (§1.5)');
|
|
962
1104
|
}
|
|
@@ -964,34 +1106,36 @@ export class RealtimeHub {
|
|
|
964
1106
|
if (record !== undefined && record.actorId !== options.actorId) {
|
|
965
1107
|
throw syncError('sync.invalid_client_id', 'clientId is bound to a different actor in this partition (§1.5)');
|
|
966
1108
|
}
|
|
967
|
-
const registrations = await this.loadRegistrations(options.partition, options.actorId, options.clientId);
|
|
968
|
-
const latestSeq = await storage.getMaxCommitSeq(options.partition);
|
|
1109
|
+
const registrations = await this.loadRegistrations(options.partition, options.actorId, options.clientId, record);
|
|
1110
|
+
const latestSeq = restored?.latestSeq ?? (await storage.getMaxCommitSeq(options.partition));
|
|
969
1111
|
const cursor = record?.cursor ?? -1;
|
|
970
|
-
const session = new RealtimeSession(this, options, registrations, cursor, latestSeq, clock, this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES, storage,
|
|
1112
|
+
const session = new RealtimeSession(this, options, registrations, cursor, latestSeq, clock, this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES, storage, logEpoch, record !== undefined && isSupportedProtocolWireVersion(record.wireVersion)
|
|
971
1113
|
? record.wireVersion
|
|
972
1114
|
: PROTOCOL_WIRE_VERSION, this.#config.events);
|
|
973
1115
|
this.#sessions.add(session);
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
helloResult
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
1116
|
+
if (restored === undefined) {
|
|
1117
|
+
const helloResult = options.send(JSON.stringify({
|
|
1118
|
+
event: 'hello',
|
|
1119
|
+
data: {
|
|
1120
|
+
protocolVersion: 1,
|
|
1121
|
+
sessionId: session.sessionId,
|
|
1122
|
+
actorId: options.actorId,
|
|
1123
|
+
clientId: options.clientId,
|
|
1124
|
+
cursor,
|
|
1125
|
+
latestCursor: latestSeq,
|
|
1126
|
+
requiresSync: record === undefined ||
|
|
1127
|
+
cursor < latestSeq ||
|
|
1128
|
+
(this.#config.schemaWindow !== undefined &&
|
|
1129
|
+
record.subscriptions.some((subscription) => !schemaWindowOf(this.#config.schema, this.#config.schemaWindow).some((schema) => schema.version === subscription.schemaVersion))),
|
|
1130
|
+
timestamp: clock(),
|
|
1131
|
+
},
|
|
1132
|
+
}));
|
|
1133
|
+
if (helloResult instanceof Promise)
|
|
1134
|
+
helloResult.catch(() => { });
|
|
1135
|
+
// §8.6.4: the newly-registered connection sees who is already present
|
|
1136
|
+
// on each key it holds (a burst of joins, after hello).
|
|
1137
|
+
session.deliverInitialPresence();
|
|
1138
|
+
}
|
|
995
1139
|
const events = this.#config.events;
|
|
996
1140
|
if (events !== undefined) {
|
|
997
1141
|
emitEvent(events, {
|
|
@@ -1087,6 +1231,22 @@ export class RealtimeHub {
|
|
|
1087
1231
|
for (const session of this.#sessions)
|
|
1088
1232
|
session.sendWake(reason);
|
|
1089
1233
|
}
|
|
1234
|
+
/**
|
|
1235
|
+
* §8.5: send `heartbeat` to every session. The host calls this on its
|
|
1236
|
+
* heartbeat interval (suggested 30 s); a client that hears nothing for twice
|
|
1237
|
+
* the interval treats the socket as lost.
|
|
1238
|
+
*/
|
|
1239
|
+
heartbeat() {
|
|
1240
|
+
for (const session of this.#sessions)
|
|
1241
|
+
session.sendHeartbeat();
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
/** The §8.5 `heartbeat` control message. */
|
|
1245
|
+
export function encodeHeartbeat(timestampMs) {
|
|
1246
|
+
return JSON.stringify({
|
|
1247
|
+
event: 'heartbeat',
|
|
1248
|
+
data: { timestamp: timestampMs },
|
|
1249
|
+
});
|
|
1090
1250
|
}
|
|
1091
1251
|
export function createRealtimeHub(config) {
|
|
1092
1252
|
return new RealtimeHub(config);
|
|
@@ -1101,7 +1261,8 @@ function errorResponseBytes(error, wireVersion, logEpoch) {
|
|
|
1101
1261
|
frames: [
|
|
1102
1262
|
{
|
|
1103
1263
|
type: 'RESP_HEADER',
|
|
1104
|
-
|
|
1264
|
+
logEpoch,
|
|
1265
|
+
resetRequired: false,
|
|
1105
1266
|
},
|
|
1106
1267
|
{
|
|
1107
1268
|
type: 'ERROR',
|