@syncular/server 0.57.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +31 -22
  2. package/dist/admin.js +8 -4
  3. package/dist/blob-handlers.d.ts +25 -31
  4. package/dist/blob-handlers.js +63 -41
  5. package/dist/blob-store.d.ts +42 -24
  6. package/dist/blob-store.js +0 -0
  7. package/dist/bucket-blob-store.d.ts +12 -8
  8. package/dist/bucket-blob-store.js +54 -45
  9. package/dist/bucket-segment-store.js +18 -16
  10. package/dist/content-encoding.d.ts +5 -2
  11. package/dist/content-encoding.js +42 -13
  12. package/dist/context.d.ts +16 -0
  13. package/dist/context.js +21 -1
  14. package/dist/d1-storage.d.ts +14 -98
  15. package/dist/d1-storage.js +223 -1111
  16. package/dist/errors.js +8 -0
  17. package/dist/events.d.ts +1 -1
  18. package/dist/handler.js +135 -103
  19. package/dist/local-authority.d.ts +2 -0
  20. package/dist/local-authority.js +17 -3
  21. package/dist/object-bucket.d.ts +2 -0
  22. package/dist/operations-realtime.js +69 -50
  23. package/dist/pg-executor-pglite.d.ts +5 -6
  24. package/dist/pg-executor-pglite.js +22 -14
  25. package/dist/pg-executor.d.ts +4 -1
  26. package/dist/postgres-storage.d.ts +1 -1
  27. package/dist/postgres-storage.js +182 -105
  28. package/dist/pull.d.ts +28 -20
  29. package/dist/pull.js +85 -87
  30. package/dist/push.js +272 -147
  31. package/dist/realtime.d.ts +68 -28
  32. package/dist/realtime.js +297 -136
  33. package/dist/relational-rows.d.ts +27 -15
  34. package/dist/relational-rows.js +38 -22
  35. package/dist/s3-blob-store.d.ts +11 -12
  36. package/dist/s3-blob-store.js +20 -13
  37. package/dist/s3-object-bucket.d.ts +8 -1
  38. package/dist/s3-object-bucket.js +16 -3
  39. package/dist/schema.d.ts +14 -0
  40. package/dist/schema.js +23 -0
  41. package/dist/scopes.d.ts +6 -2
  42. package/dist/scopes.js +22 -4
  43. package/dist/seed.js +1 -2
  44. package/dist/segment-download.js +18 -4
  45. package/dist/signed-url.d.ts +2 -2
  46. package/dist/signed-url.js +9 -2
  47. package/dist/sigv4.d.ts +8 -2
  48. package/dist/sigv4.js +8 -4
  49. package/dist/sqlite-blob-store.d.ts +1 -1
  50. package/dist/sqlite-blob-store.js +23 -22
  51. package/dist/sqlite-dialect.d.ts +150 -47
  52. package/dist/sqlite-dialect.js +987 -88
  53. package/dist/sqlite-lease-store.js +13 -18
  54. package/dist/sqlite-storage.d.ts +9 -62
  55. package/dist/sqlite-storage.js +293 -1117
  56. package/dist/storage.d.ts +32 -12
  57. package/package.json +2 -2
  58. package/src/admin.ts +16 -2
  59. package/src/blob-handlers.ts +91 -72
  60. package/src/blob-store.ts +0 -0
  61. package/src/bucket-blob-store.ts +75 -48
  62. package/src/bucket-segment-store.ts +20 -15
  63. package/src/content-encoding.ts +48 -16
  64. package/src/context.ts +32 -1
  65. package/src/d1-storage.ts +326 -1692
  66. package/src/errors.ts +8 -0
  67. package/src/events.ts +1 -1
  68. package/src/handler.ts +178 -127
  69. package/src/local-authority.ts +17 -3
  70. package/src/object-bucket.ts +2 -0
  71. package/src/operations-realtime.ts +85 -61
  72. package/src/pg-executor-pglite.ts +27 -23
  73. package/src/pg-executor.ts +5 -1
  74. package/src/postgres-storage.ts +231 -138
  75. package/src/pull.ts +133 -147
  76. package/src/push.ts +356 -205
  77. package/src/realtime.ts +380 -179
  78. package/src/relational-rows.ts +54 -27
  79. package/src/s3-blob-store.ts +23 -13
  80. package/src/s3-object-bucket.ts +20 -2
  81. package/src/schema.ts +31 -0
  82. package/src/scopes.ts +23 -3
  83. package/src/seed.ts +1 -2
  84. package/src/segment-download.ts +25 -4
  85. package/src/signed-url.ts +10 -4
  86. package/src/sigv4.ts +17 -4
  87. package/src/sqlite-blob-store.ts +27 -26
  88. package/src/sqlite-dialect.ts +1607 -114
  89. package/src/sqlite-lease-store.ts +12 -23
  90. package/src/sqlite-storage.ts +360 -1811
  91. package/src/storage.ts +33 -12
package/dist/realtime.js CHANGED
@@ -16,8 +16,8 @@
16
16
  * one handler, two framings); the only JSON data-plane server event is
17
17
  * the `sync` wake-up (§8.3).
18
18
  */
19
- import { DecodeError, decodeMessage, encodeMessage, encodePresenceError, encodePresenceFanout, isSupportedProtocolWireVersion, MessageStreamScanner, PROTOCOL_WIRE_VERSION, parseRealtimePresencePublish, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
20
- import { REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
19
+ import { DecodeError, decodeMessage, encodeMessage, encodePresenceError, encodePresenceFanout, isSupportedProtocolWireVersion, MessageStreamScanner, PROTOCOL_WIRE_VERSION, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
20
+ import { limitsOf, REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
21
21
  import { adapterSyncError, reportError, SyncError, syncError, } from './errors.js';
22
22
  import { emitEvent } from './events.js';
23
23
  import { composeEvents } from './events-ring.js';
@@ -26,18 +26,18 @@ import { compileSchema, } from './schema.js';
26
26
  import { schemaWindowOf, projectRowPayload } from './schema-window.js';
27
27
  import { computeEffective, matchesEffective, } from './scopes.js';
28
28
  const DEFAULT_MAX_DELTA_BYTES = 1024 * 1024;
29
- const DEFAULT_MAX_PRESENCE_BYTES = 16 * 1024;
29
+ const DEFAULT_MAX_PRESENCE_BYTES = 8192;
30
+ const DEFAULT_MAX_UNACKED_DELTAS = 256;
31
+ const DEFAULT_MAX_UNACKED_DELTA_BYTES = 4 * 1024 * 1024;
30
32
  /**
31
- * §8.6 presence registry — pure in-memory ephemeral state, keyed per
32
- * `(partition, scopeKey)` → the set of present sessions and their
33
- * documents. It never touches `ServerStorage`; a server restart loses all
34
- * presence (§8.6.1). Fanout is scoped to registered peers only — the
35
- * privacy floor (§8.6.3).
33
+ * §8.6 presence index: per `(partition, scopeKey)`, the sessions holding a
34
+ * document there. The documents live with their session (§8.6.1). Pure
35
+ * in-memory state that never touches `ServerStorage`.
36
36
  */
37
37
  class PresenceRegistry {
38
- /** partition → scopeKey → session → document. */
38
+ /** partition → scopeKey → sessions with a document on the key. */
39
39
  #byKey = new Map();
40
- #keyMap(partition, scopeKey) {
40
+ add(partition, scopeKey, session) {
41
41
  let byScope = this.#byKey.get(partition);
42
42
  if (byScope === undefined) {
43
43
  byScope = new Map();
@@ -45,60 +45,22 @@ class PresenceRegistry {
45
45
  }
46
46
  let sessions = byScope.get(scopeKey);
47
47
  if (sessions === undefined) {
48
- sessions = new Map();
48
+ sessions = new Set();
49
49
  byScope.set(scopeKey, sessions);
50
50
  }
51
- return sessions;
51
+ sessions.add(session);
52
52
  }
53
- /** Current documents on a key, excluding one session (the snapshot,
54
- * §8.6.4). */
55
- snapshot(partition, scopeKey, exclude) {
53
+ delete(partition, scopeKey, session) {
56
54
  const sessions = this.#byKey.get(partition)?.get(scopeKey);
57
55
  if (sessions === undefined)
58
- return [];
59
- const out = [];
60
- for (const [session, doc] of sessions) {
61
- if (session !== exclude)
62
- out.push({ session, doc });
63
- }
64
- return out;
65
- }
66
- /** Store/replace a session's document for a key. Returns whether this is
67
- * the session's FIRST document on the key (join) or a replacement
68
- * (update). */
69
- set(partition, scopeKey, session, doc) {
70
- const sessions = this.#keyMap(partition, scopeKey);
71
- const kind = sessions.has(session) ? 'update' : 'join';
72
- sessions.set(session, doc);
73
- return kind;
74
- }
75
- /** Remove a session's document for a key. Returns true if one existed. */
76
- clear(partition, scopeKey, session) {
77
- const sessions = this.#byKey.get(partition)?.get(scopeKey);
78
- if (sessions === undefined)
79
- return false;
80
- const existed = sessions.delete(session);
56
+ return;
57
+ sessions.delete(session);
81
58
  if (sessions.size === 0)
82
59
  this.#byKey.get(partition)?.delete(scopeKey);
83
- return existed;
84
60
  }
85
- /** Every key a session currently holds a document on (for leave-on-drop
86
- * and registration-change re-derivation). */
87
- keysOf(partition, session) {
88
- const byScope = this.#byKey.get(partition);
89
- if (byScope === undefined)
90
- return [];
91
- const keys = [];
92
- for (const [scopeKey, sessions] of byScope) {
93
- if (sessions.has(session))
94
- keys.push(scopeKey);
95
- }
96
- return keys;
97
- }
98
- /** Sessions currently registered as peers on a key (for fanout). */
61
+ /** Sessions holding a document on a key. */
99
62
  peers(partition, scopeKey) {
100
- const sessions = this.#byKey.get(partition)?.get(scopeKey);
101
- return sessions === undefined ? [] : [...sessions.keys()];
63
+ return [...(this.#byKey.get(partition)?.get(scopeKey) ?? [])];
102
64
  }
103
65
  }
104
66
  /** The scope keys a set of registrations covers (§3.1: `prefix:value`). A
@@ -135,6 +97,11 @@ export class RealtimeSession {
135
97
  lastKnownSeq;
136
98
  /** Replaced at socket-round completion (§8.7); initial set from §8.1. */
137
99
  registrations;
100
+ /** The scope keys `registrations` cover: the presence grant (§8.6.3). */
101
+ scopeKeys;
102
+ /** Sent deltas the client has not acknowledged, in commit order (§8.2). */
103
+ #unacked = [];
104
+ #unackedBytes = 0;
138
105
  /** Epoch-ms (hub clock) at registration, for `realtime.closed`. */
139
106
  openedAtMs;
140
107
  #send;
@@ -148,6 +115,8 @@ export class RealtimeSession {
148
115
  * token identifies the owning round so a finished round's cleanup
149
116
  * never clobbers a follow-up round's in-flight marker. */
150
117
  #requestScanner;
118
+ /** Bytes the scanner holds, against `maxRequestBytes` (§1.7). */
119
+ #requestBytes = 0;
151
120
  #activeRound;
152
121
  /** Deltas wait for every round response and registration refresh to finish. */
153
122
  #roundsRefreshing = 0;
@@ -155,6 +124,11 @@ export class RealtimeSession {
155
124
  #deferredBytes = 0;
156
125
  #deferredRegistrations;
157
126
  #closed = false;
127
+ /** §8.6.1: this connection's presence documents by scope key, and their
128
+ * summed charge against `maxPresenceBytes` (§8.6.2). */
129
+ #presence = new Map();
130
+ #presenceBytes = 0;
131
+ #onPresence;
158
132
  /** §8.6.4 rate cap: per-scope-key last-fanout time + a pending latest
159
133
  * document coalesced while over the cap. */
160
134
  #presenceRate = new Map();
@@ -180,9 +154,11 @@ export class RealtimeSession {
180
154
  this.wakePending = cursor < latestSeq;
181
155
  this.#wakeRequiredThroughSeq = cursor < latestSeq ? latestSeq : 0;
182
156
  this.registrations = registrations;
157
+ this.scopeKeys = hub.scopeKeysOf(registrations);
183
158
  this.openedAtMs = clock();
184
159
  this.#send = options.send;
185
160
  this.#closeSocket = options.closeSocket;
161
+ this.#onPresence = options.onPresence;
186
162
  this.#hub = hub;
187
163
  this.#clock = clock;
188
164
  this.#maxDeltaBytes = maxDeltaBytes;
@@ -204,7 +180,7 @@ export class RealtimeSession {
204
180
  // §8.6.2: client→server presence carries `event: 'presence'` (server
205
181
  // control messages carry `event`; the ack carries `type`, §8.1).
206
182
  if (parsed.event === 'presence') {
207
- this.#handlePresence(text);
183
+ this.#handlePresence(parsed.data);
208
184
  return;
209
185
  }
210
186
  if (parsed.type !== 'ack')
@@ -212,6 +188,9 @@ export class RealtimeSession {
212
188
  const cursor = parsed.cursor;
213
189
  if (typeof cursor !== 'number' || !Number.isSafeInteger(cursor))
214
190
  return;
191
+ while ((this.#unacked[0]?.commitSeq ?? Infinity) <= cursor) {
192
+ this.#unackedBytes -= this.#unacked.shift().bytes;
193
+ }
215
194
  this.cursor = Math.max(this.cursor, cursor);
216
195
  // The client may have caught up through another binding (§8.4). Its ack
217
196
  // proves those commits no longer need socket notifications, so the next
@@ -236,22 +215,28 @@ export class RealtimeSession {
236
215
  throw failure;
237
216
  });
238
217
  }
239
- /** §8.6.2 inbound presence publish/leave from the client. */
240
- #handlePresence(text) {
241
- let publish;
242
- try {
243
- publish = parseRealtimePresencePublish(text);
244
- }
245
- catch {
218
+ /**
219
+ * §8.6.2 inbound presence publish/leave from the client: the `data` of the
220
+ * control message `handleMessage` parsed, a non-empty `scopeKey` and a
221
+ * `doc` that is an object or `null`.
222
+ */
223
+ #handlePresence(data) {
224
+ const publish = typeof data === 'object' && data !== null && !Array.isArray(data)
225
+ ? data
226
+ : undefined;
227
+ const scopeKey = publish?.scopeKey;
228
+ const doc = publish?.doc;
229
+ if (typeof scopeKey !== 'string' ||
230
+ scopeKey.length === 0 ||
231
+ (doc !== null && (typeof doc !== 'object' || Array.isArray(doc)))) {
246
232
  // A malformed known event is a parse error (§8.1) — a broken client;
247
233
  // drop the connection loudly rather than silently ignore.
248
234
  this.#violation('malformed presence control message (§8.6.2)');
249
235
  return;
250
236
  }
251
- const { scopeKey, doc } = publish.data;
252
237
  // §8.6.3 authorization: the publish key must be in the connection's
253
238
  // registered effective scopes.
254
- if (!this.#currentScopeKeys().has(scopeKey)) {
239
+ if (!this.scopeKeys.has(scopeKey)) {
255
240
  this.#sendSafe(encodePresenceError(scopeKey, 'presence.forbidden', this.#clock()));
256
241
  return;
257
242
  }
@@ -259,17 +244,34 @@ export class RealtimeSession {
259
244
  this.#leavePresence(scopeKey);
260
245
  return;
261
246
  }
262
- // §8.6.2 size cap: fail loud to the publisher, fan out nothing.
263
- const serialized = JSON.stringify(doc);
264
- if (serialized.length > this.#hub.maxPresenceBytes) {
265
- this.#sendSafe(encodePresenceError(scopeKey, 'presence.too_large', this.#clock()));
266
- return;
247
+ const entry = this.#presenceEntry(scopeKey, doc);
248
+ if (entry !== undefined)
249
+ this.#publishPresence(scopeKey, entry);
250
+ }
251
+ /**
252
+ * §8.6.2 size cap per connection: the key's and document's UTF-8 bytes,
253
+ * added to the connection's documents on its other keys, stay within
254
+ * `maxPresenceBytes`. Over the cap the publisher alone hears
255
+ * `presence.too_large` and nothing changes.
256
+ */
257
+ #presenceEntry(scopeKey, doc) {
258
+ const json = JSON.stringify(doc);
259
+ const budget = this.#hub.maxPresenceBytes -
260
+ this.#presenceBytes +
261
+ (this.#presence.get(scopeKey)?.bytes ?? 0);
262
+ // A string never has fewer UTF-8 bytes than UTF-16 code units, so the
263
+ // unit count rejects first.
264
+ if (scopeKey.length + json.length <= budget) {
265
+ const bytes = new TextEncoder().encode(scopeKey + json).byteLength;
266
+ if (bytes <= budget)
267
+ return { doc, json, bytes };
267
268
  }
268
- this.#publishPresence(scopeKey, doc);
269
+ this.#sendSafe(encodePresenceError(scopeKey, 'presence.too_large', this.#clock()));
270
+ return undefined;
269
271
  }
270
272
  /** §8.6.4: store + fan out a publish, honoring the rate cap (latest-wins
271
273
  * coalesce). */
272
- #publishPresence(scopeKey, doc) {
274
+ #publishPresence(scopeKey, entry) {
273
275
  const minInterval = this.#hub.presenceMinIntervalMs;
274
276
  if (minInterval > 0) {
275
277
  const now = this.#clock();
@@ -277,24 +279,45 @@ export class RealtimeSession {
277
279
  if (rate !== undefined && now - rate.lastFanoutMs < minInterval) {
278
280
  // Over the cap: keep only the latest document, fan it out at the
279
281
  // window edge (never an error, never a stale/lost latest, §8.6.4).
280
- rate.pending = doc;
282
+ rate.pending = entry.doc;
281
283
  if (rate.timer === undefined) {
282
284
  const delay = minInterval - (now - rate.lastFanoutMs);
283
285
  rate.timer = setTimeout(() => {
284
286
  delete rate.timer;
285
287
  const pending = rate.pending;
286
288
  delete rate.pending;
287
- if (pending !== undefined)
288
- this.#fanoutPublish(scopeKey, pending);
289
+ // The connection's other keys may have grown since: the cap
290
+ // holds when the document is stored.
291
+ const latest = pending === undefined
292
+ ? undefined
293
+ : this.#presenceEntry(scopeKey, pending);
294
+ if (latest !== undefined)
295
+ this.#fanoutPublish(scopeKey, latest);
289
296
  }, delay);
290
297
  }
291
298
  return;
292
299
  }
293
300
  }
294
- this.#fanoutPublish(scopeKey, doc);
301
+ this.#fanoutPublish(scopeKey, entry);
295
302
  }
296
- #fanoutPublish(scopeKey, doc) {
297
- const kind = this.#hub.presence.set(this.partition, scopeKey, this, doc);
303
+ #fanoutPublish(scopeKey, entry) {
304
+ // §8.6.1: the peer identity is (actorId, clientId). Another session of
305
+ // this identity (a stale socket of a reconnected client) loses its
306
+ // document silently, so its later close announces nothing.
307
+ let replaced = false;
308
+ for (const peer of this.#hub.presence.peers(this.partition, scopeKey)) {
309
+ if (peer !== this &&
310
+ peer.actorId === this.actorId &&
311
+ peer.clientId === this.clientId) {
312
+ replaced = peer.forgetPresence(scopeKey) || replaced;
313
+ }
314
+ }
315
+ const held = this.#presence.get(scopeKey);
316
+ const kind = replaced || held !== undefined ? 'update' : 'join';
317
+ this.#presenceBytes += entry.bytes - (held?.bytes ?? 0);
318
+ this.#presence.set(scopeKey, entry);
319
+ this.#hub.presence.add(this.partition, scopeKey, this);
320
+ this.#reportPresence();
298
321
  let rate = this.#presenceRate.get(scopeKey);
299
322
  if (rate === undefined) {
300
323
  rate = { lastFanoutMs: this.#clock() };
@@ -303,34 +326,61 @@ export class RealtimeSession {
303
326
  else {
304
327
  rate.lastFanoutMs = this.#clock();
305
328
  }
306
- this.#hub.fanoutPresence(this, scopeKey, kind, doc);
329
+ this.#hub.fanoutPresence(this, scopeKey, kind, entry.doc);
330
+ }
331
+ /** §8.6.1: hand the host this connection's documents after a change. */
332
+ #reportPresence() {
333
+ if (this.#closed || this.#onPresence === undefined)
334
+ return;
335
+ this.#onPresence(new Map([...this.#presence].map(([key, entry]) => [key, entry.json])));
336
+ }
337
+ /** §8.6.1: hold the documents a host kept for this connection across an
338
+ * eviction, without fanout: peers already saw them. */
339
+ seedPresence(presence) {
340
+ for (const [scopeKey, json] of presence) {
341
+ const bytes = new TextEncoder().encode(scopeKey + json).byteLength;
342
+ this.#presence.set(scopeKey, {
343
+ doc: JSON.parse(json),
344
+ json,
345
+ bytes,
346
+ });
347
+ this.#presenceBytes += bytes;
348
+ this.#hub.presence.add(this.partition, scopeKey, this);
349
+ }
307
350
  }
308
351
  /** §8.6.1/§8.6.3: clear this session's document for a key and fan a
309
352
  * leave to the remaining peers. */
310
353
  #leavePresence(scopeKey) {
311
- const existed = this.#hub.presence.clear(this.partition, scopeKey, this);
354
+ if (this.forgetPresence(scopeKey))
355
+ this.#hub.fanoutPresence(this, scopeKey, 'leave', null);
356
+ }
357
+ /** Clear this session's document for a key without fanout, including a
358
+ * coalesced publish still pending. Returns whether a document existed. */
359
+ forgetPresence(scopeKey) {
312
360
  const rate = this.#presenceRate.get(scopeKey);
313
361
  if (rate?.timer !== undefined)
314
362
  clearTimeout(rate.timer);
315
363
  this.#presenceRate.delete(scopeKey);
316
- if (existed)
317
- this.#hub.fanoutPresence(this, scopeKey, 'leave', null);
364
+ const held = this.#presence.get(scopeKey);
365
+ if (held === undefined)
366
+ return false;
367
+ this.#presence.delete(scopeKey);
368
+ this.#presenceBytes -= held.bytes;
369
+ this.#hub.presence.delete(this.partition, scopeKey, this);
370
+ this.#reportPresence();
371
+ return true;
318
372
  }
319
373
  /** Deliver a fanout event to this session (called by the hub for peers,
320
374
  * §8.6.3 receive authorization checked by the caller). */
321
375
  receivePresence(scopeKey, kind, actorId, clientId, doc) {
322
376
  this.#sendSafe(encodePresenceFanout(scopeKey, kind, actorId, clientId, doc, this.#clock()));
323
377
  }
324
- /** The scope keys this connection currently holds (§8.6.3). */
325
- #currentScopeKeys() {
326
- return this.#hub.scopeKeysOf(this.registrations);
327
- }
328
378
  /** §8.6.3: drop presence on keys the connection no longer holds and
329
379
  * deliver the snapshot on keys it newly holds. Called after a
330
380
  * registration change (§8.7 round end, §8.1 reconnect handled by
331
381
  * connect's snapshot). */
332
382
  reconcilePresence(previousKeys) {
333
- const currentKeys = this.#currentScopeKeys();
383
+ const currentKeys = this.scopeKeys;
334
384
  // Keys lost: leave my own published docs; presence delivery stops
335
385
  // naturally (peers no longer fan to me — I am not registered).
336
386
  for (const key of previousKeys) {
@@ -346,8 +396,11 @@ export class RealtimeSession {
346
396
  /** §8.6.4 snapshot: a burst of `join`s for peers already present on a
347
397
  * key, delivered to this session only. */
348
398
  #deliverSnapshot(scopeKey) {
349
- for (const peer of this.#hub.presence.snapshot(this.partition, scopeKey, this)) {
350
- this.receivePresence(scopeKey, 'join', peer.session.actorId, peer.session.clientId, peer.doc);
399
+ for (const peer of this.#hub.presence.peers(this.partition, scopeKey)) {
400
+ const held = peer.#presence.get(scopeKey);
401
+ if (peer === this || held === undefined)
402
+ continue;
403
+ this.receivePresence(scopeKey, 'join', peer.actorId, peer.clientId, held.doc);
351
404
  }
352
405
  }
353
406
  /** §8.6.1: on disconnect, leave every key this session was present on. */
@@ -356,9 +409,8 @@ export class RealtimeSession {
356
409
  this.#deferredDeltas = [];
357
410
  this.#deferredBytes = 0;
358
411
  this.#deferredRegistrations = undefined;
359
- for (const key of this.#hub.presence.keysOf(this.partition, this)) {
412
+ for (const key of [...this.#presence.keys()])
360
413
  this.#leavePresence(key);
361
- }
362
414
  for (const rate of this.#presenceRate.values()) {
363
415
  if (rate.timer !== undefined)
364
416
  clearTimeout(rate.timer);
@@ -368,7 +420,7 @@ export class RealtimeSession {
368
420
  /** Snapshot delivery on connect (§8.6.4) — the newly-registered
369
421
  * connection sees who is already present on each of its keys. */
370
422
  deliverInitialPresence() {
371
- for (const key of this.#currentScopeKeys())
423
+ for (const key of this.scopeKeys)
372
424
  this.#deliverSnapshot(key);
373
425
  }
374
426
  /**
@@ -394,6 +446,17 @@ export class RealtimeSession {
394
446
  return;
395
447
  }
396
448
  const chunk = bytes.subarray(1);
449
+ // §1.7: the assembled request counts against `maxRequestBytes`; past it
450
+ // the stream has no acceptable end, so the connection drops (§8.7).
451
+ this.#requestBytes =
452
+ this.#requestScanner === undefined
453
+ ? chunk.length
454
+ : this.#requestBytes + chunk.length;
455
+ if (this.#requestBytes > this.#hub.limits.maxRequestBytes) {
456
+ this.#requestScanner = undefined;
457
+ this.#violation('request exceeds maxRequestBytes (§1.7)');
458
+ return;
459
+ }
397
460
  this.#requestScanner ??= new MessageStreamScanner();
398
461
  let done;
399
462
  try {
@@ -559,7 +622,7 @@ export class RealtimeSession {
559
622
  const generation = ++this.#registrationGeneration;
560
623
  let registrations;
561
624
  try {
562
- registrations = await this.#hub.loadRegistrations(this.partition, this.actorId, this.clientId);
625
+ registrations = await this.#hub.loadRegistrations(this.partition, this.actorId, this.clientId, await this.#storage.getClientRecord(this.partition, this.clientId));
563
626
  }
564
627
  catch {
565
628
  if (!failClosed) {
@@ -577,8 +640,9 @@ export class RealtimeSession {
577
640
  // §8.6.3: a registration change re-derives the presence grant — capture
578
641
  // the keys before, reconcile after (leaves on lost keys, snapshots on
579
642
  // gained keys).
580
- const previousKeys = this.#currentScopeKeys();
643
+ const previousKeys = this.scopeKeys;
581
644
  this.registrations = registrations;
645
+ this.scopeKeys = this.#hub.scopeKeysOf(registrations);
582
646
  this.reconcilePresence(previousKeys);
583
647
  }
584
648
  /** §8.7 protocol violation: drop the connection, fail loud. The
@@ -615,10 +679,7 @@ export class RealtimeSession {
615
679
  }
616
680
  }
617
681
  sendHeartbeat() {
618
- this.#sendSafe(JSON.stringify({
619
- event: 'heartbeat',
620
- data: { timestamp: this.#clock() },
621
- }));
682
+ this.#sendSafe(encodeHeartbeat(this.#clock()));
622
683
  }
623
684
  sendWake(reason) {
624
685
  this.#deferredDeltas = [];
@@ -708,9 +769,8 @@ export class RealtimeSession {
708
769
  const frames = [
709
770
  {
710
771
  type: 'RESP_HEADER',
711
- ...(this.wireVersion >= 2
712
- ? { logEpoch: this.logEpoch, resetRequired: false }
713
- : {}),
772
+ logEpoch: this.logEpoch,
773
+ resetRequired: false,
714
774
  },
715
775
  ];
716
776
  for (const section of sections) {
@@ -762,6 +822,20 @@ export class RealtimeSession {
762
822
  this.#sendDelta(tagged, commit.commitSeq, changeCount);
763
823
  }
764
824
  #sendDelta(tagged, commitSeq, changes) {
825
+ // A wake earlier in a deferred flush suppresses the rest of it.
826
+ if (this.wakePending)
827
+ return;
828
+ // §8.2 flow control: a client this far behind on acks catches up by pull;
829
+ // its acknowledgement after that round lifts the suppression. One delta
830
+ // always fits, so `maxDeltaBytes` alone bounds a single message.
831
+ if (this.#unacked.length >= this.#hub.maxUnackedDeltas ||
832
+ (this.#unacked.length > 0 &&
833
+ this.#unackedBytes + tagged.length > this.#hub.maxUnackedDeltaBytes)) {
834
+ this.sendWake('catchup-required');
835
+ return;
836
+ }
837
+ this.#unacked.push({ commitSeq, bytes: tagged.length });
838
+ this.#unackedBytes += tagged.length;
765
839
  this.#sendSafe(tagged);
766
840
  this.cursor = Math.max(this.cursor, commitSeq);
767
841
  const events = this.#events;
@@ -786,6 +860,8 @@ export class RealtimeSession {
786
860
  export class RealtimeHub {
787
861
  #config;
788
862
  #schema;
863
+ /** The validated configured limits (§1.7); a bad value throws here. */
864
+ limits;
789
865
  #sessions = new Set();
790
866
  /** §8.6 presence registry — ephemeral, in-memory, never persisted. */
791
867
  presence = new PresenceRegistry();
@@ -793,6 +869,7 @@ export class RealtimeHub {
793
869
  this.#config = config;
794
870
  this.#schema = compileSchema(config.schema);
795
871
  schemaWindowOf(config.schema, config.schemaWindow);
872
+ this.limits = limitsOf(config);
796
873
  }
797
874
  projectPayload(registration, payload) {
798
875
  return projectRowPayload(this.#schema.tables.get(registration.table), registration.codec, payload);
@@ -827,6 +904,14 @@ export class RealtimeHub {
827
904
  get maxPresenceBytes() {
828
905
  return this.#config.maxPresenceBytes ?? DEFAULT_MAX_PRESENCE_BYTES;
829
906
  }
907
+ /** §8.2 flow control: unacknowledged deltas per session. */
908
+ get maxUnackedDeltas() {
909
+ return this.#config.maxUnackedDeltas ?? DEFAULT_MAX_UNACKED_DELTAS;
910
+ }
911
+ /** §8.2 flow control: unacknowledged delta bytes per session. */
912
+ get maxUnackedDeltaBytes() {
913
+ return this.#config.maxUnackedDeltaBytes ?? DEFAULT_MAX_UNACKED_DELTA_BYTES;
914
+ }
830
915
  /** §8.6.4 presence rate cap (ms); 0 = off (reference default). */
831
916
  get presenceMinIntervalMs() {
832
917
  return this.#config.presenceMinIntervalMs ?? 0;
@@ -847,7 +932,7 @@ export class RealtimeHub {
847
932
  if (session.partition !== origin.partition)
848
933
  continue;
849
934
  // Receive authorization (§8.6.3): the receiver must itself hold the key.
850
- if (!this.scopeKeysOf(session.registrations).has(scopeKey))
935
+ if (!session.scopeKeys.has(scopeKey))
851
936
  continue;
852
937
  session.receivePresence(scopeKey, kind, origin.actorId, origin.clientId, doc);
853
938
  }
@@ -856,8 +941,7 @@ export class RealtimeHub {
856
941
  * Resolve the client record's subscription list into per-connection
857
942
  * registrations (§8.1 at upgrade; §8.7 at socket-round end).
858
943
  */
859
- async loadRegistrations(partition, actorId, clientId) {
860
- const record = await this.#config.storage.getClientRecord(partition, clientId);
944
+ async loadRegistrations(partition, actorId, clientId, record) {
861
945
  let resolved;
862
946
  try {
863
947
  const allowed = await this.#config.resolveScopes({
@@ -899,7 +983,7 @@ export class RealtimeHub {
899
983
  const keysValid = Object.keys(subscription.scopes).every((key) => table.declaredVariables.has(key));
900
984
  if (!keysValid)
901
985
  continue;
902
- const outcome = computeEffective(subscription.scopes, resolved);
986
+ const outcome = computeEffective(table.declaredVariables, subscription.scopes, resolved);
903
987
  if (outcome.status !== 'active')
904
988
  continue;
905
989
  registrations.push({
@@ -949,14 +1033,72 @@ export class RealtimeHub {
949
1033
  * Register a connected socket (§8.1): load the client's last pull's
950
1034
  * subscription list, resolve + intersect scopes, send `hello`.
951
1035
  */
952
- async connect(options) {
1036
+ connect(options) {
1037
+ return this.#open(options, undefined);
1038
+ }
1039
+ /**
1040
+ * Rebuild the sessions of sockets that outlived their host's memory, such as
1041
+ * a hibernated Durable Object (§8.1). Each client saw `hello` and the
1042
+ * presence snapshot at its real upgrade, so neither is sent again. The
1043
+ * partition is touched and its latest sequence read once per partition, and
1044
+ * each client record once. The caller holds back commits until this
1045
+ * resolves, so the shared latest sequence stays current.
1046
+ *
1047
+ * `presence` is the connection's last `onPresence` report; the session
1048
+ * holds it again without fanout (§8.6.1). Once every session is back, a
1049
+ * key the reloaded registrations no longer cover leaves, and so does every
1050
+ * key of a socket whose session could not be rebuilt.
1051
+ */
1052
+ async restore(sockets) {
1053
+ const partitions = new Map();
1054
+ const results = [];
1055
+ for (const options of sockets) {
1056
+ try {
1057
+ let shared = partitions.get(options.partition);
1058
+ if (shared === undefined) {
1059
+ const { logEpoch } = await touchAuthenticatedPartition({
1060
+ ...this.#config,
1061
+ partition: options.partition,
1062
+ actorId: options.actorId,
1063
+ });
1064
+ shared = {
1065
+ logEpoch,
1066
+ latestSeq: await this.#config.storage.getMaxCommitSeq(options.partition),
1067
+ };
1068
+ partitions.set(options.partition, shared);
1069
+ }
1070
+ const value = await this.#open(options, shared);
1071
+ if (options.presence !== undefined)
1072
+ value.seedPresence(options.presence);
1073
+ results.push({ status: 'fulfilled', value });
1074
+ }
1075
+ catch (reason) {
1076
+ results.push({ status: 'rejected', reason });
1077
+ }
1078
+ }
1079
+ for (const [i, result] of results.entries()) {
1080
+ const held = [...(sockets[i].presence?.keys() ?? [])];
1081
+ if (result.status === 'fulfilled') {
1082
+ const session = result.value;
1083
+ session.reconcilePresence(new Set([...session.scopeKeys, ...held]));
1084
+ }
1085
+ else {
1086
+ for (const key of held) {
1087
+ this.fanoutPresence(sockets[i], key, 'leave', null);
1088
+ }
1089
+ }
1090
+ }
1091
+ return results;
1092
+ }
1093
+ async #open(options, restored) {
953
1094
  const { storage } = this.#config;
954
1095
  const clock = this.#config.clock ?? Date.now;
955
- const registry = await touchAuthenticatedPartition({
956
- ...this.#config,
957
- partition: options.partition,
958
- actorId: options.actorId,
959
- });
1096
+ const logEpoch = restored?.logEpoch ??
1097
+ (await touchAuthenticatedPartition({
1098
+ ...this.#config,
1099
+ partition: options.partition,
1100
+ actorId: options.actorId,
1101
+ })).logEpoch;
960
1102
  if (options.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
961
1103
  throw syncError('sync.invalid_client_id', 'clientId uses a reserved server-command namespace (§1.5)');
962
1104
  }
@@ -964,34 +1106,36 @@ export class RealtimeHub {
964
1106
  if (record !== undefined && record.actorId !== options.actorId) {
965
1107
  throw syncError('sync.invalid_client_id', 'clientId is bound to a different actor in this partition (§1.5)');
966
1108
  }
967
- const registrations = await this.loadRegistrations(options.partition, options.actorId, options.clientId);
968
- const latestSeq = await storage.getMaxCommitSeq(options.partition);
1109
+ const registrations = await this.loadRegistrations(options.partition, options.actorId, options.clientId, record);
1110
+ const latestSeq = restored?.latestSeq ?? (await storage.getMaxCommitSeq(options.partition));
969
1111
  const cursor = record?.cursor ?? -1;
970
- const session = new RealtimeSession(this, options, registrations, cursor, latestSeq, clock, this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES, storage, registry.logEpoch, record !== undefined && isSupportedProtocolWireVersion(record.wireVersion)
1112
+ const session = new RealtimeSession(this, options, registrations, cursor, latestSeq, clock, this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES, storage, logEpoch, record !== undefined && isSupportedProtocolWireVersion(record.wireVersion)
971
1113
  ? record.wireVersion
972
1114
  : PROTOCOL_WIRE_VERSION, this.#config.events);
973
1115
  this.#sessions.add(session);
974
- const helloResult = options.send(JSON.stringify({
975
- event: 'hello',
976
- data: {
977
- protocolVersion: 1,
978
- sessionId: session.sessionId,
979
- actorId: options.actorId,
980
- clientId: options.clientId,
981
- cursor,
982
- latestCursor: latestSeq,
983
- requiresSync: record === undefined ||
984
- cursor < latestSeq ||
985
- (this.#config.schemaWindow !== undefined &&
986
- record.subscriptions.some((subscription) => !schemaWindowOf(this.#config.schema, this.#config.schemaWindow).some((schema) => schema.version === subscription.schemaVersion))),
987
- timestamp: clock(),
988
- },
989
- }));
990
- if (helloResult instanceof Promise)
991
- helloResult.catch(() => { });
992
- // §8.6.4: the newly-registered connection sees who is already present
993
- // on each key it holds (a burst of joins, after hello).
994
- session.deliverInitialPresence();
1116
+ if (restored === undefined) {
1117
+ const helloResult = options.send(JSON.stringify({
1118
+ event: 'hello',
1119
+ data: {
1120
+ protocolVersion: 1,
1121
+ sessionId: session.sessionId,
1122
+ actorId: options.actorId,
1123
+ clientId: options.clientId,
1124
+ cursor,
1125
+ latestCursor: latestSeq,
1126
+ requiresSync: record === undefined ||
1127
+ cursor < latestSeq ||
1128
+ (this.#config.schemaWindow !== undefined &&
1129
+ record.subscriptions.some((subscription) => !schemaWindowOf(this.#config.schema, this.#config.schemaWindow).some((schema) => schema.version === subscription.schemaVersion))),
1130
+ timestamp: clock(),
1131
+ },
1132
+ }));
1133
+ if (helloResult instanceof Promise)
1134
+ helloResult.catch(() => { });
1135
+ // §8.6.4: the newly-registered connection sees who is already present
1136
+ // on each key it holds (a burst of joins, after hello).
1137
+ session.deliverInitialPresence();
1138
+ }
995
1139
  const events = this.#config.events;
996
1140
  if (events !== undefined) {
997
1141
  emitEvent(events, {
@@ -1087,6 +1231,22 @@ export class RealtimeHub {
1087
1231
  for (const session of this.#sessions)
1088
1232
  session.sendWake(reason);
1089
1233
  }
1234
+ /**
1235
+ * §8.5: send `heartbeat` to every session. The host calls this on its
1236
+ * heartbeat interval (suggested 30 s); a client that hears nothing for twice
1237
+ * the interval treats the socket as lost.
1238
+ */
1239
+ heartbeat() {
1240
+ for (const session of this.#sessions)
1241
+ session.sendHeartbeat();
1242
+ }
1243
+ }
1244
+ /** The §8.5 `heartbeat` control message. */
1245
+ export function encodeHeartbeat(timestampMs) {
1246
+ return JSON.stringify({
1247
+ event: 'heartbeat',
1248
+ data: { timestamp: timestampMs },
1249
+ });
1090
1250
  }
1091
1251
  export function createRealtimeHub(config) {
1092
1252
  return new RealtimeHub(config);
@@ -1101,7 +1261,8 @@ function errorResponseBytes(error, wireVersion, logEpoch) {
1101
1261
  frames: [
1102
1262
  {
1103
1263
  type: 'RESP_HEADER',
1104
- ...(wireVersion >= 2 ? { logEpoch, resetRequired: false } : {}),
1264
+ logEpoch,
1265
+ resetRequired: false,
1105
1266
  },
1106
1267
  {
1107
1268
  type: 'ERROR',