@edryslabs/genericprovider 1.0.4 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/README.md +129 -5
  2. package/dist/index.d.ts +893 -60
  3. package/dist/index.d.ts.map +1 -1
  4. package/dist/index.js +2683 -428
  5. package/dist/index.js.map +1 -1
  6. package/dist/lib.d.ts +1 -0
  7. package/dist/lib.d.ts.map +1 -1
  8. package/dist/lib.js +3 -0
  9. package/dist/lib.js.map +1 -1
  10. package/dist/providers/ably/index.d.ts +203 -0
  11. package/dist/providers/ably/index.d.ts.map +1 -0
  12. package/dist/providers/ably/index.js +561 -0
  13. package/dist/providers/ably/index.js.map +1 -0
  14. package/dist/providers/chunking.d.ts +26 -0
  15. package/dist/providers/chunking.d.ts.map +1 -0
  16. package/dist/providers/chunking.js +52 -0
  17. package/dist/providers/chunking.js.map +1 -0
  18. package/dist/providers/dummy/index.d.ts +138 -2
  19. package/dist/providers/dummy/index.d.ts.map +1 -1
  20. package/dist/providers/dummy/index.js +266 -4
  21. package/dist/providers/dummy/index.js.map +1 -1
  22. package/dist/providers/gun/index.d.ts +11 -4
  23. package/dist/providers/gun/index.d.ts.map +1 -1
  24. package/dist/providers/gun/index.js +104 -22
  25. package/dist/providers/gun/index.js.map +1 -1
  26. package/dist/providers/indexeddb/index.d.ts +44 -20
  27. package/dist/providers/indexeddb/index.d.ts.map +1 -1
  28. package/dist/providers/indexeddb/index.js +85 -71
  29. package/dist/providers/indexeddb/index.js.map +1 -1
  30. package/dist/providers/matrix/index.d.ts +6 -1
  31. package/dist/providers/matrix/index.d.ts.map +1 -1
  32. package/dist/providers/matrix/index.js +44 -6
  33. package/dist/providers/matrix/index.js.map +1 -1
  34. package/dist/providers/nostr/index.d.ts +69 -5
  35. package/dist/providers/nostr/index.d.ts.map +1 -1
  36. package/dist/providers/nostr/index.js +204 -36
  37. package/dist/providers/nostr/index.js.map +1 -1
  38. package/dist/providers/peerjs/index.d.ts +11 -1
  39. package/dist/providers/peerjs/index.d.ts.map +1 -1
  40. package/dist/providers/peerjs/index.js +32 -2
  41. package/dist/providers/peerjs/index.js.map +1 -1
  42. package/dist/providers/pubnub/index.d.ts +35 -1
  43. package/dist/providers/pubnub/index.d.ts.map +1 -1
  44. package/dist/providers/pubnub/index.js +56 -35
  45. package/dist/providers/pubnub/index.js.map +1 -1
  46. package/dist/providers/simple-peer/index.d.ts +8 -43
  47. package/dist/providers/simple-peer/index.d.ts.map +1 -1
  48. package/dist/providers/simple-peer/index.js +57 -161
  49. package/dist/providers/simple-peer/index.js.map +1 -1
  50. package/dist/providers/supabase/index.d.ts +43 -1
  51. package/dist/providers/supabase/index.d.ts.map +1 -1
  52. package/dist/providers/supabase/index.js +211 -18
  53. package/dist/providers/supabase/index.js.map +1 -1
  54. package/dist/providers/trystero/index.d.ts +19 -1
  55. package/dist/providers/trystero/index.d.ts.map +1 -1
  56. package/dist/providers/trystero/index.js +37 -2
  57. package/dist/providers/trystero/index.js.map +1 -1
  58. package/dist/providers/websocket/index.d.ts +9 -1
  59. package/dist/providers/websocket/index.d.ts.map +1 -1
  60. package/dist/providers/websocket/index.js +7 -1
  61. package/dist/providers/websocket/index.js.map +1 -1
  62. package/dist/transport.d.ts +69 -16
  63. package/dist/transport.d.ts.map +1 -1
  64. package/package.json +35 -24
package/dist/index.js CHANGED
@@ -10,10 +10,83 @@ const MESSAGE_SYNC = 0;
10
10
  const MESSAGE_AWARENESS = 1;
11
11
  const MESSAGE_PUBSUB = 2;
12
12
  const MESSAGE_SYNC_VERIFIED = 3; // Sync message with hash verification
13
- const MESSAGE_PUBSUB_TARGETED = 4; // Pub/sub message aimed at a single target
14
- // Sub-channel inside a MESSAGE_AWARENESS frame: [opcode 1][channel][update].
15
- const AWARENESS_CHANNEL_MAIN = 0;
16
- const AWARENESS_CHANNEL_APP = 1;
13
+ const MESSAGE_BATCH = 4; // Envelope for N independently-typed sub-messages sent as one wire message
14
+ // Digest beacon: replaces SyncStep1 on the wire. [version][flags][sender
15
+ // clientID][state vector][delete-set hash]. Receivers reply only when the
16
+ // sender is behind them or the delete-set hashes differ (SyncStep2, as
17
+ // before), or - on a JOIN-flagged beacon with equal state - with their own
18
+ // beacon as an ack; otherwise not at all. See
19
+ // docs/superpowers/specs/2026-09-05-digest-beacon-design.md and
20
+ // _handleDigest(). Versions are append-only: receivers read the fields
21
+ // they know and ignore trailing bytes.
22
+ const MESSAGE_SYNC_DIGEST = 5;
23
+ const DIGEST_VERSION = 1;
24
+ const DIGEST_FLAG_JOIN = 1; // bit 0: "I just joined - send me your presence and confirm my state"
25
+ // bit 1: this message is an ACK, not a request. It echoes the state vector
26
+ // and delete-set hash of the JOIN beacon it answers ("I hold exactly this
27
+ // state too"), never the sender's own state - so it is never read as a
28
+ // request and never collects a SyncStep2 from anyone. Receivers whose state
29
+ // equals the echoed digest mark themselves synced (the joiner it was for,
30
+ // and everyone else in that state); receivers holding a pending ack for the
31
+ // same digest drop it. See _handleDigest(). Before this flag existed the ack
32
+ // was the acker's own beacon; an acker that was itself behind the room (a
33
+ // joiner acking another joiner) made every peer ahead of it reply with a
34
+ // SyncStep2, and acks landing after an edit burst had started did the same
35
+ // (Task 3c in the design doc).
36
+ const DIGEST_FLAG_ACK = 2;
37
+ // bit 2: "ack me if our states are equal" WITHOUT the presence request of
38
+ // DIGEST_FLAG_JOIN. Sent only by the response-wait retry
39
+ // (_armResponseWait): a joiner that gets neither a SyncStep2 nor an ack
40
+ // within the wait has lost one message or the other and asks again - reply
41
+ // suppression deliberately leaves ~1 reply per request, so a single lost
42
+ // reply would otherwise strand the joiner's `synced` until the next
43
+ // periodic beacon (never, with syncInterval 0). Measured: the last joiner
44
+ // of a simultaneous 5-peer burst at 10% loss failed to reach `synced` in 1
45
+ // of 600 runs before this existed. Resync beacons do NOT request acks: in
46
+ // an equal room every peer would answer, and at high latency the
47
+ // suppression window cannot thin those replies (measured 4-5x more acks).
48
+ const DIGEST_FLAG_CONFIRM = 4;
49
+ // bit 3: the sender is CONFIRMED - it has received a SyncStep2, or an equal
50
+ // digest from a confirmed peer, or its own join asked three times and got
51
+ // nothing better (so it is the room). A joiner's response wait is satisfied
52
+ // only by data (SyncStep2) or by an equal digest carrying this bit: an
53
+ // equal ack from a fellow joiner says nothing about whether the room holds
54
+ // more than both of us, and treating it as an answer left a joiner whose
55
+ // SyncStep2 was lost with an empty document (measured: 1 of 150 lossy
56
+ // 15-peer joins; research doc item 13 follow-up in the phase-1b design).
57
+ const DIGEST_FLAG_SETTLED = 8;
58
+ // Full-state push (connect()/syncNow()): the whole document as one update,
59
+ // no hash, no sequence number, applied and nothing else. Before this type a
60
+ // push was a MESSAGE_SYNC_VERIFIED update whose hash was the PUSHER's state
61
+ // - every peer holding more data read that as divergence and scheduled a
62
+ // resync of its own (research doc item 12), the seed of the cascade in item
63
+ // 13. Not a SyncStep2 either: receiving one must not flip `synced` (an
64
+ // empty joiner's push says nothing about the room's content). See
65
+ // docs/superpowers/specs/2026-09-05-resync-cascade-design.md.
66
+ const MESSAGE_SYNC_PUSH = 6;
67
+ // Pub/sub message aimed at a single target: [target][topic][message]. Every
68
+ // provider whose `localId` differs drops it. On transports with sendTo it is
69
+ // unicast; elsewhere it is broadcast and filtered on receipt.
70
+ //
71
+ // NOTE: this was type 4 before the v1.5.0 merge, which upstream had taken for
72
+ // MESSAGE_BATCH. Renumbered to 7 (the first free slot) - the two are not wire
73
+ // compatible, so all peers of a room must run the same version, as the README
74
+ // already requires.
75
+ const MESSAGE_PUBSUB_TARGETED = 7;
76
+ // A second, independent awareness channel for application/module state
77
+ // (cursors, module presence), kept off MESSAGE_AWARENESS deliberately.
78
+ //
79
+ // Upstream's MESSAGE_AWARENESS receive path is now core presence machinery:
80
+ // it feeds _knownPeers, sets _presenceCovered, and cancels pending removal
81
+ // broadcasts. App awareness is written by untrusted third-party classroom
82
+ // modules, so routing it through that path would let module cursor traffic
83
+ // drive the room's peer bookkeeping - a module setting a state would mark
84
+ // phantom peers present and suppress real removals. A separate type keeps
85
+ // the two entirely disjoint: same throttle policy, no shared presence state.
86
+ //
87
+ // (Dev encoded this as a channel varint inside MESSAGE_AWARENESS, which was
88
+ // safe when that path did nothing but applyAwarenessUpdate. It no longer is.)
89
+ const MESSAGE_AWARENESS_APP = 8;
17
90
  /**
18
91
  * CRC32 lookup table for fast computation.
19
92
  * Generated once and reused for all CRC calculations.
@@ -80,25 +153,105 @@ function unwrapAndVerifyMessage(wrapped) {
80
153
  }
81
154
  return message;
82
155
  }
156
+ /** Byte-wise equality of two Uint8Arrays. */
157
+ function bytesEqual(a, b) {
158
+ if (a.length !== b.length)
159
+ return false;
160
+ for (let i = 0; i < a.length; i++) {
161
+ if (a[i] !== b[i])
162
+ return false;
163
+ }
164
+ return true;
165
+ }
83
166
  /**
84
- * Compute a cheap hash of document state for desync detection.
85
- *
86
- * Hashes the state VECTOR, not encodeStateAsUpdate, for two independent reasons:
87
- *
88
- * 1. CORRECTNESS: the full update byte stream is NOT canonical across
89
- * CRDT-convergent replicas (client-block and tombstone ordering differ per
90
- * peer), so hashing it flags false divergence and triggers an endless
91
- * re-sync loop. The state vector (clientID -> clock) is serialized in sorted
92
- * clientID order by Yjs, so two convergent docs hash identically, while a
93
- * missed update still shows up as a differing clock exactly the "did we
94
- * fall behind?" signal this check exists to provide.
95
- * 2. COST: O(number of distinct clients) instead of O(document content size),
96
- * so this no longer re-serializes the entire document on every update.
167
+ * Componentwise minimum of two encoded state vectors: the state a
168
+ * SyncStep2 must start from to serve both requesters. Clients missing from
169
+ * one side count as clock 0 and are omitted (omitted = 0 on the wire).
170
+ */
171
+ function minStateVector(a, b) {
172
+ const ma = Y.decodeStateVector(a);
173
+ const mb = Y.decodeStateVector(b);
174
+ const out = new Map();
175
+ for (const [client, clock] of ma) {
176
+ const m = Math.min(clock, mb.get(client) ?? 0);
177
+ if (m > 0)
178
+ out.set(client, m);
179
+ }
180
+ return Y.encodeStateVector(out);
181
+ }
182
+ /**
183
+ * Whether this runtime has the Compression Streams API (Node 18+, all
184
+ * evergreen browsers). Checked once at module load; compressionThresholdBytes
185
+ * falls back to sending uncompressed (still flag-byte-prefixed, flag=0) if
186
+ * this is false, rather than throwing.
187
+ */
188
+ const COMPRESSION_AVAILABLE = typeof CompressionStream !== 'undefined' &&
189
+ typeof DecompressionStream !== 'undefined';
190
+ /** Drain a ReadableStream<Uint8Array> into a single concatenated Uint8Array. */
191
+ async function _readAllChunks(readable) {
192
+ const chunks = [];
193
+ const reader = readable.getReader();
194
+ for (;;) {
195
+ const { done, value } = await reader.read();
196
+ if (done)
197
+ break;
198
+ chunks.push(value);
199
+ }
200
+ const total = chunks.reduce((sum, c) => sum + c.length, 0);
201
+ const out = new Uint8Array(total);
202
+ let offset = 0;
203
+ for (const chunk of chunks) {
204
+ out.set(chunk, offset);
205
+ offset += chunk.length;
206
+ }
207
+ return out;
208
+ }
209
+ /**
210
+ * Compress with deflate-raw (no gzip header/trailer - see
211
+ * compressionThresholdBytes's doc comment for why deflate-raw over gzip).
212
+ */
213
+ async function compressDeflateRaw(data) {
214
+ const cs = new CompressionStream('deflate-raw');
215
+ const writer = cs.writable.getWriter();
216
+ // The writer promises reject too when the stream errors; the reader side
217
+ // already surfaces that error, and an unhandled rejection here crashes
218
+ // Node (seen with a corrupt deflate stream in the Nostr end-to-end test).
219
+ writer.write(data).catch(() => { });
220
+ writer.close().catch(() => { });
221
+ return _readAllChunks(cs.readable);
222
+ }
223
+ /** Inverse of compressDeflateRaw(). */
224
+ async function decompressDeflateRaw(data) {
225
+ const ds = new DecompressionStream('deflate-raw');
226
+ const writer = ds.writable.getWriter();
227
+ writer.write(data).catch(() => { });
228
+ writer.close().catch(() => { });
229
+ return _readAllChunks(ds.readable);
230
+ }
231
+ /**
232
+ * Prepend a 1-byte compressed(1)/uncompressed(0) flag. Only used when
233
+ * compressionThresholdBytes is configured - see that option's doc comment
234
+ * for why this is a deliberate, opt-in wire-format change.
235
+ */
236
+ function prefixCompressionFlag(flag, data) {
237
+ const out = new Uint8Array(1 + data.length);
238
+ out[0] = flag;
239
+ out.set(data, 1);
240
+ return out;
241
+ }
242
+ /**
243
+ * Compute a cheap hash of document state for verification.
97
244
  *
98
- * Two peers can only reach the same state vector by having applied the same set
99
- * of operations, so real content divergence is still caught. (CRC32 already
100
- * guards wire corruption, and sequence tracking guards reordering/loss — this
101
- * hash is the last line of defense against logical divergence between peers.)
245
+ * Hashes the state VECTOR (each client's clock), not the full document
246
+ * content O(number of distinct clients) instead of O(document content
247
+ * size). `Y.encodeStateVector()` writes entries sorted by clientID, so the
248
+ * result is deterministic regardless of the internal Map's iteration order.
249
+ * Two peers can only reach the same state vector by having applied the same
250
+ * set of operations, so this still catches real content divergence; it just
251
+ * no longer re-serializes the entire document on every single update.
252
+ * (CRC32 already guards against wire corruption, and sequence tracking
253
+ * guards against reordering/loss — this hash is the last line of defense
254
+ * against logical divergence between peers.)
102
255
  */
103
256
  function computeDocHash(doc) {
104
257
  const state = Y.encodeStateVector(doc);
@@ -108,6 +261,109 @@ function computeDocHash(doc) {
108
261
  }
109
262
  return hash;
110
263
  }
264
+ /**
265
+ * Cheap, peer-deterministic hash of the document's delete set - the half of
266
+ * a Yjs document's identity that the state vector does NOT cover (yjs
267
+ * INTERNALS.md: "deletions are tracked in the DeleteSet, and do not update
268
+ * the state vector"). Two docs that differ only by a lost delete-only
269
+ * update have identical state vectors, so `computeDocHash` can never
270
+ * detect that divergence; this hash can, at heartbeat granularity (see
271
+ * `_encodeSyncStep1()` / `_handleDigest()`).
272
+ *
273
+ * Cost: `Y.createDeleteSetFromStructStore` walks every struct (Yjs keeps no
274
+ * incremental delete set), so this is O(items) - fine once per heartbeat
275
+ * (every empty SyncStep2 already did this exact walk inside
276
+ * `encodeStateAsUpdate`), NOT fine per update; hence the cache in
277
+ * `_deleteSetHash()`. Per-client runs come out already sorted and merged;
278
+ * the only per-peer non-determinism is `Map` insertion order, fixed by
279
+ * sorting client IDs before hashing.
280
+ *
281
+ * Exported for the property check in test/dummy/bench-idle-room.ts.
282
+ * @internal
283
+ */
284
+ export function computeDeleteSetHash(doc) {
285
+ const ds = Y.createDeleteSetFromStructStore(doc.store);
286
+ const encoder = encoding.createEncoder();
287
+ const clients = Array.from(ds.clients.keys()).sort((x, y) => x - y);
288
+ for (const client of clients) {
289
+ encoding.writeVarUint(encoder, client);
290
+ for (const item of ds.clients.get(client)) {
291
+ encoding.writeVarUint(encoder, item.clock);
292
+ encoding.writeVarUint(encoder, item.len);
293
+ }
294
+ }
295
+ return computeCRC32(encoding.toUint8Array(encoder));
296
+ }
297
+ /**
298
+ * The Yjs updates a CRC-wrapped frame (as handed to `Transport.send`)
299
+ * carries: the update of a MESSAGE_SYNC_VERIFIED or MESSAGE_SYNC
300
+ * Update/SyncStep2 message, the whole document of a MESSAGE_SYNC_PUSH,
301
+ * each such sub-message of a MESSAGE_BATCH - and nothing for awareness,
302
+ * pub/sub, digest beacons and SyncStep1 requests, which carry no document
303
+ * state. For persistence transports (`providers/indexeddb`, LiaScript's
304
+ * Dexie cache): store only what this returns, and only this. Storing whole
305
+ * frames and replaying them on the next load resurrects the previous
306
+ * session's clientID as a phantom peer (its presence, its beacons - which
307
+ * the provider then answers into the store, multiplying rows) and keeps
308
+ * ~10x the bytes (a keystroke's frame carries its cursor). A frame this
309
+ * cannot parse yields `[]`, never a partial read. Frames of a provider with
310
+ * `compressionThresholdBytes` set (a leading flag byte) are not supported.
311
+ * Round 7, item 6; measured in test/dummy/bench-persist-log.ts.
312
+ */
313
+ export function extractDocUpdates(frame) {
314
+ const updates = [];
315
+ const walk = (message) => {
316
+ const decoder = decoding.createDecoder(message);
317
+ const type = decoding.readVarUint(decoder);
318
+ switch (type) {
319
+ case MESSAGE_BATCH:
320
+ while (decoding.hasContent(decoder))
321
+ walk(decoding.readVarUint8Array(decoder));
322
+ break;
323
+ case MESSAGE_SYNC_VERIFIED:
324
+ case MESSAGE_SYNC: {
325
+ if (type === MESSAGE_SYNC_VERIFIED) {
326
+ decoding.readVarUint(decoder); // sequence number
327
+ decoding.readVarUint(decoder); // sender clientID
328
+ }
329
+ const sub = decoding.readVarUint(decoder);
330
+ if (sub === syncProtocol.messageYjsSyncStep2 ||
331
+ sub === syncProtocol.messageYjsUpdate) {
332
+ updates.push(decoding.readVarUint8Array(decoder));
333
+ }
334
+ break;
335
+ }
336
+ case MESSAGE_SYNC_PUSH:
337
+ updates.push(decoding.readVarUint8Array(decoder));
338
+ break;
339
+ default:
340
+ break;
341
+ }
342
+ };
343
+ try {
344
+ if (frame.length > 4)
345
+ walk(frame.subarray(4));
346
+ }
347
+ catch {
348
+ return [];
349
+ }
350
+ return updates;
351
+ }
352
+ /**
353
+ * The counterpart of `extractDocUpdates()` for the load path of a
354
+ * persistence transport: wraps one (merged) update as a CRC-wrapped
355
+ * MESSAGE_SYNC SyncStep2 frame. Handed to the `onMessage` callback, the
356
+ * provider applies it as the answer to its own request - `synced` fires,
357
+ * nothing is sent back - exactly what a local copy is: the peer that had
358
+ * our document.
359
+ */
360
+ export function frameDocUpdate(update) {
361
+ const encoder = encoding.createEncoder();
362
+ encoding.writeVarUint(encoder, MESSAGE_SYNC);
363
+ encoding.writeVarUint(encoder, syncProtocol.messageYjsSyncStep2);
364
+ encoding.writeVarUint8Array(encoder, update);
365
+ return wrapMessageWithChecksum(encoding.toUint8Array(encoder));
366
+ }
111
367
  /**
112
368
  * PubSub channel for real-time messaging alongside Yjs.
113
369
  * Allows sending ephemeral messages that don't need CRDT properties.
@@ -207,6 +463,18 @@ export class PubSubChannel extends Observable {
207
463
  * ```
208
464
  */
209
465
  export class GenericProvider extends Observable {
466
+ get appAwareness() {
467
+ if (!this._appAwareness) {
468
+ // Created on first use, not in the constructor: every y-protocols
469
+ // Awareness starts its own setInterval that only destroy() clears, so
470
+ // an eagerly-built second instance cost one live timer per provider
471
+ // for the majority of consumers that never touch this channel
472
+ // (bench-reload-phantoms part 2 counts exactly that).
473
+ this._appAwareness = new awarenessProtocol.Awareness(this.doc);
474
+ this._attachAppAwareness(this._appAwareness);
475
+ }
476
+ return this._appAwareness;
477
+ }
210
478
  /**
211
479
  * Create a new generic provider.
212
480
  *
@@ -219,6 +487,10 @@ export class GenericProvider extends Observable {
219
487
  this._status = { state: 'disconnected' };
220
488
  this._synced = false;
221
489
  this._destroying = false;
490
+ this._lastActivityTime = Date.now();
491
+ this._lastPeriodicTickTime = Date.now();
492
+ this._equalBeaconsHeard = 0;
493
+ this._beaconForced = false;
222
494
  // BroadcastChannel state for cross-tab sync
223
495
  this._bcChannel = '';
224
496
  this._bcConnected = false;
@@ -237,8 +509,15 @@ export class GenericProvider extends Observable {
237
509
  // own. See _requestResync().
238
510
  this._resyncAttemptCount = 0;
239
511
  this._lastResyncAttemptTime = 0;
240
- // Rate limiting for sync requests
512
+ // Rate limiting for sync traffic - two budgets since phase 1b: one for
513
+ // what we ask for (beacons, pushes, syncNow), one for what we owe
514
+ // (SyncStep2 replies, acks). With a single shared budget a join burst's
515
+ // replies spent the slots a peer needed for its own recovery (measured:
516
+ // joiners arriving within 10 s of a burst converged in 13 s once, and a
517
+ // 50-peer room's periodic beacons ran at a third of their rate for 10 s
518
+ // after every join burst). Same size, same window, independent.
241
519
  this._syncRequestTimes = [];
520
+ this._syncReplyTimes = [];
242
521
  // SyncStep2 reply suppression (NACK-suppression style): delay a reply to
243
522
  // a SyncStep1 request briefly, and drop it if another peer's reply is
244
523
  // overheard first - since every reply is broadcast to the whole room
@@ -248,6 +527,105 @@ export class GenericProvider extends Observable {
248
527
  // peers there's no "someone else" to rely on, so replies go out
249
528
  // immediately as before.
250
529
  this._pendingSyncReply = null;
530
+ // Whether _pendingSyncReply is a digest-beacon ack (see _handleDigest())
531
+ // rather than a SyncStep2. An overheard beacon with a digest equal to ours
532
+ // makes a pending ACK redundant (the joiner it was for has received that
533
+ // same beacon and is synced by it) but says nothing about a pending
534
+ // SyncStep2, which carries data - so only acks are cancelled on that
535
+ // signal. Measured in test/dummy/bench-user-scaling.ts: without this, acks
536
+ // were ~94% of a 50-peer join burst's messages (Task 3b in the design doc).
537
+ this._pendingSyncReplyIsAck = false;
538
+ // The requester's state vector the pending SyncStep2 answers (null for
539
+ // acks and for replies to plain SyncStep1s). A later request with the
540
+ // same state vector is the same question: the pending reply's bytes are
541
+ // refreshed to the current document and the timer kept, instead of the
542
+ // old reply being flushed as "a different request" - measured: with one
543
+ // peer typing while K empty peers join, keystroke and JOIN-beacon
544
+ // arrivals interleave at random, every keystroke in between changed the
545
+ // reply bytes, and each settled peer flushed up to K replies (Task 7 in
546
+ // the phase-1b design doc).
547
+ this._pendingSyncReplyTargetSv = null;
548
+ this._responseWaitAttempts = 0;
549
+ this._responseSeen = false;
550
+ // Phase 1e: an equal ack or equal beacon from an UNSETTLED peer arrived
551
+ // during the current response wait. Not a response (a settled peer with
552
+ // content may still answer), but evidence that the room is a fresh one
553
+ // whose peers are all in our state - see _armResponseWait().
554
+ this._equalUnsettledSeen = false;
555
+ this._responseWaitFlags = 0; // flags for the retry beacon: CONFIRM after a JOIN, 0 after a resync
556
+ this._behindSv = null;
557
+ // Round-trip estimate from our own requests (JOIN/resync beacon -> first
558
+ // SyncStep2 or ack): the minimum of the last 8 samples, because a sample
559
+ // includes the responder's random suppression delay and the fastest
560
+ // reply had the least of it. Drives _replySuppressionMaxDelay() (a
561
+ // suppression window shorter than the one-way latency suppresses
562
+ // nothing: at 250-350 ms latency every peer ahead of a requester replied
563
+ // before any reply could be overheard, ~20 replies per request at N=100)
564
+ // and _armResponseWait()'s first delay (a fixed 1 s fired premature
565
+ // retries on the Matrix profile). See the phase-1b design doc, 1c.
566
+ this._rttSamples = [];
567
+ this._requestSentAt = 0;
568
+ // See DIGEST_FLAG_SETTLED. Reset on connect (a re-joining peer is a joiner).
569
+ this._confirmed = false;
570
+ // ClientIDs we have heard from (beacon and verified-update senders, and
571
+ // the ids a relayed presence table names), each with the time we last
572
+ // heard it. The reply-suppression gate ("is there someone else who could
573
+ // answer?") used awareness alone, and in a join burst the awareness
574
+ // messages trail the beacons - so the gate was still closed exactly when
575
+ // 49 requests arrived at once, and every one got an immediate reply
576
+ // (phase-1b design, item 3). Cleared on disconnect, pruned by the lease
577
+ // sweep after a lease of silence (round 7, item 3: on a relay transport a
578
+ // reload's old clientID never says goodbye, and every one of them counted
579
+ // in _peerCount() forever).
580
+ this._knownPeers = new Map();
581
+ // Transport address (the `from` of Transport.onMessage) per remote
582
+ // clientID, learned from beacons and verified updates; lets replies, acks
583
+ // and presence responses go to the requester alone when the transport
584
+ // has sendTo (phase-1c design, item B). Cleared on disconnect, pruned
585
+ // with _knownPeers.
586
+ this._peerAddress = new Map();
587
+ // Requesters whose JOIN presence request the pending presence-response
588
+ // timer covers (see _schedulePresenceResponse).
589
+ this._presencePending = new Set();
590
+ // Phase 1e: an awareness message carrying OUR state at our current clock
591
+ // arrived since the presence-response timer was armed - the room's
592
+ // relayer (see _schedulePresenceResponse) already told the joiner about
593
+ // us, our own response would repeat it.
594
+ this._presenceCovered = false;
595
+ // Same NACK-style suppression as _pendingSyncReply above, applied to
596
+ // awareness updates that are a pure timeout-triggered removal (see
597
+ // _scheduleAwarenessRemoval()) - every OTHER connected peer runs its own
598
+ // independent 30s outdatedTimeout sweep (y-protocols/awareness.js), so
599
+ // one peer going silent (crash/dirty drop, not a clean disconnect())
600
+ // causes an O(N) simultaneous "peer X is gone" broadcast burst without
601
+ // this. See docs/superpowers/specs/2026-09-04-sync-optimization-round-3-ideas.md
602
+ // item 7 and test/dummy/bench-awareness-removal-burst.ts.
603
+ this._pendingAwarenessRemoval = null;
604
+ // Peers whose channel opened since the debounce timer was armed (the
605
+ // `from`/sendTo address). Round 5, item 5: on a transport with sendTo
606
+ // each gets a plain beacon addressed to it - it answers with what we
607
+ // lack and its own JOIN beacon makes us answer it - instead of the
608
+ // full-state push + beacon broadcast to EVERY connection that
609
+ // `_syncNow(0)` did (the O(N^2) mesh-join burst CLAUDE.md warns about).
610
+ this._pendingPeerConnectIds = new Set();
611
+ // Round 5, item 5: the state vector and delete-set hash the room last
612
+ // confirmed as equal to ours (an equal digest from any peer, see
613
+ // _handleDigest). A reconnect's push then carries only what we produced
614
+ // since - our offline edits, the one thing a push exists for (round 2:
615
+ // one message must survive alone) - instead of the whole document, which
616
+ // on a chunking transport was several messages per reconnect. A room
617
+ // that has been replaced meanwhile shows up behind in its own JOIN
618
+ // beacons and is answered like any late joiner. Null until the first
619
+ // confirmation: the first connect still pushes everything.
620
+ this._confirmedSv = null;
621
+ this._confirmedDsHash = null;
622
+ // connect() is awaiting ConnectionConfig.waitFor: doc updates are the local load.
623
+ this._loading = false;
624
+ // Cached computeDeleteSetHash(doc); null = stale. Invalidated on every
625
+ // doc 'update' (deletes are content changes, so this is exact; inserts
626
+ // invalidate needlessly but cheaply). See computeDeleteSetHash's doc for
627
+ // why this must not be recomputed per update.
628
+ this._dsHashCache = null;
251
629
  // Sequence numbers for causal ordering
252
630
  this._localSeqNum = 0; // Our sequence number counter
253
631
  // Per-sender sequence tracking for reordering-tolerant gap detection.
@@ -257,41 +635,77 @@ export class GenericProvider extends Observable {
257
635
  // positives from mere network reordering. See _trackRemoteSeq().
258
636
  this._remoteSeqInfo = new Map();
259
637
  this._gapCheckTimers = new Map();
260
- // Update batching/debouncing
261
- this._batchUpdates = 0; // milliseconds delay (0 = disabled)
638
+ // Update batching/debouncing. `_batchUpdates` 0 (the default) no longer
639
+ // means a synchronous send from inside the Y.Doc 'update' event: the
640
+ // update is merged into `_pendingUpdate` and flushed at the end of the
641
+ // current task (a microtask - no timer, no measurable delay), so that
642
+ // (a) the several transactions an editor binding can emit for one input
643
+ // event leave as one message and (b) the cursor awareness the binding
644
+ // sets right after the text change in the same task rides along in the
645
+ // same wire message (round 5, item 1 - see _flushPendingUpdate()).
646
+ // `_batchTimeoutId` is set only for the timed (`batchUpdates > 0`) flush;
647
+ // `_flushScheduled` covers both.
648
+ this._batchUpdates = 0; // milliseconds delay (0 = end of current task)
649
+ // Origins whose updates are never sent to the transport (local-only txns).
650
+ this._excludeOrigins = new Set();
651
+ // 'pull' never pushes local state unasked - for read-mostly replicas that
652
+ // must not write into the room.
653
+ this._syncMode = 'push-pull';
262
654
  this._pendingUpdate = null;
655
+ this._flushScheduled = false;
263
656
  // Awareness throttling - prevents awareness from flooding document sync
264
- this._awarenessInterval = 100; // ms between awareness broadcasts
657
+ this._awarenessInterval = 100; // ms between awareness broadcasts, or 'auto' (round 6, item 9)
265
658
  this._pendingAwarenessClients = new Set();
266
659
  this._lastAwarenessTime = 0;
267
- // Independent throttle state for the app awareness channel.
660
+ // Independent throttle state for the app awareness channel, so module
661
+ // cursor churn never delays or coalesces with core presence.
268
662
  this._pendingAppAwarenessClients = new Set();
269
663
  this._lastAppAwarenessTime = 0;
270
- // Origins whose updates are never sent to the transport (local-only txns).
271
- this._excludeOrigins = new Set();
272
- // Connect-time sync strategy: 'push-pull' (default) sends full local state
273
- // then requests remote; 'pull' only requests remote state.
274
- this._syncMode = 'push-pull';
275
664
  this.doc = doc;
276
665
  this.transport = transport;
277
666
  this.pubsub = new PubSubChannel(this);
278
667
  this.awareness = options.awareness || new awarenessProtocol.Awareness(doc);
279
- this.appAwareness =
280
- options.appAwareness || new awarenessProtocol.Awareness(doc);
668
+ // Only when supplied - otherwise the getter builds it on first use.
669
+ if (options.appAwareness) {
670
+ this._appAwareness = options.appAwareness;
671
+ }
281
672
  this._syncInterval = options.syncInterval ?? 5000;
282
673
  this._verifyUpdates = options.verifyUpdates ?? true;
283
674
  this._batchUpdates =
284
675
  options.batchUpdates ?? transport.preferredBatchMs ?? 0;
285
676
  this._disableBc = options.disableBc ?? false;
286
- this._awarenessInterval = options.awarenessInterval ?? 100;
677
+ this._awarenessInterval =
678
+ options.awarenessInterval ?? transport.preferredAwarenessMs ?? 100;
287
679
  this._excludeOrigins = new Set(options.excludeOrigins ?? []);
288
680
  this._localId = options.localId;
289
681
  this._syncMode = options.syncMode ?? 'push-pull';
290
682
  this._maxSyncRequestsPerWindow = options.maxSyncRequestsPerWindow ?? 20;
291
683
  this._syncRequestWindowMs = options.syncRequestWindowMs ?? 10000;
292
684
  this._syncReplySuppressionMs = options.syncReplySuppressionMs ?? 30;
685
+ this._peerConnectDebounceMs = options.peerConnectDebounceMs ?? 50;
293
686
  this._gapGraceMs = options.gapGraceMs ?? 300;
294
687
  this._seqWindowSize = options.seqWindowSize ?? 64;
688
+ // Explicit 0 disables even when the transport hints a floor.
689
+ this._compressionThresholdBytes =
690
+ (options.compressionThresholdBytes ?? transport.preferredCompressMinBytes) ||
691
+ undefined;
692
+ this._idleBackoffEnabled = options.idleBackoffEnabled ?? true;
693
+ this._idleBackoffMaxMs = options.idleBackoffMaxMs ?? 60000;
694
+ this._trickleK = options.trickleK ?? 1;
695
+ this._ownsAwareness = !options.awareness;
696
+ this._awarenessTimeoutMs =
697
+ options.awarenessTimeoutMs ??
698
+ (typeof transport.onPeerDisconnect === 'function' ? 300000 : 30000);
699
+ // Silence y-protocols' own 3 s sweep on an awareness we created - the
700
+ // lease sweep that replaces it is armed in connect() and cleared in
701
+ // disconnect() (round 7, item 2: it used to start here and outlive
702
+ // disconnect(), keeping a dropped provider alive through its timer).
703
+ if (this._ownsAwareness) {
704
+ const aw = this.awareness;
705
+ if (aw._checkInterval !== undefined)
706
+ clearInterval(aw._checkInterval);
707
+ }
708
+ this._currentSyncIntervalMs = this._syncInterval;
295
709
  this._setupDocumentSync();
296
710
  this._setupAwarenessSync();
297
711
  }
@@ -317,18 +731,21 @@ export class GenericProvider extends Observable {
317
731
  try {
318
732
  // Setup BroadcastChannel for cross-tab sync (if enabled and available)
319
733
  this._setupBroadcastChannel(config);
320
- // Connect the transport
321
- await this.transport.connect(config);
322
- // Register for incoming messages
323
- this._unsubscribeTransport = this.transport.onMessage((data) => {
324
- this._handleIncomingMessage(data);
734
+ // Register for incoming messages and new-peer notifications BEFORE
735
+ // connecting the transport. Some transports (e.g. PeerJS for a
736
+ // joining, non-coordinator peer) establish and fully open their first
737
+ // connection *inside* transport.connect() itself — so a peer-connect
738
+ // notification or an immediate reply from the other side can arrive
739
+ // before that promise resolves. Registering after the await left
740
+ // exactly that window uncovered: whatever arrived during it was
741
+ // silently dropped since neither callback was wired up yet.
742
+ this._unsubscribeTransport = this.transport.onMessage((data, from) => {
743
+ this._handleIncomingMessage(data, from);
325
744
  });
326
- // When a new WebRTC peer channel opens, immediately push our full state
327
- // so peers that reconnected after offline edits receive our changes.
328
745
  if (this.transport.onPeerConnect) {
329
- const unsubPeer = this.transport.onPeerConnect((_peerId) => {
746
+ const unsubPeer = this.transport.onPeerConnect((peerId) => {
330
747
  if (!this._destroying)
331
- this.syncNow();
748
+ this._schedulePeerConnectSync(peerId);
332
749
  });
333
750
  const originalUnsub = this._unsubscribeTransport;
334
751
  this._unsubscribeTransport = () => {
@@ -336,31 +753,127 @@ export class GenericProvider extends Observable {
336
753
  unsubPeer();
337
754
  };
338
755
  }
756
+ if (this.transport.onPeerDisconnect) {
757
+ const unsubLeave = this.transport.onPeerDisconnect((peerId) => {
758
+ if (!this._destroying)
759
+ this._handlePeerLeave(peerId);
760
+ });
761
+ const originalUnsub = this._unsubscribeTransport;
762
+ this._unsubscribeTransport = () => {
763
+ originalUnsub?.();
764
+ unsubLeave();
765
+ };
766
+ }
767
+ // Connect the transport
768
+ await this.transport.connect(config);
339
769
  this._setStatus({ state: 'connected' });
340
- // Send initial sync. In 'push-pull' mode, syncNow() pushes our local
341
- // state (so offline edits reach currently-connected peers) then requests
342
- // remote state. In 'pull' mode, only request remote state — a relay/server
343
- // holds authoritative state and we adopt it rather than pushing a
344
- // competing local copy on every (re)connect.
345
- if (this._syncMode === 'pull') {
346
- this._sendSyncStep1();
770
+ this._startAwarenessSweep();
771
+ // Seed the round-trip estimate from the transport's hint (see
772
+ // Transport.expectedRttMs); the minimum-of-8 rule lets real samples
773
+ // take over as soon as they arrive.
774
+ if (this.transport.expectedRttMs) {
775
+ this._rttSamples = [this.transport.expectedRttMs];
347
776
  }
348
- else {
349
- this.syncNow();
777
+ // Persistence first (round 5, item 7): let a local copy load before
778
+ // the first beacon says what we have. While it loads, the doc
779
+ // updates it produces are the load, not edits - they are not
780
+ // broadcast - and afterwards the loaded state counts as confirmed:
781
+ // no full-state push. The beacon reconciles: peers behind us (our
782
+ // offline edits) see themselves behind and ask; peers ahead of us
783
+ // answer. Measured in bench-reconnect-push part 2: a 50 KB copy that
784
+ // loaded 100 ms after connect cost the room a 50 KB SyncStep2, and a
785
+ // first version of waitFor that only delayed the beacon cost 100 KB
786
+ // (the load's broadcast plus the push). See ConnectionConfig.waitFor.
787
+ if (config.waitFor) {
788
+ this._loading = true;
789
+ try {
790
+ await config.waitFor;
791
+ }
792
+ catch {
793
+ // The local load failed; sync from the room as if there were none.
794
+ }
795
+ this._loading = false;
796
+ if (this._destroying || !this.transport.isConnected)
797
+ return;
798
+ this._confirmedSv = Y.encodeStateVector(this.doc);
799
+ this._confirmedDsHash = this._deleteSetHash();
800
+ }
801
+ // Send initial sync pushing our local state plus requesting remote state.
802
+ // syncNow() is used instead of _sendSyncStep1() so that any offline edits
803
+ // made before this connect() call are pushed to currently-connected peers
804
+ // (e.g. same-browser tabs via BroadcastChannel).
805
+ this.syncNow();
806
+ // (Local awareness goes out inside syncNow()'s batch, or via its
807
+ // throttled fallback - a second broadcast here was a duplicate 100ms
808
+ // later.)
809
+ // The app channel has no equivalent path into syncNow()'s batch, so
810
+ // its local state is announced here when there is any.
811
+ if (this._appAwareness?.getLocalState() != null) {
812
+ this._broadcastAppAwareness([this.doc.clientID]);
350
813
  }
351
- // Broadcast local awareness state
352
- this._broadcastAwareness([this.doc.clientID]);
353
- this._broadcastAwareness([this.doc.clientID], AWARENESS_CHANNEL_APP);
354
814
  // Start periodic sync to handle packet loss
355
815
  // Just request sync without sending full state (avoid redundant broadcasts)
356
816
  // _sendSyncStep1() already checks the shared rate limiter internally
357
817
  // and silently drops the request if it's exceeded.
818
+ //
819
+ // Uses a recursive setTimeout (re-jittered by ~20% each tick) rather
820
+ // than a plain setInterval so peers that connect() within a short
821
+ // window of each other - the common case: everyone joining a room
822
+ // near session start, or reconnecting together after a shared network
823
+ // blip - don't end up with near-synchronized periodic timers that all
824
+ // fire in the same few milliseconds every syncInterval. This doesn't
825
+ // reduce total periodic-sync traffic, only spreads it out so a room's
826
+ // background traffic is smooth instead of bursty.
358
827
  if (this._syncInterval > 0) {
359
- this._syncIntervalId = setInterval(() => {
360
- if (this.transport.isConnected && !this._destroying) {
361
- this._sendSyncStep1();
362
- }
363
- }, this._syncInterval);
828
+ // Reset idle-backoff state on every (re)connect so a reconnect
829
+ // always starts its first tick at the base interval, never
830
+ // inheriting a backed-off value left over from a previous session.
831
+ this._currentSyncIntervalMs = this._syncInterval;
832
+ this._lastActivityTime = Date.now();
833
+ this._lastPeriodicTickTime = Date.now();
834
+ this._equalBeaconsHeard = 0;
835
+ this._beaconForced = false;
836
+ const scheduleNextPeriodicSync = (delayMs) => {
837
+ this._syncIntervalId = setTimeout(() => {
838
+ const tickTime = Date.now();
839
+ if (this._idleBackoffEnabled) {
840
+ // "Activity" = anything _markActivity() call sites observed
841
+ // (incoming message via transport or BroadcastChannel, local
842
+ // or remote doc change, local or remote awareness change)
843
+ // since the LAST tick fired - not since backoff started, so a
844
+ // single quiet tick after a burst of activity still resets to
845
+ // base rather than needing a full quiet cycle to catch up.
846
+ const hadActivitySinceLastTick = this._lastActivityTime > this._lastPeriodicTickTime;
847
+ this._currentSyncIntervalMs = hadActivitySinceLastTick
848
+ ? this._syncInterval
849
+ : Math.min(this._idleBackoffMaxMs, this._currentSyncIntervalMs * 2);
850
+ }
851
+ this._lastPeriodicTickTime = tickTime;
852
+ if (this.transport.isConnected && !this._destroying) {
853
+ // Beacon only. Presence is no longer re-announced per tick on
854
+ // any transport: a joiner requests it via DIGEST_FLAG_JOIN
855
+ // (see _handleDigest()), and y-protocols/awareness renews the
856
+ // local state itself every outdatedTimeout/2 = 15s
857
+ // (awareness.js _checkInterval), which the awareness update
858
+ // handler broadcasts. Measured in
859
+ // test/dummy/bench-idle-room.ts: the per-tick re-announce was
860
+ // ~40% of an idle room's deliveries.
861
+ // Trickle (round 5, item 3): silent if enough equal beacons
862
+ // were overheard since the last tick - see _equalBeaconsHeard.
863
+ const suppressed = this._trickleK > 0 &&
864
+ !this._beaconForced &&
865
+ this._equalBeaconsHeard >= this._trickleK;
866
+ if (!suppressed)
867
+ this._sendSyncStep1();
868
+ this._equalBeaconsHeard = 0;
869
+ this._beaconForced = false;
870
+ }
871
+ if (!this._destroying)
872
+ scheduleNextPeriodicSync();
873
+ }, delayMs ?? this._jitteredSyncInterval());
874
+ };
875
+ this._periodicScheduler = scheduleNextPeriodicSync;
876
+ scheduleNextPeriodicSync();
364
877
  }
365
878
  }
366
879
  catch (error) {
@@ -378,9 +891,15 @@ export class GenericProvider extends Observable {
378
891
  disconnect() {
379
892
  // Stop periodic sync
380
893
  if (this._syncIntervalId !== undefined) {
381
- clearInterval(this._syncIntervalId);
894
+ clearTimeout(this._syncIntervalId);
382
895
  this._syncIntervalId = undefined;
383
896
  }
897
+ this._periodicScheduler = undefined;
898
+ // Stop the awareness lease sweep (armed in connect(); round 7, item 2).
899
+ if (this._awarenessSweepId !== undefined) {
900
+ clearTimeout(this._awarenessSweepId);
901
+ this._awarenessSweepId = undefined;
902
+ }
384
903
  // Reset resync escalation tracking
385
904
  this._resyncAttemptCount = 0;
386
905
  this._lastResyncAttemptTime = 0;
@@ -391,26 +910,73 @@ export class GenericProvider extends Observable {
391
910
  clearTimeout(this._pendingResyncTimeoutId);
392
911
  this._pendingResyncTimeoutId = undefined;
393
912
  }
913
+ // Stop any pending gap-check timers and forget per-sender sequence
914
+ // tracking. Without this, a gap-check timer armed before this
915
+ // disconnect() keeps running in the background and can fire
916
+ // _requestResync() after reconnect using sequence-number bookkeeping
917
+ // from the PREVIOUS connection - a spurious resync race disconnected
918
+ // from anything actually missing in the new session. Mirrors the
919
+ // equivalent cleanup in destroy().
920
+ for (const timer of this._gapCheckTimers.values()) {
921
+ clearTimeout(timer);
922
+ }
923
+ this._gapCheckTimers.clear();
924
+ this._remoteSeqInfo.clear();
925
+ // Cancel any pending debounced onPeerConnect sync - a reconnect gets a
926
+ // fresh burst of onPeerConnect events (if the transport supports it) and
927
+ // shouldn't fire a stale one left over from before this disconnect.
928
+ if (this._pendingPeerConnectSyncTimeoutId !== undefined) {
929
+ clearTimeout(this._pendingPeerConnectSyncTimeoutId);
930
+ this._pendingPeerConnectSyncTimeoutId = undefined;
931
+ }
932
+ this._pendingPeerConnectIds.clear();
394
933
  // Reset the sync rate-limit budget. Without this, a reconnect inherits
395
934
  // whatever budget was left over from before the disconnect - and since
396
935
  // syncNow()'s full-state push now shares this same limiter (see
397
936
  // _tryReserveSyncSlot()), a rate-limited reconnect could silently skip
398
937
  // the very push that delivers edits made while offline.
399
938
  this._syncRequestTimes = [];
939
+ this._syncReplyTimes = [];
940
+ this._knownPeers.clear();
941
+ this._peerAddress.clear();
942
+ this._presencePending.clear();
943
+ // A pending response-wait belongs to a request on the old connection.
944
+ if (this._responseWaitTimer !== undefined) {
945
+ clearTimeout(this._responseWaitTimer);
946
+ this._responseWaitTimer = undefined;
947
+ }
948
+ if (this._pendingCheckTimer !== undefined) {
949
+ clearTimeout(this._pendingCheckTimer);
950
+ this._pendingCheckTimer = undefined;
951
+ }
952
+ if (this._behindCheckTimer !== undefined) {
953
+ clearTimeout(this._behindCheckTimer);
954
+ this._behindCheckTimer = undefined;
955
+ }
956
+ this._behindSv = null;
957
+ if (this._presenceResponseTimer !== undefined) {
958
+ clearTimeout(this._presenceResponseTimer);
959
+ this._presenceResponseTimer = undefined;
960
+ }
961
+ this._responseWaitAttempts = 0;
962
+ this._responseSeen = false;
963
+ this._equalUnsettledSeen = false;
964
+ this._rttSamples = [];
965
+ this._requestSentAt = 0;
966
+ this._confirmed = false;
400
967
  // Drop any pending suppressed sync reply - safe to simply discard (not
401
968
  // flush/send like batched updates/awareness below), since a suppressed
402
969
  // reply is by design redundant with whatever the room already has.
403
970
  this._cancelPendingSyncReply();
404
- // Flush any pending batched updates before disconnecting
405
- if (this._batchTimeoutId !== undefined) {
406
- clearTimeout(this._batchTimeoutId);
407
- this._batchTimeoutId = undefined;
408
- // Send pending update if transport is still connected
409
- if (this._pendingUpdate && this.transport.isConnected) {
410
- this._sendUpdate(this._pendingUpdate);
411
- }
412
- this._pendingUpdate = null;
413
- }
971
+ // Same reasoning for a pending suppressed awareness-removal broadcast -
972
+ // every other surviving peer is independently running the same
973
+ // suppression for the same departure, so dropping ours on disconnect
974
+ // (rather than flushing it through a transport that's about to go
975
+ // down) is safe.
976
+ this._cancelPendingAwarenessRemoval();
977
+ // Flush any pending batched updates before disconnecting (sent only if
978
+ // the transport is still connected; dropped otherwise, as before)
979
+ this._flushPendingUpdate();
414
980
  // Flush pending awareness updates before disconnecting
415
981
  if (this._awarenessTimeoutId !== undefined) {
416
982
  clearTimeout(this._awarenessTimeoutId);
@@ -423,13 +989,13 @@ export class GenericProvider extends Observable {
423
989
  }
424
990
  }
425
991
  this._pendingAwarenessClients.clear();
426
- // Flush pending app awareness updates before disconnecting
992
+ // Same for the app channel's independent throttle.
427
993
  if (this._appAwarenessTimeoutId !== undefined) {
428
994
  clearTimeout(this._appAwarenessTimeoutId);
429
995
  this._appAwarenessTimeoutId = undefined;
430
996
  if (this._pendingAppAwarenessClients.size > 0 &&
431
997
  this.transport.isConnected) {
432
- this._sendAwarenessNow(Array.from(this._pendingAppAwarenessClients), AWARENESS_CHANNEL_APP);
998
+ this._sendAppAwarenessNow(Array.from(this._pendingAppAwarenessClients));
433
999
  }
434
1000
  }
435
1001
  this._pendingAppAwarenessClients.clear();
@@ -441,7 +1007,6 @@ export class GenericProvider extends Observable {
441
1007
  }
442
1008
  // Mark local client as offline in awareness
443
1009
  awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'disconnect');
444
- awarenessProtocol.removeAwarenessStates(this.appAwareness, [this.doc.clientID], 'disconnect');
445
1010
  this.transport.disconnect();
446
1011
  this._synced = false;
447
1012
  this._setStatus({ state: 'disconnected' });
@@ -454,7 +1019,7 @@ export class GenericProvider extends Observable {
454
1019
  this._destroying = true;
455
1020
  // Stop periodic sync (disconnect() will also do this, but be explicit)
456
1021
  if (this._syncIntervalId !== undefined) {
457
- clearInterval(this._syncIntervalId);
1022
+ clearTimeout(this._syncIntervalId);
458
1023
  this._syncIntervalId = undefined;
459
1024
  }
460
1025
  // Stop any pending gap-check timers
@@ -465,38 +1030,42 @@ export class GenericProvider extends Observable {
465
1030
  // Drop any pending suppressed sync reply (disconnect() will also do
466
1031
  // this, but be explicit)
467
1032
  this._cancelPendingSyncReply();
1033
+ // Same, for a pending suppressed awareness-removal broadcast
1034
+ // (disconnect() will also do this, but be explicit)
1035
+ this._cancelPendingAwarenessRemoval();
468
1036
  // Flush any pending batched updates before destroying
469
- if (this._batchTimeoutId !== undefined) {
470
- clearTimeout(this._batchTimeoutId);
471
- this._batchTimeoutId = undefined;
472
- // Send pending update if transport is still connected
473
- if (this._pendingUpdate && this.transport.isConnected) {
474
- this._sendUpdate(this._pendingUpdate);
475
- }
476
- this._pendingUpdate = null;
477
- }
1037
+ this._flushPendingUpdate();
478
1038
  this.disconnect();
479
1039
  // Remove document update listener
480
1040
  if (this._updateHandler) {
481
1041
  this.doc.off('update', this._updateHandler);
482
1042
  this._updateHandler = undefined;
483
1043
  }
484
- // Remove awareness update listeners
1044
+ // Remove awareness update listener
485
1045
  if (this._awarenessUpdateHandler) {
486
- this.awareness.off('update', this._awarenessUpdateHandler);
1046
+ this.awareness.off(this._ownsAwareness ? 'change' : 'update', this._awarenessUpdateHandler);
487
1047
  this._awarenessUpdateHandler = undefined;
488
1048
  }
489
1049
  if (this._appAwarenessUpdateHandler) {
490
- this.appAwareness.off('update', this._appAwarenessUpdateHandler);
1050
+ this._appAwareness?.off('update', this._appAwarenessUpdateHandler);
491
1051
  this._appAwarenessUpdateHandler = undefined;
492
1052
  }
1053
+ if (this._appAwarenessTimeoutId !== undefined) {
1054
+ clearTimeout(this._appAwarenessTimeoutId);
1055
+ this._appAwarenessTimeoutId = undefined;
1056
+ }
1057
+ if (this._awarenessSweepId !== undefined) {
1058
+ clearTimeout(this._awarenessSweepId);
1059
+ this._awarenessSweepId = undefined;
1060
+ }
493
1061
  // Remove beforeunload handler
494
1062
  if (this._beforeUnloadHandler && typeof window !== 'undefined') {
495
1063
  window.removeEventListener('beforeunload', this._beforeUnloadHandler);
496
1064
  this._beforeUnloadHandler = undefined;
497
1065
  }
498
1066
  this.awareness.destroy();
499
- this.appAwareness.destroy();
1067
+ // Only if one was ever built - never construct one just to destroy it.
1068
+ this._appAwareness?.destroy();
500
1069
  super.destroy();
501
1070
  }
502
1071
  /**
@@ -524,14 +1093,29 @@ export class GenericProvider extends Observable {
524
1093
  return this._synced;
525
1094
  }
526
1095
  /**
527
- * Force an immediate sync with remote peers.
528
- * Useful after network interruptions or to manually trigger re-sync.
1096
+ * Push local state + request remote state, gated by the shared rate
1097
+ * limiter. Returns whether it actually reserved a slot and sent anything
1098
+ * - `false` means the caller was rate-limited right now. Extracted out of
1099
+ * `syncNow()` so `_requestResync()`'s scheduled retry (see below) can tell
1100
+ * the difference between "sent" and "silently skipped" and react to it,
1101
+ * instead of assuming a resync always succeeds once it fires.
1102
+ *
1103
+ * @param push - whether to also broadcast full local document state.
1104
+ * `_requestResync()`'s retry passes `false` (pull-only is enough for a
1105
+ * resync trigger - see its call site).
1106
+ * @param buildExtra - optional callback, invoked ONLY once a rate-limit
1107
+ * slot is actually reserved (so it never runs, and never mutates
1108
+ * whatever state it touches, on a call that ends up rate-limited),
1109
+ * returning additional already-encoded sub-messages to fold into the SAME
1110
+ * batched wire send as the push/pull messages below - e.g. an awareness
1111
+ * update that's ready to go out "now" anyway (see
1112
+ * `_tryImmediateAwarenessMessage()`). Pure wire-framing: whether a caller
1113
+ * passes this never changes whether/when the push+pull half itself sends,
1114
+ * only how many separate `transport.send()`/`bc.publish()` calls it costs.
1115
+ * @param flags - digest beacon flags: DIGEST_FLAG_JOIN from syncNow(), 0
1116
+ * from the peer-connect debounce and the resync retry (see _syncNow()).
529
1117
  */
530
- syncNow() {
531
- if (!this.transport.isConnected) {
532
- console.warn('Cannot sync: transport not connected');
533
- return;
534
- }
1118
+ _trySyncPushPull(push = true, buildExtra, flags = 0) {
535
1119
  // Push (full document state) and pull (SyncStep1 request) share a
536
1120
  // single rate-limit reservation. syncNow() is called from several
537
1121
  // triggers that can all fire in a short window when many peers are
@@ -543,21 +1127,308 @@ export class GenericProvider extends Observable {
543
1127
  // test/dummy/bench-user-scaling.ts: at 100 simulated users this drove
544
1128
  // message counts to 20-200x the theoretical linear cost. See
545
1129
  // docs/superpowers/specs/2026-07-26-dummy-benchmark-scaling-design.md.
546
- if (this._tryReserveSyncSlot()) {
547
- // Send our current document state to all peers
548
- // This ensures any changes made while offline are transmitted
549
- const update = Y.encodeStateAsUpdate(this.doc);
550
- if (update.length > 0) {
551
- this._sendUpdate(update);
1130
+ if (!this._tryReserveSyncSlot())
1131
+ return false;
1132
+ const messages = [];
1133
+ // Send our document state to all peers - everything on the first
1134
+ // connect, only what we produced since the room last confirmed our
1135
+ // state afterwards (see _confirmedSv). This is what carries edits made
1136
+ // while offline.
1137
+ if (push) {
1138
+ const update = Y.encodeStateAsUpdate(this.doc, this._confirmedSv ?? undefined);
1139
+ let worthPushing = update.length > 0;
1140
+ if (worthPushing && this._confirmedSv !== null) {
1141
+ // A diff against a confirmed state is never byte-empty (it always
1142
+ // carries the delete set): push it only if it holds structs, or
1143
+ // deletes the room has not confirmed.
1144
+ try {
1145
+ worthPushing =
1146
+ Y.parseUpdateMeta(update).to.size > 0 ||
1147
+ this._deleteSetHash() !== this._confirmedDsHash;
1148
+ }
1149
+ catch {
1150
+ worthPushing = true;
1151
+ }
1152
+ }
1153
+ if (worthPushing) {
1154
+ messages.push(this._encodePush(update));
1155
+ }
1156
+ }
1157
+ // Send sync request to get updates from others
1158
+ messages.push(this._encodeSyncStep1(flags));
1159
+ if (buildExtra) {
1160
+ messages.push(...buildExtra());
1161
+ }
1162
+ // Batched into one wire message (MESSAGE_BATCH) instead of one
1163
+ // transport.send()/bc.publish() call per sub-message - see _sendBatch().
1164
+ this._sendBatch(messages);
1165
+ if (flags & DIGEST_FLAG_JOIN)
1166
+ this._armResponseWait(DIGEST_FLAG_CONFIRM);
1167
+ return true;
1168
+ }
1169
+ /**
1170
+ * Force an immediate sync with remote peers.
1171
+ * Useful after network interruptions or to manually trigger re-sync.
1172
+ * Sends the beacon with DIGEST_FLAG_JOIN: peers answer with their
1173
+ * presence and, if our state already matches theirs, with an ack beacon
1174
+ * so `synced` flips without a data round trip.
1175
+ */
1176
+ syncNow() {
1177
+ this._syncNow(DIGEST_FLAG_JOIN);
1178
+ }
1179
+ /**
1180
+ * syncNow() body. `flags` = 0 for callers that must NOT request presence:
1181
+ * `_schedulePeerConnectSync()` (mesh transports already re-broadcast
1182
+ * presence to a newcomer via their own onPeerConnect -> syncNow()).
1183
+ */
1184
+ _syncNow(flags) {
1185
+ if (!this.transport.isConnected) {
1186
+ console.warn('Cannot sync: transport not connected');
1187
+ return;
1188
+ }
1189
+ if (flags & DIGEST_FLAG_JOIN)
1190
+ this._confirmed = false; // a (re-)joiner until answered
1191
+ // Try to fold the awareness broadcast into the same wire send as the
1192
+ // sync push+pull below. _tryImmediateAwarenessMessage() only returns
1193
+ // non-null (and only mutates awareness-throttle state) when the
1194
+ // throttle would have let an immediate send through anyway - so this
1195
+ // never changes awareness throttle semantics, only whether it travels
1196
+ // as its own message or bundled with the sync message going out "now"
1197
+ // too. Built inside buildExtra so it's only even attempted once a sync
1198
+ // rate-limit slot is confirmed reserved (see _trySyncPushPull's doc).
1199
+ let awarenessBatched = false;
1200
+ const sent = this._trySyncPushPull(
1201
+ // 'pull': send the beacon but never the full-state push. A relay or
1202
+ // server holds authoritative state and we adopt it rather than pushing
1203
+ // a competing local copy on every (re)connect. Peers that are actually
1204
+ // behind us still ask, and we still answer - the beacon reconciles.
1205
+ this._syncMode !== 'pull', () => {
1206
+ const msg = this._tryImmediateAwarenessMessage([this.doc.clientID]);
1207
+ if (msg) {
1208
+ awarenessBatched = true;
1209
+ return [msg];
1210
+ }
1211
+ return [];
1212
+ }, flags);
1213
+ // Awareness broadcasting is independently throttled and explicitly NOT
1214
+ // gated by the sync rate limiter above - preserve that exactly: it
1215
+ // always ends up broadcast one way or another (batched above, or via
1216
+ // its own throttled path here), regardless of whether the sync half
1217
+ // above was rate-limited.
1218
+ if (!sent || !awarenessBatched) {
1219
+ this._broadcastAwareness([this.doc.clientID]);
1220
+ }
1221
+ }
1222
+ /**
1223
+ * Compute the next periodic-sync delay, jittered by ~+/-20% around
1224
+ * `_currentSyncIntervalMs` (== `_syncInterval` unless `idleBackoffEnabled`
1225
+ * has backed it off - see that option's doc comment). Re-jittered fresh
1226
+ * each tick (not computed once per connect()) so a room's peers - which
1227
+ * commonly all connect() within a short window of each other - drift
1228
+ * apart over time instead of staying loosely synchronized. Extracted to
1229
+ * its own method purely so benchmarks can shadow it to compare against
1230
+ * the unjittered baseline.
1231
+ */
1232
+ _jitteredSyncInterval() {
1233
+ const jitter = 1 + (Math.random() * 2 - 1) * 0.2; // +/-20%
1234
+ return this._currentSyncIntervalMs * jitter;
1235
+ }
1236
+ /**
1237
+ * Record that "activity" happened right now, for `idleBackoffEnabled`'s
1238
+ * benefit. Cheap (one timestamp write) and called unconditionally
1239
+ * regardless of whether idle backoff is enabled, so there's no behavioral
1240
+ * branch to keep in sync - the backoff decision in connect()'s periodic
1241
+ * tick is the only place that actually reads this.
1242
+ *
1243
+ * Call sites are deliberately NOT "any inbound wire message" - an earlier
1244
+ * version of this hooked `_handleIncomingMessage()` unconditionally, which
1245
+ * made the periodic tick's OWN SyncStep1 request and the SyncStep2 reply
1246
+ * answering it (empty payload - nothing to sync) each count as "activity",
1247
+ * permanently resetting the backoff on every single tick and making the
1248
+ * whole feature a no-op (caught by this bench script's own first run: ON
1249
+ * and OFF produced statistically indistinguishable message counts). Both
1250
+ * Yjs's `doc.emit('update', ...)` and y-protocols' `awareness.emit('update', ...)`
1251
+ * already only fire when something with actual content changed
1252
+ * (`hasContent`/non-empty added+updated+removed - confirmed by reading
1253
+ * yjs's `Transaction.js` and y-protocols' `awareness.js` directly), so
1254
+ * hooking THOSE instead is exactly "local or remote document/awareness
1255
+ * change" with no extra filtering needed - a no-op SyncStep2 reply, a
1256
+ * digest beacon, or a duplicate/no-change awareness re-announce (e.g. a
1257
+ * JOIN-triggered presence response that changed nothing) never reaches
1258
+ * these handlers. A corrupted (CRC32
1259
+ * mismatch) message is real evidence of wire activity that neither
1260
+ * handler would ever see (it's rejected before decoding) - see the
1261
+ * explicit call in `_processWrappedMessage()`'s corruption branch.
1262
+ *
1263
+ * Call sites: `_setupDocumentSync()`'s update handler (LOCAL document
1264
+ * edits only, since phase 1e) and `_processWrappedMessage()`'s
1265
+ * corrupted-message branch (wire noise, not silence).
1266
+ */
1267
+ _markActivity() {
1268
+ this._lastActivityTime = Date.now();
1269
+ // A backed-off periodic timer is re-armed at the base interval NOW, not
1270
+ // at its next tick (which may be idleBackoffMaxMs away). Design D of
1271
+ // the phase-1d doc rests on it: the peer that just had activity beacons
1272
+ // within one base interval, and a peer that lost that activity's
1273
+ // message learns from that beacon that it is behind (design A) - its
1274
+ // own backed-off interval no longer bounds the recovery. Measured in
1275
+ // test/dummy/bench-idle-backoff.ts. At most once per idle stretch.
1276
+ if (this._idleBackoffEnabled &&
1277
+ this._currentSyncIntervalMs !== this._syncInterval &&
1278
+ this._syncIntervalId !== undefined &&
1279
+ this._periodicScheduler !== undefined) {
1280
+ clearTimeout(this._syncIntervalId);
1281
+ this._currentSyncIntervalMs = this._syncInterval;
1282
+ // Phase 1e: at a random point inside the base interval, not a full
1283
+ // one (the phase-1d note): the loser's recovery chain starts with
1284
+ // this beacon - so Trickle never suppresses it (round 5, item 3).
1285
+ this._beaconForced = true;
1286
+ this._periodicScheduler(Math.random() * this._syncInterval);
1287
+ }
1288
+ }
1289
+ /**
1290
+ * The lease sweep: y-protocols' own (awareness.js `_checkInterval`: renew
1291
+ * at outdatedTimeout/2, remove at outdatedTimeout, every
1292
+ * outdatedTimeout/10) replaced by the same loop at `_awarenessTimeoutMs`,
1293
+ * the period jittered so a room that joined together does not renew in
1294
+ * one burst (measured: all 49 listeners of a 50-peer room renewed inside
1295
+ * the same 10 s window). The renew/remove half runs only on an awareness
1296
+ * we created (`_ownsAwareness`); the peer-table prune (round 7, item 3)
1297
+ * runs regardless. Armed by connect(), cleared by disconnect() - round 7,
1298
+ * item 2: started from the constructor it outlived disconnect(), ticking
1299
+ * ~20 times a minute and keeping the dropped provider reachable
1300
+ * (test/dummy/bench-reload-phantoms.ts, part 2).
1301
+ */
1302
+ _startAwarenessSweep() {
1303
+ if (this._awarenessSweepId !== undefined)
1304
+ return;
1305
+ const lease = this._awarenessTimeoutMs;
1306
+ const arm = () => {
1307
+ this._awarenessSweepId = setTimeout(tick, (lease / 10) * (0.8 + Math.random() * 0.4));
1308
+ };
1309
+ const tick = () => {
1310
+ const now = Date.now();
1311
+ if (this._ownsAwareness) {
1312
+ const mine = this.awareness.meta.get(this.doc.clientID);
1313
+ if (this.awareness.getLocalState() !== null &&
1314
+ mine !== undefined &&
1315
+ lease / 2 <= now - mine.lastUpdated) {
1316
+ this.awareness.setLocalState(this.awareness.getLocalState()); // renew: bumps the clock
1317
+ this._broadcastAwareness([this.doc.clientID]); // 'change' does not fire for an equal state (item 8)
1318
+ }
1319
+ const remove = [];
1320
+ this.awareness.meta.forEach((meta, clientID) => {
1321
+ if (clientID !== this.doc.clientID &&
1322
+ lease <= now - meta.lastUpdated &&
1323
+ this.awareness.getStates().has(clientID)) {
1324
+ remove.push(clientID);
1325
+ }
1326
+ });
1327
+ if (remove.length > 0) {
1328
+ awarenessProtocol.removeAwarenessStates(this.awareness, remove, 'timeout');
1329
+ }
1330
+ }
1331
+ // Round 7, item 3: forget peers not heard from for a lease. A live
1332
+ // peer is heard at least every lease/2 through its presence renewal
1333
+ // (every receiver scans it, above at MESSAGE_AWARENESS); a pruned
1334
+ // peer that speaks again is simply learned again. Measured before
1335
+ // this: 30 reloads in a 20-peer relay room left 49 known peers for
1336
+ // good, and every cursor moved at a 50-peer room's 'auto' interval
1337
+ // (test/dummy/bench-reload-phantoms.ts). awareness.meta is left to
1338
+ // y-protocols: it holds the clock a late message is checked against.
1339
+ for (const [id, heardAt] of this._knownPeers) {
1340
+ if (lease <= now - heardAt) {
1341
+ this._knownPeers.delete(id);
1342
+ this._peerAddress.delete(id);
1343
+ this._remoteSeqInfo.delete(id);
1344
+ }
552
1345
  }
553
- // Send sync request to get updates from others
554
- this._writeSyncStep1();
1346
+ // Re-arm only while connected: a disconnect() from inside a listener
1347
+ // above has just cleared the timer, and must stay cleared.
1348
+ if (!this._destroying && this._status.state === 'connected')
1349
+ arm();
1350
+ else
1351
+ this._awarenessSweepId = undefined;
1352
+ };
1353
+ arm();
1354
+ }
1355
+ /**
1356
+ * A digest, verified update or ack from `clientID` (or one we are about
1357
+ * to send, for our own id) is proof of presence: refresh the lease the
1358
+ * sweep above checks. Only for ids with a state - a departed peer's
1359
+ * `meta` entry survives its removal (y-protocols keeps it for the clock)
1360
+ * and must not be revived by a late message.
1361
+ */
1362
+ _touchPeer(clientID) {
1363
+ const meta = this.awareness.meta.get(clientID);
1364
+ if (meta !== undefined && this.awareness.getStates().has(clientID)) {
1365
+ meta.lastUpdated = Date.now();
1366
+ }
1367
+ }
1368
+ /**
1369
+ * Transport.onPeerDisconnect: the peer at `peerId` is gone. Forget its
1370
+ * address and id, drop its awareness state with origin 'peer-left': the
1371
+ * broadcast goes through the same suppression as a timeout removal, but
1372
+ * with a long window - every peer gets the leave signal in the same
1373
+ * millisecond, and at the reply window (~170 ms at N=50) 28 of 49
1374
+ * survivors broadcast before the first broadcast could be overheard
1375
+ * (bench-awareness-removal-burst, DUMMY_PEER_EVENTS=1). One broadcast
1376
+ * room-wide is still worth having: it corrects a joiner that received
1377
+ * this peer in a relayed presence table but had no channel to it yet.
1378
+ */
1379
+ _handlePeerLeave(peerId) {
1380
+ const gone = [];
1381
+ for (const [clientID, address] of this._peerAddress) {
1382
+ if (address === peerId)
1383
+ gone.push(clientID);
1384
+ }
1385
+ for (const id of gone) {
1386
+ this._peerAddress.delete(id);
1387
+ this._knownPeers.delete(id);
1388
+ }
1389
+ const present = gone.filter((id) => this.awareness.getStates().has(id));
1390
+ if (present.length > 0) {
1391
+ awarenessProtocol.removeAwarenessStates(this.awareness, present, 'peer-left');
1392
+ }
1393
+ }
1394
+ /** Cached delete-set hash - see computeDeleteSetHash(). */
1395
+ _deleteSetHash() {
1396
+ if (this._dsHashCache === null) {
1397
+ this._dsHashCache = computeDeleteSetHash(this.doc);
555
1398
  }
556
- // Broadcast current awareness state - independently throttled and much
557
- // cheaper than a full document push, so it isn't gated by the sync
558
- // rate limiter above even when the sync half is skipped.
559
- this._broadcastAwareness([this.doc.clientID]);
560
- this._broadcastAwareness([this.doc.clientID], AWARENESS_CHANNEL_APP);
1399
+ return this._dsHashCache;
1400
+ }
1401
+ /**
1402
+ * Debounce onPeerConnect-triggered syncNow() calls. A burst of connect
1403
+ * events within `_peerConnectDebounceMs` collapses into one call instead
1404
+ * of one per event - without this, N peers joining a mesh in a short
1405
+ * window each independently broadcast full state to everyone already
1406
+ * connected (O(N^2) traffic), since onPeerConnect fires once per
1407
+ * newly-opened peer connection with no coalescing of its own.
1408
+ */
1409
+ _schedulePeerConnectSync(peerId) {
1410
+ if (peerId !== undefined)
1411
+ this._pendingPeerConnectIds.add(peerId);
1412
+ if (this._pendingPeerConnectSyncTimeoutId !== undefined)
1413
+ return;
1414
+ this._pendingPeerConnectSyncTimeoutId = setTimeout(() => {
1415
+ this._pendingPeerConnectSyncTimeoutId = undefined;
1416
+ const ids = Array.from(this._pendingPeerConnectIds);
1417
+ this._pendingPeerConnectIds.clear();
1418
+ if (!this.transport.isConnected || this._destroying)
1419
+ return;
1420
+ if (typeof this.transport.sendTo === 'function' && ids.length > 0) {
1421
+ // Round 5, item 5: one plain beacon to each new peer, nothing to
1422
+ // the rest of the mesh. Not rate-limited as a request: it answers
1423
+ // a channel that just opened, and the peer's own JOIN beacon is
1424
+ // the fallback if it is lost.
1425
+ const beacon = wrapMessageWithChecksum(this._encodeSyncStep1(0));
1426
+ for (const id of ids)
1427
+ this._sendToTransport(beacon, id);
1428
+ return;
1429
+ }
1430
+ this._syncNow(0);
1431
+ }, this._peerConnectDebounceMs);
561
1432
  }
562
1433
  /**
563
1434
  * Setup automatic document synchronization.
@@ -566,27 +1437,36 @@ export class GenericProvider extends Observable {
566
1437
  */
567
1438
  _setupDocumentSync() {
568
1439
  this._updateHandler = (update, origin) => {
569
- // Don't send updates that originated from this provider
570
- // This prevents infinite loops when receiving updates
571
- if (origin === this)
572
- return;
573
- // Don't send updates from excluded origins (local-only txns)
574
- if (this._excludeOrigins.has(origin))
575
- return;
576
- if (this._batchUpdates > 0) {
577
- // Batch mode: merge updates and debounce
1440
+ this._dsHashCache = null;
1441
+ this._equalBeaconsHeard = 0; // our digest changed - see the Trickle fields
1442
+ // Fires for BOTH local edits and remotely-applied updates (the latter
1443
+ // go through doc.transact with origin=this) - see _markActivity()'s
1444
+ // doc comment. Phase 1e: only a LOCAL edit counts as activity for
1445
+ // idle backoff - a listener has nothing a beacon would announce, and
1446
+ // the typist's base-interval beacon heals any listener that lost the
1447
+ // keystroke (phase 1d design A). Before this, one typist kept all N
1448
+ // peers at the base cadence: N*(N-1) deliveries per interval against
1449
+ // N-1 per keystroke.
1450
+ // Don't send updates that originated from this provider (received
1451
+ // from the wire) or from the local load connect() is waiting for.
1452
+ if (origin !== this && !this._loading) {
1453
+ // Local-only transaction origins (e.g. a rollback the peer must not
1454
+ // replicate) never reach the transport.
1455
+ if (this._excludeOrigins.has(origin))
1456
+ return;
1457
+ this._markActivity();
1458
+ // 'pull' replicas answer requests but never push unasked.
1459
+ if (this._syncMode === 'pull')
1460
+ return;
578
1461
  this._batchUpdate(update);
579
1462
  }
580
- else {
581
- // Immediate mode: send right away
582
- this._sendUpdate(update);
583
- }
584
1463
  };
585
1464
  this.doc.on('update', this._updateHandler);
586
1465
  }
587
1466
  /**
588
- * Batch/debounce updates to reduce network traffic.
589
- * Merges multiple updates and sends after delay.
1467
+ * Merge a local update into the pending batch and schedule its flush:
1468
+ * after `batchUpdates` ms (debounced) when that is > 0, otherwise at the
1469
+ * end of the current task via queueMicrotask - see `_pendingUpdate`.
590
1470
  */
591
1471
  _batchUpdate(update) {
592
1472
  // Merge with pending update if exists
@@ -606,18 +1486,38 @@ export class GenericProvider extends Observable {
606
1486
  else {
607
1487
  this._pendingUpdate = update;
608
1488
  }
609
- // Clear existing timeout
1489
+ if (this._batchUpdates > 0) {
1490
+ // Debounce: restart the timer on every update.
1491
+ if (this._batchTimeoutId !== undefined)
1492
+ clearTimeout(this._batchTimeoutId);
1493
+ this._batchTimeoutId = setTimeout(() => {
1494
+ this._batchTimeoutId = undefined;
1495
+ this._flushPendingUpdate();
1496
+ }, this._batchUpdates);
1497
+ this._flushScheduled = true;
1498
+ return;
1499
+ }
1500
+ if (this._flushScheduled)
1501
+ return;
1502
+ this._flushScheduled = true;
1503
+ queueMicrotask(() => this._flushPendingUpdate());
1504
+ }
1505
+ /**
1506
+ * Send the pending update batch as one wire message, carrying any
1507
+ * awareness change the throttle is holding (see _takePendingAwareness).
1508
+ * Shared by the microtask flush, the timed flush, and the
1509
+ * disconnect()/destroy() flush.
1510
+ */
1511
+ _flushPendingUpdate() {
1512
+ this._flushScheduled = false;
610
1513
  if (this._batchTimeoutId !== undefined) {
611
1514
  clearTimeout(this._batchTimeoutId);
612
- }
613
- // Set new timeout to send after delay
614
- this._batchTimeoutId = setTimeout(() => {
615
- if (this._pendingUpdate) {
616
- this._sendUpdate(this._pendingUpdate);
617
- this._pendingUpdate = null;
618
- }
619
1515
  this._batchTimeoutId = undefined;
620
- }, this._batchUpdates);
1516
+ }
1517
+ const update = this._pendingUpdate;
1518
+ this._pendingUpdate = null;
1519
+ if (update && this.transport.isConnected)
1520
+ this._sendUpdate(update);
621
1521
  }
622
1522
  /**
623
1523
  * Setup automatic awareness synchronization.
@@ -625,21 +1525,85 @@ export class GenericProvider extends Observable {
625
1525
  */
626
1526
  _setupAwarenessSync() {
627
1527
  this._awarenessUpdateHandler = ({ added, updated, removed, }, origin) => {
628
- // Broadcast awareness changes (unless they came from remote)
1528
+ // Not _markActivity(): awareness changes are not something a beacon
1529
+ // announces (phase 1e; see the idleBackoffEnabled option).
1530
+ // Broadcast awareness changes, UNLESS they came from remote (this
1531
+ // comment described the intent since the very first commit, but the
1532
+ // actual origin check was never implemented until now - confirmed by
1533
+ // `git log -p` on this handler). `origin === this` is exactly the
1534
+ // signature the MESSAGE_AWARENESS handler stamps on an update applied
1535
+ // from an incoming wire message (`applyAwarenessUpdate(this.awareness,
1536
+ // ..., this)`, below). Every transport this project targets is a
1537
+ // full-room relay (websocket/pubnub/gun/matrix/ably/supabase) or a
1538
+ // full mesh (peerjs/simple-peer/trystero - see CLAUDE.md), so the
1539
+ // sender's own broadcast already reached every other peer directly;
1540
+ // re-broadcasting it here on receipt is pure redundant traffic that
1541
+ // compounds across every OTHER receiver doing the same thing.
1542
+ // Verified with a throwaway probe: a single awareness field change in
1543
+ // an N-peer room cost N*(N-1) wire deliveries before this check (one
1544
+ // echo per receiver, each reaching N-1 peers) vs. N-1 after - e.g.
1545
+ // N=20: 380 -> 19.
1546
+ //
1547
+ // Carve-out: `applyAwarenessUpdate` has its own defense against a
1548
+ // remote peer incorrectly removing OUR OWN state (a stale/racy
1549
+ // timeout-removal from someone else's clock) - it bumps our clock
1550
+ // instead of deleting our state, but (a quirk of that function) still
1551
+ // reports it via `removed` including our own clientID. That specific
1552
+ // case must still be broadcast so the room's stale belief that we're
1553
+ // gone gets corrected promptly, instead of only self-healing on our
1554
+ // next unrelated state change/renewal (up to ~15s later).
1555
+ if (origin === this) {
1556
+ // An incoming removal might be the SAME departure we have a
1557
+ // suppressed broadcast queued for (see _scheduleAwarenessRemoval())
1558
+ // - someone else already told the room, drop ours.
1559
+ if (removed.length > 0) {
1560
+ this._cancelPendingAwarenessRemovalIfOverlaps(removed);
1561
+ }
1562
+ if (!removed.includes(this.awareness.clientID)) {
1563
+ return;
1564
+ }
1565
+ }
629
1566
  const changedClients = added.concat(updated).concat(removed);
1567
+ // A pure timeout-triggered removal ('timeout' is the exact origin
1568
+ // string y-protocols/awareness.js's own _checkInterval passes to
1569
+ // removeAwarenessStates()) is redundant across the whole room: every
1570
+ // OTHER connected peer runs the identical 30s-timeout sweep
1571
+ // independently, so all of them detect and would broadcast the SAME
1572
+ // departure within the same ~3s tick - measured as O(N-1) broadcasts
1573
+ // / O((N-1)(N-2)) deliveries for ONE departure in
1574
+ // test/dummy/bench-awareness-removal-burst.ts (e.g. N=50: 2352
1575
+ // deliveries). Delay + drop-if-overheard, exactly mirroring
1576
+ // _scheduleSyncReply()/_cancelPendingSyncReply()'s suppression of
1577
+ // redundant SyncStep2 replies. Deliberately NOT applied to
1578
+ // 'disconnect'/'window unload' removals (a single broadcaster, not
1579
+ // redundant) or to added/updated clients (every sender's
1580
+ // cursor/presence data is meaningfully different and must never be
1581
+ // suppressed).
1582
+ if (origin === 'timeout' || origin === 'peer-left') {
1583
+ this._scheduleAwarenessRemoval(changedClients, origin);
1584
+ return;
1585
+ }
630
1586
  this._broadcastAwareness(changedClients);
631
1587
  };
632
- this.awareness.on('update', this._awarenessUpdateHandler);
633
- this._appAwarenessUpdateHandler = ({ added, updated, removed, }, _origin) => {
634
- const changedClients = added.concat(updated).concat(removed);
635
- this._broadcastAwareness(changedClients, AWARENESS_CHANNEL_APP);
636
- };
637
- this.appAwareness.on('update', this._appAwarenessUpdateHandler);
1588
+ // Round 5, item 8: broadcast on 'change' (y-protocols filters updates
1589
+ // whose state deep-equals the previous one) rather than 'update'
1590
+ // (every setLocalState call) - an app that re-sets unchanged state no
1591
+ // longer costs a broadcast per call. The renewal is the one
1592
+ // equal-state update that must go out; the provider's own sweep sends
1593
+ // it explicitly (_startAwarenessSweep). With an app-supplied Awareness
1594
+ // y-protocols' own sweep renews through 'update' only, so that case
1595
+ // keeps listening on 'update'.
1596
+ this.awareness.on(this._ownsAwareness ? 'change' : 'update', this._awarenessUpdateHandler);
1597
+ // App channel: attached here only if one was supplied to the
1598
+ // constructor; otherwise the getter attaches on first use. Reading
1599
+ // `this.appAwareness` here would defeat the lazy construction.
1600
+ if (this._appAwareness) {
1601
+ this._attachAppAwareness(this._appAwareness);
1602
+ }
638
1603
  // Cleanup: mark as offline and disconnect BC when page unloads
639
1604
  if (typeof window !== 'undefined') {
640
1605
  this._beforeUnloadHandler = () => {
641
1606
  awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'window unload');
642
- awarenessProtocol.removeAwarenessStates(this.appAwareness, [this.doc.clientID], 'window unload');
643
1607
  // Disconnect BroadcastChannel to notify other tabs
644
1608
  this._disconnectBroadcastChannel();
645
1609
  };
@@ -647,17 +1611,81 @@ export class GenericProvider extends Observable {
647
1611
  }
648
1612
  }
649
1613
  /**
650
- * Handle incoming messages from the transport.
651
- * Verifies message integrity with CRC32 before processing.
652
- * Corrupt messages are rejected immediately without attempting to decode.
1614
+ * Handle incoming messages from the transport (or BroadcastChannel).
1615
+ *
1616
+ * When compressionThresholdBytes is disabled (the default), this is a
1617
+ * fully synchronous fast path, byte-for-byte the same behavior as before
1618
+ * that option existed: straight into _processWrappedMessage().
1619
+ *
1620
+ * When enabled, every message - from the network transport AND from
1621
+ * BroadcastChannel (see _send()) - carries a leading compressed(1)/
1622
+ * uncompressed(0) flag byte ahead of the usual CRC32 wrapper. Reading
1623
+ * that flag and, if set, decompressing is inherently async (the
1624
+ * Compression Streams API has no synchronous form), so this method
1625
+ * dispatches to a promise chain instead of processing inline in that
1626
+ * case. This means a large (compressed) message and a small (uncompressed
1627
+ * or below-threshold) message that arrive back-to-back can finish
1628
+ * processing out of arrival order - acceptable here because Yjs updates
1629
+ * are idempotent/commutative (see MESSAGE_SYNC_VERIFIED's handling below)
1630
+ * and because the compression threshold keeps this path almost entirely
1631
+ * to large, full-state syncs, not the per-keystroke incremental updates
1632
+ * that per-sender gap detection actually relies on ordering-sensitive
1633
+ * heuristics for.
1634
+ */
1635
+ _handleIncomingMessage(data, from) {
1636
+ if (!this._compressionThresholdBytes) {
1637
+ this._processWrappedMessage(data, from);
1638
+ return;
1639
+ }
1640
+ if (data.length < 1) {
1641
+ console.warn('[GenericProvider] Dropping empty message (missing compression flag byte)');
1642
+ return;
1643
+ }
1644
+ const flag = data[0];
1645
+ const rest = data.subarray(1);
1646
+ if (flag === 0) {
1647
+ this._processWrappedMessage(rest, from);
1648
+ return;
1649
+ }
1650
+ if (!COMPRESSION_AVAILABLE) {
1651
+ console.warn('[GenericProvider] Received a compressed message but this runtime has no DecompressionStream - dropping it.');
1652
+ return;
1653
+ }
1654
+ decompressDeflateRaw(rest)
1655
+ .then((wrapped) => this._processWrappedMessage(wrapped, from))
1656
+ .catch((error) => {
1657
+ // Treat decompression failure the same as a CRC32 mismatch on the
1658
+ // uncompressed path: request a resync rather than silently dropping.
1659
+ console.warn('[GenericProvider] Failed to decompress incoming message, treating as corrupted:', error);
1660
+ this._requestResync();
1661
+ });
1662
+ }
1663
+ /**
1664
+ * Verify message integrity with CRC32 and decode. Corrupt messages are
1665
+ * rejected immediately without attempting to decode. Operates on bytes
1666
+ * that have already had any compression flag/decompression handled by
1667
+ * _handleIncomingMessage() - this is the pre-compression-feature
1668
+ * implementation, unchanged.
653
1669
  */
654
- _handleIncomingMessage(data) {
1670
+ _processWrappedMessage(data, from) {
655
1671
  // Verify message integrity with CRC32 checksum
656
1672
  const message = unwrapAndVerifyMessage(data);
657
1673
  if (message === null) {
658
- // Message is corrupted - reject it immediately
1674
+ // Message is corrupted - reject it immediately. Note: if this was a
1675
+ // MESSAGE_BATCH envelope, the CRC32 wrap covers the WHOLE batch, so a
1676
+ // single corrupted bit here loses every sub-message it contained, not
1677
+ // just one - a deliberate tradeoff of batching multiple logical
1678
+ // messages behind one wire message/one checksum. See _sendBatch()'s
1679
+ // doc comment for why this was chosen over per-sub-message checksums,
1680
+ // and this project's benchmark suite (bench-corruption-storm.ts,
1681
+ // bench-packet-loss.ts) for how that tradeoff was measured.
659
1682
  console.warn(`[GenericProvider] 💥 Corrupted message rejected: CRC32 checksum mismatch. ` +
660
1683
  `This is expected if data corruption simulation is enabled.`);
1684
+ // Wire noise, not silence - counts as activity for idleBackoffEnabled
1685
+ // even though nothing here reaches the doc/awareness update handlers
1686
+ // (the message is rejected before decoding). See _markActivity()'s
1687
+ // doc comment.
1688
+ this._markActivity();
661
1689
  // Request re-sync to recover any lost data - routed through the
662
1690
  // shared coordinator so this doesn't stack an independent timer on
663
1691
  // top of any hash-mismatch/gap-confirmed resync already pending.
@@ -666,168 +1694,7 @@ export class GenericProvider extends Observable {
666
1694
  }
667
1695
  // Message integrity verified - safe to decode
668
1696
  try {
669
- const decoder = decoding.createDecoder(message);
670
- const messageType = decoding.readVarUint(decoder);
671
- switch (messageType) {
672
- case MESSAGE_SYNC: {
673
- const encoder = encoding.createEncoder();
674
- encoding.writeVarUint(encoder, MESSAGE_SYNC);
675
- const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this);
676
- if (syncMessageType === syncProtocol.messageYjsSyncStep2) {
677
- // If we received SyncStep2, we're synced
678
- if (!this._synced) {
679
- this._synced = true;
680
- this.emit('synced', [true]);
681
- }
682
- // Someone else's SyncStep2 reply just arrived - our own pending
683
- // reply (if any) is now most likely redundant.
684
- this._cancelPendingSyncReply();
685
- }
686
- // Send reply if needed. Suppression only engages with genuine
687
- // redundancy (>=2 other known peers via awareness) - below that,
688
- // there's no "someone else" to rely on, so reply immediately
689
- // (still rate-limited via _sendSyncReply() as a hard backstop).
690
- if (encoding.length(encoder) > 1) {
691
- if (this.awareness.getStates().size >= 3) {
692
- this._scheduleSyncReply(encoding.toUint8Array(encoder));
693
- }
694
- else {
695
- this._sendSyncReply(encoding.toUint8Array(encoder));
696
- }
697
- }
698
- break;
699
- }
700
- case MESSAGE_AWARENESS: {
701
- const channel = decoding.readVarUint(decoder);
702
- awarenessProtocol.applyAwarenessUpdate(channel === AWARENESS_CHANNEL_APP
703
- ? this.appAwareness
704
- : this.awareness, decoding.readVarUint8Array(decoder), this);
705
- break;
706
- }
707
- case MESSAGE_PUBSUB: {
708
- // Read topic
709
- const topic = decoding.readVarString(decoder);
710
- // Read message payload
711
- const payloadBytes = decoding.readVarUint8Array(decoder);
712
- try {
713
- // Decode JSON payload
714
- const decoder = new TextDecoder();
715
- const payloadStr = decoder.decode(payloadBytes);
716
- const message = JSON.parse(payloadStr);
717
- // Emit to pubsub channel
718
- this.pubsub._handleMessage(topic, message);
719
- }
720
- catch (error) {
721
- console.error('Error decoding pub/sub message:', error);
722
- }
723
- break;
724
- }
725
- case MESSAGE_PUBSUB_TARGETED: {
726
- const target = decoding.readVarString(decoder);
727
- const topic = decoding.readVarString(decoder);
728
- const payloadBytes = decoding.readVarUint8Array(decoder);
729
- // Drop messages aimed at someone else (broadcast-and-filter path).
730
- if (this._localId !== undefined && target !== this._localId) {
731
- break;
732
- }
733
- try {
734
- const message = JSON.parse(new TextDecoder().decode(payloadBytes));
735
- this.pubsub._handleMessage(topic, message);
736
- }
737
- catch (error) {
738
- console.error('Error decoding targeted pub/sub message:', error);
739
- }
740
- break;
741
- }
742
- case MESSAGE_SYNC_VERIFIED: {
743
- // Sync message with sequence number and hash verification
744
- // Read sequence number and clientID first
745
- const seqNum = decoding.readVarUint(decoder);
746
- const senderClientID = decoding.readVarUint(decoder);
747
- // Track for gap detection only — does NOT gate whether we apply
748
- // the update below (see _trackRemoteSeq() for why).
749
- this._trackRemoteSeq(senderClientID, seqNum);
750
- // Always apply the update. Yjs updates are idempotent/commutative,
751
- // so re-applying an already-seen update is a harmless no-op.
752
- // Under reordering, a merely-late (not actually duplicate) update
753
- // must still be applied here — the old "skip if seqNum <= last
754
- // seen" logic silently dropped such updates forever whenever a
755
- // later-numbered message happened to arrive first.
756
- const encoder = encoding.createEncoder();
757
- encoding.writeVarUint(encoder, MESSAGE_SYNC);
758
- const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this);
759
- // Someone else's SyncStep2 reply just arrived - our own pending
760
- // reply (if any) is now most likely redundant. Mirrors the
761
- // MESSAGE_SYNC case: the reply encoded above is always a plain
762
- // MESSAGE_SYNC-typed message regardless of which message type
763
- // triggered it, so the same suppression scheme applies here too.
764
- if (syncMessageType === syncProtocol.messageYjsSyncStep2) {
765
- this._cancelPendingSyncReply();
766
- }
767
- // Read the expected hash from sender (signed integer)
768
- const expectedHash = decoding.readVarInt(decoder);
769
- // Compute our local hash after applying the update
770
- const localHash = computeDocHash(this.doc);
771
- // Verify hash match
772
- if (localHash !== expectedHash) {
773
- // If we already know this sender has a suspected reordering gap
774
- // (see _trackRemoteSeq()/_scheduleGapCheck()), a hash mismatch
775
- // right now is the *expected* transient state — we're missing a
776
- // piece that's very likely still in flight, not actually
777
- // diverged. Let the pending gap-check grace period resolve it
778
- // instead of also escalating the hash-mismatch backoff: under
779
- // heavy reordering this previously caused a burst of mismatches
780
- // to rack up the exponential backoff to its 10s cap within a
781
- // single edit burst, purely from timing, not real divergence.
782
- // A hash mismatch with NO pending gap (in-order, but still
783
- // wrong) is not explained by reordering and still escalates
784
- // normally below.
785
- const reorderingSuspected = this._gapCheckTimers.has(senderClientID);
786
- if (!reorderingSuspected) {
787
- // Push our full state AND request theirs (syncNow() does
788
- // both). A hash mismatch means the two peers have diverged -
789
- // one side may have edits the other lacks. Routed through the
790
- // shared coordinator so this doesn't stack an independent
791
- // timer on top of any corrupted-message/gap-confirmed resync
792
- // already pending.
793
- this._requestResync();
794
- // Logged with the shared attempt counter (kept as "#N" for
795
- // compatibility with existing tooling/benchmarks that grep
796
- // for this exact "Hash mismatch #" pattern) - it now reflects
797
- // the unified resync-attempt count rather than a
798
- // hash-mismatch-specific one, since the two escalation
799
- // counters were merged.
800
- console.warn(`[GenericProvider] Hash mismatch #${this._resyncAttemptCount} detected! Local: ${localHash}, Expected: ${expectedHash}`);
801
- }
802
- }
803
- // If we received SyncStep2, we're synced (unless hash mismatched)
804
- if (syncMessageType === syncProtocol.messageYjsSyncStep2 &&
805
- !this._synced &&
806
- localHash === expectedHash) {
807
- this._synced = true;
808
- this.emit('synced', [true]);
809
- }
810
- // Send reply if needed (as standard MESSAGE_SYNC). Suppression
811
- // only engages with genuine redundancy (>=2 other known peers via
812
- // awareness) - below that, reply immediately (still rate-limited
813
- // via _sendSyncReply() as a hard backstop). Matches the
814
- // MESSAGE_SYNC case's gate exactly; without this, a hash-mismatch
815
- // resync burst under packet loss bypassed suppression entirely,
816
- // since every peer answering a post-mismatch SyncStep1 replied
817
- // immediately via this path.
818
- if (encoding.length(encoder) > 1) {
819
- if (this.awareness.getStates().size >= 3) {
820
- this._scheduleSyncReply(encoding.toUint8Array(encoder));
821
- }
822
- else {
823
- this._sendSyncReply(encoding.toUint8Array(encoder));
824
- }
825
- }
826
- break;
827
- }
828
- default:
829
- console.warn('Unknown message type:', messageType);
830
- }
1697
+ this._dispatchMessage(message, from);
831
1698
  }
832
1699
  catch (error) {
833
1700
  // This should only happen for logic errors, not corruption
@@ -836,35 +1703,609 @@ export class GenericProvider extends Observable {
836
1703
  }
837
1704
  }
838
1705
  /**
839
- * Schedule a SyncStep2 reply after a short random delay instead of
840
- * sending immediately. If another peer's reply is overheard in the
841
- * meantime (`_cancelPendingSyncReply`), this reply is dropped as
842
- * redundant - the requester likely already got what it needed.
843
- *
844
- * A reply that is already pending when this is called answers a
845
- * *different* SyncStep1 request (e.g. peer A's request, followed 5ms
846
- * later by peer B's) - it must not be silently overwritten by the new
847
- * one. Flush it immediately, then schedule the new reply fresh. The only
848
- * sanctioned way a reply gets dropped is `_cancelPendingSyncReply()`,
849
- * because we overheard someone else's SyncStep2 for the SAME request.
1706
+ * Decode and act on one already-integrity-verified, already-decompressed
1707
+ * message. Split out of `_processWrappedMessage()` so `MESSAGE_BATCH`
1708
+ * (see `_sendBatch()`) can recurse into this for each sub-message it
1709
+ * unwraps, running the EXACT SAME per-message-type logic used for a
1710
+ * top-level message rather than a parallel reimplementation. A thrown
1711
+ * error partway through a batch's sub-messages aborts the REST of that
1712
+ * batch (propagates up to `_processWrappedMessage()`'s catch) - same as
1713
+ * a logic error aborting a single top-level message today, just now
1714
+ * scoped to "the rest of this batch" instead of "this one message".
850
1715
  */
851
- _scheduleSyncReply(reply) {
852
- if (this._pendingSyncReplyTimeoutId !== undefined) {
853
- if (this._pendingSyncReply) {
854
- this._sendSyncReply(this._pendingSyncReply);
1716
+ _dispatchMessage(message, from) {
1717
+ const decoder = decoding.createDecoder(message);
1718
+ const messageType = decoding.readVarUint(decoder);
1719
+ switch (messageType) {
1720
+ case MESSAGE_BATCH: {
1721
+ // Payload is N length-prefixed sub-messages (writeVarUint8Array
1722
+ // per sub-message, mirroring MESSAGE_AWARENESS's own framing).
1723
+ // Each was NOT individually CRC32-wrapped - see _sendBatch()'s doc
1724
+ // comment - so just decode and dispatch each one directly.
1725
+ while (decoding.hasContent(decoder)) {
1726
+ const subMessage = decoding.readVarUint8Array(decoder);
1727
+ this._dispatchMessage(subMessage, from);
1728
+ }
1729
+ break;
855
1730
  }
856
- clearTimeout(this._pendingSyncReplyTimeoutId);
857
- this._pendingSyncReplyTimeoutId = undefined;
858
- }
859
- this._pendingSyncReply = reply;
860
- const delay = Math.random() * this._syncReplySuppressionMs;
861
- this._pendingSyncReplyTimeoutId = setTimeout(() => {
862
- this._pendingSyncReplyTimeoutId = undefined;
863
- if (this._pendingSyncReply) {
864
- this._sendSyncReply(this._pendingSyncReply);
865
- this._pendingSyncReply = null;
1731
+ case MESSAGE_SYNC_DIGEST: {
1732
+ this._handleDigest(decoder, from);
1733
+ break;
866
1734
  }
867
- }, delay);
1735
+ case MESSAGE_SYNC_PUSH: {
1736
+ // Somebody's whole document: apply it, nothing else (see the
1737
+ // constant's comment for why no hash check and no synced flip).
1738
+ Y.applyUpdate(this.doc, decoding.readVarUint8Array(decoder), this);
1739
+ this._checkPendingAfterReply();
1740
+ break;
1741
+ }
1742
+ case MESSAGE_SYNC: {
1743
+ const encoder = encoding.createEncoder();
1744
+ encoding.writeVarUint(encoder, MESSAGE_SYNC);
1745
+ const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this);
1746
+ if (syncMessageType === syncProtocol.messageYjsSyncStep2) {
1747
+ // If we received SyncStep2, we're synced (and confirmed: we have
1748
+ // heard from a peer that had data for us)
1749
+ this._confirmed = true;
1750
+ this._noteResponse(true);
1751
+ this._markSynced();
1752
+ // Design E (phase 1d): a SyncStep2 carries its encoder's own pending
1753
+ // structs, so a responder missing a struct hands us the same hole.
1754
+ // Our response wait just ended - make sure something still asks.
1755
+ this._checkPendingAfterReply();
1756
+ // Someone else's SyncStep2 reply just arrived - our own pending
1757
+ // reply (if any) is now most likely redundant.
1758
+ this._cancelPendingSyncReply();
1759
+ }
1760
+ // Send reply if needed. Suppression only engages with genuine
1761
+ // redundancy (>=2 other known peers via awareness) - below that,
1762
+ // there's no "someone else" to rely on, so reply immediately
1763
+ // (still rate-limited via _sendSyncReply() as a hard backstop).
1764
+ if (encoding.length(encoder) > 1) {
1765
+ this._replyToSyncRequest(encoding.toUint8Array(encoder));
1766
+ }
1767
+ break;
1768
+ }
1769
+ case MESSAGE_AWARENESS: {
1770
+ const payload = decoding.readVarUint8Array(decoder);
1771
+ // Someone else's removal broadcast just arrived - cancel our own
1772
+ // suppressed removal for the same clientID(s), if pending (see
1773
+ // _scheduleAwarenessRemoval()). This MUST be checked here, at the
1774
+ // wire-message level, before calling applyAwarenessUpdate() below -
1775
+ // by the time this arrives we've very likely already independently
1776
+ // detected and applied the SAME removal ourselves (every peer's
1777
+ // 30s outdatedTimeout sweep fires near-simultaneously, well before
1778
+ // this message's network delay elapses - see
1779
+ // test/dummy/bench-awareness-removal-burst.ts), so
1780
+ // applyAwarenessUpdate() below will see a clock it already knows
1781
+ // and emit no 'update' event at all - mirrors exactly how the
1782
+ // MESSAGE_SYNC/MESSAGE_SYNC_VERIFIED case above cancels
1783
+ // _pendingSyncReply on seeing a SyncStep2 message TYPE arrive, not
1784
+ // on whether it changed anything locally.
1785
+ const scan = this._scanAwarenessPayload(payload);
1786
+ if (scan.removed.length > 0) {
1787
+ this._cancelPendingAwarenessRemovalIfOverlaps(scan.removed);
1788
+ }
1789
+ if (scan.coversUs)
1790
+ this._presenceCovered = true;
1791
+ // Round 5, item 3: with Trickle, settled peers rarely beacon, so a
1792
+ // joiner would learn them only from the few phase-winning beacons
1793
+ // per interval. The relayed presence table names everyone - ids
1794
+ // only, addresses stay beacon-learned (unicast needs a `from`).
1795
+ const heardAt = Date.now();
1796
+ for (const id of scan.present) {
1797
+ if (id !== this.doc.clientID)
1798
+ this._knownPeers.set(id, heardAt);
1799
+ }
1800
+ awarenessProtocol.applyAwarenessUpdate(this.awareness, payload, this);
1801
+ break;
1802
+ }
1803
+ case MESSAGE_PUBSUB: {
1804
+ // Read topic
1805
+ const topic = decoding.readVarString(decoder);
1806
+ // Read message payload
1807
+ const payloadBytes = decoding.readVarUint8Array(decoder);
1808
+ try {
1809
+ // Decode JSON payload
1810
+ const decoder = new TextDecoder();
1811
+ const payloadStr = decoder.decode(payloadBytes);
1812
+ const message = JSON.parse(payloadStr);
1813
+ // Emit to pubsub channel
1814
+ this.pubsub._handleMessage(topic, message);
1815
+ }
1816
+ catch (error) {
1817
+ console.error('Error decoding pub/sub message:', error);
1818
+ }
1819
+ break;
1820
+ }
1821
+ case MESSAGE_AWARENESS_APP: {
1822
+ // Deliberately none of the presence bookkeeping the core awareness
1823
+ // case does (_knownPeers, _presenceCovered, removal cancellation):
1824
+ // this channel is written by untrusted modules and must not be able
1825
+ // to influence the room's view of who is present.
1826
+ awarenessProtocol.applyAwarenessUpdate(this.appAwareness, decoding.readVarUint8Array(decoder), this);
1827
+ break;
1828
+ }
1829
+ case MESSAGE_PUBSUB_TARGETED: {
1830
+ const target = decoding.readVarString(decoder);
1831
+ const topic = decoding.readVarString(decoder);
1832
+ const payloadBytes = decoding.readVarUint8Array(decoder);
1833
+ // Drop messages aimed at someone else (broadcast-and-filter path).
1834
+ if (this._localId !== undefined && target !== this._localId) {
1835
+ break;
1836
+ }
1837
+ try {
1838
+ const message = JSON.parse(new TextDecoder().decode(payloadBytes));
1839
+ this.pubsub._handleMessage(topic, message);
1840
+ }
1841
+ catch (error) {
1842
+ console.error('Error decoding targeted pub/sub message:', error);
1843
+ }
1844
+ break;
1845
+ }
1846
+ case MESSAGE_SYNC_VERIFIED: {
1847
+ // Sync message with sequence number and hash verification
1848
+ // Read sequence number and clientID first
1849
+ const seqNum = decoding.readVarUint(decoder);
1850
+ const senderClientID = decoding.readVarUint(decoder);
1851
+ // Track for gap detection only — does NOT gate whether we apply
1852
+ // the update below (see _trackRemoteSeq() for why).
1853
+ this._trackRemoteSeq(senderClientID, seqNum);
1854
+ this._knownPeers.set(senderClientID, Date.now());
1855
+ if (from !== undefined)
1856
+ this._peerAddress.set(senderClientID, from);
1857
+ this._touchPeer(senderClientID);
1858
+ // Always apply the update. Yjs updates are idempotent/commutative,
1859
+ // so re-applying an already-seen update is a harmless no-op.
1860
+ // Under reordering, a merely-late (not actually duplicate) update
1861
+ // must still be applied here — the old "skip if seqNum <= last
1862
+ // seen" logic silently dropped such updates forever whenever a
1863
+ // later-numbered message happened to arrive first.
1864
+ // Peek at the sync sub-message's payload (SyncStep2/Update carry
1865
+ // an update as a varUint8Array right after the sub-type) without
1866
+ // consuming the decoder - _isLateUpdate() needs the bytes after
1867
+ // readSyncMessage() has applied them.
1868
+ const updateBytes = this._peekSyncUpdate(decoder);
1869
+ const encoder = encoding.createEncoder();
1870
+ encoding.writeVarUint(encoder, MESSAGE_SYNC);
1871
+ const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this);
1872
+ // Someone else's SyncStep2 reply just arrived - our own pending
1873
+ // reply (if any) is now most likely redundant. Mirrors the
1874
+ // MESSAGE_SYNC case: the reply encoded above is always a plain
1875
+ // MESSAGE_SYNC-typed message regardless of which message type
1876
+ // triggered it, so the same suppression scheme applies here too.
1877
+ if (syncMessageType === syncProtocol.messageYjsSyncStep2) {
1878
+ this._cancelPendingSyncReply();
1879
+ this._confirmed = true;
1880
+ this._checkPendingAfterReply();
1881
+ this._noteResponse(true);
1882
+ }
1883
+ // Read the expected hash from sender (signed integer)
1884
+ const expectedHash = decoding.readVarInt(decoder);
1885
+ // Compute our local hash after applying the update
1886
+ const localHash = computeDocHash(this.doc);
1887
+ // Verify hash match
1888
+ if (localHash !== expectedHash) {
1889
+ // If we already know this sender has a suspected reordering gap
1890
+ // (see _trackRemoteSeq()/_scheduleGapCheck()), a hash mismatch
1891
+ // right now is the *expected* transient state — we're missing a
1892
+ // piece that's very likely still in flight, not actually
1893
+ // diverged. Let the pending gap-check grace period resolve it
1894
+ // instead of also escalating the hash-mismatch backoff: under
1895
+ // heavy reordering this previously caused a burst of mismatches
1896
+ // to rack up the exponential backoff to its 10s cap within a
1897
+ // single edit burst, purely from timing, not real divergence.
1898
+ // A hash mismatch with NO pending gap (in-order, but still
1899
+ // wrong) is not explained by reordering and still escalates
1900
+ // normally below.
1901
+ const reorderingSuspected = this._gapCheckTimers.has(senderClientID);
1902
+ // A late update - one whose content we had already been past
1903
+ // when it arrived (its sender's clock in the update is below
1904
+ // ours, because a SyncStep2 or a reordered later update got here
1905
+ // first) - carries a hash of a state we have legitimately moved
1906
+ // beyond. Its mismatch says nothing about anything we lack; it
1907
+ // was the other half of the item-13 cascade (a resync's reply
1908
+ // fast-forwards a peer, then every in-flight keystroke behind it
1909
+ // mismatches). Cheap to detect from the update's own metadata.
1910
+ const lateUpdate = this._isLateUpdate(updateBytes);
1911
+ // Yjs's own verdict: if the update could not be fully integrated
1912
+ // because a causal dependency is missing, the struct store holds
1913
+ // it as pending. That is the ONE mismatch that is evidence of a
1914
+ // gap - and under jitter it is usually a reordering that the
1915
+ // next few ms resolve, so it gets the same grace a sequence gap
1916
+ // gets before a beacon goes out (_schedulePendingCheck). It also
1917
+ // covers the case the sequence anchor no longer does: since the
1918
+ // connect push carries no sequence number (MESSAGE_SYNC_PUSH), a
1919
+ // peer's first keystroke can be the first numbered message we see
1920
+ // from it, and a reordered first burst has no earlier number to
1921
+ // open a gap against.
1922
+ const pending = this.doc.store.pendingStructs !== null ||
1923
+ this.doc.store.pendingDs !== null;
1924
+ if (pending) {
1925
+ this._schedulePendingCheck();
1926
+ }
1927
+ else if (!reorderingSuspected && !lateUpdate) {
1928
+ // Push our full state AND request theirs (syncNow() does
1929
+ // both). A hash mismatch means the two peers have diverged -
1930
+ // one side may have edits the other lacks. Routed through the
1931
+ // shared coordinator so this doesn't stack an independent
1932
+ // timer on top of any corrupted-message/gap-confirmed resync
1933
+ // already pending.
1934
+ this._requestResync();
1935
+ // Logged with the shared attempt counter (kept as "#N" for
1936
+ // compatibility with existing tooling/benchmarks that grep
1937
+ // for this exact "Hash mismatch #" pattern) - it now reflects
1938
+ // the unified resync-attempt count rather than a
1939
+ // hash-mismatch-specific one, since the two escalation
1940
+ // counters were merged.
1941
+ console.warn(`[GenericProvider] Hash mismatch #${this._resyncAttemptCount} detected! Local: ${localHash}, Expected: ${expectedHash}`);
1942
+ }
1943
+ }
1944
+ // If we received SyncStep2, we're synced (unless hash mismatched)
1945
+ if (syncMessageType === syncProtocol.messageYjsSyncStep2 &&
1946
+ localHash === expectedHash) {
1947
+ this._markSynced();
1948
+ }
1949
+ // Send reply if needed (as standard MESSAGE_SYNC). Suppression
1950
+ // only engages with genuine redundancy (>=2 other known peers via
1951
+ // awareness) - below that, reply immediately (still rate-limited
1952
+ // via _sendSyncReply() as a hard backstop). Matches the
1953
+ // MESSAGE_SYNC case's gate exactly; without this, a hash-mismatch
1954
+ // resync burst under packet loss bypassed suppression entirely,
1955
+ // since every peer answering a post-mismatch SyncStep1 replied
1956
+ // immediately via this path.
1957
+ if (encoding.length(encoder) > 1) {
1958
+ this._replyToSyncRequest(encoding.toUint8Array(encoder));
1959
+ }
1960
+ break;
1961
+ }
1962
+ default:
1963
+ console.warn('Unknown message type:', messageType);
1964
+ }
1965
+ }
1966
+ /**
1967
+ * Handle a digest beacon (MESSAGE_SYNC_DIGEST). Reply rule (design doc
1968
+ * §3): SyncStep2 if the sender is behind us or its delete-set hash
1969
+ * differs from ours (the SyncStep2 always carries our full delete set, so
1970
+ * it also heals a lost delete on their side - and their beacon does the
1971
+ * same for us, symmetrically, within one interval); our own beacon as an
1972
+ * ack if the beacon is JOIN-flagged and states are equal; nothing
1973
+ * otherwise - which is what removes the ~5-12 empty replies per heartbeat
1974
+ * measured at N=50 in test/dummy/bench-idle-room.ts. "Sender is ahead of
1975
+ * us" triggers no reply: our own next beacon fetches it. Nothing here
1976
+ * removes a recovery path (the round-2 lesson in
1977
+ * 2026-09-04-resync-message-reduction-design.md's addendum), only
1978
+ * replies that carry no information.
1979
+ *
1980
+ * `synced`: a beacon we are not behind, with equal delete-set hash, is a
1981
+ * stronger statement than the empty SyncStep2 it replaces ("you lack
1982
+ * nothing I have"), so it marks us synced too - this is what keeps two
1983
+ * fresh peers, or a whole concurrent join burst, converging to `synced`
1984
+ * with no acks needing to survive the rate limiter.
1985
+ */
1986
+ _handleDigest(decoder, from) {
1987
+ decoding.readVarUint(decoder); // DIGEST_VERSION - append-only, nothing to branch on yet
1988
+ const flags = decoding.readVarUint(decoder);
1989
+ const senderClientID = decoding.readVarUint(decoder);
1990
+ this._knownPeers.set(senderClientID, Date.now());
1991
+ if (from !== undefined)
1992
+ this._peerAddress.set(senderClientID, from);
1993
+ this._touchPeer(senderClientID);
1994
+ const remoteSv = decoding.readVarUint8Array(decoder);
1995
+ const remoteDsHash = decoding.readVarUint(decoder);
1996
+ // Any trailing bytes belong to a newer version; ignored by design.
1997
+ const remote = Y.decodeStateVector(remoteSv);
1998
+ const local = Y.decodeStateVector(Y.encodeStateVector(this.doc));
1999
+ let senderBehind = false;
2000
+ for (const [client, clock] of local) {
2001
+ if ((remote.get(client) ?? 0) < clock) {
2002
+ senderBehind = true;
2003
+ break;
2004
+ }
2005
+ }
2006
+ let weBehind = false;
2007
+ for (const [client, clock] of remote) {
2008
+ if ((local.get(client) ?? 0) < clock) {
2009
+ weBehind = true;
2010
+ break;
2011
+ }
2012
+ }
2013
+ const dsEqual = remoteDsHash === this._deleteSetHash();
2014
+ const equal = !senderBehind && !weBehind && dsEqual;
2015
+ if (equal) {
2016
+ // Any peer holding exactly our state has confirmed it - see _confirmedSv.
2017
+ this._confirmedSv = remoteSv;
2018
+ this._confirmedDsHash = remoteDsHash;
2019
+ }
2020
+ if (flags & DIGEST_FLAG_ACK) {
2021
+ // Somebody confirmed the echoed state (see DIGEST_FLAG_ACK). If it is
2022
+ // ours, we're synced; if we were about to confirm the same state, we
2023
+ // no longer need to. Never a request: no reply, no presence.
2024
+ if (equal) {
2025
+ if (flags & DIGEST_FLAG_SETTLED) {
2026
+ this._confirmed = true;
2027
+ this._noteResponse(true);
2028
+ }
2029
+ else {
2030
+ this._equalUnsettledSeen = true;
2031
+ }
2032
+ this._markSynced();
2033
+ this._cancelPendingAck();
2034
+ }
2035
+ return;
2036
+ }
2037
+ // An equal-digest beacon from a settled peer's periodic tick has just
2038
+ // told the room (including whoever our pending ack was for) that this
2039
+ // state is confirmed. Our ack would say the same thing again. Task 3b
2040
+ // in the design doc: without this, acks were ~94% of a 50-peer join
2041
+ // burst's messages, throttled only by the rate limiter. An equal JOIN
2042
+ // beacon (another joiner in the same burst) asks for an ack too; our
2043
+ // pending ack - identical bytes, since it echoes that same state -
2044
+ // answers it as well, and _scheduleSyncReply() dedupes it below.
2045
+ if (equal && !(flags & (DIGEST_FLAG_JOIN | DIGEST_FLAG_CONFIRM))) {
2046
+ // A peer's periodic/resync beacon in our state: it makes our pending
2047
+ // ack redundant, and - if that peer is confirmed - it answers our
2048
+ // own outstanding join as well as any ack would. It also counts for
2049
+ // Trickle: the room has just compared itself against our digest.
2050
+ this._equalBeaconsHeard++;
2051
+ if (flags & DIGEST_FLAG_SETTLED) {
2052
+ this._confirmed = true;
2053
+ this._noteResponse(false);
2054
+ }
2055
+ else {
2056
+ this._equalUnsettledSeen = true;
2057
+ }
2058
+ this._cancelPendingAck();
2059
+ }
2060
+ if (senderBehind || !dsEqual) {
2061
+ const encoder = encoding.createEncoder();
2062
+ encoding.writeVarUint(encoder, MESSAGE_SYNC);
2063
+ syncProtocol.writeSyncStep2(encoder, this.doc, remoteSv);
2064
+ let reply = encoding.toUint8Array(encoder);
2065
+ if (!dsEqual) {
2066
+ // Round 5: a delete moves no clock, so the hash cannot say which of
2067
+ // us lacks a delete - and a SyncStep2 heals only its receiver. The
2068
+ // loser of a delete-only update was healed only by its OWN next
2069
+ // beacon, which idle backoff parks for up to 60 s (bench-idle-room
2070
+ // part b failed its 11 s cap at a 5 s interval). So the reply asks
2071
+ // back: our beacon rides in the same batch, and if the sender is
2072
+ // the one ahead it answers with its delete set. Terminates: a peer
2073
+ // healed by the SyncStep2 half sees an equal digest in the beacon
2074
+ // half and stays silent. (A behind-check timer on the mismatch was
2075
+ // tried first: every peer that overheard the loser's beacon armed
2076
+ // one and asked - ~2x the deliveries of a lossy edit burst on the
2077
+ // Matrix profile.)
2078
+ reply = this._encodeBatch([reply, this._encodeSyncStep1(0)]);
2079
+ }
2080
+ this._replyToSyncRequest(reply, false, remoteSv, senderClientID);
2081
+ }
2082
+ else if (flags & (DIGEST_FLAG_JOIN | DIGEST_FLAG_CONFIRM)) {
2083
+ this._replyToSyncRequest(this._encodeAck(remoteSv, remoteDsHash), true, null, senderClientID);
2084
+ }
2085
+ if (!weBehind && dsEqual) {
2086
+ this._markSynced();
2087
+ }
2088
+ if (weBehind) {
2089
+ // The sender has something we lack. Not a request yet - the update
2090
+ // may still be in flight (see _scheduleBehindCheck).
2091
+ this._scheduleBehindCheck(remoteSv);
2092
+ }
2093
+ if (flags & DIGEST_FLAG_JOIN && this.awareness.getLocalState() !== null) {
2094
+ // Presence on demand: the joiner asked. One broadcast per burst of
2095
+ // joiners (see _schedulePresenceResponse), never suppressed (each
2096
+ // responder's state is distinct). Skipped when we have no state to
2097
+ // announce.
2098
+ this._schedulePresenceResponse(senderClientID);
2099
+ }
2100
+ }
2101
+ /**
2102
+ * Answer a JOIN beacon's presence request once for all JOIN beacons that
2103
+ * arrive within `clamp(2 * minRTT, 100, 500)` ms of the first - long
2104
+ * enough to cover a join burst spread by latency, short enough that a
2105
+ * lone joiner sees the room's presence within a few round trips.
2106
+ */
2107
+ _schedulePresenceResponse(requester) {
2108
+ this._presencePending.add(requester);
2109
+ if (this._presenceResponseTimer !== undefined)
2110
+ return;
2111
+ this._presenceCovered = false;
2112
+ const rtt = this._rttMinMs();
2113
+ // Phase 1e, relay path: one peer per 2 s bucket - the first-ranked
2114
+ // for a constant requester - relays the whole awareness table at once
2115
+ // (the way y-websocket's server does; clocks travel with the states,
2116
+ // a peer's own echoed state at an equal clock is ignored by
2117
+ // applyAwarenessUpdate). Everyone else waits the usual window, and
2118
+ // stays silent if that table carried their state (_presenceCovered).
2119
+ // Presence per late join: ~N deliveries instead of (N-1)^2 - it was
2120
+ // 82% of a late join into a 100-peer room (bench-join-census). Bytes
2121
+ // are unchanged (the table goes to everyone). Peers that cannot yet
2122
+ // tell (no RTT estimate on a slow link, so the table may arrive after
2123
+ // their window) fall back to the broadcast of their own state, as
2124
+ // before. The unicast path below is untouched.
2125
+ // Needs a view of the room: in a fresh burst the first JOIN arrives
2126
+ // before any peer is known, everyone would rank first and relay a
2127
+ // table each - the broadcast fallback is right for that case.
2128
+ const relayer = !this._canUnicast(requester) &&
2129
+ this._knownPeers.size >= 3 &&
2130
+ this._responderRank(0, 1) === 0;
2131
+ const delay = relayer ? 0 : Math.min(500, Math.max(100, rtt === null ? 0 : 2 * rtt));
2132
+ this._presenceResponseTimer = setTimeout(() => {
2133
+ this._presenceResponseTimer = undefined;
2134
+ const requesters = Array.from(this._presencePending);
2135
+ this._presencePending.clear();
2136
+ if (this._destroying || !this.transport.isConnected)
2137
+ return;
2138
+ if (this.awareness.getLocalState() === null)
2139
+ return;
2140
+ // Every joiner covered by this timer is addressable: one unicast
2141
+ // each ((N-1) deliveries per joiner room-wide) instead of one
2142
+ // broadcast ((N-1)^2). Otherwise the broadcast, as before.
2143
+ if (requesters.every((id) => this._canUnicast(id))) {
2144
+ const msg = this._encodeAwareness([this.doc.clientID]);
2145
+ for (const id of requesters)
2146
+ this._sendDirect(id, msg);
2147
+ }
2148
+ else if (relayer) {
2149
+ this._sendAwarenessNow(Array.from(this.awareness.getStates().keys()));
2150
+ }
2151
+ else if (!this._presenceCovered) {
2152
+ this._broadcastAwareness([this.doc.clientID]);
2153
+ }
2154
+ }, delay);
2155
+ }
2156
+ /**
2157
+ * Max random delay (ms) before replying to a SyncStep1 request, scaled by
2158
+ * a room-size signal already available (`this.awareness.getStates().size`
2159
+ * - the same signal read at the `>= 3` suppression gate). A fixed window
2160
+ * (the pre-fix behavior: always `_syncReplySuppressionMs`) doesn't scale
2161
+ * with room size, so a larger room has more independent repliers racing
2162
+ * to answer the same request within the same window - more of them lose
2163
+ * the race and get silently dropped by the `_sendSyncReply()` rate-limit
2164
+ * backstop instead of never sending in the first place. Measured in
2165
+ * test/dummy/bench-corruption-storm.ts: the SyncStep2/SyncStep1 ratio (
2166
+ * ideally ~1 if suppression alone were sufficient) grew from ~1.1-1.3 at
2167
+ * N=2 to ~4.5-5.9 at N=10 with the fixed 30ms window.
2168
+ *
2169
+ * `min(cap, base * log2(peerCount))` - log2 growth spreads replies over a
2170
+ * wider window as the room grows without the delay exploding at very high
2171
+ * N. Capped at 200ms: the slowest-profile round trip this project
2172
+ * benchmarks against (Matrix, ~350ms one-way) already tolerates hundreds
2173
+ * of ms of latency, so 200ms of extra requester-perceived delay stays
2174
+ * well inside that budget while still giving a 100-peer room roughly
2175
+ * 6-7x the base window instead of an unbounded one.
2176
+ */
2177
+ /**
2178
+ * How many peers we believe are in the room: awareness states (includes
2179
+ * ourselves) or, if larger, the distinct beacon/update senders we have
2180
+ * heard plus ourselves. See `_knownPeers`.
2181
+ */
2182
+ _peerCount() {
2183
+ return Math.max(this.awareness.getStates().size, this._knownPeers.size + 1);
2184
+ }
2185
+ /**
2186
+ * Resolves `_awarenessInterval` to a concrete ms value: the configured
2187
+ * fixed number, or (round 6, item 9) `max(transport hint ?? 100,
2188
+ * AWARENESS_AUTO_MS_PER_PEER * peerCount)` when set to `'auto'`.
2189
+ */
2190
+ _effectiveAwarenessInterval() {
2191
+ if (this._awarenessInterval !== 'auto')
2192
+ return this._awarenessInterval;
2193
+ const hint = this.transport.preferredAwarenessMs ?? 100;
2194
+ return Math.max(hint, GenericProvider.AWARENESS_AUTO_MS_PER_PEER * this._peerCount());
2195
+ }
2196
+ _replySuppressionMaxDelay() {
2197
+ const peerCount = this._peerCount();
2198
+ const byRoomSize = Math.min(200, this._syncReplySuppressionMs * Math.log2(Math.max(2, peerCount)));
2199
+ // Phase 1b: the window must exceed the one-way latency or nobody
2200
+ // overhears anybody in time (see _rttSamples). 1.5x the smallest
2201
+ // observed round trip, capped at 2 s - on a 350 ms profile that is
2202
+ // ~1 s of extra requester-perceived delay in exchange for ~1 reply
2203
+ // instead of ~20.
2204
+ const rtt = this._rttMinMs();
2205
+ return rtt === null ? byRoomSize : Math.min(2000, Math.max(byRoomSize, 1.5 * rtt));
2206
+ }
2207
+ /**
2208
+ * Schedule a SyncStep2 reply after a short random delay instead of
2209
+ * sending immediately. If another peer's reply is overheard in the
2210
+ * meantime (`_cancelPendingSyncReply`), this reply is dropped as
2211
+ * redundant - the requester likely already got what it needed.
2212
+ *
2213
+ * A reply that is already pending when this is called answers a
2214
+ * *different* request (e.g. peer A's request, followed 5ms later by
2215
+ * peer B's) - it must not be silently overwritten by the new one. Flush
2216
+ * it immediately, then schedule the new reply fresh. The only sanctioned
2217
+ * ways a reply gets dropped are `_cancelPendingSyncReply()` (we overheard
2218
+ * someone else's SyncStep2 for the SAME request), `_cancelPendingAck()`,
2219
+ * and the identical-bytes case below.
2220
+ *
2221
+ * Identical-bytes case (Task 3c in the design doc): K peers with the same
2222
+ * state asking at once (K empty joiners in a burst) get K byte-identical
2223
+ * SyncStep2s from us - the same full document K times, one flushed
2224
+ * immediately per arriving request, each burning a rate-limit slot. If
2225
+ * the new reply's bytes equal the pending reply's bytes, the pending one
2226
+ * already answers this request too: keep it (same delay, same
2227
+ * suppression) and drop the new one. Measured in
2228
+ * test/dummy/bench-join-after-burst.ts.
2229
+ */
2230
+ _scheduleSyncReply(reply, isAck = false, targetSv = null, requester = null) {
2231
+ if (this._pendingSyncReplyTimeoutId !== undefined && this._pendingSyncReply !== null) {
2232
+ if (bytesEqual(this._pendingSyncReply, reply)) {
2233
+ return; // identical answer already scheduled
2234
+ }
2235
+ if (targetSv !== null &&
2236
+ this._pendingSyncReplyTargetSv !== null &&
2237
+ bytesEqual(this._pendingSyncReplyTargetSv, targetSv)) {
2238
+ // Same question (same requester state), newer document: refresh
2239
+ // the answer, keep the timer. See _pendingSyncReplyTargetSv.
2240
+ this._pendingSyncReply = reply;
2241
+ return;
2242
+ }
2243
+ if (!isAck &&
2244
+ !this._pendingSyncReplyIsAck &&
2245
+ targetSv !== null &&
2246
+ this._pendingSyncReplyTargetSv !== null) {
2247
+ // Two requesters, both behind, different states: one SyncStep2
2248
+ // from the componentwise minimum of both state vectors contains
2249
+ // everything either of them lacks. Keep the pending reply's timer
2250
+ // and widen its content instead of flushing it. The flush (the
2251
+ // rule below, kept for acks and legacy SyncStep1s) sent an
2252
+ // unsuppressed broadcast for every second request that arrived
2253
+ // inside the suppression window; with the window at 1.5x RTT on a
2254
+ // 350 ms link and several peers behind after a lossy edit burst
2255
+ // that was ~4-5 broadcast replies per beacon (phase-1c results).
2256
+ const merged = minStateVector(this._pendingSyncReplyTargetSv, targetSv);
2257
+ const encoder = encoding.createEncoder();
2258
+ encoding.writeVarUint(encoder, MESSAGE_SYNC);
2259
+ syncProtocol.writeSyncStep2(encoder, this.doc, merged);
2260
+ this._pendingSyncReply = encoding.toUint8Array(encoder);
2261
+ this._pendingSyncReplyTargetSv = merged;
2262
+ return;
2263
+ }
2264
+ // Phase 1e: an ack and a SyncStep2 never flush each other. A room
2265
+ // whose JOIN waits expire together sends N CONFIRMs in the same
2266
+ // millisecond; while a lossy edit burst is still healing, some of
2267
+ // them find us equal (ack) and some behind (SyncStep2), and the flush
2268
+ // rule below turned every type change into an immediate broadcast -
2269
+ // ~650 replies in 200 ms at N=50, 5 % loss (probe timeline in the
2270
+ // phase-1e design doc). An ack adds nothing to a pending reply of
2271
+ // either kind (the requester's wait retries, or an equal peer's
2272
+ // SETTLED ack confirms it); a SyncStep2 replaces a pending ack.
2273
+ if (isAck)
2274
+ return;
2275
+ if (this._pendingSyncReplyIsAck) {
2276
+ clearTimeout(this._pendingSyncReplyTimeoutId);
2277
+ this._pendingSyncReplyTimeoutId = undefined;
2278
+ }
2279
+ }
2280
+ if (this._pendingSyncReplyTimeoutId !== undefined) {
2281
+ // Only a legacy plain SyncStep1 (no target state vector) still
2282
+ // flushes a pending SyncStep2.
2283
+ if (this._pendingSyncReply) {
2284
+ this._sendSyncReply(this._pendingSyncReply);
2285
+ }
2286
+ clearTimeout(this._pendingSyncReplyTimeoutId);
2287
+ this._pendingSyncReplyTimeoutId = undefined;
2288
+ }
2289
+ this._pendingSyncReply = reply;
2290
+ this._pendingSyncReplyIsAck = isAck;
2291
+ this._pendingSyncReplyTargetSv = targetSv;
2292
+ // Acks keep the uniform window: they carry no data, so their delay
2293
+ // costs nothing but a few ms on a joiner's `synced` flip, and in a
2294
+ // join burst one pending ack answers every equal JOIN that arrives
2295
+ // inside that window (identical bytes, deduped above). Ranked, rank 0
2296
+ // fired at once for every requester - measured: fresh-burst acks
2297
+ // 6,039 -> 15,147 at Gun N=100, join-after-burst Matrix 147 -> 686.
2298
+ const delay = isAck
2299
+ ? Math.random() * this._replySuppressionMaxDelay()
2300
+ : this._replyDelay(requester);
2301
+ this._pendingSyncReplyTimeoutId = setTimeout(() => {
2302
+ this._pendingSyncReplyTimeoutId = undefined;
2303
+ if (this._pendingSyncReply) {
2304
+ this._sendSyncReply(this._pendingSyncReply);
2305
+ this._pendingSyncReply = null;
2306
+ this._pendingSyncReplyTargetSv = null;
2307
+ }
2308
+ }, delay);
868
2309
  }
869
2310
  /** Cancel a pending suppressed reply, if any. */
870
2311
  _cancelPendingSyncReply() {
@@ -873,6 +2314,497 @@ export class GenericProvider extends Observable {
873
2314
  this._pendingSyncReplyTimeoutId = undefined;
874
2315
  }
875
2316
  this._pendingSyncReply = null;
2317
+ this._pendingSyncReplyIsAck = false;
2318
+ this._pendingSyncReplyTargetSv = null;
2319
+ }
2320
+ /**
2321
+ * Cancel a pending reply only if it is a digest ack - see
2322
+ * `_pendingSyncReplyIsAck`. Called from `_handleDigest()` on every
2323
+ * overheard beacon whose digest equals ours.
2324
+ */
2325
+ _cancelPendingAck() {
2326
+ if (this._pendingSyncReplyIsAck) {
2327
+ this._cancelPendingSyncReply();
2328
+ }
2329
+ }
2330
+ /**
2331
+ * Route a SyncStep2 (or digest-ack) reply through the redundancy
2332
+ * suppression when there's genuine redundancy (>= 2 other known peers via
2333
+ * awareness - below that there's no "someone else" to rely on), else send
2334
+ * immediately. Both paths are rate-limited by `_sendSyncReply()`. Shared
2335
+ * by the MESSAGE_SYNC, MESSAGE_SYNC_VERIFIED and MESSAGE_SYNC_DIGEST cases.
2336
+ */
2337
+ _replyToSyncRequest(reply, isAck = false, targetSv = null, toClientID) {
2338
+ // A peer that knows it is incomplete does not answer. A SyncStep2 is
2339
+ // encoded from integrated structs only, so with structs (or a delete
2340
+ // set) still pending ours would be provably partial - and the
2341
+ // requester's response wait ends on the first SyncStep2 it gets, so a
2342
+ // partial answer strands it until its next trigger (phase-1c gates:
2343
+ // 5 s resync backoff, or a stall with syncInterval 0). In relay mode a
2344
+ // partial broadcast also cancels the complete replies other peers had
2345
+ // pending. Let them answer; the requester retries if nobody does, and
2346
+ // in unicast mode the rank bucket rotates the responders every 2 s.
2347
+ // An ack from us would likewise confirm a state we do not trust.
2348
+ if (this.doc.store.pendingStructs !== null ||
2349
+ this.doc.store.pendingDs !== null) {
2350
+ return;
2351
+ }
2352
+ // Unicast path (transport has sendTo and we know the requester's
2353
+ // address): nobody overhears a unicast, so the delay-and-cancel
2354
+ // suppression below cannot thin the replies. Instead each candidate
2355
+ // responder decides for itself whether it is one of ~3 that answer
2356
+ // (_selectedResponder), and answers at once - no suppression delay,
2357
+ // one delivery. The requester's response wait retries if all ~3 are
2358
+ // lost. Phase-1c design, item B.
2359
+ if (toClientID !== undefined && this._canUnicast(toClientID)) {
2360
+ if (!this._selectedResponder(toClientID))
2361
+ return;
2362
+ if (!this._tryReserveReplySlot())
2363
+ return;
2364
+ this._sendDirect(toClientID, reply);
2365
+ return;
2366
+ }
2367
+ // Acks ALWAYS take the delayed/suppressible path: they carry no data,
2368
+ // so the only cost of delaying one is a few ms on the joiner's `synced`
2369
+ // flip (measured before this rule: 37,240 of a 50-peer join burst's
2370
+ // 40,915 deliveries were immediate acks). Everything else goes through
2371
+ // suppression once there is someone else who could answer - counted
2372
+ // from beacon/update senders as well as awareness, see _peerCount().
2373
+ if (isAck || this._peerCount() >= 3) {
2374
+ this._scheduleSyncReply(reply, isAck, targetSv, toClientID ?? null);
2375
+ }
2376
+ else {
2377
+ this._sendSyncReply(reply);
2378
+ }
2379
+ }
2380
+ /** Whether a reply to `clientID` can go over Transport.sendTo. */
2381
+ _canUnicast(clientID) {
2382
+ return (typeof this.transport.sendTo === 'function' &&
2383
+ this._peerAddress.has(clientID));
2384
+ }
2385
+ /**
2386
+ * Responder self-selection for unicast replies: the three peers whose
2387
+ * hash for this requester ranks lowest among the peers we know answer
2388
+ * it. Every candidate ranks itself against the same known set, so the
2389
+ * sets agree wherever the views agree, and the peer that ranks first in
2390
+ * the true order always ranks first in its own view - the selection is
2391
+ * never empty. A 2 s time bucket in the hash rotates the ranking, so
2392
+ * three departed peers at the top only delay a reply until the
2393
+ * requester's next attempt. Everyone answers in rooms of four or fewer.
2394
+ * (A first cut chose each responder independently with probability 3/N;
2395
+ * ~5 % of requests then selected nobody and waited for the 1 s retry.)
2396
+ */
2397
+ _selectedResponder(requester) {
2398
+ if (this._peerCount() < 4)
2399
+ return true;
2400
+ return this._responderRank(requester, 3) < 3;
2401
+ }
2402
+ /**
2403
+ * How many known peers rank below us for `requester` in the current 2 s
2404
+ * bucket (counting stops at `cap`). Shared by unicast self-selection
2405
+ * (rank < 3 answers) and, since phase 1e, the relay-mode reply delay
2406
+ * (rank r waits r slots, see _replyDelay()).
2407
+ */
2408
+ _responderRank(requester, cap) {
2409
+ const bucket = Math.floor(Date.now() / 2000);
2410
+ const rank = (id) => (Math.imul(requester ^ bucket, 0x9e3779b1) ^ Math.imul(id, 0x85ebca6b)) >>> 0;
2411
+ const mine = rank(this.doc.clientID);
2412
+ let better = 0;
2413
+ for (const id of this._knownPeers.keys()) {
2414
+ if (id === requester || id === this.doc.clientID)
2415
+ continue;
2416
+ if (rank(id) < mine && ++better >= cap)
2417
+ break;
2418
+ }
2419
+ return better;
2420
+ }
2421
+ /**
2422
+ * Delay before a suppressible reply goes out (relay path). Phase 1e:
2423
+ * ranked, not uniform. A uniform draw from [0, W] lets ~N * L / W
2424
+ * repliers fire before the first reply is overheard (L = one-way
2425
+ * latency): 10-27 SyncStep2 sends per request at N=100 in
2426
+ * test/dummy/bench-join-census.ts, and the WebRTC join-burst cell's
2427
+ * 16-34k spread. With the responder rank (the same hash the unicast
2428
+ * self-selection uses) rank 0 answers at once and rank r waits r
2429
+ * windows (W = _replySuppressionMaxDelay(), 1.5x the minimum round
2430
+ * trip: with request arrival spread 2jL and reply flight L(1+j), rank 1
2431
+ * has overheard rank 0 iff the slot is >= L(1+3j), which 3L(1-j) covers
2432
+ * up to j~0.33). Ranks >= 8 add a random window on top so a room whose
2433
+ * first eight ranked peers are all gone does not answer in one
2434
+ * avalanche. Without an RTT sample or a requester id (legacy SyncStep1)
2435
+ * the uniform window stays.
2436
+ */
2437
+ _replyDelay(requester) {
2438
+ const window = this._replySuppressionMaxDelay();
2439
+ const rtt = this._rttMinMs();
2440
+ if (requester === null || rtt === null)
2441
+ return Math.random() * window;
2442
+ // Half a window per rank (0.75 x the minimum round trip): a rank that
2443
+ // stays silent (pending structs during a lossy burst, phase 1d B)
2444
+ // costs the requester half a window, not a whole one - at a full
2445
+ // window the Matrix 5 % loss fan-out's median convergence doubled
2446
+ // (1.1 -> 2.2 s, worst 7.6 s); the price is an occasional second
2447
+ // reply where the jitter exceeds ~1/3 (probe numbers in the design doc).
2448
+ const slot = Math.max(this._syncReplySuppressionMs, 0.75 * rtt);
2449
+ const rank = this._responderRank(requester, 8);
2450
+ return rank * slot + (rank >= 8 ? Math.random() * window : 0);
2451
+ }
2452
+ /**
2453
+ * Send one already-encoded message to a single peer over
2454
+ * Transport.sendTo, with the same CRC32 wrapping and optional compression
2455
+ * as a broadcast. Not mirrored to BroadcastChannel (a same-browser tab
2456
+ * never appears as an addressable peer). Returns false if the peer's
2457
+ * address is unknown or the transport cannot unicast.
2458
+ */
2459
+ _sendDirect(clientID, data) {
2460
+ const address = this._peerAddress.get(clientID);
2461
+ if (address === undefined || typeof this.transport.sendTo !== 'function') {
2462
+ return false;
2463
+ }
2464
+ if (!this.transport.isConnected)
2465
+ return false;
2466
+ this._sendToTransport(wrapMessageWithChecksum(data), address);
2467
+ return true;
2468
+ }
2469
+ /**
2470
+ * Re-check Yjs's pending-struct store after the gap grace period and
2471
+ * request a resync (a beacon, see _requestResync) only if something is
2472
+ * still missing. One timer; a check scheduled while one is pending is
2473
+ * absorbed. Cleared on disconnect/destroy.
2474
+ */
2475
+ /**
2476
+ * A beacon (a peer's periodic tick, or its request) just showed its
2477
+ * sender ahead of us. Until phase 1d nothing happened with that: a peer
2478
+ * whose last update was lost (no later message to open a sequence gap
2479
+ * against), or whose request was answered by a responder that was
2480
+ * itself behind, waited for its OWN next periodic beacon - up to
2481
+ * syncInterval, up to idleBackoffMaxMs with idle backoff on. Now we
2482
+ * check again after a grace and, if still behind that state, ask through
2483
+ * the resync coordinator (coalesced, backed off, rate-limited).
2484
+ *
2485
+ * The grace is what keeps this quiet during typing: at Matrix latency
2486
+ * almost every receiver of a periodic beacon is "behind" by a keystroke
2487
+ * that is still in flight (jitter +-140 ms); max(gapGraceMs, 2 x minRTT)
2488
+ * later it has arrived and the check finds nothing to do. A lost
2489
+ * keystroke that opened a sequence gap is already being requested by the
2490
+ * gap check - the outstanding response wait tells us so, and we stay
2491
+ * quiet. One timer, the newest state vector: a later beacon that shows us
2492
+ * behind by more replaces the reference, the timer keeps running.
2493
+ */
2494
+ _scheduleBehindCheck(remoteSv) {
2495
+ this._behindSv = remoteSv;
2496
+ if (this._behindCheckTimer !== undefined)
2497
+ return;
2498
+ const rtt = this._rttMinMs();
2499
+ const delay = Math.max(this._gapGraceMs, rtt === null ? 0 : 2 * rtt);
2500
+ this._behindCheckTimer = setTimeout(() => {
2501
+ this._behindCheckTimer = undefined;
2502
+ const sv = this._behindSv;
2503
+ this._behindSv = null;
2504
+ if (sv === null || this._destroying || !this.transport.isConnected)
2505
+ return;
2506
+ // A request of ours sent within the grace is still being answered.
2507
+ // An OLDER outstanding wait is not a reason to stay behind: a new
2508
+ // room's first peers keep their JOIN wait parked for seconds (three
2509
+ // retries, nobody SETTLED yet), and measured against it the check
2510
+ // never fired - bench-idle-backoff's recovery stayed at one
2511
+ // backed-off interval (1,846 ms) with this line reading
2512
+ // `_responseWaitTimer !== undefined`.
2513
+ if (this._requestSentAt > 0 && Date.now() - this._requestSentAt < delay)
2514
+ return;
2515
+ const remote = Y.decodeStateVector(sv);
2516
+ const local = Y.decodeStateVector(Y.encodeStateVector(this.doc));
2517
+ for (const [client, clock] of remote) {
2518
+ if ((local.get(client) ?? 0) < clock) {
2519
+ this._requestResync();
2520
+ return;
2521
+ }
2522
+ }
2523
+ }, delay);
2524
+ }
2525
+ /** Design E: after a reply or push, pending structs mean the sender had the same hole - arm the grace check. */
2526
+ _checkPendingAfterReply() {
2527
+ if (this.doc.store.pendingStructs !== null ||
2528
+ this.doc.store.pendingDs !== null) {
2529
+ this._schedulePendingCheck();
2530
+ }
2531
+ }
2532
+ _schedulePendingCheck() {
2533
+ if (this._pendingCheckTimer !== undefined)
2534
+ return;
2535
+ this._pendingCheckTimer = setTimeout(() => {
2536
+ this._pendingCheckTimer = undefined;
2537
+ if (this._destroying || !this.transport.isConnected)
2538
+ return;
2539
+ // A request of ours may already be outstanding (JOIN or resync beacon
2540
+ // with its response wait running): whatever is pending is what that
2541
+ // request is fetching, and the response wait retries if it is lost.
2542
+ // Asking again here just raced the responders' suppression window
2543
+ // (measured: a joiner's keystrokes-before-content check fired at
2544
+ // 300 ms while the settled peers' replies were still delayed by a
2545
+ // window of up to ~300 ms, and every such re-beacon collected
2546
+ // another round of replies).
2547
+ // ... but only a request that went out within the last grace, whose
2548
+ // answer may still be on its way. Deferring to ANY outstanding
2549
+ // response wait (the rule until phase 1d) parked every pending check
2550
+ // behind a new room's JOIN wait - three retries with nobody SETTLED
2551
+ // yet, 19.6 s on a 700 ms link - so a peer that lost a keystroke in
2552
+ // such a room asked only once an overheard reply happened to clear
2553
+ // that wait (phase-1d design doc, "After Task 4", the fresh-budget
2554
+ // Matrix fan-out). Look again once the recent request is answered or
2555
+ // given up - dropping the check here left a joiner whose SyncStep2
2556
+ // predated the keystrokes it had received with nine pending updates
2557
+ // and no trigger (measured: 2 of 150 lossy 15-peer joins).
2558
+ const rtt = this._rttMinMs();
2559
+ const recent = Math.max(this._gapGraceMs, rtt === null ? 0 : 2 * rtt);
2560
+ if (this._requestSentAt > 0 && Date.now() - this._requestSentAt < recent) {
2561
+ this._schedulePendingCheck();
2562
+ return;
2563
+ }
2564
+ if (this.doc.store.pendingStructs !== null ||
2565
+ this.doc.store.pendingDs !== null) {
2566
+ this._requestResync();
2567
+ }
2568
+ }, this._gapGraceMs);
2569
+ }
2570
+ /**
2571
+ * Return the update payload of a SyncStep2/Update sync sub-message
2572
+ * without advancing `decoder` (null for SyncStep1 or malformed input).
2573
+ * y-protocols frames both as [subType varUint][update varUint8Array].
2574
+ */
2575
+ _peekSyncUpdate(decoder) {
2576
+ const peek = decoding.clone(decoder);
2577
+ try {
2578
+ const subType = decoding.readVarUint(peek);
2579
+ if (subType === syncProtocol.messageYjsSyncStep1)
2580
+ return null;
2581
+ return decoding.readVarUint8Array(peek);
2582
+ }
2583
+ catch {
2584
+ return null;
2585
+ }
2586
+ }
2587
+ /**
2588
+ * Whether an update we just applied was already superseded here: every
2589
+ * client it touches ends at a clock we were at or beyond BEFORE this
2590
+ * update (i.e. it added nothing). Uses the update's own metadata
2591
+ * (`Y.parseUpdateMeta`), O(clients in the update).
2592
+ */
2593
+ _isLateUpdate(updateBytes) {
2594
+ if (!updateBytes)
2595
+ return false;
2596
+ try {
2597
+ const { to } = Y.parseUpdateMeta(updateBytes);
2598
+ if (to.size === 0)
2599
+ return false;
2600
+ const local = Y.decodeStateVector(Y.encodeStateVector(this.doc));
2601
+ for (const [client, clock] of to) {
2602
+ // `to` is the exclusive end clock of the update's range for that
2603
+ // client; our state vector is exclusive too. Equal means the update
2604
+ // brought us exactly here (not late); greater means we were past it.
2605
+ if ((local.get(client) ?? 0) <= clock)
2606
+ return false;
2607
+ }
2608
+ return true;
2609
+ }
2610
+ catch {
2611
+ return false;
2612
+ }
2613
+ }
2614
+ /** Flip `synced` once and emit; idempotent. */
2615
+ _markSynced() {
2616
+ if (!this._synced) {
2617
+ this._synced = true;
2618
+ this.emit('synced', [true]);
2619
+ }
2620
+ }
2621
+ /**
2622
+ * Wait for a response to the JOIN or resync beacon we just sent. If
2623
+ * neither a SyncStep2 nor an equal ack/beacon arrives within 1s (then
2624
+ * 2s, 4s), ask again - with a CONFIRM beacon after a JOIN (so an equal
2625
+ * room acks), with a plain beacon after a resync (only peers ahead of us
2626
+ * need to answer; an equal room's silence is the correct answer and its
2627
+ * periodic beacons end the wait) - three times at most; after that the
2628
+ * periodic beacon is the fallback, as before. Requester-side retry is how the protocol
2629
+ * stays loss-tolerant now that reply suppression leaves ~1 reply per
2630
+ * request; N-fold redundant replies were the old (accidental) way.
2631
+ */
2632
+ _armResponseWait(retryFlags) {
2633
+ if (this._responseWaitTimer !== undefined)
2634
+ return;
2635
+ this._responseSeen = false;
2636
+ if (this._responseWaitAttempts === 0)
2637
+ this._equalUnsettledSeen = false;
2638
+ this._responseWaitFlags = retryFlags;
2639
+ this._requestSentAt = Date.now();
2640
+ const rtt = this._rttMinMs();
2641
+ const delay = Math.max(1000, rtt === null ? 0 : 4 * rtt) *
2642
+ Math.pow(2, this._responseWaitAttempts);
2643
+ this._responseWaitTimer = setTimeout(() => {
2644
+ this._responseWaitTimer = undefined;
2645
+ // Phase 1e: a fresh room - equal but unsettled peers answered both
2646
+ // the JOIN and one CONFIRM retry, nobody settled did. Two rounds
2647
+ // instead of three; the first confirmed peers' acks then carry
2648
+ // SETTLED for everyone after them. Measured in
2649
+ // test/dummy/bench-join-census.ts (b): the third round was ~a third
2650
+ // of a fresh N=100 room's 92k-delivery join burst.
2651
+ const freshRoomDone = this._equalUnsettledSeen && this._responseWaitAttempts >= 1;
2652
+ if (this._responseSeen ||
2653
+ this._responseWaitAttempts >= 3 ||
2654
+ freshRoomDone ||
2655
+ !this.transport.isConnected ||
2656
+ this._destroying) {
2657
+ // Asked three times, nobody had more for us: we are the room's
2658
+ // state (or its first peer). Bootstraps DIGEST_FLAG_SETTLED in a
2659
+ // brand-new room so later joiners are confirmed by our acks.
2660
+ if (this._responseWaitAttempts >= 3 || freshRoomDone)
2661
+ this._confirmed = true;
2662
+ this._responseWaitAttempts = 0;
2663
+ return;
2664
+ }
2665
+ this._responseWaitAttempts++;
2666
+ this._sendSyncStep1(this._responseWaitFlags); // rate-limited; a dropped attempt is simply retried next round
2667
+ this._armResponseWait(this._responseWaitFlags);
2668
+ }, delay);
2669
+ }
2670
+ /**
2671
+ * A SyncStep2 or an equal ack/beacon arrived - whatever we asked for is
2672
+ * answered. `sample` = it was a direct reply (SyncStep2/ack), so its
2673
+ * timing is a round-trip sample; an equal periodic beacon from a settled
2674
+ * peer also ends the wait but says nothing about latency.
2675
+ */
2676
+ _noteResponse(sample) {
2677
+ if (sample && this._requestSentAt > 0) {
2678
+ this._rttSamples.push(Date.now() - this._requestSentAt);
2679
+ if (this._rttSamples.length > 8)
2680
+ this._rttSamples.shift();
2681
+ }
2682
+ this._requestSentAt = 0;
2683
+ this._responseSeen = true;
2684
+ this._responseWaitAttempts = 0;
2685
+ if (this._responseWaitTimer !== undefined) {
2686
+ clearTimeout(this._responseWaitTimer);
2687
+ this._responseWaitTimer = undefined;
2688
+ }
2689
+ }
2690
+ /** Minimum of the recent round-trip samples, or null before the first reply. */
2691
+ _rttMinMs() {
2692
+ return this._rttSamples.length === 0 ? null : Math.min(...this._rttSamples);
2693
+ }
2694
+ /**
2695
+ * Delay a pure timeout-removal awareness broadcast and drop it if
2696
+ * another peer's broadcast of the SAME removal is overheard first (see
2697
+ * the `origin === this` branch in `_setupAwarenessSync()`'s handler,
2698
+ * which calls `_cancelPendingAwarenessRemovalIfOverlaps()`) - the exact
2699
+ * same NACK-style suppression `_scheduleSyncReply()` already applies to
2700
+ * SyncStep2 replies, reusing the same room-size-scaled delay
2701
+ * (`_replySuppressionMaxDelay()`).
2702
+ *
2703
+ * A pending removal already queued when this is called is for a
2704
+ * DIFFERENT departure (two peers timing out, or leaving, within the same
2705
+ * window): since round 5 the ids are merged into the pending set and its
2706
+ * timer kept - one broadcast carries both - instead of flushing the
2707
+ * first as an unsuppressed broadcast (with the long leave window below a
2708
+ * burst of departures would have flushed on every peer). An overheard
2709
+ * broadcast trims only the ids it covers from the pending set
2710
+ * (`_cancelPendingAwarenessRemovalIfOverlaps()`).
2711
+ *
2712
+ * Window: the reply-suppression window for timeouts (sweeps are spread
2713
+ * over seconds anyway); for leaves reported by the transport - all
2714
+ * survivors learn of them in the same millisecond - ten times that,
2715
+ * at least a second, so the first broadcast is overheard before the
2716
+ * rest fire. A departure is not urgent: every peer already dropped the
2717
+ * state locally.
2718
+ */
2719
+ _scheduleAwarenessRemoval(clients, origin = 'timeout') {
2720
+ if (this._pendingAwarenessRemovalTimeoutId !== undefined) {
2721
+ const pending = this._pendingAwarenessRemoval ?? [];
2722
+ for (const id of clients)
2723
+ if (!pending.includes(id))
2724
+ pending.push(id);
2725
+ this._pendingAwarenessRemoval = pending;
2726
+ return;
2727
+ }
2728
+ this._pendingAwarenessRemoval = clients.slice();
2729
+ const window = this._replySuppressionMaxDelay();
2730
+ const delay = Math.random() * (origin === 'peer-left' ? Math.max(1000, 10 * window) : window);
2731
+ this._pendingAwarenessRemovalTimeoutId = setTimeout(() => {
2732
+ this._pendingAwarenessRemovalTimeoutId = undefined;
2733
+ if (this._pendingAwarenessRemoval) {
2734
+ this._broadcastAwareness(this._pendingAwarenessRemoval);
2735
+ this._pendingAwarenessRemoval = null;
2736
+ }
2737
+ }, delay);
2738
+ }
2739
+ /**
2740
+ * Peek at an awareness-update payload (still in
2741
+ * `awarenessProtocol.encodeAwarenessUpdate()`'s wire encoding) for
2742
+ * clientIDs whose state is `null` (a removal), without applying it.
2743
+ * y-protocols/awareness.js doesn't export a standalone decoder for this,
2744
+ * only `applyAwarenessUpdate()` (which also mutates state) and
2745
+ * `modifyAwarenessUpdate()` (which re-encodes) - so this mirrors the
2746
+ * format by hand: varUint length, then per entry
2747
+ * [varUint clientID][varUint clock][varString JSON state]. Used to cancel
2748
+ * a pending suppressed removal (see `_scheduleAwarenessRemoval()`) at the
2749
+ * wire-message level, before `applyAwarenessUpdate()` runs - see the
2750
+ * `MESSAGE_AWARENESS` case's comment for why timing matters here.
2751
+ */
2752
+ _scanAwarenessPayload(payload) {
2753
+ const removed = [];
2754
+ const present = [];
2755
+ let coversUs = false;
2756
+ try {
2757
+ const d = decoding.createDecoder(payload);
2758
+ const len = decoding.readVarUint(d);
2759
+ const ourClock = this.awareness.meta.get(this.awareness.clientID)?.clock ?? 0;
2760
+ for (let i = 0; i < len; i++) {
2761
+ const clientID = decoding.readVarUint(d);
2762
+ const clock = decoding.readVarUint(d);
2763
+ const state = JSON.parse(decoding.readVarString(d));
2764
+ if (state === null)
2765
+ removed.push(clientID);
2766
+ else {
2767
+ present.push(clientID);
2768
+ if (clientID === this.awareness.clientID && clock >= ourClock)
2769
+ coversUs = true;
2770
+ }
2771
+ }
2772
+ }
2773
+ catch {
2774
+ // Malformed payload - let applyAwarenessUpdate() below be the one
2775
+ // that deals with it (or throws); suppression is a pure optimization,
2776
+ // never worth failing the actual message handling over.
2777
+ }
2778
+ return { removed, present, coversUs };
2779
+ }
2780
+ /**
2781
+ * Trim a pending suppressed removal broadcast by `removedClientIds` -
2782
+ * someone else already broadcast those departures; what they did not
2783
+ * cover stays queued.
2784
+ */
2785
+ _cancelPendingAwarenessRemovalIfOverlaps(removedClientIds) {
2786
+ if (!this._pendingAwarenessRemoval)
2787
+ return;
2788
+ const rest = this._pendingAwarenessRemoval.filter((id) => !removedClientIds.includes(id));
2789
+ if (rest.length === this._pendingAwarenessRemoval.length)
2790
+ return;
2791
+ if (rest.length > 0) {
2792
+ this._pendingAwarenessRemoval = rest; // someone covered part of it; the rest stays queued
2793
+ return;
2794
+ }
2795
+ if (this._pendingAwarenessRemovalTimeoutId !== undefined) {
2796
+ clearTimeout(this._pendingAwarenessRemovalTimeoutId);
2797
+ this._pendingAwarenessRemovalTimeoutId = undefined;
2798
+ }
2799
+ this._pendingAwarenessRemoval = null;
2800
+ }
2801
+ /** Cancel a pending suppressed awareness-removal broadcast, if any. */
2802
+ _cancelPendingAwarenessRemoval() {
2803
+ if (this._pendingAwarenessRemovalTimeoutId !== undefined) {
2804
+ clearTimeout(this._pendingAwarenessRemovalTimeoutId);
2805
+ this._pendingAwarenessRemovalTimeoutId = undefined;
2806
+ }
2807
+ this._pendingAwarenessRemoval = null;
876
2808
  }
877
2809
  /**
878
2810
  * Send a SyncStep2 reply, gated by the same shared per-peer budget as
@@ -894,7 +2826,7 @@ export class GenericProvider extends Observable {
894
2826
  * would itself become log spam exactly when things are already noisy.
895
2827
  */
896
2828
  _sendSyncReply(reply) {
897
- if (!this._tryReserveSyncSlot()) {
2829
+ if (!this._tryReserveReplySlot()) {
898
2830
  return; // Rate limited - drop the reply silently
899
2831
  }
900
2832
  this._send(reply);
@@ -1011,8 +2943,27 @@ export class GenericProvider extends Observable {
1011
2943
  console.warn(`[GenericProvider] Resync scheduled in ${delay}ms (attempt #${this._resyncAttemptCount})...`);
1012
2944
  this._pendingResyncTimeoutId = setTimeout(() => {
1013
2945
  this._pendingResyncTimeoutId = undefined;
1014
- if (this.transport.isConnected && !this._destroying) {
1015
- this.syncNow();
2946
+ if (!this.transport.isConnected || this._destroying)
2947
+ return;
2948
+ // A resync trigger means "I may be missing something" - never "the
2949
+ // room is missing my data" (my updates travel on their own, and the
2950
+ // connect-time push covers offline edits). So: ask with a 12-byte
2951
+ // beacon; the peers ahead of me reply with exactly the diff (a
2952
+ // SyncStep2 against my state vector, suppressed as any reply), the
2953
+ // rest stay silent. Until phase 1b this pushed the WHOLE document to
2954
+ // the room on every trigger, and that push - a hashed update of my
2955
+ // state - failed the hash check at every peer ahead of me, which
2956
+ // scheduled a resync of its own: the cascade of research doc item 13
2957
+ // (198,990 deliveries for 10 keystrokes at N=100 on the Gun profile,
2958
+ // the rate limiter's ceiling). If the beacon or its reply is lost,
2959
+ // _armResponseWait() re-beacons (1s/2s/4s); the periodic beacon is
2960
+ // the fallback after that. A rate-limited attempt re-arms through
2961
+ // this same coordinator so a stranded peer keeps retrying.
2962
+ if (this._sendSyncStep1(0)) {
2963
+ this._armResponseWait(0);
2964
+ }
2965
+ else {
2966
+ this._requestResync();
1016
2967
  }
1017
2968
  }, delay);
1018
2969
  }
@@ -1025,48 +2976,90 @@ export class GenericProvider extends Observable {
1025
2976
  * its own uncapped or separately-capped allowance.
1026
2977
  */
1027
2978
  _tryReserveSyncSlot() {
2979
+ return this._tryReserveSlot(this._syncRequestTimes);
2980
+ }
2981
+ /** Same limiter, separate budget, for SyncStep2 replies and acks. */
2982
+ _tryReserveReplySlot() {
2983
+ return this._tryReserveSlot(this._syncReplyTimes);
2984
+ }
2985
+ _tryReserveSlot(times) {
1028
2986
  const now = Date.now();
1029
- // Clean up old entries outside the rate limit window
1030
- this._syncRequestTimes = this._syncRequestTimes.filter((t) => now - t < this._syncRequestWindowMs);
1031
- if (this._syncRequestTimes.length >= this._maxSyncRequestsPerWindow) {
2987
+ // Drop entries outside the rolling window (in place: the arrays are
2988
+ // referenced from two fields)
2989
+ let keep = 0;
2990
+ for (const t of times) {
2991
+ if (now - t < this._syncRequestWindowMs)
2992
+ times[keep++] = t;
2993
+ }
2994
+ times.length = keep;
2995
+ if (times.length >= this._maxSyncRequestsPerWindow) {
1032
2996
  return false;
1033
2997
  }
1034
- this._syncRequestTimes.push(now);
2998
+ times.push(now);
1035
2999
  return true;
1036
3000
  }
1037
3001
  /**
1038
- * Encode and send a SyncStep1 message requesting missing updates.
1039
- * Does not check the rate limiter itself - callers must reserve a slot
1040
- * via `_tryReserveSyncSlot()` first.
3002
+ * Encode the digest beacon that replaces SyncStep1 (see
3003
+ * MESSAGE_SYNC_DIGEST). Still the one place every "request sync" path
3004
+ * goes through (connect()'s syncNow(), the periodic tick,
3005
+ * _requestResync()'s retry), so they all switched together.
1041
3006
  */
1042
- _writeSyncStep1() {
3007
+ _encodeSyncStep1(flags = 0) {
3008
+ if (this._confirmed)
3009
+ flags |= DIGEST_FLAG_SETTLED;
3010
+ this._touchPeer(this.doc.clientID);
1043
3011
  const encoder = encoding.createEncoder();
1044
- // SyncStep1 is always sent as standard MESSAGE_SYNC (no verification)
1045
- // It's just a request, not an assertion of state
1046
- encoding.writeVarUint(encoder, MESSAGE_SYNC);
1047
- syncProtocol.writeSyncStep1(encoder, this.doc);
1048
- this._send(encoding.toUint8Array(encoder));
3012
+ encoding.writeVarUint(encoder, MESSAGE_SYNC_DIGEST);
3013
+ encoding.writeVarUint(encoder, DIGEST_VERSION);
3014
+ encoding.writeVarUint(encoder, flags);
3015
+ encoding.writeVarUint(encoder, this.doc.clientID);
3016
+ encoding.writeVarUint8Array(encoder, Y.encodeStateVector(this.doc));
3017
+ encoding.writeVarUint(encoder, this._deleteSetHash());
3018
+ return encoding.toUint8Array(encoder);
1049
3019
  }
1050
3020
  /**
1051
- * Send SyncStep1 message to request missing updates.
1052
- * This is sent when first connecting to sync with remote peers.
1053
- * Note: SyncStep1 is just a request and doesn't include hash verification.
1054
- * Rate limited to prevent spam.
3021
+ * Encode an ack for a JOIN beacon: same framing as a beacon, DIGEST_FLAG_ACK
3022
+ * set, and the JOINER's state vector + delete-set hash echoed back instead
3023
+ * of ours (see DIGEST_FLAG_ACK for why it must never carry our own state).
1055
3024
  */
1056
- _sendSyncStep1() {
3025
+ _encodeAck(ackedSv, ackedDsHash) {
3026
+ this._touchPeer(this.doc.clientID);
3027
+ const encoder = encoding.createEncoder();
3028
+ encoding.writeVarUint(encoder, MESSAGE_SYNC_DIGEST);
3029
+ encoding.writeVarUint(encoder, DIGEST_VERSION);
3030
+ encoding.writeVarUint(encoder, DIGEST_FLAG_ACK | (this._confirmed ? DIGEST_FLAG_SETTLED : 0));
3031
+ encoding.writeVarUint(encoder, this.doc.clientID);
3032
+ encoding.writeVarUint8Array(encoder, ackedSv);
3033
+ encoding.writeVarUint(encoder, ackedDsHash);
3034
+ return encoding.toUint8Array(encoder);
3035
+ }
3036
+ /**
3037
+ * Send the periodic digest beacon. Rate limited to prevent spam. Returns
3038
+ * whether it actually sent (false means rate-limited).
3039
+ */
3040
+ _sendSyncStep1(flags = 0) {
1057
3041
  if (!this._tryReserveSyncSlot()) {
1058
3042
  console.warn(`[GenericProvider] Sync rate limit exceeded (${this._maxSyncRequestsPerWindow} requests per ${this._syncRequestWindowMs / 1000}s), throttling...`);
1059
- return; // Drop the request
3043
+ return false; // Drop the request
1060
3044
  }
1061
- this._writeSyncStep1();
3045
+ this._send(this._encodeSyncStep1(flags));
3046
+ return true;
1062
3047
  }
1063
3048
  /**
1064
- * Send a document update to the transport.
1065
- * If verifyUpdates is enabled, includes sequence number and document hash for ordering and desync detection.
3049
+ * Encode a document update, without sending it. Extracted from the old
3050
+ * `_sendUpdate()` so `_trySyncPushPull()` can fold the push half into a
3051
+ * batched wire send (see `_sendBatch()`); `_sendUpdate()` below is the
3052
+ * send-immediately form still used by every other update-emitting path
3053
+ * (the doc-update handler, batch-flush, disconnect/destroy flush) since
3054
+ * those aren't part of this batching effort's scope.
3055
+ *
3056
+ * NOTE: has a side effect (`_localSeqNum++`) - call exactly once per
3057
+ * logical update, same as before.
1066
3058
  */
1067
- _sendUpdate(update) {
3059
+ _encodeUpdate(update) {
1068
3060
  const encoder = encoding.createEncoder();
1069
3061
  if (this._verifyUpdates) {
3062
+ this._touchPeer(this.doc.clientID);
1070
3063
  // Use verified sync protocol with sequence number and hash
1071
3064
  encoding.writeVarUint(encoder, MESSAGE_SYNC_VERIFIED);
1072
3065
  // Include sequence number and clientID for causal ordering
@@ -1082,6 +3075,35 @@ export class GenericProvider extends Observable {
1082
3075
  encoding.writeVarUint(encoder, MESSAGE_SYNC);
1083
3076
  syncProtocol.writeUpdate(encoder, update);
1084
3077
  }
3078
+ return encoding.toUint8Array(encoder);
3079
+ }
3080
+ /**
3081
+ * Encode a full-state push (see MESSAGE_SYNC_PUSH): the document as one
3082
+ * update, deliberately without the hash and sequence number that
3083
+ * `_encodeUpdate()` adds to incremental updates.
3084
+ */
3085
+ _encodePush(update) {
3086
+ const encoder = encoding.createEncoder();
3087
+ encoding.writeVarUint(encoder, MESSAGE_SYNC_PUSH);
3088
+ encoding.writeVarUint8Array(encoder, update);
3089
+ return encoding.toUint8Array(encoder);
3090
+ }
3091
+ /**
3092
+ * Send a document update to the transport, with whatever awareness change
3093
+ * the throttle is holding folded into the same wire message (round 5,
3094
+ * item 1). If verifyUpdates is enabled, the update carries a sequence
3095
+ * number and document hash for ordering and desync detection.
3096
+ */
3097
+ _sendUpdate(update) {
3098
+ this._sendBatch([this._encodeUpdate(update), ...this._takePendingAwareness()]);
3099
+ }
3100
+ /**
3101
+ * Send awareness update to the transport.
3102
+ */
3103
+ _sendAwarenessUpdate(changedClients) {
3104
+ const encoder = encoding.createEncoder();
3105
+ encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
3106
+ encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, changedClients));
1085
3107
  this._send(encoding.toUint8Array(encoder));
1086
3108
  }
1087
3109
  /**
@@ -1112,10 +3134,11 @@ export class GenericProvider extends Observable {
1112
3134
  }
1113
3135
  }
1114
3136
  /**
1115
- * Send a targeted pub/sub message.
1116
- * Uses transport.sendTo when available (direct delivery), otherwise
1117
- * broadcasts a targeted frame that non-target providers drop.
1118
- * Internal method called by PubSubChannel.
3137
+ * Send a pub/sub message to a single target.
3138
+ *
3139
+ * With `Transport.sendTo` the frame is unicast to that peer; without it the
3140
+ * frame is broadcast with the target embedded and dropped on receipt by
3141
+ * every provider whose `localId` differs.
1119
3142
  */
1120
3143
  _sendPubSubTo(target, topic, message) {
1121
3144
  if (!this.transport.isConnected) {
@@ -1154,63 +3177,201 @@ export class GenericProvider extends Observable {
1154
3177
  * Throttled to prevent awareness updates from flooding document sync.
1155
3178
  * Multiple rapid updates are batched together.
1156
3179
  */
1157
- _broadcastAwareness(clients, channel = AWARENESS_CHANNEL_MAIN) {
3180
+ _broadcastAwareness(clients) {
1158
3181
  if (clients.length === 0)
1159
3182
  return;
1160
- const isApp = channel === AWARENESS_CHANNEL_APP;
1161
- const pending = isApp
1162
- ? this._pendingAppAwarenessClients
1163
- : this._pendingAwarenessClients;
3183
+ const interval = this._effectiveAwarenessInterval();
1164
3184
  // If throttling is disabled, send immediately
1165
- if (this._awarenessInterval <= 0) {
1166
- this._sendAwarenessNow(clients, channel);
3185
+ if (interval <= 0) {
3186
+ this._sendAwarenessNow(clients);
1167
3187
  return;
1168
3188
  }
1169
3189
  // Add clients to pending set
1170
3190
  for (const client of clients) {
1171
- pending.add(client);
3191
+ this._pendingAwarenessClients.add(client);
1172
3192
  }
1173
3193
  // If we already have a scheduled broadcast, let it handle the batched clients
1174
- if ((isApp ? this._appAwarenessTimeoutId : this._awarenessTimeoutId) !==
1175
- undefined) {
3194
+ if (this._awarenessTimeoutId !== undefined) {
1176
3195
  return;
1177
3196
  }
1178
3197
  // Calculate delay - respect minimum interval since last broadcast
1179
3198
  const now = Date.now();
1180
- const lastTime = isApp ? this._lastAppAwarenessTime : this._lastAwarenessTime;
1181
- const delay = Math.max(0, this._awarenessInterval - (now - lastTime));
3199
+ const timeSinceLastBroadcast = now - this._lastAwarenessTime;
3200
+ const delay = Math.max(0, interval - timeSinceLastBroadcast);
1182
3201
  // Schedule the batched broadcast
1183
- const timeoutId = setTimeout(() => {
1184
- if (isApp) {
1185
- this._appAwarenessTimeoutId = undefined;
1186
- this._lastAppAwarenessTime = Date.now();
1187
- }
1188
- else {
1189
- this._awarenessTimeoutId = undefined;
1190
- this._lastAwarenessTime = Date.now();
3202
+ this._awarenessTimeoutId = setTimeout(() => {
3203
+ this._awarenessTimeoutId = undefined;
3204
+ this._lastAwarenessTime = Date.now();
3205
+ // Send all pending clients in one message - and a pending timed
3206
+ // update batch (`batchUpdates > 0`) with them, update first.
3207
+ const clientsToSend = Array.from(this._pendingAwarenessClients);
3208
+ this._pendingAwarenessClients.clear();
3209
+ if (clientsToSend.length > 0) {
3210
+ this._sendBatch([...this._takePendingUpdate(), this._encodeAwareness(clientsToSend)]);
1191
3211
  }
1192
- // Send all pending clients in one message
1193
- const clientsToSend = Array.from(pending);
1194
- pending.clear();
3212
+ }, delay);
3213
+ }
3214
+ /**
3215
+ * Register the app channel's update listener. Same echo suppression as the
3216
+ * core channel (never re-broadcast what came off the wire), but none of its
3217
+ * presence/removal handling - this channel has no bearing on who the room
3218
+ * thinks is present.
3219
+ */
3220
+ _attachAppAwareness(aw) {
3221
+ if (this._appAwarenessUpdateHandler)
3222
+ return;
3223
+ this._appAwarenessUpdateHandler = ({ added, updated, removed, }, origin) => {
3224
+ if (origin === this)
3225
+ return;
3226
+ const changedClients = [...added, ...updated, ...removed];
3227
+ if (changedClients.length === 0)
3228
+ return;
3229
+ this._broadcastAppAwareness(changedClients);
3230
+ };
3231
+ aw.on('update', this._appAwarenessUpdateHandler);
3232
+ }
3233
+ /**
3234
+ * Broadcast app-channel awareness. Mirrors `_broadcastAwareness()`'s
3235
+ * throttle, against its own pending set and timer, and deliberately does
3236
+ * NOT piggyback on the core sync batch: module cursor churn must not pull
3237
+ * document or presence traffic onto its cadence (or vice versa).
3238
+ */
3239
+ _broadcastAppAwareness(clients) {
3240
+ if (clients.length === 0)
3241
+ return;
3242
+ const interval = this._effectiveAwarenessInterval();
3243
+ if (interval <= 0) {
3244
+ this._sendAppAwarenessNow(clients);
3245
+ return;
3246
+ }
3247
+ for (const client of clients) {
3248
+ this._pendingAppAwarenessClients.add(client);
3249
+ }
3250
+ if (this._appAwarenessTimeoutId !== undefined) {
3251
+ return;
3252
+ }
3253
+ const delay = Math.max(0, interval - (Date.now() - this._lastAppAwarenessTime));
3254
+ this._appAwarenessTimeoutId = setTimeout(() => {
3255
+ this._appAwarenessTimeoutId = undefined;
3256
+ this._lastAppAwarenessTime = Date.now();
3257
+ const clientsToSend = Array.from(this._pendingAppAwarenessClients);
3258
+ this._pendingAppAwarenessClients.clear();
1195
3259
  if (clientsToSend.length > 0) {
1196
- this._sendAwarenessNow(clientsToSend, channel);
3260
+ this._sendAppAwarenessNow(clientsToSend);
1197
3261
  }
1198
3262
  }, delay);
1199
- if (isApp)
1200
- this._appAwarenessTimeoutId = timeoutId;
1201
- else
1202
- this._awarenessTimeoutId = timeoutId;
3263
+ }
3264
+ /** Encode and send an app-channel awareness update immediately. */
3265
+ _sendAppAwarenessNow(clients) {
3266
+ const encoder = encoding.createEncoder();
3267
+ encoding.writeVarUint(encoder, MESSAGE_AWARENESS_APP);
3268
+ encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.appAwareness, clients));
3269
+ this._send(encoding.toUint8Array(encoder));
1203
3270
  }
1204
3271
  /**
1205
- * Send awareness update immediately without throttling.
3272
+ * Round 5, item 1: the awareness change the throttle is holding rides
3273
+ * along with a wire message that is leaving anyway. Returns the encoded
3274
+ * awareness sub-message (or nothing) and commits the throttle state
3275
+ * exactly as the timer's own flush would. A piggybacked broadcast costs
3276
+ * no message, only its payload bytes, so it goes out early instead of as
3277
+ * its own message up to `_awarenessInterval` later. Measured in
3278
+ * test/dummy/bench-typing-census.ts: a keystroke in an editor binding is
3279
+ * a text insert plus a cursor update - two broadcasts per keystroke
3280
+ * before this, one after. Broadcast paths only: `_sendDirect` and the
3281
+ * BroadcastChannel-only publishes never call this.
3282
+ */
3283
+ _takePendingAwareness() {
3284
+ if (this._awarenessTimeoutId === undefined)
3285
+ return [];
3286
+ clearTimeout(this._awarenessTimeoutId);
3287
+ this._awarenessTimeoutId = undefined;
3288
+ const clients = Array.from(this._pendingAwarenessClients);
3289
+ this._pendingAwarenessClients.clear();
3290
+ if (clients.length === 0)
3291
+ return [];
3292
+ this._lastAwarenessTime = Date.now();
3293
+ return [this._encodeAwareness(clients)];
3294
+ }
3295
+ /**
3296
+ * The counterpart for the timed batch: a `batchUpdates > 0` batch that is
3297
+ * still waiting rides along with an awareness flush (Matrix: both
3298
+ * default to 2 s, so a typist's cursor and text leave as one PUT).
1206
3299
  */
1207
- _sendAwarenessNow(clients, channel = AWARENESS_CHANNEL_MAIN) {
1208
- const aw = channel === AWARENESS_CHANNEL_APP ? this.appAwareness : this.awareness;
3300
+ _takePendingUpdate() {
3301
+ if (this._batchTimeoutId === undefined || !this._pendingUpdate)
3302
+ return [];
3303
+ clearTimeout(this._batchTimeoutId);
3304
+ this._batchTimeoutId = undefined;
3305
+ this._flushScheduled = false;
3306
+ const update = this._pendingUpdate;
3307
+ this._pendingUpdate = null;
3308
+ return [this._encodeUpdate(update)];
3309
+ }
3310
+ /**
3311
+ * Encode an awareness update, without sending it. Extracted from the old
3312
+ * `_sendAwarenessNow()` so `_tryImmediateAwarenessMessage()` can fold it
3313
+ * into a batched wire send instead of always sending it as its own
3314
+ * message.
3315
+ */
3316
+ _encodeAwareness(clients) {
1209
3317
  const encoder = encoding.createEncoder();
1210
3318
  encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
1211
- encoding.writeVarUint(encoder, channel);
1212
- encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(aw, clients));
1213
- this._send(encoding.toUint8Array(encoder));
3319
+ encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, clients));
3320
+ return encoding.toUint8Array(encoder);
3321
+ }
3322
+ /**
3323
+ * Send awareness update immediately without throttling.
3324
+ */
3325
+ _sendAwarenessNow(clients) {
3326
+ this._send(this._encodeAwareness(clients));
3327
+ }
3328
+ /**
3329
+ * Attempt to build an awareness broadcast message for immediate
3330
+ * inclusion in the same wire send as a sync message a caller is about to
3331
+ * send anyway (see `_trySyncPushPull`'s `buildExtra` parameter), instead
3332
+ * of going through
3333
+ * `_broadcastAwareness()`'s independent debounce.
3334
+ *
3335
+ * Only returns non-null when the throttle would have let an immediate
3336
+ * send through anyway - i.e. no debounced broadcast is already pending
3337
+ * AND (throttling is disabled, or at least `_awarenessInterval` ms have
3338
+ * passed since the last broadcast) - so this never changes awareness
3339
+ * throttle semantics, only whether the resulting message travels as its
3340
+ * own wire send or bundled with a sync message that happens to be going
3341
+ * out "now" too.
3342
+ *
3343
+ * Mutates the same state `_broadcastAwareness()`'s own immediate-send
3344
+ * branches mutate (`_pendingAwarenessClients`, `_lastAwarenessTime`) -
3345
+ * once this returns non-null, the state is already committed as "sent
3346
+ * now", so the caller MUST actually send the returned message (bundled
3347
+ * or standalone) rather than discarding it.
3348
+ */
3349
+ _tryImmediateAwarenessMessage(clients) {
3350
+ if (clients.length === 0)
3351
+ return null;
3352
+ // A debounced broadcast is already scheduled - let it handle these
3353
+ // clients via the normal pending-set path below, don't race it with an
3354
+ // immediate send here.
3355
+ if (this._awarenessTimeoutId !== undefined)
3356
+ return null;
3357
+ const interval = this._effectiveAwarenessInterval();
3358
+ if (interval > 0) {
3359
+ const timeSinceLastBroadcast = Date.now() - this._lastAwarenessTime;
3360
+ if (timeSinceLastBroadcast < interval)
3361
+ return null;
3362
+ }
3363
+ // Merge with anything already pending (normally empty here since no
3364
+ // timer is scheduled, but merge for safety) and commit to sending now -
3365
+ // mirrors exactly what _broadcastAwareness()'s own immediate
3366
+ // (_awarenessInterval <= 0) and debounced-timer-fired branches do.
3367
+ for (const c of clients)
3368
+ this._pendingAwarenessClients.add(c);
3369
+ const clientsToSend = Array.from(this._pendingAwarenessClients);
3370
+ this._pendingAwarenessClients.clear();
3371
+ if (clientsToSend.length === 0)
3372
+ return null;
3373
+ this._lastAwarenessTime = Date.now();
3374
+ return this._encodeAwareness(clientsToSend);
1214
3375
  }
1215
3376
  /**
1216
3377
  * Setup BroadcastChannel for cross-tab communication.
@@ -1244,30 +3405,22 @@ export class GenericProvider extends Observable {
1244
3405
  const encoderSync = encoding.createEncoder();
1245
3406
  encoding.writeVarUint(encoderSync, MESSAGE_SYNC);
1246
3407
  syncProtocol.writeSyncStep1(encoderSync, this.doc);
1247
- bc.publish(this._bcChannel, wrapMessageWithChecksum(encoding.toUint8Array(encoderSync)), this);
3408
+ this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoderSync)));
1248
3409
  // Broadcast local state via BroadcastChannel (wrapped with CRC32)
1249
3410
  const encoderState = encoding.createEncoder();
1250
3411
  encoding.writeVarUint(encoderState, MESSAGE_SYNC);
1251
3412
  syncProtocol.writeSyncStep2(encoderState, this.doc);
1252
- bc.publish(this._bcChannel, wrapMessageWithChecksum(encoding.toUint8Array(encoderState)), this);
3413
+ this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoderState)));
1253
3414
  // Broadcast local awareness state via BroadcastChannel (wrapped with CRC32)
1254
3415
  if (this.awareness.getLocalState() !== null) {
1255
- this._publishAwarenessToBroadcastChannel(this.awareness, AWARENESS_CHANNEL_MAIN);
1256
- }
1257
- if (this.appAwareness.getLocalState() !== null) {
1258
- this._publishAwarenessToBroadcastChannel(this.appAwareness, AWARENESS_CHANNEL_APP);
3416
+ const encoderAwareness = encoding.createEncoder();
3417
+ encoding.writeVarUint(encoderAwareness, MESSAGE_AWARENESS);
3418
+ encoding.writeVarUint8Array(encoderAwareness, awarenessProtocol.encodeAwarenessUpdate(this.awareness, [
3419
+ this.doc.clientID,
3420
+ ]));
3421
+ this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoderAwareness)));
1259
3422
  }
1260
3423
  }
1261
- /**
1262
- * Encode and publish an awareness update for the local client to the BroadcastChannel.
1263
- */
1264
- _publishAwarenessToBroadcastChannel(awareness, channel) {
1265
- const encoder = encoding.createEncoder();
1266
- encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
1267
- encoding.writeVarUint(encoder, channel);
1268
- encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(awareness, [this.doc.clientID]));
1269
- bc.publish(this._bcChannel, wrapMessageWithChecksum(encoding.toUint8Array(encoder)), this);
1270
- }
1271
3424
  /**
1272
3425
  * Disconnect from BroadcastChannel and mark local client as offline.
1273
3426
  */
@@ -1276,21 +3429,71 @@ export class GenericProvider extends Observable {
1276
3429
  return;
1277
3430
  }
1278
3431
  // Broadcast awareness state with null (indicating disconnect) - wrapped with CRC32
1279
- for (const [awareness, channel] of [
1280
- [this.awareness, AWARENESS_CHANNEL_MAIN],
1281
- [this.appAwareness, AWARENESS_CHANNEL_APP],
1282
- ]) {
1283
- const encoder = encoding.createEncoder();
1284
- encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
1285
- encoding.writeVarUint(encoder, channel);
1286
- encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(awareness, [this.doc.clientID], new Map()));
1287
- bc.publish(this._bcChannel, wrapMessageWithChecksum(encoding.toUint8Array(encoder)), this);
1288
- }
3432
+ const encoder = encoding.createEncoder();
3433
+ encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
3434
+ encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, [this.doc.clientID], new Map()));
3435
+ this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoder)));
1289
3436
  // Unsubscribe from channel
1290
3437
  bc.unsubscribe(this._bcChannel, this._bcSubscriber);
1291
3438
  this._bcConnected = false;
1292
3439
  this._bcSubscriber = undefined;
1293
3440
  }
3441
+ /**
3442
+ * Send N already-encoded, already-typed sub-messages as ONE wire message
3443
+ * instead of N separate `transport.send()`/`bc.publish()` calls, when
3444
+ * there's more than one to send. Used at trigger points that
3445
+ * conceptually produce a single event but historically sent multiple
3446
+ * independent messages for it (sync push, sync pull, awareness) - see
3447
+ * `_trySyncPushPull()` (connect-time push + digest beacon + awareness in
3448
+ * one wire message).
3449
+ *
3450
+ * Design (see the task's framing requirements):
3451
+ * - Each sub-message is length-prefixed with `writeVarUint8Array`,
3452
+ * consistent with how this codebase already frames variable-length
3453
+ * payloads elsewhere (e.g. MESSAGE_AWARENESS). On receipt,
3454
+ * `_dispatchMessage()`'s `MESSAGE_BATCH` case unwraps and re-dispatches
3455
+ * each one through the EXACT SAME per-message-type logic used for a
3456
+ * top-level message - no parallel reimplementation.
3457
+ * - Sub-messages are NOT individually CRC32-wrapped here - the whole
3458
+ * batch envelope goes through the normal, single `_send()` pipeline
3459
+ * below, which wraps the WHOLE envelope in exactly one CRC32 checksum
3460
+ * (and, if `compressionThresholdBytes` is configured, one compression
3461
+ * pass) - built and computed exactly like any other outgoing message,
3462
+ * so this composes with the existing compression pipeline for free
3463
+ * rather than fighting it with a second, nested wrap/compress step.
3464
+ * The tradeoff: a single corrupted bit anywhere in a batched wire
3465
+ * message now invalidates every sub-message it carried, not just one -
3466
+ * per-sub-message CRC32s would avoid that, at the cost of ~4 extra
3467
+ * bytes per sub-message for a benefit that only matters under active
3468
+ * corruption. This tradeoff is exactly what
3469
+ * test/dummy/bench-corruption-storm.ts and bench-packet-loss.ts exist
3470
+ * to measure empirically, per this task's validation requirements,
3471
+ * rather than deciding it by design argument alone.
3472
+ * - BroadcastChannel (cross-tab) traffic is NOT specially batched beyond
3473
+ * whatever `_send()` already does per call - same-tab-group cross-tab
3474
+ * traffic is local/cheap, and `_send()` already only issues one
3475
+ * `bc.publish()` per call regardless, so a batch of N sub-messages
3476
+ * already becomes exactly one `bc.publish()` call for free once routed
3477
+ * through here - no separate BC-specific batching logic needed.
3478
+ */
3479
+ _sendBatch(messages) {
3480
+ if (messages.length === 0)
3481
+ return;
3482
+ if (messages.length === 1) {
3483
+ this._send(messages[0]);
3484
+ return;
3485
+ }
3486
+ this._send(this._encodeBatch(messages));
3487
+ }
3488
+ /** The MESSAGE_BATCH envelope of `_sendBatch`, without sending it. */
3489
+ _encodeBatch(messages) {
3490
+ const encoder = encoding.createEncoder();
3491
+ encoding.writeVarUint(encoder, MESSAGE_BATCH);
3492
+ for (const message of messages) {
3493
+ encoding.writeVarUint8Array(encoder, message);
3494
+ }
3495
+ return encoding.toUint8Array(encoder);
3496
+ }
1294
3497
  /**
1295
3498
  * Send data through both BroadcastChannel (if connected) and transport.
1296
3499
  * All messages are wrapped with CRC32 checksum for integrity verification.
@@ -1299,16 +3502,66 @@ export class GenericProvider extends Observable {
1299
3502
  _send(data) {
1300
3503
  // Wrap message with CRC32 checksum
1301
3504
  const wrappedData = wrapMessageWithChecksum(data);
1302
- // Send via BroadcastChannel to other tabs first
1303
- if (this._bcConnected) {
1304
- bc.publish(this._bcChannel, wrappedData, this);
1305
- }
3505
+ // Send via BroadcastChannel to other tabs first.
3506
+ if (this._bcConnected)
3507
+ this._bcPublish(wrappedData);
1306
3508
  // Send via network transport
1307
3509
  if (!this.transport.isConnected) {
1308
3510
  return;
1309
3511
  }
3512
+ this._sendToTransport(wrappedData);
3513
+ }
3514
+ /**
3515
+ * Publish already-CRC32-wrapped bytes to the other tabs. BroadcastChannel
3516
+ * is same-process - never worth compressing - but when
3517
+ * compressionThresholdBytes is enabled every message still needs the
3518
+ * leading flag byte _handleIncomingMessage() expects regardless of
3519
+ * source, so this sends flag=0 in that case. The ONE place for every BC
3520
+ * publish: the connect-time burst in _setupBroadcastChannel() used to
3521
+ * publish without the flag, and with compression on (the transport
3522
+ * hints made that a default) the other tab read a CRC byte as the flag
3523
+ * and failed to inflate three messages per join (Nostr playground,
3524
+ * 2026-09-06).
3525
+ */
3526
+ _bcPublish(wrappedData) {
3527
+ bc.publish(this._bcChannel, this._compressionThresholdBytes
3528
+ ? prefixCompressionFlag(0, wrappedData)
3529
+ : wrappedData, this);
3530
+ }
3531
+ /**
3532
+ * Send already-CRC32-wrapped bytes to the network transport, compressing
3533
+ * first if compressionThresholdBytes is configured and this payload
3534
+ * clears it. See that option's doc comment for the size threshold
3535
+ * reasoning and the wire-format compatibility tradeoff of enabling it.
3536
+ */
3537
+ _sendToTransport(wrappedData, to) {
3538
+ const threshold = this._compressionThresholdBytes;
3539
+ if (!threshold) {
3540
+ this._dispatchToTransport(wrappedData, to);
3541
+ return;
3542
+ }
3543
+ if (!COMPRESSION_AVAILABLE || wrappedData.length < threshold) {
3544
+ this._dispatchToTransport(prefixCompressionFlag(0, wrappedData), to);
3545
+ return;
3546
+ }
3547
+ compressDeflateRaw(wrappedData)
3548
+ .then((compressed) => {
3549
+ this._dispatchToTransport(prefixCompressionFlag(1, compressed), to);
3550
+ })
3551
+ .catch((error) => {
3552
+ console.error('[GenericProvider] Compression failed, sending uncompressed:', error);
3553
+ this._dispatchToTransport(prefixCompressionFlag(0, wrappedData), to);
3554
+ });
3555
+ }
3556
+ /**
3557
+ * Hand fully-framed bytes to transport.send() - or transport.sendTo() when
3558
+ * a peer address is given - tolerating a sync or async result.
3559
+ */
3560
+ _dispatchToTransport(data, to) {
1310
3561
  try {
1311
- const result = this.transport.send(wrappedData);
3562
+ const result = to !== undefined && typeof this.transport.sendTo === 'function'
3563
+ ? this.transport.sendTo(to, data)
3564
+ : this.transport.send(data);
1312
3565
  // Handle async send
1313
3566
  if (result instanceof Promise) {
1314
3567
  result.catch((error) => {
@@ -1344,4 +3597,6 @@ export class GenericProvider extends Observable {
1344
3597
  return this._localSeqNum;
1345
3598
  }
1346
3599
  }
3600
+ // ms of throttle added per known peer under `awarenessInterval: 'auto'`
3601
+ GenericProvider.AWARENESS_AUTO_MS_PER_PEER = 20;
1347
3602
  //# sourceMappingURL=index.js.map