@edryslabs/genericprovider 1.0.3 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +129 -5
- package/dist/index.d.ts +1009 -42
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2936 -392
- package/dist/index.js.map +1 -1
- package/dist/lib.d.ts +1 -0
- package/dist/lib.d.ts.map +1 -1
- package/dist/lib.js +3 -0
- package/dist/lib.js.map +1 -1
- package/dist/providers/ably/index.d.ts +203 -0
- package/dist/providers/ably/index.d.ts.map +1 -0
- package/dist/providers/ably/index.js +561 -0
- package/dist/providers/ably/index.js.map +1 -0
- package/dist/providers/chunking.d.ts +26 -0
- package/dist/providers/chunking.d.ts.map +1 -0
- package/dist/providers/chunking.js +52 -0
- package/dist/providers/chunking.js.map +1 -0
- package/dist/providers/dummy/index.d.ts +138 -2
- package/dist/providers/dummy/index.d.ts.map +1 -1
- package/dist/providers/dummy/index.js +266 -4
- package/dist/providers/dummy/index.js.map +1 -1
- package/dist/providers/gun/index.d.ts +29 -4
- package/dist/providers/gun/index.d.ts.map +1 -1
- package/dist/providers/gun/index.js +165 -34
- package/dist/providers/gun/index.js.map +1 -1
- package/dist/providers/indexeddb/index.d.ts +44 -20
- package/dist/providers/indexeddb/index.d.ts.map +1 -1
- package/dist/providers/indexeddb/index.js +85 -71
- package/dist/providers/indexeddb/index.js.map +1 -1
- package/dist/providers/matrix/index.d.ts +11 -0
- package/dist/providers/matrix/index.d.ts.map +1 -1
- package/dist/providers/matrix/index.js +49 -5
- package/dist/providers/matrix/index.js.map +1 -1
- package/dist/providers/nostr/index.d.ts +76 -5
- package/dist/providers/nostr/index.d.ts.map +1 -1
- package/dist/providers/nostr/index.js +211 -36
- package/dist/providers/nostr/index.js.map +1 -1
- package/dist/providers/peerjs/index.d.ts +11 -1
- package/dist/providers/peerjs/index.d.ts.map +1 -1
- package/dist/providers/peerjs/index.js +32 -2
- package/dist/providers/peerjs/index.js.map +1 -1
- package/dist/providers/pubnub/index.d.ts +35 -1
- package/dist/providers/pubnub/index.d.ts.map +1 -1
- package/dist/providers/pubnub/index.js +56 -35
- package/dist/providers/pubnub/index.js.map +1 -1
- package/dist/providers/simple-peer/index.d.ts +8 -14
- package/dist/providers/simple-peer/index.d.ts.map +1 -1
- package/dist/providers/simple-peer/index.js +45 -68
- package/dist/providers/simple-peer/index.js.map +1 -1
- package/dist/providers/supabase/index.d.ts +43 -1
- package/dist/providers/supabase/index.d.ts.map +1 -1
- package/dist/providers/supabase/index.js +211 -18
- package/dist/providers/supabase/index.js.map +1 -1
- package/dist/providers/trystero/index.d.ts +19 -1
- package/dist/providers/trystero/index.d.ts.map +1 -1
- package/dist/providers/trystero/index.js +37 -2
- package/dist/providers/trystero/index.js.map +1 -1
- package/dist/providers/websocket/index.d.ts +9 -1
- package/dist/providers/websocket/index.d.ts.map +1 -1
- package/dist/providers/websocket/index.js +7 -1
- package/dist/providers/websocket/index.js.map +1 -1
- package/dist/transport.d.ts +80 -16
- package/dist/transport.d.ts.map +1 -1
- package/package.json +35 -24
package/dist/index.js
CHANGED
|
@@ -10,10 +10,83 @@ const MESSAGE_SYNC = 0;
|
|
|
10
10
|
const MESSAGE_AWARENESS = 1;
|
|
11
11
|
const MESSAGE_PUBSUB = 2;
|
|
12
12
|
const MESSAGE_SYNC_VERIFIED = 3; // Sync message with hash verification
|
|
13
|
-
const
|
|
14
|
-
//
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
const MESSAGE_BATCH = 4; // Envelope for N independently-typed sub-messages sent as one wire message
|
|
14
|
+
// Digest beacon: replaces SyncStep1 on the wire. [version][flags][sender
|
|
15
|
+
// clientID][state vector][delete-set hash]. Receivers reply only when the
|
|
16
|
+
// sender is behind them or the delete-set hashes differ (SyncStep2, as
|
|
17
|
+
// before), or - on a JOIN-flagged beacon with equal state - with their own
|
|
18
|
+
// beacon as an ack; otherwise not at all. See
|
|
19
|
+
// docs/superpowers/specs/2026-09-05-digest-beacon-design.md and
|
|
20
|
+
// _handleDigest(). Versions are append-only: receivers read the fields
|
|
21
|
+
// they know and ignore trailing bytes.
|
|
22
|
+
const MESSAGE_SYNC_DIGEST = 5;
|
|
23
|
+
const DIGEST_VERSION = 1;
|
|
24
|
+
const DIGEST_FLAG_JOIN = 1; // bit 0: "I just joined - send me your presence and confirm my state"
|
|
25
|
+
// bit 1: this message is an ACK, not a request. It echoes the state vector
|
|
26
|
+
// and delete-set hash of the JOIN beacon it answers ("I hold exactly this
|
|
27
|
+
// state too"), never the sender's own state - so it is never read as a
|
|
28
|
+
// request and never collects a SyncStep2 from anyone. Receivers whose state
|
|
29
|
+
// equals the echoed digest mark themselves synced (the joiner it was for,
|
|
30
|
+
// and everyone else in that state); receivers holding a pending ack for the
|
|
31
|
+
// same digest drop it. See _handleDigest(). Before this flag existed the ack
|
|
32
|
+
// was the acker's own beacon; an acker that was itself behind the room (a
|
|
33
|
+
// joiner acking another joiner) made every peer ahead of it reply with a
|
|
34
|
+
// SyncStep2, and acks landing after an edit burst had started did the same
|
|
35
|
+
// (Task 3c in the design doc).
|
|
36
|
+
const DIGEST_FLAG_ACK = 2;
|
|
37
|
+
// bit 2: "ack me if our states are equal" WITHOUT the presence request of
|
|
38
|
+
// DIGEST_FLAG_JOIN. Sent only by the response-wait retry
|
|
39
|
+
// (_armResponseWait): a joiner that gets neither a SyncStep2 nor an ack
|
|
40
|
+
// within the wait has lost one message or the other and asks again - reply
|
|
41
|
+
// suppression deliberately leaves ~1 reply per request, so a single lost
|
|
42
|
+
// reply would otherwise strand the joiner's `synced` until the next
|
|
43
|
+
// periodic beacon (never, with syncInterval 0). Measured: the last joiner
|
|
44
|
+
// of a simultaneous 5-peer burst at 10% loss failed to reach `synced` in 1
|
|
45
|
+
// of 600 runs before this existed. Resync beacons do NOT request acks: in
|
|
46
|
+
// an equal room every peer would answer, and at high latency the
|
|
47
|
+
// suppression window cannot thin those replies (measured 4-5x more acks).
|
|
48
|
+
const DIGEST_FLAG_CONFIRM = 4;
|
|
49
|
+
// bit 3: the sender is CONFIRMED - it has received a SyncStep2, or an equal
|
|
50
|
+
// digest from a confirmed peer, or its own join asked three times and got
|
|
51
|
+
// nothing better (so it is the room). A joiner's response wait is satisfied
|
|
52
|
+
// only by data (SyncStep2) or by an equal digest carrying this bit: an
|
|
53
|
+
// equal ack from a fellow joiner says nothing about whether the room holds
|
|
54
|
+
// more than both of us, and treating it as an answer left a joiner whose
|
|
55
|
+
// SyncStep2 was lost with an empty document (measured: 1 of 150 lossy
|
|
56
|
+
// 15-peer joins; research doc item 13 follow-up in the phase-1b design).
|
|
57
|
+
const DIGEST_FLAG_SETTLED = 8;
|
|
58
|
+
// Full-state push (connect()/syncNow()): the whole document as one update,
|
|
59
|
+
// no hash, no sequence number, applied and nothing else. Before this type a
|
|
60
|
+
// push was a MESSAGE_SYNC_VERIFIED update whose hash was the PUSHER's state
|
|
61
|
+
// - every peer holding more data read that as divergence and scheduled a
|
|
62
|
+
// resync of its own (research doc item 12), the seed of the cascade in item
|
|
63
|
+
// 13. Not a SyncStep2 either: receiving one must not flip `synced` (an
|
|
64
|
+
// empty joiner's push says nothing about the room's content). See
|
|
65
|
+
// docs/superpowers/specs/2026-09-05-resync-cascade-design.md.
|
|
66
|
+
const MESSAGE_SYNC_PUSH = 6;
|
|
67
|
+
// Pub/sub message aimed at a single target: [target][topic][message]. Every
|
|
68
|
+
// provider whose `localId` differs drops it. On transports with sendTo it is
|
|
69
|
+
// unicast; elsewhere it is broadcast and filtered on receipt.
|
|
70
|
+
//
|
|
71
|
+
// NOTE: this was type 4 before the v1.5.0 merge, which upstream had taken for
|
|
72
|
+
// MESSAGE_BATCH. Renumbered to 7 (the first free slot) - the two are not wire
|
|
73
|
+
// compatible, so all peers of a room must run the same version, as the README
|
|
74
|
+
// already requires.
|
|
75
|
+
const MESSAGE_PUBSUB_TARGETED = 7;
|
|
76
|
+
// A second, independent awareness channel for application/module state
|
|
77
|
+
// (cursors, module presence), kept off MESSAGE_AWARENESS deliberately.
|
|
78
|
+
//
|
|
79
|
+
// Upstream's MESSAGE_AWARENESS receive path is now core presence machinery:
|
|
80
|
+
// it feeds _knownPeers, sets _presenceCovered, and cancels pending removal
|
|
81
|
+
// broadcasts. App awareness is written by untrusted third-party classroom
|
|
82
|
+
// modules, so routing it through that path would let module cursor traffic
|
|
83
|
+
// drive the room's peer bookkeeping - a module setting a state would mark
|
|
84
|
+
// phantom peers present and suppress real removals. A separate type keeps
|
|
85
|
+
// the two entirely disjoint: same throttle policy, no shared presence state.
|
|
86
|
+
//
|
|
87
|
+
// (Dev encoded this as a channel varint inside MESSAGE_AWARENESS, which was
|
|
88
|
+
// safe when that path did nothing but applyAwarenessUpdate. It no longer is.)
|
|
89
|
+
const MESSAGE_AWARENESS_APP = 8;
|
|
17
90
|
/**
|
|
18
91
|
* CRC32 lookup table for fast computation.
|
|
19
92
|
* Generated once and reused for all CRC calculations.
|
|
@@ -80,17 +153,105 @@ function unwrapAndVerifyMessage(wrapped) {
|
|
|
80
153
|
}
|
|
81
154
|
return message;
|
|
82
155
|
}
|
|
156
|
+
/** Byte-wise equality of two Uint8Arrays. */
|
|
157
|
+
function bytesEqual(a, b) {
|
|
158
|
+
if (a.length !== b.length)
|
|
159
|
+
return false;
|
|
160
|
+
for (let i = 0; i < a.length; i++) {
|
|
161
|
+
if (a[i] !== b[i])
|
|
162
|
+
return false;
|
|
163
|
+
}
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Componentwise minimum of two encoded state vectors: the state a
|
|
168
|
+
* SyncStep2 must start from to serve both requesters. Clients missing from
|
|
169
|
+
* one side count as clock 0 and are omitted (omitted = 0 on the wire).
|
|
170
|
+
*/
|
|
171
|
+
function minStateVector(a, b) {
|
|
172
|
+
const ma = Y.decodeStateVector(a);
|
|
173
|
+
const mb = Y.decodeStateVector(b);
|
|
174
|
+
const out = new Map();
|
|
175
|
+
for (const [client, clock] of ma) {
|
|
176
|
+
const m = Math.min(clock, mb.get(client) ?? 0);
|
|
177
|
+
if (m > 0)
|
|
178
|
+
out.set(client, m);
|
|
179
|
+
}
|
|
180
|
+
return Y.encodeStateVector(out);
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Whether this runtime has the Compression Streams API (Node 18+, all
|
|
184
|
+
* evergreen browsers). Checked once at module load; compressionThresholdBytes
|
|
185
|
+
* falls back to sending uncompressed (still flag-byte-prefixed, flag=0) if
|
|
186
|
+
* this is false, rather than throwing.
|
|
187
|
+
*/
|
|
188
|
+
const COMPRESSION_AVAILABLE = typeof CompressionStream !== 'undefined' &&
|
|
189
|
+
typeof DecompressionStream !== 'undefined';
|
|
190
|
+
/** Drain a ReadableStream<Uint8Array> into a single concatenated Uint8Array. */
|
|
191
|
+
async function _readAllChunks(readable) {
|
|
192
|
+
const chunks = [];
|
|
193
|
+
const reader = readable.getReader();
|
|
194
|
+
for (;;) {
|
|
195
|
+
const { done, value } = await reader.read();
|
|
196
|
+
if (done)
|
|
197
|
+
break;
|
|
198
|
+
chunks.push(value);
|
|
199
|
+
}
|
|
200
|
+
const total = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
201
|
+
const out = new Uint8Array(total);
|
|
202
|
+
let offset = 0;
|
|
203
|
+
for (const chunk of chunks) {
|
|
204
|
+
out.set(chunk, offset);
|
|
205
|
+
offset += chunk.length;
|
|
206
|
+
}
|
|
207
|
+
return out;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Compress with deflate-raw (no gzip header/trailer - see
|
|
211
|
+
* compressionThresholdBytes's doc comment for why deflate-raw over gzip).
|
|
212
|
+
*/
|
|
213
|
+
async function compressDeflateRaw(data) {
|
|
214
|
+
const cs = new CompressionStream('deflate-raw');
|
|
215
|
+
const writer = cs.writable.getWriter();
|
|
216
|
+
// The writer promises reject too when the stream errors; the reader side
|
|
217
|
+
// already surfaces that error, and an unhandled rejection here crashes
|
|
218
|
+
// Node (seen with a corrupt deflate stream in the Nostr end-to-end test).
|
|
219
|
+
writer.write(data).catch(() => { });
|
|
220
|
+
writer.close().catch(() => { });
|
|
221
|
+
return _readAllChunks(cs.readable);
|
|
222
|
+
}
|
|
223
|
+
/** Inverse of compressDeflateRaw(). */
|
|
224
|
+
async function decompressDeflateRaw(data) {
|
|
225
|
+
const ds = new DecompressionStream('deflate-raw');
|
|
226
|
+
const writer = ds.writable.getWriter();
|
|
227
|
+
writer.write(data).catch(() => { });
|
|
228
|
+
writer.close().catch(() => { });
|
|
229
|
+
return _readAllChunks(ds.readable);
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Prepend a 1-byte compressed(1)/uncompressed(0) flag. Only used when
|
|
233
|
+
* compressionThresholdBytes is configured - see that option's doc comment
|
|
234
|
+
* for why this is a deliberate, opt-in wire-format change.
|
|
235
|
+
*/
|
|
236
|
+
function prefixCompressionFlag(flag, data) {
|
|
237
|
+
const out = new Uint8Array(1 + data.length);
|
|
238
|
+
out[0] = flag;
|
|
239
|
+
out.set(data, 1);
|
|
240
|
+
return out;
|
|
241
|
+
}
|
|
83
242
|
/**
|
|
84
|
-
* Compute a
|
|
85
|
-
* Uses a fast non-cryptographic hash for performance.
|
|
243
|
+
* Compute a cheap hash of document state for verification.
|
|
86
244
|
*
|
|
87
|
-
* Hashes the state VECTOR, not
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
245
|
+
* Hashes the state VECTOR (each client's clock), not the full document
|
|
246
|
+
* content — O(number of distinct clients) instead of O(document content
|
|
247
|
+
* size). `Y.encodeStateVector()` writes entries sorted by clientID, so the
|
|
248
|
+
* result is deterministic regardless of the internal Map's iteration order.
|
|
249
|
+
* Two peers can only reach the same state vector by having applied the same
|
|
250
|
+
* set of operations, so this still catches real content divergence; it just
|
|
251
|
+
* no longer re-serializes the entire document on every single update.
|
|
252
|
+
* (CRC32 already guards against wire corruption, and sequence tracking
|
|
253
|
+
* guards against reordering/loss — this hash is the last line of defense
|
|
254
|
+
* against logical divergence between peers.)
|
|
94
255
|
*/
|
|
95
256
|
function computeDocHash(doc) {
|
|
96
257
|
const state = Y.encodeStateVector(doc);
|
|
@@ -100,6 +261,109 @@ function computeDocHash(doc) {
|
|
|
100
261
|
}
|
|
101
262
|
return hash;
|
|
102
263
|
}
|
|
264
|
+
/**
|
|
265
|
+
* Cheap, peer-deterministic hash of the document's delete set - the half of
|
|
266
|
+
* a Yjs document's identity that the state vector does NOT cover (yjs
|
|
267
|
+
* INTERNALS.md: "deletions are tracked in the DeleteSet, and do not update
|
|
268
|
+
* the state vector"). Two docs that differ only by a lost delete-only
|
|
269
|
+
* update have identical state vectors, so `computeDocHash` can never
|
|
270
|
+
* detect that divergence; this hash can, at heartbeat granularity (see
|
|
271
|
+
* `_encodeSyncStep1()` / `_handleDigest()`).
|
|
272
|
+
*
|
|
273
|
+
* Cost: `Y.createDeleteSetFromStructStore` walks every struct (Yjs keeps no
|
|
274
|
+
* incremental delete set), so this is O(items) - fine once per heartbeat
|
|
275
|
+
* (every empty SyncStep2 already did this exact walk inside
|
|
276
|
+
* `encodeStateAsUpdate`), NOT fine per update; hence the cache in
|
|
277
|
+
* `_deleteSetHash()`. Per-client runs come out already sorted and merged;
|
|
278
|
+
* the only per-peer non-determinism is `Map` insertion order, fixed by
|
|
279
|
+
* sorting client IDs before hashing.
|
|
280
|
+
*
|
|
281
|
+
* Exported for the property check in test/dummy/bench-idle-room.ts.
|
|
282
|
+
* @internal
|
|
283
|
+
*/
|
|
284
|
+
export function computeDeleteSetHash(doc) {
|
|
285
|
+
const ds = Y.createDeleteSetFromStructStore(doc.store);
|
|
286
|
+
const encoder = encoding.createEncoder();
|
|
287
|
+
const clients = Array.from(ds.clients.keys()).sort((x, y) => x - y);
|
|
288
|
+
for (const client of clients) {
|
|
289
|
+
encoding.writeVarUint(encoder, client);
|
|
290
|
+
for (const item of ds.clients.get(client)) {
|
|
291
|
+
encoding.writeVarUint(encoder, item.clock);
|
|
292
|
+
encoding.writeVarUint(encoder, item.len);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
return computeCRC32(encoding.toUint8Array(encoder));
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* The Yjs updates a CRC-wrapped frame (as handed to `Transport.send`)
|
|
299
|
+
* carries: the update of a MESSAGE_SYNC_VERIFIED or MESSAGE_SYNC
|
|
300
|
+
* Update/SyncStep2 message, the whole document of a MESSAGE_SYNC_PUSH,
|
|
301
|
+
* each such sub-message of a MESSAGE_BATCH - and nothing for awareness,
|
|
302
|
+
* pub/sub, digest beacons and SyncStep1 requests, which carry no document
|
|
303
|
+
* state. For persistence transports (`providers/indexeddb`, LiaScript's
|
|
304
|
+
* Dexie cache): store only what this returns, and only this. Storing whole
|
|
305
|
+
* frames and replaying them on the next load resurrects the previous
|
|
306
|
+
* session's clientID as a phantom peer (its presence, its beacons - which
|
|
307
|
+
* the provider then answers into the store, multiplying rows) and keeps
|
|
308
|
+
* ~10x the bytes (a keystroke's frame carries its cursor). A frame this
|
|
309
|
+
* cannot parse yields `[]`, never a partial read. Frames of a provider with
|
|
310
|
+
* `compressionThresholdBytes` set (a leading flag byte) are not supported.
|
|
311
|
+
* Round 7, item 6; measured in test/dummy/bench-persist-log.ts.
|
|
312
|
+
*/
|
|
313
|
+
export function extractDocUpdates(frame) {
|
|
314
|
+
const updates = [];
|
|
315
|
+
const walk = (message) => {
|
|
316
|
+
const decoder = decoding.createDecoder(message);
|
|
317
|
+
const type = decoding.readVarUint(decoder);
|
|
318
|
+
switch (type) {
|
|
319
|
+
case MESSAGE_BATCH:
|
|
320
|
+
while (decoding.hasContent(decoder))
|
|
321
|
+
walk(decoding.readVarUint8Array(decoder));
|
|
322
|
+
break;
|
|
323
|
+
case MESSAGE_SYNC_VERIFIED:
|
|
324
|
+
case MESSAGE_SYNC: {
|
|
325
|
+
if (type === MESSAGE_SYNC_VERIFIED) {
|
|
326
|
+
decoding.readVarUint(decoder); // sequence number
|
|
327
|
+
decoding.readVarUint(decoder); // sender clientID
|
|
328
|
+
}
|
|
329
|
+
const sub = decoding.readVarUint(decoder);
|
|
330
|
+
if (sub === syncProtocol.messageYjsSyncStep2 ||
|
|
331
|
+
sub === syncProtocol.messageYjsUpdate) {
|
|
332
|
+
updates.push(decoding.readVarUint8Array(decoder));
|
|
333
|
+
}
|
|
334
|
+
break;
|
|
335
|
+
}
|
|
336
|
+
case MESSAGE_SYNC_PUSH:
|
|
337
|
+
updates.push(decoding.readVarUint8Array(decoder));
|
|
338
|
+
break;
|
|
339
|
+
default:
|
|
340
|
+
break;
|
|
341
|
+
}
|
|
342
|
+
};
|
|
343
|
+
try {
|
|
344
|
+
if (frame.length > 4)
|
|
345
|
+
walk(frame.subarray(4));
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
return [];
|
|
349
|
+
}
|
|
350
|
+
return updates;
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* The counterpart of `extractDocUpdates()` for the load path of a
|
|
354
|
+
* persistence transport: wraps one (merged) update as a CRC-wrapped
|
|
355
|
+
* MESSAGE_SYNC SyncStep2 frame. Handed to the `onMessage` callback, the
|
|
356
|
+
* provider applies it as the answer to its own request - `synced` fires,
|
|
357
|
+
* nothing is sent back - exactly what a local copy is: the peer that had
|
|
358
|
+
* our document.
|
|
359
|
+
*/
|
|
360
|
+
export function frameDocUpdate(update) {
|
|
361
|
+
const encoder = encoding.createEncoder();
|
|
362
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
363
|
+
encoding.writeVarUint(encoder, syncProtocol.messageYjsSyncStep2);
|
|
364
|
+
encoding.writeVarUint8Array(encoder, update);
|
|
365
|
+
return wrapMessageWithChecksum(encoding.toUint8Array(encoder));
|
|
366
|
+
}
|
|
103
367
|
/**
|
|
104
368
|
* PubSub channel for real-time messaging alongside Yjs.
|
|
105
369
|
* Allows sending ephemeral messages that don't need CRDT properties.
|
|
@@ -199,6 +463,18 @@ export class PubSubChannel extends Observable {
|
|
|
199
463
|
* ```
|
|
200
464
|
*/
|
|
201
465
|
export class GenericProvider extends Observable {
|
|
466
|
+
get appAwareness() {
|
|
467
|
+
if (!this._appAwareness) {
|
|
468
|
+
// Created on first use, not in the constructor: every y-protocols
|
|
469
|
+
// Awareness starts its own setInterval that only destroy() clears, so
|
|
470
|
+
// an eagerly-built second instance cost one live timer per provider
|
|
471
|
+
// for the majority of consumers that never touch this channel
|
|
472
|
+
// (bench-reload-phantoms part 2 counts exactly that).
|
|
473
|
+
this._appAwareness = new awarenessProtocol.Awareness(this.doc);
|
|
474
|
+
this._attachAppAwareness(this._appAwareness);
|
|
475
|
+
}
|
|
476
|
+
return this._appAwareness;
|
|
477
|
+
}
|
|
202
478
|
/**
|
|
203
479
|
* Create a new generic provider.
|
|
204
480
|
*
|
|
@@ -211,51 +487,225 @@ export class GenericProvider extends Observable {
|
|
|
211
487
|
this._status = { state: 'disconnected' };
|
|
212
488
|
this._synced = false;
|
|
213
489
|
this._destroying = false;
|
|
490
|
+
this._lastActivityTime = Date.now();
|
|
491
|
+
this._lastPeriodicTickTime = Date.now();
|
|
492
|
+
this._equalBeaconsHeard = 0;
|
|
493
|
+
this._beaconForced = false;
|
|
214
494
|
// BroadcastChannel state for cross-tab sync
|
|
215
495
|
this._bcChannel = '';
|
|
216
496
|
this._bcConnected = false;
|
|
217
|
-
//
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
//
|
|
497
|
+
// Unified resync-request coordinator. Previously hash-mismatch,
|
|
498
|
+
// corrupted-message, and gap-confirmed triggers each coalesced only
|
|
499
|
+
// against themselves (three separate pending-timer fields, three
|
|
500
|
+
// separate escalation counters), so under sustained wire corruption they
|
|
501
|
+
// could each independently burn through the shared _tryReserveSyncSlot()
|
|
502
|
+
// budget in the same window - a resync storm that grew combinatorially
|
|
503
|
+
// with peer count (see test/dummy/bench-corruption-storm.ts: at 10
|
|
504
|
+
// simulated peers, 5% per-link corruption drove message volume to ~11x
|
|
505
|
+
// the corruption-free baseline). Now there is exactly ONE pending timer
|
|
506
|
+
// and ONE shared escalation counter for all three triggers - only one
|
|
507
|
+
// resync is ever in flight at a time, and any trigger that fires while
|
|
508
|
+
// one is already pending is absorbed into it instead of scheduling its
|
|
509
|
+
// own. See _requestResync().
|
|
510
|
+
this._resyncAttemptCount = 0;
|
|
511
|
+
this._lastResyncAttemptTime = 0;
|
|
512
|
+
// Rate limiting for sync traffic - two budgets since phase 1b: one for
|
|
513
|
+
// what we ask for (beacons, pushes, syncNow), one for what we owe
|
|
514
|
+
// (SyncStep2 replies, acks). With a single shared budget a join burst's
|
|
515
|
+
// replies spent the slots a peer needed for its own recovery (measured:
|
|
516
|
+
// joiners arriving within 10 s of a burst converged in 13 s once, and a
|
|
517
|
+
// 50-peer room's periodic beacons ran at a third of their rate for 10 s
|
|
518
|
+
// after every join burst). Same size, same window, independent.
|
|
221
519
|
this._syncRequestTimes = [];
|
|
222
|
-
this.
|
|
223
|
-
|
|
520
|
+
this._syncReplyTimes = [];
|
|
521
|
+
// SyncStep2 reply suppression (NACK-suppression style): delay a reply to
|
|
522
|
+
// a SyncStep1 request briefly, and drop it if another peer's reply is
|
|
523
|
+
// overheard first - since every reply is broadcast to the whole room
|
|
524
|
+
// anyway, this avoids every peer answering the same request redundantly.
|
|
525
|
+
// Only engages when there's genuine redundancy (see _handleIncomingMessage's
|
|
526
|
+
// MESSAGE_SYNC and MESSAGE_SYNC_VERIFIED cases) - with 0-1 other known
|
|
527
|
+
// peers there's no "someone else" to rely on, so replies go out
|
|
528
|
+
// immediately as before.
|
|
529
|
+
this._pendingSyncReply = null;
|
|
530
|
+
// Whether _pendingSyncReply is a digest-beacon ack (see _handleDigest())
|
|
531
|
+
// rather than a SyncStep2. An overheard beacon with a digest equal to ours
|
|
532
|
+
// makes a pending ACK redundant (the joiner it was for has received that
|
|
533
|
+
// same beacon and is synced by it) but says nothing about a pending
|
|
534
|
+
// SyncStep2, which carries data - so only acks are cancelled on that
|
|
535
|
+
// signal. Measured in test/dummy/bench-user-scaling.ts: without this, acks
|
|
536
|
+
// were ~94% of a 50-peer join burst's messages (Task 3b in the design doc).
|
|
537
|
+
this._pendingSyncReplyIsAck = false;
|
|
538
|
+
// The requester's state vector the pending SyncStep2 answers (null for
|
|
539
|
+
// acks and for replies to plain SyncStep1s). A later request with the
|
|
540
|
+
// same state vector is the same question: the pending reply's bytes are
|
|
541
|
+
// refreshed to the current document and the timer kept, instead of the
|
|
542
|
+
// old reply being flushed as "a different request" - measured: with one
|
|
543
|
+
// peer typing while K empty peers join, keystroke and JOIN-beacon
|
|
544
|
+
// arrivals interleave at random, every keystroke in between changed the
|
|
545
|
+
// reply bytes, and each settled peer flushed up to K replies (Task 7 in
|
|
546
|
+
// the phase-1b design doc).
|
|
547
|
+
this._pendingSyncReplyTargetSv = null;
|
|
548
|
+
this._responseWaitAttempts = 0;
|
|
549
|
+
this._responseSeen = false;
|
|
550
|
+
// Phase 1e: an equal ack or equal beacon from an UNSETTLED peer arrived
|
|
551
|
+
// during the current response wait. Not a response (a settled peer with
|
|
552
|
+
// content may still answer), but evidence that the room is a fresh one
|
|
553
|
+
// whose peers are all in our state - see _armResponseWait().
|
|
554
|
+
this._equalUnsettledSeen = false;
|
|
555
|
+
this._responseWaitFlags = 0; // flags for the retry beacon: CONFIRM after a JOIN, 0 after a resync
|
|
556
|
+
this._behindSv = null;
|
|
557
|
+
// Round-trip estimate from our own requests (JOIN/resync beacon -> first
|
|
558
|
+
// SyncStep2 or ack): the minimum of the last 8 samples, because a sample
|
|
559
|
+
// includes the responder's random suppression delay and the fastest
|
|
560
|
+
// reply had the least of it. Drives _replySuppressionMaxDelay() (a
|
|
561
|
+
// suppression window shorter than the one-way latency suppresses
|
|
562
|
+
// nothing: at 250-350 ms latency every peer ahead of a requester replied
|
|
563
|
+
// before any reply could be overheard, ~20 replies per request at N=100)
|
|
564
|
+
// and _armResponseWait()'s first delay (a fixed 1 s fired premature
|
|
565
|
+
// retries on the Matrix profile). See the phase-1b design doc, 1c.
|
|
566
|
+
this._rttSamples = [];
|
|
567
|
+
this._requestSentAt = 0;
|
|
568
|
+
// See DIGEST_FLAG_SETTLED. Reset on connect (a re-joining peer is a joiner).
|
|
569
|
+
this._confirmed = false;
|
|
570
|
+
// ClientIDs we have heard from (beacon and verified-update senders, and
|
|
571
|
+
// the ids a relayed presence table names), each with the time we last
|
|
572
|
+
// heard it. The reply-suppression gate ("is there someone else who could
|
|
573
|
+
// answer?") used awareness alone, and in a join burst the awareness
|
|
574
|
+
// messages trail the beacons - so the gate was still closed exactly when
|
|
575
|
+
// 49 requests arrived at once, and every one got an immediate reply
|
|
576
|
+
// (phase-1b design, item 3). Cleared on disconnect, pruned by the lease
|
|
577
|
+
// sweep after a lease of silence (round 7, item 3: on a relay transport a
|
|
578
|
+
// reload's old clientID never says goodbye, and every one of them counted
|
|
579
|
+
// in _peerCount() forever).
|
|
580
|
+
this._knownPeers = new Map();
|
|
581
|
+
// Transport address (the `from` of Transport.onMessage) per remote
|
|
582
|
+
// clientID, learned from beacons and verified updates; lets replies, acks
|
|
583
|
+
// and presence responses go to the requester alone when the transport
|
|
584
|
+
// has sendTo (phase-1c design, item B). Cleared on disconnect, pruned
|
|
585
|
+
// with _knownPeers.
|
|
586
|
+
this._peerAddress = new Map();
|
|
587
|
+
// Requesters whose JOIN presence request the pending presence-response
|
|
588
|
+
// timer covers (see _schedulePresenceResponse).
|
|
589
|
+
this._presencePending = new Set();
|
|
590
|
+
// Phase 1e: an awareness message carrying OUR state at our current clock
|
|
591
|
+
// arrived since the presence-response timer was armed - the room's
|
|
592
|
+
// relayer (see _schedulePresenceResponse) already told the joiner about
|
|
593
|
+
// us, our own response would repeat it.
|
|
594
|
+
this._presenceCovered = false;
|
|
595
|
+
// Same NACK-style suppression as _pendingSyncReply above, applied to
|
|
596
|
+
// awareness updates that are a pure timeout-triggered removal (see
|
|
597
|
+
// _scheduleAwarenessRemoval()) - every OTHER connected peer runs its own
|
|
598
|
+
// independent 30s outdatedTimeout sweep (y-protocols/awareness.js), so
|
|
599
|
+
// one peer going silent (crash/dirty drop, not a clean disconnect())
|
|
600
|
+
// causes an O(N) simultaneous "peer X is gone" broadcast burst without
|
|
601
|
+
// this. See docs/superpowers/specs/2026-09-04-sync-optimization-round-3-ideas.md
|
|
602
|
+
// item 7 and test/dummy/bench-awareness-removal-burst.ts.
|
|
603
|
+
this._pendingAwarenessRemoval = null;
|
|
604
|
+
// Peers whose channel opened since the debounce timer was armed (the
|
|
605
|
+
// `from`/sendTo address). Round 5, item 5: on a transport with sendTo
|
|
606
|
+
// each gets a plain beacon addressed to it - it answers with what we
|
|
607
|
+
// lack and its own JOIN beacon makes us answer it - instead of the
|
|
608
|
+
// full-state push + beacon broadcast to EVERY connection that
|
|
609
|
+
// `_syncNow(0)` did (the O(N^2) mesh-join burst CLAUDE.md warns about).
|
|
610
|
+
this._pendingPeerConnectIds = new Set();
|
|
611
|
+
// Round 5, item 5: the state vector and delete-set hash the room last
|
|
612
|
+
// confirmed as equal to ours (an equal digest from any peer, see
|
|
613
|
+
// _handleDigest). A reconnect's push then carries only what we produced
|
|
614
|
+
// since - our offline edits, the one thing a push exists for (round 2:
|
|
615
|
+
// one message must survive alone) - instead of the whole document, which
|
|
616
|
+
// on a chunking transport was several messages per reconnect. A room
|
|
617
|
+
// that has been replaced meanwhile shows up behind in its own JOIN
|
|
618
|
+
// beacons and is answered like any late joiner. Null until the first
|
|
619
|
+
// confirmation: the first connect still pushes everything.
|
|
620
|
+
this._confirmedSv = null;
|
|
621
|
+
this._confirmedDsHash = null;
|
|
622
|
+
// connect() is awaiting ConnectionConfig.waitFor: doc updates are the local load.
|
|
623
|
+
this._loading = false;
|
|
624
|
+
// Cached computeDeleteSetHash(doc); null = stale. Invalidated on every
|
|
625
|
+
// doc 'update' (deletes are content changes, so this is exact; inserts
|
|
626
|
+
// invalidate needlessly but cheaply). See computeDeleteSetHash's doc for
|
|
627
|
+
// why this must not be recomputed per update.
|
|
628
|
+
this._dsHashCache = null;
|
|
224
629
|
// Sequence numbers for causal ordering
|
|
225
630
|
this._localSeqNum = 0; // Our sequence number counter
|
|
226
|
-
|
|
227
|
-
//
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
//
|
|
231
|
-
this.
|
|
631
|
+
// Per-sender sequence tracking for reordering-tolerant gap detection.
|
|
632
|
+
// Applying a Yjs update is always safe even for duplicates or out-of-order
|
|
633
|
+
// arrivals (Yjs updates are idempotent/commutative) — this state exists
|
|
634
|
+
// only to detect genuine gaps (likely packet loss) without false
|
|
635
|
+
// positives from mere network reordering. See _trackRemoteSeq().
|
|
636
|
+
this._remoteSeqInfo = new Map();
|
|
637
|
+
this._gapCheckTimers = new Map();
|
|
638
|
+
// Update batching/debouncing. `_batchUpdates` 0 (the default) no longer
|
|
639
|
+
// means a synchronous send from inside the Y.Doc 'update' event: the
|
|
640
|
+
// update is merged into `_pendingUpdate` and flushed at the end of the
|
|
641
|
+
// current task (a microtask - no timer, no measurable delay), so that
|
|
642
|
+
// (a) the several transactions an editor binding can emit for one input
|
|
643
|
+
// event leave as one message and (b) the cursor awareness the binding
|
|
644
|
+
// sets right after the text change in the same task rides along in the
|
|
645
|
+
// same wire message (round 5, item 1 - see _flushPendingUpdate()).
|
|
646
|
+
// `_batchTimeoutId` is set only for the timed (`batchUpdates > 0`) flush;
|
|
647
|
+
// `_flushScheduled` covers both.
|
|
648
|
+
this._batchUpdates = 0; // milliseconds delay (0 = end of current task)
|
|
649
|
+
// Origins whose updates are never sent to the transport (local-only txns).
|
|
650
|
+
this._excludeOrigins = new Set();
|
|
651
|
+
// 'pull' never pushes local state unasked - for read-mostly replicas that
|
|
652
|
+
// must not write into the room.
|
|
653
|
+
this._syncMode = 'push-pull';
|
|
232
654
|
this._pendingUpdate = null;
|
|
655
|
+
this._flushScheduled = false;
|
|
233
656
|
// Awareness throttling - prevents awareness from flooding document sync
|
|
234
|
-
this._awarenessInterval = 100; // ms between awareness broadcasts
|
|
657
|
+
this._awarenessInterval = 100; // ms between awareness broadcasts, or 'auto' (round 6, item 9)
|
|
235
658
|
this._pendingAwarenessClients = new Set();
|
|
236
659
|
this._lastAwarenessTime = 0;
|
|
237
|
-
// Independent throttle state for the app awareness channel
|
|
660
|
+
// Independent throttle state for the app awareness channel, so module
|
|
661
|
+
// cursor churn never delays or coalesces with core presence.
|
|
238
662
|
this._pendingAppAwarenessClients = new Set();
|
|
239
663
|
this._lastAppAwarenessTime = 0;
|
|
240
|
-
// Origins whose updates are never sent to the transport (local-only txns).
|
|
241
|
-
this._excludeOrigins = new Set();
|
|
242
|
-
// Connect-time sync strategy: 'push-pull' (default) sends full local state
|
|
243
|
-
// then requests remote; 'pull' only requests remote state.
|
|
244
|
-
this._syncMode = 'push-pull';
|
|
245
664
|
this.doc = doc;
|
|
246
665
|
this.transport = transport;
|
|
247
666
|
this.pubsub = new PubSubChannel(this);
|
|
248
667
|
this.awareness = options.awareness || new awarenessProtocol.Awareness(doc);
|
|
249
|
-
|
|
250
|
-
|
|
668
|
+
// Only when supplied - otherwise the getter builds it on first use.
|
|
669
|
+
if (options.appAwareness) {
|
|
670
|
+
this._appAwareness = options.appAwareness;
|
|
671
|
+
}
|
|
251
672
|
this._syncInterval = options.syncInterval ?? 5000;
|
|
252
673
|
this._verifyUpdates = options.verifyUpdates ?? true;
|
|
253
|
-
this._batchUpdates =
|
|
674
|
+
this._batchUpdates =
|
|
675
|
+
options.batchUpdates ?? transport.preferredBatchMs ?? 0;
|
|
254
676
|
this._disableBc = options.disableBc ?? false;
|
|
255
|
-
this._awarenessInterval =
|
|
677
|
+
this._awarenessInterval =
|
|
678
|
+
options.awarenessInterval ?? transport.preferredAwarenessMs ?? 100;
|
|
256
679
|
this._excludeOrigins = new Set(options.excludeOrigins ?? []);
|
|
257
680
|
this._localId = options.localId;
|
|
258
681
|
this._syncMode = options.syncMode ?? 'push-pull';
|
|
682
|
+
this._maxSyncRequestsPerWindow = options.maxSyncRequestsPerWindow ?? 20;
|
|
683
|
+
this._syncRequestWindowMs = options.syncRequestWindowMs ?? 10000;
|
|
684
|
+
this._syncReplySuppressionMs = options.syncReplySuppressionMs ?? 30;
|
|
685
|
+
this._peerConnectDebounceMs = options.peerConnectDebounceMs ?? 50;
|
|
686
|
+
this._gapGraceMs = options.gapGraceMs ?? 300;
|
|
687
|
+
this._seqWindowSize = options.seqWindowSize ?? 64;
|
|
688
|
+
// Explicit 0 disables even when the transport hints a floor.
|
|
689
|
+
this._compressionThresholdBytes =
|
|
690
|
+
(options.compressionThresholdBytes ?? transport.preferredCompressMinBytes) ||
|
|
691
|
+
undefined;
|
|
692
|
+
this._idleBackoffEnabled = options.idleBackoffEnabled ?? true;
|
|
693
|
+
this._idleBackoffMaxMs = options.idleBackoffMaxMs ?? 60000;
|
|
694
|
+
this._trickleK = options.trickleK ?? 1;
|
|
695
|
+
this._ownsAwareness = !options.awareness;
|
|
696
|
+
this._awarenessTimeoutMs =
|
|
697
|
+
options.awarenessTimeoutMs ??
|
|
698
|
+
(typeof transport.onPeerDisconnect === 'function' ? 300000 : 30000);
|
|
699
|
+
// Silence y-protocols' own 3 s sweep on an awareness we created - the
|
|
700
|
+
// lease sweep that replaces it is armed in connect() and cleared in
|
|
701
|
+
// disconnect() (round 7, item 2: it used to start here and outlive
|
|
702
|
+
// disconnect(), keeping a dropped provider alive through its timer).
|
|
703
|
+
if (this._ownsAwareness) {
|
|
704
|
+
const aw = this.awareness;
|
|
705
|
+
if (aw._checkInterval !== undefined)
|
|
706
|
+
clearInterval(aw._checkInterval);
|
|
707
|
+
}
|
|
708
|
+
this._currentSyncIntervalMs = this._syncInterval;
|
|
259
709
|
this._setupDocumentSync();
|
|
260
710
|
this._setupAwarenessSync();
|
|
261
711
|
}
|
|
@@ -281,18 +731,21 @@ export class GenericProvider extends Observable {
|
|
|
281
731
|
try {
|
|
282
732
|
// Setup BroadcastChannel for cross-tab sync (if enabled and available)
|
|
283
733
|
this._setupBroadcastChannel(config);
|
|
284
|
-
//
|
|
285
|
-
|
|
286
|
-
//
|
|
287
|
-
|
|
288
|
-
|
|
734
|
+
// Register for incoming messages and new-peer notifications BEFORE
|
|
735
|
+
// connecting the transport. Some transports (e.g. PeerJS for a
|
|
736
|
+
// joining, non-coordinator peer) establish and fully open their first
|
|
737
|
+
// connection *inside* transport.connect() itself — so a peer-connect
|
|
738
|
+
// notification or an immediate reply from the other side can arrive
|
|
739
|
+
// before that promise resolves. Registering after the await left
|
|
740
|
+
// exactly that window uncovered: whatever arrived during it was
|
|
741
|
+
// silently dropped since neither callback was wired up yet.
|
|
742
|
+
this._unsubscribeTransport = this.transport.onMessage((data, from) => {
|
|
743
|
+
this._handleIncomingMessage(data, from);
|
|
289
744
|
});
|
|
290
|
-
// When a new WebRTC peer channel opens, immediately push our full state
|
|
291
|
-
// so peers that reconnected after offline edits receive our changes.
|
|
292
745
|
if (this.transport.onPeerConnect) {
|
|
293
|
-
const unsubPeer = this.transport.onPeerConnect((
|
|
746
|
+
const unsubPeer = this.transport.onPeerConnect((peerId) => {
|
|
294
747
|
if (!this._destroying)
|
|
295
|
-
this.
|
|
748
|
+
this._schedulePeerConnectSync(peerId);
|
|
296
749
|
});
|
|
297
750
|
const originalUnsub = this._unsubscribeTransport;
|
|
298
751
|
this._unsubscribeTransport = () => {
|
|
@@ -300,35 +753,127 @@ export class GenericProvider extends Observable {
|
|
|
300
753
|
unsubPeer();
|
|
301
754
|
};
|
|
302
755
|
}
|
|
756
|
+
if (this.transport.onPeerDisconnect) {
|
|
757
|
+
const unsubLeave = this.transport.onPeerDisconnect((peerId) => {
|
|
758
|
+
if (!this._destroying)
|
|
759
|
+
this._handlePeerLeave(peerId);
|
|
760
|
+
});
|
|
761
|
+
const originalUnsub = this._unsubscribeTransport;
|
|
762
|
+
this._unsubscribeTransport = () => {
|
|
763
|
+
originalUnsub?.();
|
|
764
|
+
unsubLeave();
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
// Connect the transport
|
|
768
|
+
await this.transport.connect(config);
|
|
303
769
|
this._setStatus({ state: 'connected' });
|
|
304
|
-
|
|
305
|
-
//
|
|
306
|
-
//
|
|
307
|
-
//
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
this._sendSyncStep1();
|
|
770
|
+
this._startAwarenessSweep();
|
|
771
|
+
// Seed the round-trip estimate from the transport's hint (see
|
|
772
|
+
// Transport.expectedRttMs); the minimum-of-8 rule lets real samples
|
|
773
|
+
// take over as soon as they arrive.
|
|
774
|
+
if (this.transport.expectedRttMs) {
|
|
775
|
+
this._rttSamples = [this.transport.expectedRttMs];
|
|
311
776
|
}
|
|
312
|
-
|
|
313
|
-
|
|
777
|
+
// Persistence first (round 5, item 7): let a local copy load before
|
|
778
|
+
// the first beacon says what we have. While it loads, the doc
|
|
779
|
+
// updates it produces are the load, not edits - they are not
|
|
780
|
+
// broadcast - and afterwards the loaded state counts as confirmed:
|
|
781
|
+
// no full-state push. The beacon reconciles: peers behind us (our
|
|
782
|
+
// offline edits) see themselves behind and ask; peers ahead of us
|
|
783
|
+
// answer. Measured in bench-reconnect-push part 2: a 50 KB copy that
|
|
784
|
+
// loaded 100 ms after connect cost the room a 50 KB SyncStep2, and a
|
|
785
|
+
// first version of waitFor that only delayed the beacon cost 100 KB
|
|
786
|
+
// (the load's broadcast plus the push). See ConnectionConfig.waitFor.
|
|
787
|
+
if (config.waitFor) {
|
|
788
|
+
this._loading = true;
|
|
789
|
+
try {
|
|
790
|
+
await config.waitFor;
|
|
791
|
+
}
|
|
792
|
+
catch {
|
|
793
|
+
// The local load failed; sync from the room as if there were none.
|
|
794
|
+
}
|
|
795
|
+
this._loading = false;
|
|
796
|
+
if (this._destroying || !this.transport.isConnected)
|
|
797
|
+
return;
|
|
798
|
+
this._confirmedSv = Y.encodeStateVector(this.doc);
|
|
799
|
+
this._confirmedDsHash = this._deleteSetHash();
|
|
800
|
+
}
|
|
801
|
+
// Send initial sync pushing our local state plus requesting remote state.
|
|
802
|
+
// syncNow() is used instead of _sendSyncStep1() so that any offline edits
|
|
803
|
+
// made before this connect() call are pushed to currently-connected peers
|
|
804
|
+
// (e.g. same-browser tabs via BroadcastChannel).
|
|
805
|
+
this.syncNow();
|
|
806
|
+
// (Local awareness goes out inside syncNow()'s batch, or via its
|
|
807
|
+
// throttled fallback - a second broadcast here was a duplicate 100ms
|
|
808
|
+
// later.)
|
|
809
|
+
// The app channel has no equivalent path into syncNow()'s batch, so
|
|
810
|
+
// its local state is announced here when there is any.
|
|
811
|
+
if (this._appAwareness?.getLocalState() != null) {
|
|
812
|
+
this._broadcastAppAwareness([this.doc.clientID]);
|
|
314
813
|
}
|
|
315
|
-
// Broadcast local awareness state
|
|
316
|
-
this._broadcastAwareness([this.doc.clientID]);
|
|
317
|
-
this._broadcastAwareness([this.doc.clientID], AWARENESS_CHANNEL_APP);
|
|
318
814
|
// Start periodic sync to handle packet loss
|
|
319
815
|
// Just request sync without sending full state (avoid redundant broadcasts)
|
|
816
|
+
// _sendSyncStep1() already checks the shared rate limiter internally
|
|
817
|
+
// and silently drops the request if it's exceeded.
|
|
818
|
+
//
|
|
819
|
+
// Uses a recursive setTimeout (re-jittered by ~20% each tick) rather
|
|
820
|
+
// than a plain setInterval so peers that connect() within a short
|
|
821
|
+
// window of each other - the common case: everyone joining a room
|
|
822
|
+
// near session start, or reconnecting together after a shared network
|
|
823
|
+
// blip - don't end up with near-synchronized periodic timers that all
|
|
824
|
+
// fire in the same few milliseconds every syncInterval. This doesn't
|
|
825
|
+
// reduce total periodic-sync traffic, only spreads it out so a room's
|
|
826
|
+
// background traffic is smooth instead of bursty.
|
|
320
827
|
if (this._syncInterval > 0) {
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
828
|
+
// Reset idle-backoff state on every (re)connect so a reconnect
|
|
829
|
+
// always starts its first tick at the base interval, never
|
|
830
|
+
// inheriting a backed-off value left over from a previous session.
|
|
831
|
+
this._currentSyncIntervalMs = this._syncInterval;
|
|
832
|
+
this._lastActivityTime = Date.now();
|
|
833
|
+
this._lastPeriodicTickTime = Date.now();
|
|
834
|
+
this._equalBeaconsHeard = 0;
|
|
835
|
+
this._beaconForced = false;
|
|
836
|
+
const scheduleNextPeriodicSync = (delayMs) => {
|
|
837
|
+
this._syncIntervalId = setTimeout(() => {
|
|
838
|
+
const tickTime = Date.now();
|
|
839
|
+
if (this._idleBackoffEnabled) {
|
|
840
|
+
// "Activity" = anything _markActivity() call sites observed
|
|
841
|
+
// (incoming message via transport or BroadcastChannel, local
|
|
842
|
+
// or remote doc change, local or remote awareness change)
|
|
843
|
+
// since the LAST tick fired - not since backoff started, so a
|
|
844
|
+
// single quiet tick after a burst of activity still resets to
|
|
845
|
+
// base rather than needing a full quiet cycle to catch up.
|
|
846
|
+
const hadActivitySinceLastTick = this._lastActivityTime > this._lastPeriodicTickTime;
|
|
847
|
+
this._currentSyncIntervalMs = hadActivitySinceLastTick
|
|
848
|
+
? this._syncInterval
|
|
849
|
+
: Math.min(this._idleBackoffMaxMs, this._currentSyncIntervalMs * 2);
|
|
328
850
|
}
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
851
|
+
this._lastPeriodicTickTime = tickTime;
|
|
852
|
+
if (this.transport.isConnected && !this._destroying) {
|
|
853
|
+
// Beacon only. Presence is no longer re-announced per tick on
|
|
854
|
+
// any transport: a joiner requests it via DIGEST_FLAG_JOIN
|
|
855
|
+
// (see _handleDigest()), and y-protocols/awareness renews the
|
|
856
|
+
// local state itself every outdatedTimeout/2 = 15s
|
|
857
|
+
// (awareness.js _checkInterval), which the awareness update
|
|
858
|
+
// handler broadcasts. Measured in
|
|
859
|
+
// test/dummy/bench-idle-room.ts: the per-tick re-announce was
|
|
860
|
+
// ~40% of an idle room's deliveries.
|
|
861
|
+
// Trickle (round 5, item 3): silent if enough equal beacons
|
|
862
|
+
// were overheard since the last tick - see _equalBeaconsHeard.
|
|
863
|
+
const suppressed = this._trickleK > 0 &&
|
|
864
|
+
!this._beaconForced &&
|
|
865
|
+
this._equalBeaconsHeard >= this._trickleK;
|
|
866
|
+
if (!suppressed)
|
|
867
|
+
this._sendSyncStep1();
|
|
868
|
+
this._equalBeaconsHeard = 0;
|
|
869
|
+
this._beaconForced = false;
|
|
870
|
+
}
|
|
871
|
+
if (!this._destroying)
|
|
872
|
+
scheduleNextPeriodicSync();
|
|
873
|
+
}, delayMs ?? this._jitteredSyncInterval());
|
|
874
|
+
};
|
|
875
|
+
this._periodicScheduler = scheduleNextPeriodicSync;
|
|
876
|
+
scheduleNextPeriodicSync();
|
|
332
877
|
}
|
|
333
878
|
}
|
|
334
879
|
catch (error) {
|
|
@@ -346,22 +891,92 @@ export class GenericProvider extends Observable {
|
|
|
346
891
|
disconnect() {
|
|
347
892
|
// Stop periodic sync
|
|
348
893
|
if (this._syncIntervalId !== undefined) {
|
|
349
|
-
|
|
894
|
+
clearTimeout(this._syncIntervalId);
|
|
350
895
|
this._syncIntervalId = undefined;
|
|
351
896
|
}
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
this.
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
897
|
+
this._periodicScheduler = undefined;
|
|
898
|
+
// Stop the awareness lease sweep (armed in connect(); round 7, item 2).
|
|
899
|
+
if (this._awarenessSweepId !== undefined) {
|
|
900
|
+
clearTimeout(this._awarenessSweepId);
|
|
901
|
+
this._awarenessSweepId = undefined;
|
|
902
|
+
}
|
|
903
|
+
// Reset resync escalation tracking
|
|
904
|
+
this._resyncAttemptCount = 0;
|
|
905
|
+
this._lastResyncAttemptTime = 0;
|
|
906
|
+
// Cancel any pending unified resync - it would otherwise still fire
|
|
907
|
+
// syncNow() after disconnect/reconnect against a transport that may be
|
|
908
|
+
// in a completely different state by then.
|
|
909
|
+
if (this._pendingResyncTimeoutId !== undefined) {
|
|
910
|
+
clearTimeout(this._pendingResyncTimeoutId);
|
|
911
|
+
this._pendingResyncTimeoutId = undefined;
|
|
364
912
|
}
|
|
913
|
+
// Stop any pending gap-check timers and forget per-sender sequence
|
|
914
|
+
// tracking. Without this, a gap-check timer armed before this
|
|
915
|
+
// disconnect() keeps running in the background and can fire
|
|
916
|
+
// _requestResync() after reconnect using sequence-number bookkeeping
|
|
917
|
+
// from the PREVIOUS connection - a spurious resync race disconnected
|
|
918
|
+
// from anything actually missing in the new session. Mirrors the
|
|
919
|
+
// equivalent cleanup in destroy().
|
|
920
|
+
for (const timer of this._gapCheckTimers.values()) {
|
|
921
|
+
clearTimeout(timer);
|
|
922
|
+
}
|
|
923
|
+
this._gapCheckTimers.clear();
|
|
924
|
+
this._remoteSeqInfo.clear();
|
|
925
|
+
// Cancel any pending debounced onPeerConnect sync - a reconnect gets a
|
|
926
|
+
// fresh burst of onPeerConnect events (if the transport supports it) and
|
|
927
|
+
// shouldn't fire a stale one left over from before this disconnect.
|
|
928
|
+
if (this._pendingPeerConnectSyncTimeoutId !== undefined) {
|
|
929
|
+
clearTimeout(this._pendingPeerConnectSyncTimeoutId);
|
|
930
|
+
this._pendingPeerConnectSyncTimeoutId = undefined;
|
|
931
|
+
}
|
|
932
|
+
this._pendingPeerConnectIds.clear();
|
|
933
|
+
// Reset the sync rate-limit budget. Without this, a reconnect inherits
|
|
934
|
+
// whatever budget was left over from before the disconnect - and since
|
|
935
|
+
// syncNow()'s full-state push now shares this same limiter (see
|
|
936
|
+
// _tryReserveSyncSlot()), a rate-limited reconnect could silently skip
|
|
937
|
+
// the very push that delivers edits made while offline.
|
|
938
|
+
this._syncRequestTimes = [];
|
|
939
|
+
this._syncReplyTimes = [];
|
|
940
|
+
this._knownPeers.clear();
|
|
941
|
+
this._peerAddress.clear();
|
|
942
|
+
this._presencePending.clear();
|
|
943
|
+
// A pending response-wait belongs to a request on the old connection.
|
|
944
|
+
if (this._responseWaitTimer !== undefined) {
|
|
945
|
+
clearTimeout(this._responseWaitTimer);
|
|
946
|
+
this._responseWaitTimer = undefined;
|
|
947
|
+
}
|
|
948
|
+
if (this._pendingCheckTimer !== undefined) {
|
|
949
|
+
clearTimeout(this._pendingCheckTimer);
|
|
950
|
+
this._pendingCheckTimer = undefined;
|
|
951
|
+
}
|
|
952
|
+
if (this._behindCheckTimer !== undefined) {
|
|
953
|
+
clearTimeout(this._behindCheckTimer);
|
|
954
|
+
this._behindCheckTimer = undefined;
|
|
955
|
+
}
|
|
956
|
+
this._behindSv = null;
|
|
957
|
+
if (this._presenceResponseTimer !== undefined) {
|
|
958
|
+
clearTimeout(this._presenceResponseTimer);
|
|
959
|
+
this._presenceResponseTimer = undefined;
|
|
960
|
+
}
|
|
961
|
+
this._responseWaitAttempts = 0;
|
|
962
|
+
this._responseSeen = false;
|
|
963
|
+
this._equalUnsettledSeen = false;
|
|
964
|
+
this._rttSamples = [];
|
|
965
|
+
this._requestSentAt = 0;
|
|
966
|
+
this._confirmed = false;
|
|
967
|
+
// Drop any pending suppressed sync reply - safe to simply discard (not
|
|
968
|
+
// flush/send like batched updates/awareness below), since a suppressed
|
|
969
|
+
// reply is by design redundant with whatever the room already has.
|
|
970
|
+
this._cancelPendingSyncReply();
|
|
971
|
+
// Same reasoning for a pending suppressed awareness-removal broadcast -
|
|
972
|
+
// every other surviving peer is independently running the same
|
|
973
|
+
// suppression for the same departure, so dropping ours on disconnect
|
|
974
|
+
// (rather than flushing it through a transport that's about to go
|
|
975
|
+
// down) is safe.
|
|
976
|
+
this._cancelPendingAwarenessRemoval();
|
|
977
|
+
// Flush any pending batched updates before disconnecting (sent only if
|
|
978
|
+
// the transport is still connected; dropped otherwise, as before)
|
|
979
|
+
this._flushPendingUpdate();
|
|
365
980
|
// Flush pending awareness updates before disconnecting
|
|
366
981
|
if (this._awarenessTimeoutId !== undefined) {
|
|
367
982
|
clearTimeout(this._awarenessTimeoutId);
|
|
@@ -374,13 +989,13 @@ export class GenericProvider extends Observable {
|
|
|
374
989
|
}
|
|
375
990
|
}
|
|
376
991
|
this._pendingAwarenessClients.clear();
|
|
377
|
-
//
|
|
992
|
+
// Same for the app channel's independent throttle.
|
|
378
993
|
if (this._appAwarenessTimeoutId !== undefined) {
|
|
379
994
|
clearTimeout(this._appAwarenessTimeoutId);
|
|
380
995
|
this._appAwarenessTimeoutId = undefined;
|
|
381
996
|
if (this._pendingAppAwarenessClients.size > 0 &&
|
|
382
997
|
this.transport.isConnected) {
|
|
383
|
-
this.
|
|
998
|
+
this._sendAppAwarenessNow(Array.from(this._pendingAppAwarenessClients));
|
|
384
999
|
}
|
|
385
1000
|
}
|
|
386
1001
|
this._pendingAppAwarenessClients.clear();
|
|
@@ -392,7 +1007,6 @@ export class GenericProvider extends Observable {
|
|
|
392
1007
|
}
|
|
393
1008
|
// Mark local client as offline in awareness
|
|
394
1009
|
awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'disconnect');
|
|
395
|
-
awarenessProtocol.removeAwarenessStates(this.appAwareness, [this.doc.clientID], 'disconnect');
|
|
396
1010
|
this.transport.disconnect();
|
|
397
1011
|
this._synced = false;
|
|
398
1012
|
this._setStatus({ state: 'disconnected' });
|
|
@@ -405,41 +1019,53 @@ export class GenericProvider extends Observable {
|
|
|
405
1019
|
this._destroying = true;
|
|
406
1020
|
// Stop periodic sync (disconnect() will also do this, but be explicit)
|
|
407
1021
|
if (this._syncIntervalId !== undefined) {
|
|
408
|
-
|
|
1022
|
+
clearTimeout(this._syncIntervalId);
|
|
409
1023
|
this._syncIntervalId = undefined;
|
|
410
1024
|
}
|
|
411
|
-
//
|
|
412
|
-
|
|
413
|
-
clearTimeout(
|
|
414
|
-
this._batchTimeoutId = undefined;
|
|
415
|
-
// Send pending update if transport is still connected
|
|
416
|
-
if (this._pendingUpdate && this.transport.isConnected) {
|
|
417
|
-
this._sendUpdate(this._pendingUpdate);
|
|
418
|
-
}
|
|
419
|
-
this._pendingUpdate = null;
|
|
1025
|
+
// Stop any pending gap-check timers
|
|
1026
|
+
for (const timer of this._gapCheckTimers.values()) {
|
|
1027
|
+
clearTimeout(timer);
|
|
420
1028
|
}
|
|
1029
|
+
this._gapCheckTimers.clear();
|
|
1030
|
+
// Drop any pending suppressed sync reply (disconnect() will also do
|
|
1031
|
+
// this, but be explicit)
|
|
1032
|
+
this._cancelPendingSyncReply();
|
|
1033
|
+
// Same, for a pending suppressed awareness-removal broadcast
|
|
1034
|
+
// (disconnect() will also do this, but be explicit)
|
|
1035
|
+
this._cancelPendingAwarenessRemoval();
|
|
1036
|
+
// Flush any pending batched updates before destroying
|
|
1037
|
+
this._flushPendingUpdate();
|
|
421
1038
|
this.disconnect();
|
|
422
1039
|
// Remove document update listener
|
|
423
1040
|
if (this._updateHandler) {
|
|
424
1041
|
this.doc.off('update', this._updateHandler);
|
|
425
1042
|
this._updateHandler = undefined;
|
|
426
1043
|
}
|
|
427
|
-
// Remove awareness update
|
|
1044
|
+
// Remove awareness update listener
|
|
428
1045
|
if (this._awarenessUpdateHandler) {
|
|
429
|
-
this.awareness.off('update', this._awarenessUpdateHandler);
|
|
1046
|
+
this.awareness.off(this._ownsAwareness ? 'change' : 'update', this._awarenessUpdateHandler);
|
|
430
1047
|
this._awarenessUpdateHandler = undefined;
|
|
431
1048
|
}
|
|
432
1049
|
if (this._appAwarenessUpdateHandler) {
|
|
433
|
-
this.
|
|
1050
|
+
this._appAwareness?.off('update', this._appAwarenessUpdateHandler);
|
|
434
1051
|
this._appAwarenessUpdateHandler = undefined;
|
|
435
1052
|
}
|
|
1053
|
+
if (this._appAwarenessTimeoutId !== undefined) {
|
|
1054
|
+
clearTimeout(this._appAwarenessTimeoutId);
|
|
1055
|
+
this._appAwarenessTimeoutId = undefined;
|
|
1056
|
+
}
|
|
1057
|
+
if (this._awarenessSweepId !== undefined) {
|
|
1058
|
+
clearTimeout(this._awarenessSweepId);
|
|
1059
|
+
this._awarenessSweepId = undefined;
|
|
1060
|
+
}
|
|
436
1061
|
// Remove beforeunload handler
|
|
437
1062
|
if (this._beforeUnloadHandler && typeof window !== 'undefined') {
|
|
438
1063
|
window.removeEventListener('beforeunload', this._beforeUnloadHandler);
|
|
439
1064
|
this._beforeUnloadHandler = undefined;
|
|
440
1065
|
}
|
|
441
1066
|
this.awareness.destroy();
|
|
442
|
-
|
|
1067
|
+
// Only if one was ever built - never construct one just to destroy it.
|
|
1068
|
+
this._appAwareness?.destroy();
|
|
443
1069
|
super.destroy();
|
|
444
1070
|
}
|
|
445
1071
|
/**
|
|
@@ -466,26 +1092,343 @@ export class GenericProvider extends Observable {
|
|
|
466
1092
|
get synced() {
|
|
467
1093
|
return this._synced;
|
|
468
1094
|
}
|
|
1095
|
+
/**
|
|
1096
|
+
* Push local state + request remote state, gated by the shared rate
|
|
1097
|
+
* limiter. Returns whether it actually reserved a slot and sent anything
|
|
1098
|
+
* - `false` means the caller was rate-limited right now. Extracted out of
|
|
1099
|
+
* `syncNow()` so `_requestResync()`'s scheduled retry (see below) can tell
|
|
1100
|
+
* the difference between "sent" and "silently skipped" and react to it,
|
|
1101
|
+
* instead of assuming a resync always succeeds once it fires.
|
|
1102
|
+
*
|
|
1103
|
+
* @param push - whether to also broadcast full local document state.
|
|
1104
|
+
* `_requestResync()`'s retry passes `false` (pull-only is enough for a
|
|
1105
|
+
* resync trigger - see its call site).
|
|
1106
|
+
* @param buildExtra - optional callback, invoked ONLY once a rate-limit
|
|
1107
|
+
* slot is actually reserved (so it never runs, and never mutates
|
|
1108
|
+
* whatever state it touches, on a call that ends up rate-limited),
|
|
1109
|
+
* returning additional already-encoded sub-messages to fold into the SAME
|
|
1110
|
+
* batched wire send as the push/pull messages below - e.g. an awareness
|
|
1111
|
+
* update that's ready to go out "now" anyway (see
|
|
1112
|
+
* `_tryImmediateAwarenessMessage()`). Pure wire-framing: whether a caller
|
|
1113
|
+
* passes this never changes whether/when the push+pull half itself sends,
|
|
1114
|
+
* only how many separate `transport.send()`/`bc.publish()` calls it costs.
|
|
1115
|
+
* @param flags - digest beacon flags: DIGEST_FLAG_JOIN from syncNow(), 0
|
|
1116
|
+
* from the peer-connect debounce and the resync retry (see _syncNow()).
|
|
1117
|
+
*/
|
|
1118
|
+
_trySyncPushPull(push = true, buildExtra, flags = 0) {
|
|
1119
|
+
// Push (full document state) and pull (SyncStep1 request) share a
|
|
1120
|
+
// single rate-limit reservation. syncNow() is called from several
|
|
1121
|
+
// triggers that can all fire in a short window when many peers are
|
|
1122
|
+
// converging at once (hash-mismatch resyncs, gap-check confirmations,
|
|
1123
|
+
// per-peer connect events on mesh transports) - without this gate the
|
|
1124
|
+
// push above had NO limit at all, so each trigger broadcast the full
|
|
1125
|
+
// document state to the whole room, and those broadcasts caused more
|
|
1126
|
+
// reordering/mismatches elsewhere, causing more triggers. Measured in
|
|
1127
|
+
// test/dummy/bench-user-scaling.ts: at 100 simulated users this drove
|
|
1128
|
+
// message counts to 20-200x the theoretical linear cost. See
|
|
1129
|
+
// docs/superpowers/specs/2026-07-26-dummy-benchmark-scaling-design.md.
|
|
1130
|
+
if (!this._tryReserveSyncSlot())
|
|
1131
|
+
return false;
|
|
1132
|
+
const messages = [];
|
|
1133
|
+
// Send our document state to all peers - everything on the first
|
|
1134
|
+
// connect, only what we produced since the room last confirmed our
|
|
1135
|
+
// state afterwards (see _confirmedSv). This is what carries edits made
|
|
1136
|
+
// while offline.
|
|
1137
|
+
if (push) {
|
|
1138
|
+
const update = Y.encodeStateAsUpdate(this.doc, this._confirmedSv ?? undefined);
|
|
1139
|
+
let worthPushing = update.length > 0;
|
|
1140
|
+
if (worthPushing && this._confirmedSv !== null) {
|
|
1141
|
+
// A diff against a confirmed state is never byte-empty (it always
|
|
1142
|
+
// carries the delete set): push it only if it holds structs, or
|
|
1143
|
+
// deletes the room has not confirmed.
|
|
1144
|
+
try {
|
|
1145
|
+
worthPushing =
|
|
1146
|
+
Y.parseUpdateMeta(update).to.size > 0 ||
|
|
1147
|
+
this._deleteSetHash() !== this._confirmedDsHash;
|
|
1148
|
+
}
|
|
1149
|
+
catch {
|
|
1150
|
+
worthPushing = true;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
if (worthPushing) {
|
|
1154
|
+
messages.push(this._encodePush(update));
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
// Send sync request to get updates from others
|
|
1158
|
+
messages.push(this._encodeSyncStep1(flags));
|
|
1159
|
+
if (buildExtra) {
|
|
1160
|
+
messages.push(...buildExtra());
|
|
1161
|
+
}
|
|
1162
|
+
// Batched into one wire message (MESSAGE_BATCH) instead of one
|
|
1163
|
+
// transport.send()/bc.publish() call per sub-message - see _sendBatch().
|
|
1164
|
+
this._sendBatch(messages);
|
|
1165
|
+
if (flags & DIGEST_FLAG_JOIN)
|
|
1166
|
+
this._armResponseWait(DIGEST_FLAG_CONFIRM);
|
|
1167
|
+
return true;
|
|
1168
|
+
}
|
|
469
1169
|
/**
|
|
470
1170
|
* Force an immediate sync with remote peers.
|
|
471
1171
|
* Useful after network interruptions or to manually trigger re-sync.
|
|
1172
|
+
* Sends the beacon with DIGEST_FLAG_JOIN: peers answer with their
|
|
1173
|
+
* presence and, if our state already matches theirs, with an ack beacon
|
|
1174
|
+
* so `synced` flips without a data round trip.
|
|
472
1175
|
*/
|
|
473
1176
|
syncNow() {
|
|
1177
|
+
this._syncNow(DIGEST_FLAG_JOIN);
|
|
1178
|
+
}
|
|
1179
|
+
/**
|
|
1180
|
+
* syncNow() body. `flags` = 0 for callers that must NOT request presence:
|
|
1181
|
+
* `_schedulePeerConnectSync()` (mesh transports already re-broadcast
|
|
1182
|
+
* presence to a newcomer via their own onPeerConnect -> syncNow()).
|
|
1183
|
+
*/
|
|
1184
|
+
_syncNow(flags) {
|
|
474
1185
|
if (!this.transport.isConnected) {
|
|
475
1186
|
console.warn('Cannot sync: transport not connected');
|
|
476
1187
|
return;
|
|
477
1188
|
}
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
1189
|
+
if (flags & DIGEST_FLAG_JOIN)
|
|
1190
|
+
this._confirmed = false; // a (re-)joiner until answered
|
|
1191
|
+
// Try to fold the awareness broadcast into the same wire send as the
|
|
1192
|
+
// sync push+pull below. _tryImmediateAwarenessMessage() only returns
|
|
1193
|
+
// non-null (and only mutates awareness-throttle state) when the
|
|
1194
|
+
// throttle would have let an immediate send through anyway - so this
|
|
1195
|
+
// never changes awareness throttle semantics, only whether it travels
|
|
1196
|
+
// as its own message or bundled with the sync message going out "now"
|
|
1197
|
+
// too. Built inside buildExtra so it's only even attempted once a sync
|
|
1198
|
+
// rate-limit slot is confirmed reserved (see _trySyncPushPull's doc).
|
|
1199
|
+
let awarenessBatched = false;
|
|
1200
|
+
const sent = this._trySyncPushPull(
|
|
1201
|
+
// 'pull': send the beacon but never the full-state push. A relay or
|
|
1202
|
+
// server holds authoritative state and we adopt it rather than pushing
|
|
1203
|
+
// a competing local copy on every (re)connect. Peers that are actually
|
|
1204
|
+
// behind us still ask, and we still answer - the beacon reconciles.
|
|
1205
|
+
this._syncMode !== 'pull', () => {
|
|
1206
|
+
const msg = this._tryImmediateAwarenessMessage([this.doc.clientID]);
|
|
1207
|
+
if (msg) {
|
|
1208
|
+
awarenessBatched = true;
|
|
1209
|
+
return [msg];
|
|
1210
|
+
}
|
|
1211
|
+
return [];
|
|
1212
|
+
}, flags);
|
|
1213
|
+
// Awareness broadcasting is independently throttled and explicitly NOT
|
|
1214
|
+
// gated by the sync rate limiter above - preserve that exactly: it
|
|
1215
|
+
// always ends up broadcast one way or another (batched above, or via
|
|
1216
|
+
// its own throttled path here), regardless of whether the sync half
|
|
1217
|
+
// above was rate-limited.
|
|
1218
|
+
if (!sent || !awarenessBatched) {
|
|
1219
|
+
this._broadcastAwareness([this.doc.clientID]);
|
|
483
1220
|
}
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
1221
|
+
}
|
|
1222
|
+
/**
|
|
1223
|
+
* Compute the next periodic-sync delay, jittered by ~+/-20% around
|
|
1224
|
+
* `_currentSyncIntervalMs` (== `_syncInterval` unless `idleBackoffEnabled`
|
|
1225
|
+
* has backed it off - see that option's doc comment). Re-jittered fresh
|
|
1226
|
+
* each tick (not computed once per connect()) so a room's peers - which
|
|
1227
|
+
* commonly all connect() within a short window of each other - drift
|
|
1228
|
+
* apart over time instead of staying loosely synchronized. Extracted to
|
|
1229
|
+
* its own method purely so benchmarks can shadow it to compare against
|
|
1230
|
+
* the unjittered baseline.
|
|
1231
|
+
*/
|
|
1232
|
+
_jitteredSyncInterval() {
|
|
1233
|
+
const jitter = 1 + (Math.random() * 2 - 1) * 0.2; // +/-20%
|
|
1234
|
+
return this._currentSyncIntervalMs * jitter;
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* Record that "activity" happened right now, for `idleBackoffEnabled`'s
|
|
1238
|
+
* benefit. Cheap (one timestamp write) and called unconditionally
|
|
1239
|
+
* regardless of whether idle backoff is enabled, so there's no behavioral
|
|
1240
|
+
* branch to keep in sync - the backoff decision in connect()'s periodic
|
|
1241
|
+
* tick is the only place that actually reads this.
|
|
1242
|
+
*
|
|
1243
|
+
* Call sites are deliberately NOT "any inbound wire message" - an earlier
|
|
1244
|
+
* version of this hooked `_handleIncomingMessage()` unconditionally, which
|
|
1245
|
+
* made the periodic tick's OWN SyncStep1 request and the SyncStep2 reply
|
|
1246
|
+
* answering it (empty payload - nothing to sync) each count as "activity",
|
|
1247
|
+
* permanently resetting the backoff on every single tick and making the
|
|
1248
|
+
* whole feature a no-op (caught by this bench script's own first run: ON
|
|
1249
|
+
* and OFF produced statistically indistinguishable message counts). Both
|
|
1250
|
+
* Yjs's `doc.emit('update', ...)` and y-protocols' `awareness.emit('update', ...)`
|
|
1251
|
+
* already only fire when something with actual content changed
|
|
1252
|
+
* (`hasContent`/non-empty added+updated+removed - confirmed by reading
|
|
1253
|
+
* yjs's `Transaction.js` and y-protocols' `awareness.js` directly), so
|
|
1254
|
+
* hooking THOSE instead is exactly "local or remote document/awareness
|
|
1255
|
+
* change" with no extra filtering needed - a no-op SyncStep2 reply, a
|
|
1256
|
+
* digest beacon, or a duplicate/no-change awareness re-announce (e.g. a
|
|
1257
|
+
* JOIN-triggered presence response that changed nothing) never reaches
|
|
1258
|
+
* these handlers. A corrupted (CRC32
|
|
1259
|
+
* mismatch) message is real evidence of wire activity that neither
|
|
1260
|
+
* handler would ever see (it's rejected before decoding) - see the
|
|
1261
|
+
* explicit call in `_processWrappedMessage()`'s corruption branch.
|
|
1262
|
+
*
|
|
1263
|
+
* Call sites: `_setupDocumentSync()`'s update handler (LOCAL document
|
|
1264
|
+
* edits only, since phase 1e) and `_processWrappedMessage()`'s
|
|
1265
|
+
* corrupted-message branch (wire noise, not silence).
|
|
1266
|
+
*/
|
|
1267
|
+
_markActivity() {
|
|
1268
|
+
this._lastActivityTime = Date.now();
|
|
1269
|
+
// A backed-off periodic timer is re-armed at the base interval NOW, not
|
|
1270
|
+
// at its next tick (which may be idleBackoffMaxMs away). Design D of
|
|
1271
|
+
// the phase-1d doc rests on it: the peer that just had activity beacons
|
|
1272
|
+
// within one base interval, and a peer that lost that activity's
|
|
1273
|
+
// message learns from that beacon that it is behind (design A) - its
|
|
1274
|
+
// own backed-off interval no longer bounds the recovery. Measured in
|
|
1275
|
+
// test/dummy/bench-idle-backoff.ts. At most once per idle stretch.
|
|
1276
|
+
if (this._idleBackoffEnabled &&
|
|
1277
|
+
this._currentSyncIntervalMs !== this._syncInterval &&
|
|
1278
|
+
this._syncIntervalId !== undefined &&
|
|
1279
|
+
this._periodicScheduler !== undefined) {
|
|
1280
|
+
clearTimeout(this._syncIntervalId);
|
|
1281
|
+
this._currentSyncIntervalMs = this._syncInterval;
|
|
1282
|
+
// Phase 1e: at a random point inside the base interval, not a full
|
|
1283
|
+
// one (the phase-1d note): the loser's recovery chain starts with
|
|
1284
|
+
// this beacon - so Trickle never suppresses it (round 5, item 3).
|
|
1285
|
+
this._beaconForced = true;
|
|
1286
|
+
this._periodicScheduler(Math.random() * this._syncInterval);
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
/**
|
|
1290
|
+
* The lease sweep: y-protocols' own (awareness.js `_checkInterval`: renew
|
|
1291
|
+
* at outdatedTimeout/2, remove at outdatedTimeout, every
|
|
1292
|
+
* outdatedTimeout/10) replaced by the same loop at `_awarenessTimeoutMs`,
|
|
1293
|
+
* the period jittered so a room that joined together does not renew in
|
|
1294
|
+
* one burst (measured: all 49 listeners of a 50-peer room renewed inside
|
|
1295
|
+
* the same 10 s window). The renew/remove half runs only on an awareness
|
|
1296
|
+
* we created (`_ownsAwareness`); the peer-table prune (round 7, item 3)
|
|
1297
|
+
* runs regardless. Armed by connect(), cleared by disconnect() - round 7,
|
|
1298
|
+
* item 2: started from the constructor it outlived disconnect(), ticking
|
|
1299
|
+
* ~20 times a minute and keeping the dropped provider reachable
|
|
1300
|
+
* (test/dummy/bench-reload-phantoms.ts, part 2).
|
|
1301
|
+
*/
|
|
1302
|
+
_startAwarenessSweep() {
|
|
1303
|
+
if (this._awarenessSweepId !== undefined)
|
|
1304
|
+
return;
|
|
1305
|
+
const lease = this._awarenessTimeoutMs;
|
|
1306
|
+
const arm = () => {
|
|
1307
|
+
this._awarenessSweepId = setTimeout(tick, (lease / 10) * (0.8 + Math.random() * 0.4));
|
|
1308
|
+
};
|
|
1309
|
+
const tick = () => {
|
|
1310
|
+
const now = Date.now();
|
|
1311
|
+
if (this._ownsAwareness) {
|
|
1312
|
+
const mine = this.awareness.meta.get(this.doc.clientID);
|
|
1313
|
+
if (this.awareness.getLocalState() !== null &&
|
|
1314
|
+
mine !== undefined &&
|
|
1315
|
+
lease / 2 <= now - mine.lastUpdated) {
|
|
1316
|
+
this.awareness.setLocalState(this.awareness.getLocalState()); // renew: bumps the clock
|
|
1317
|
+
this._broadcastAwareness([this.doc.clientID]); // 'change' does not fire for an equal state (item 8)
|
|
1318
|
+
}
|
|
1319
|
+
const remove = [];
|
|
1320
|
+
this.awareness.meta.forEach((meta, clientID) => {
|
|
1321
|
+
if (clientID !== this.doc.clientID &&
|
|
1322
|
+
lease <= now - meta.lastUpdated &&
|
|
1323
|
+
this.awareness.getStates().has(clientID)) {
|
|
1324
|
+
remove.push(clientID);
|
|
1325
|
+
}
|
|
1326
|
+
});
|
|
1327
|
+
if (remove.length > 0) {
|
|
1328
|
+
awarenessProtocol.removeAwarenessStates(this.awareness, remove, 'timeout');
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
// Round 7, item 3: forget peers not heard from for a lease. A live
|
|
1332
|
+
// peer is heard at least every lease/2 through its presence renewal
|
|
1333
|
+
// (every receiver scans it, above at MESSAGE_AWARENESS); a pruned
|
|
1334
|
+
// peer that speaks again is simply learned again. Measured before
|
|
1335
|
+
// this: 30 reloads in a 20-peer relay room left 49 known peers for
|
|
1336
|
+
// good, and every cursor moved at a 50-peer room's 'auto' interval
|
|
1337
|
+
// (test/dummy/bench-reload-phantoms.ts). awareness.meta is left to
|
|
1338
|
+
// y-protocols: it holds the clock a late message is checked against.
|
|
1339
|
+
for (const [id, heardAt] of this._knownPeers) {
|
|
1340
|
+
if (lease <= now - heardAt) {
|
|
1341
|
+
this._knownPeers.delete(id);
|
|
1342
|
+
this._peerAddress.delete(id);
|
|
1343
|
+
this._remoteSeqInfo.delete(id);
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
// Re-arm only while connected: a disconnect() from inside a listener
|
|
1347
|
+
// above has just cleared the timer, and must stay cleared.
|
|
1348
|
+
if (!this._destroying && this._status.state === 'connected')
|
|
1349
|
+
arm();
|
|
1350
|
+
else
|
|
1351
|
+
this._awarenessSweepId = undefined;
|
|
1352
|
+
};
|
|
1353
|
+
arm();
|
|
1354
|
+
}
|
|
1355
|
+
/**
|
|
1356
|
+
* A digest, verified update or ack from `clientID` (or one we are about
|
|
1357
|
+
* to send, for our own id) is proof of presence: refresh the lease the
|
|
1358
|
+
* sweep above checks. Only for ids with a state - a departed peer's
|
|
1359
|
+
* `meta` entry survives its removal (y-protocols keeps it for the clock)
|
|
1360
|
+
* and must not be revived by a late message.
|
|
1361
|
+
*/
|
|
1362
|
+
_touchPeer(clientID) {
|
|
1363
|
+
const meta = this.awareness.meta.get(clientID);
|
|
1364
|
+
if (meta !== undefined && this.awareness.getStates().has(clientID)) {
|
|
1365
|
+
meta.lastUpdated = Date.now();
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
/**
|
|
1369
|
+
* Transport.onPeerDisconnect: the peer at `peerId` is gone. Forget its
|
|
1370
|
+
* address and id, drop its awareness state with origin 'peer-left': the
|
|
1371
|
+
* broadcast goes through the same suppression as a timeout removal, but
|
|
1372
|
+
* with a long window - every peer gets the leave signal in the same
|
|
1373
|
+
* millisecond, and at the reply window (~170 ms at N=50) 28 of 49
|
|
1374
|
+
* survivors broadcast before the first broadcast could be overheard
|
|
1375
|
+
* (bench-awareness-removal-burst, DUMMY_PEER_EVENTS=1). One broadcast
|
|
1376
|
+
* room-wide is still worth having: it corrects a joiner that received
|
|
1377
|
+
* this peer in a relayed presence table but had no channel to it yet.
|
|
1378
|
+
*/
|
|
1379
|
+
_handlePeerLeave(peerId) {
|
|
1380
|
+
const gone = [];
|
|
1381
|
+
for (const [clientID, address] of this._peerAddress) {
|
|
1382
|
+
if (address === peerId)
|
|
1383
|
+
gone.push(clientID);
|
|
1384
|
+
}
|
|
1385
|
+
for (const id of gone) {
|
|
1386
|
+
this._peerAddress.delete(id);
|
|
1387
|
+
this._knownPeers.delete(id);
|
|
1388
|
+
}
|
|
1389
|
+
const present = gone.filter((id) => this.awareness.getStates().has(id));
|
|
1390
|
+
if (present.length > 0) {
|
|
1391
|
+
awarenessProtocol.removeAwarenessStates(this.awareness, present, 'peer-left');
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
/** Cached delete-set hash - see computeDeleteSetHash(). */
|
|
1395
|
+
_deleteSetHash() {
|
|
1396
|
+
if (this._dsHashCache === null) {
|
|
1397
|
+
this._dsHashCache = computeDeleteSetHash(this.doc);
|
|
1398
|
+
}
|
|
1399
|
+
return this._dsHashCache;
|
|
1400
|
+
}
|
|
1401
|
+
/**
|
|
1402
|
+
* Debounce onPeerConnect-triggered syncNow() calls. A burst of connect
|
|
1403
|
+
* events within `_peerConnectDebounceMs` collapses into one call instead
|
|
1404
|
+
* of one per event - without this, N peers joining a mesh in a short
|
|
1405
|
+
* window each independently broadcast full state to everyone already
|
|
1406
|
+
* connected (O(N^2) traffic), since onPeerConnect fires once per
|
|
1407
|
+
* newly-opened peer connection with no coalescing of its own.
|
|
1408
|
+
*/
|
|
1409
|
+
_schedulePeerConnectSync(peerId) {
|
|
1410
|
+
if (peerId !== undefined)
|
|
1411
|
+
this._pendingPeerConnectIds.add(peerId);
|
|
1412
|
+
if (this._pendingPeerConnectSyncTimeoutId !== undefined)
|
|
1413
|
+
return;
|
|
1414
|
+
this._pendingPeerConnectSyncTimeoutId = setTimeout(() => {
|
|
1415
|
+
this._pendingPeerConnectSyncTimeoutId = undefined;
|
|
1416
|
+
const ids = Array.from(this._pendingPeerConnectIds);
|
|
1417
|
+
this._pendingPeerConnectIds.clear();
|
|
1418
|
+
if (!this.transport.isConnected || this._destroying)
|
|
1419
|
+
return;
|
|
1420
|
+
if (typeof this.transport.sendTo === 'function' && ids.length > 0) {
|
|
1421
|
+
// Round 5, item 5: one plain beacon to each new peer, nothing to
|
|
1422
|
+
// the rest of the mesh. Not rate-limited as a request: it answers
|
|
1423
|
+
// a channel that just opened, and the peer's own JOIN beacon is
|
|
1424
|
+
// the fallback if it is lost.
|
|
1425
|
+
const beacon = wrapMessageWithChecksum(this._encodeSyncStep1(0));
|
|
1426
|
+
for (const id of ids)
|
|
1427
|
+
this._sendToTransport(beacon, id);
|
|
1428
|
+
return;
|
|
1429
|
+
}
|
|
1430
|
+
this._syncNow(0);
|
|
1431
|
+
}, this._peerConnectDebounceMs);
|
|
489
1432
|
}
|
|
490
1433
|
/**
|
|
491
1434
|
* Setup automatic document synchronization.
|
|
@@ -494,27 +1437,36 @@ export class GenericProvider extends Observable {
|
|
|
494
1437
|
*/
|
|
495
1438
|
_setupDocumentSync() {
|
|
496
1439
|
this._updateHandler = (update, origin) => {
|
|
497
|
-
|
|
498
|
-
//
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
//
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
1440
|
+
this._dsHashCache = null;
|
|
1441
|
+
this._equalBeaconsHeard = 0; // our digest changed - see the Trickle fields
|
|
1442
|
+
// Fires for BOTH local edits and remotely-applied updates (the latter
|
|
1443
|
+
// go through doc.transact with origin=this) - see _markActivity()'s
|
|
1444
|
+
// doc comment. Phase 1e: only a LOCAL edit counts as activity for
|
|
1445
|
+
// idle backoff - a listener has nothing a beacon would announce, and
|
|
1446
|
+
// the typist's base-interval beacon heals any listener that lost the
|
|
1447
|
+
// keystroke (phase 1d design A). Before this, one typist kept all N
|
|
1448
|
+
// peers at the base cadence: N*(N-1) deliveries per interval against
|
|
1449
|
+
// N-1 per keystroke.
|
|
1450
|
+
// Don't send updates that originated from this provider (received
|
|
1451
|
+
// from the wire) or from the local load connect() is waiting for.
|
|
1452
|
+
if (origin !== this && !this._loading) {
|
|
1453
|
+
// Local-only transaction origins (e.g. a rollback the peer must not
|
|
1454
|
+
// replicate) never reach the transport.
|
|
1455
|
+
if (this._excludeOrigins.has(origin))
|
|
1456
|
+
return;
|
|
1457
|
+
this._markActivity();
|
|
1458
|
+
// 'pull' replicas answer requests but never push unasked.
|
|
1459
|
+
if (this._syncMode === 'pull')
|
|
1460
|
+
return;
|
|
506
1461
|
this._batchUpdate(update);
|
|
507
1462
|
}
|
|
508
|
-
else {
|
|
509
|
-
// Immediate mode: send right away
|
|
510
|
-
this._sendUpdate(update);
|
|
511
|
-
}
|
|
512
1463
|
};
|
|
513
1464
|
this.doc.on('update', this._updateHandler);
|
|
514
1465
|
}
|
|
515
1466
|
/**
|
|
516
|
-
*
|
|
517
|
-
*
|
|
1467
|
+
* Merge a local update into the pending batch and schedule its flush:
|
|
1468
|
+
* after `batchUpdates` ms (debounced) when that is > 0, otherwise at the
|
|
1469
|
+
* end of the current task via queueMicrotask - see `_pendingUpdate`.
|
|
518
1470
|
*/
|
|
519
1471
|
_batchUpdate(update) {
|
|
520
1472
|
// Merge with pending update if exists
|
|
@@ -534,18 +1486,38 @@ export class GenericProvider extends Observable {
|
|
|
534
1486
|
else {
|
|
535
1487
|
this._pendingUpdate = update;
|
|
536
1488
|
}
|
|
537
|
-
|
|
1489
|
+
if (this._batchUpdates > 0) {
|
|
1490
|
+
// Debounce: restart the timer on every update.
|
|
1491
|
+
if (this._batchTimeoutId !== undefined)
|
|
1492
|
+
clearTimeout(this._batchTimeoutId);
|
|
1493
|
+
this._batchTimeoutId = setTimeout(() => {
|
|
1494
|
+
this._batchTimeoutId = undefined;
|
|
1495
|
+
this._flushPendingUpdate();
|
|
1496
|
+
}, this._batchUpdates);
|
|
1497
|
+
this._flushScheduled = true;
|
|
1498
|
+
return;
|
|
1499
|
+
}
|
|
1500
|
+
if (this._flushScheduled)
|
|
1501
|
+
return;
|
|
1502
|
+
this._flushScheduled = true;
|
|
1503
|
+
queueMicrotask(() => this._flushPendingUpdate());
|
|
1504
|
+
}
|
|
1505
|
+
/**
|
|
1506
|
+
* Send the pending update batch as one wire message, carrying any
|
|
1507
|
+
* awareness change the throttle is holding (see _takePendingAwareness).
|
|
1508
|
+
* Shared by the microtask flush, the timed flush, and the
|
|
1509
|
+
* disconnect()/destroy() flush.
|
|
1510
|
+
*/
|
|
1511
|
+
_flushPendingUpdate() {
|
|
1512
|
+
this._flushScheduled = false;
|
|
538
1513
|
if (this._batchTimeoutId !== undefined) {
|
|
539
1514
|
clearTimeout(this._batchTimeoutId);
|
|
540
|
-
}
|
|
541
|
-
// Set new timeout to send after delay
|
|
542
|
-
this._batchTimeoutId = setTimeout(() => {
|
|
543
|
-
if (this._pendingUpdate) {
|
|
544
|
-
this._sendUpdate(this._pendingUpdate);
|
|
545
|
-
this._pendingUpdate = null;
|
|
546
|
-
}
|
|
547
1515
|
this._batchTimeoutId = undefined;
|
|
548
|
-
}
|
|
1516
|
+
}
|
|
1517
|
+
const update = this._pendingUpdate;
|
|
1518
|
+
this._pendingUpdate = null;
|
|
1519
|
+
if (update && this.transport.isConnected)
|
|
1520
|
+
this._sendUpdate(update);
|
|
549
1521
|
}
|
|
550
1522
|
/**
|
|
551
1523
|
* Setup automatic awareness synchronization.
|
|
@@ -553,21 +1525,85 @@ export class GenericProvider extends Observable {
|
|
|
553
1525
|
*/
|
|
554
1526
|
_setupAwarenessSync() {
|
|
555
1527
|
this._awarenessUpdateHandler = ({ added, updated, removed, }, origin) => {
|
|
556
|
-
//
|
|
1528
|
+
// Not _markActivity(): awareness changes are not something a beacon
|
|
1529
|
+
// announces (phase 1e; see the idleBackoffEnabled option).
|
|
1530
|
+
// Broadcast awareness changes, UNLESS they came from remote (this
|
|
1531
|
+
// comment described the intent since the very first commit, but the
|
|
1532
|
+
// actual origin check was never implemented until now - confirmed by
|
|
1533
|
+
// `git log -p` on this handler). `origin === this` is exactly the
|
|
1534
|
+
// signature the MESSAGE_AWARENESS handler stamps on an update applied
|
|
1535
|
+
// from an incoming wire message (`applyAwarenessUpdate(this.awareness,
|
|
1536
|
+
// ..., this)`, below). Every transport this project targets is a
|
|
1537
|
+
// full-room relay (websocket/pubnub/gun/matrix/ably/supabase) or a
|
|
1538
|
+
// full mesh (peerjs/simple-peer/trystero - see CLAUDE.md), so the
|
|
1539
|
+
// sender's own broadcast already reached every other peer directly;
|
|
1540
|
+
// re-broadcasting it here on receipt is pure redundant traffic that
|
|
1541
|
+
// compounds across every OTHER receiver doing the same thing.
|
|
1542
|
+
// Verified with a throwaway probe: a single awareness field change in
|
|
1543
|
+
// an N-peer room cost N*(N-1) wire deliveries before this check (one
|
|
1544
|
+
// echo per receiver, each reaching N-1 peers) vs. N-1 after - e.g.
|
|
1545
|
+
// N=20: 380 -> 19.
|
|
1546
|
+
//
|
|
1547
|
+
// Carve-out: `applyAwarenessUpdate` has its own defense against a
|
|
1548
|
+
// remote peer incorrectly removing OUR OWN state (a stale/racy
|
|
1549
|
+
// timeout-removal from someone else's clock) - it bumps our clock
|
|
1550
|
+
// instead of deleting our state, but (a quirk of that function) still
|
|
1551
|
+
// reports it via `removed` including our own clientID. That specific
|
|
1552
|
+
// case must still be broadcast so the room's stale belief that we're
|
|
1553
|
+
// gone gets corrected promptly, instead of only self-healing on our
|
|
1554
|
+
// next unrelated state change/renewal (up to ~15s later).
|
|
1555
|
+
if (origin === this) {
|
|
1556
|
+
// An incoming removal might be the SAME departure we have a
|
|
1557
|
+
// suppressed broadcast queued for (see _scheduleAwarenessRemoval())
|
|
1558
|
+
// - someone else already told the room, drop ours.
|
|
1559
|
+
if (removed.length > 0) {
|
|
1560
|
+
this._cancelPendingAwarenessRemovalIfOverlaps(removed);
|
|
1561
|
+
}
|
|
1562
|
+
if (!removed.includes(this.awareness.clientID)) {
|
|
1563
|
+
return;
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
557
1566
|
const changedClients = added.concat(updated).concat(removed);
|
|
1567
|
+
// A pure timeout-triggered removal ('timeout' is the exact origin
|
|
1568
|
+
// string y-protocols/awareness.js's own _checkInterval passes to
|
|
1569
|
+
// removeAwarenessStates()) is redundant across the whole room: every
|
|
1570
|
+
// OTHER connected peer runs the identical 30s-timeout sweep
|
|
1571
|
+
// independently, so all of them detect and would broadcast the SAME
|
|
1572
|
+
// departure within the same ~3s tick - measured as O(N-1) broadcasts
|
|
1573
|
+
// / O((N-1)(N-2)) deliveries for ONE departure in
|
|
1574
|
+
// test/dummy/bench-awareness-removal-burst.ts (e.g. N=50: 2352
|
|
1575
|
+
// deliveries). Delay + drop-if-overheard, exactly mirroring
|
|
1576
|
+
// _scheduleSyncReply()/_cancelPendingSyncReply()'s suppression of
|
|
1577
|
+
// redundant SyncStep2 replies. Deliberately NOT applied to
|
|
1578
|
+
// 'disconnect'/'window unload' removals (a single broadcaster, not
|
|
1579
|
+
// redundant) or to added/updated clients (every sender's
|
|
1580
|
+
// cursor/presence data is meaningfully different and must never be
|
|
1581
|
+
// suppressed).
|
|
1582
|
+
if (origin === 'timeout' || origin === 'peer-left') {
|
|
1583
|
+
this._scheduleAwarenessRemoval(changedClients, origin);
|
|
1584
|
+
return;
|
|
1585
|
+
}
|
|
558
1586
|
this._broadcastAwareness(changedClients);
|
|
559
1587
|
};
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
1588
|
+
// Round 5, item 8: broadcast on 'change' (y-protocols filters updates
|
|
1589
|
+
// whose state deep-equals the previous one) rather than 'update'
|
|
1590
|
+
// (every setLocalState call) - an app that re-sets unchanged state no
|
|
1591
|
+
// longer costs a broadcast per call. The renewal is the one
|
|
1592
|
+
// equal-state update that must go out; the provider's own sweep sends
|
|
1593
|
+
// it explicitly (_startAwarenessSweep). With an app-supplied Awareness
|
|
1594
|
+
// y-protocols' own sweep renews through 'update' only, so that case
|
|
1595
|
+
// keeps listening on 'update'.
|
|
1596
|
+
this.awareness.on(this._ownsAwareness ? 'change' : 'update', this._awarenessUpdateHandler);
|
|
1597
|
+
// App channel: attached here only if one was supplied to the
|
|
1598
|
+
// constructor; otherwise the getter attaches on first use. Reading
|
|
1599
|
+
// `this.appAwareness` here would defeat the lazy construction.
|
|
1600
|
+
if (this._appAwareness) {
|
|
1601
|
+
this._attachAppAwareness(this._appAwareness);
|
|
1602
|
+
}
|
|
566
1603
|
// Cleanup: mark as offline and disconnect BC when page unloads
|
|
567
1604
|
if (typeof window !== 'undefined') {
|
|
568
1605
|
this._beforeUnloadHandler = () => {
|
|
569
1606
|
awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'window unload');
|
|
570
|
-
awarenessProtocol.removeAwarenessStates(this.appAwareness, [this.doc.clientID], 'window unload');
|
|
571
1607
|
// Disconnect BroadcastChannel to notify other tabs
|
|
572
1608
|
this._disconnectBroadcastChannel();
|
|
573
1609
|
};
|
|
@@ -575,213 +1611,1459 @@ export class GenericProvider extends Observable {
|
|
|
575
1611
|
}
|
|
576
1612
|
}
|
|
577
1613
|
/**
|
|
578
|
-
* Handle incoming messages from the transport.
|
|
579
|
-
*
|
|
580
|
-
*
|
|
1614
|
+
* Handle incoming messages from the transport (or BroadcastChannel).
|
|
1615
|
+
*
|
|
1616
|
+
* When compressionThresholdBytes is disabled (the default), this is a
|
|
1617
|
+
* fully synchronous fast path, byte-for-byte the same behavior as before
|
|
1618
|
+
* that option existed: straight into _processWrappedMessage().
|
|
1619
|
+
*
|
|
1620
|
+
* When enabled, every message - from the network transport AND from
|
|
1621
|
+
* BroadcastChannel (see _send()) - carries a leading compressed(1)/
|
|
1622
|
+
* uncompressed(0) flag byte ahead of the usual CRC32 wrapper. Reading
|
|
1623
|
+
* that flag and, if set, decompressing is inherently async (the
|
|
1624
|
+
* Compression Streams API has no synchronous form), so this method
|
|
1625
|
+
* dispatches to a promise chain instead of processing inline in that
|
|
1626
|
+
* case. This means a large (compressed) message and a small (uncompressed
|
|
1627
|
+
* or below-threshold) message that arrive back-to-back can finish
|
|
1628
|
+
* processing out of arrival order - acceptable here because Yjs updates
|
|
1629
|
+
* are idempotent/commutative (see MESSAGE_SYNC_VERIFIED's handling below)
|
|
1630
|
+
* and because the compression threshold keeps this path almost entirely
|
|
1631
|
+
* to large, full-state syncs, not the per-keystroke incremental updates
|
|
1632
|
+
* that per-sender gap detection actually relies on ordering-sensitive
|
|
1633
|
+
* heuristics for.
|
|
1634
|
+
*/
|
|
1635
|
+
_handleIncomingMessage(data, from) {
|
|
1636
|
+
if (!this._compressionThresholdBytes) {
|
|
1637
|
+
this._processWrappedMessage(data, from);
|
|
1638
|
+
return;
|
|
1639
|
+
}
|
|
1640
|
+
if (data.length < 1) {
|
|
1641
|
+
console.warn('[GenericProvider] Dropping empty message (missing compression flag byte)');
|
|
1642
|
+
return;
|
|
1643
|
+
}
|
|
1644
|
+
const flag = data[0];
|
|
1645
|
+
const rest = data.subarray(1);
|
|
1646
|
+
if (flag === 0) {
|
|
1647
|
+
this._processWrappedMessage(rest, from);
|
|
1648
|
+
return;
|
|
1649
|
+
}
|
|
1650
|
+
if (!COMPRESSION_AVAILABLE) {
|
|
1651
|
+
console.warn('[GenericProvider] Received a compressed message but this runtime has no DecompressionStream - dropping it.');
|
|
1652
|
+
return;
|
|
1653
|
+
}
|
|
1654
|
+
decompressDeflateRaw(rest)
|
|
1655
|
+
.then((wrapped) => this._processWrappedMessage(wrapped, from))
|
|
1656
|
+
.catch((error) => {
|
|
1657
|
+
// Treat decompression failure the same as a CRC32 mismatch on the
|
|
1658
|
+
// uncompressed path: request a resync rather than silently dropping.
|
|
1659
|
+
console.warn('[GenericProvider] Failed to decompress incoming message, treating as corrupted:', error);
|
|
1660
|
+
this._requestResync();
|
|
1661
|
+
});
|
|
1662
|
+
}
|
|
1663
|
+
/**
|
|
1664
|
+
* Verify message integrity with CRC32 and decode. Corrupt messages are
|
|
1665
|
+
* rejected immediately without attempting to decode. Operates on bytes
|
|
1666
|
+
* that have already had any compression flag/decompression handled by
|
|
1667
|
+
* _handleIncomingMessage() - this is the pre-compression-feature
|
|
1668
|
+
* implementation, unchanged.
|
|
581
1669
|
*/
|
|
582
|
-
|
|
1670
|
+
_processWrappedMessage(data, from) {
|
|
583
1671
|
// Verify message integrity with CRC32 checksum
|
|
584
1672
|
const message = unwrapAndVerifyMessage(data);
|
|
585
1673
|
if (message === null) {
|
|
586
|
-
// Message is corrupted - reject it immediately
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
//
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
console.warn(`[GenericProvider] 💥 Corrupted message rejected
|
|
1674
|
+
// Message is corrupted - reject it immediately. Note: if this was a
|
|
1675
|
+
// MESSAGE_BATCH envelope, the CRC32 wrap covers the WHOLE batch, so a
|
|
1676
|
+
// single corrupted bit here loses every sub-message it contained, not
|
|
1677
|
+
// just one - a deliberate tradeoff of batching multiple logical
|
|
1678
|
+
// messages behind one wire message/one checksum. See _sendBatch()'s
|
|
1679
|
+
// doc comment for why this was chosen over per-sub-message checksums,
|
|
1680
|
+
// and this project's benchmark suite (bench-corruption-storm.ts,
|
|
1681
|
+
// bench-packet-loss.ts) for how that tradeoff was measured.
|
|
1682
|
+
console.warn(`[GenericProvider] 💥 Corrupted message rejected: CRC32 checksum mismatch. ` +
|
|
595
1683
|
`This is expected if data corruption simulation is enabled.`);
|
|
596
|
-
//
|
|
597
|
-
//
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
1684
|
+
// Wire noise, not silence - counts as activity for idleBackoffEnabled
|
|
1685
|
+
// even though nothing here reaches the doc/awareness update handlers
|
|
1686
|
+
// (the message is rejected before decoding). See _markActivity()'s
|
|
1687
|
+
// doc comment.
|
|
1688
|
+
this._markActivity();
|
|
1689
|
+
// Request re-sync to recover any lost data - routed through the
|
|
1690
|
+
// shared coordinator so this doesn't stack an independent timer on
|
|
1691
|
+
// top of any hash-mismatch/gap-confirmed resync already pending.
|
|
1692
|
+
this._requestResync();
|
|
604
1693
|
return; // Don't process corrupted message
|
|
605
1694
|
}
|
|
606
1695
|
// Message integrity verified - safe to decode
|
|
607
1696
|
try {
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
1697
|
+
this._dispatchMessage(message, from);
|
|
1698
|
+
}
|
|
1699
|
+
catch (error) {
|
|
1700
|
+
// This should only happen for logic errors, not corruption
|
|
1701
|
+
// (corruption is caught by CRC32 check above)
|
|
1702
|
+
console.error('[GenericProvider] Error handling message:', error);
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
/**
|
|
1706
|
+
* Decode and act on one already-integrity-verified, already-decompressed
|
|
1707
|
+
* message. Split out of `_processWrappedMessage()` so `MESSAGE_BATCH`
|
|
1708
|
+
* (see `_sendBatch()`) can recurse into this for each sub-message it
|
|
1709
|
+
* unwraps, running the EXACT SAME per-message-type logic used for a
|
|
1710
|
+
* top-level message rather than a parallel reimplementation. A thrown
|
|
1711
|
+
* error partway through a batch's sub-messages aborts the REST of that
|
|
1712
|
+
* batch (propagates up to `_processWrappedMessage()`'s catch) - same as
|
|
1713
|
+
* a logic error aborting a single top-level message today, just now
|
|
1714
|
+
* scoped to "the rest of this batch" instead of "this one message".
|
|
1715
|
+
*/
|
|
1716
|
+
_dispatchMessage(message, from) {
|
|
1717
|
+
const decoder = decoding.createDecoder(message);
|
|
1718
|
+
const messageType = decoding.readVarUint(decoder);
|
|
1719
|
+
switch (messageType) {
|
|
1720
|
+
case MESSAGE_BATCH: {
|
|
1721
|
+
// Payload is N length-prefixed sub-messages (writeVarUint8Array
|
|
1722
|
+
// per sub-message, mirroring MESSAGE_AWARENESS's own framing).
|
|
1723
|
+
// Each was NOT individually CRC32-wrapped - see _sendBatch()'s doc
|
|
1724
|
+
// comment - so just decode and dispatch each one directly.
|
|
1725
|
+
while (decoding.hasContent(decoder)) {
|
|
1726
|
+
const subMessage = decoding.readVarUint8Array(decoder);
|
|
1727
|
+
this._dispatchMessage(subMessage, from);
|
|
626
1728
|
}
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
1729
|
+
break;
|
|
1730
|
+
}
|
|
1731
|
+
case MESSAGE_SYNC_DIGEST: {
|
|
1732
|
+
this._handleDigest(decoder, from);
|
|
1733
|
+
break;
|
|
1734
|
+
}
|
|
1735
|
+
case MESSAGE_SYNC_PUSH: {
|
|
1736
|
+
// Somebody's whole document: apply it, nothing else (see the
|
|
1737
|
+
// constant's comment for why no hash check and no synced flip).
|
|
1738
|
+
Y.applyUpdate(this.doc, decoding.readVarUint8Array(decoder), this);
|
|
1739
|
+
this._checkPendingAfterReply();
|
|
1740
|
+
break;
|
|
1741
|
+
}
|
|
1742
|
+
case MESSAGE_SYNC: {
|
|
1743
|
+
const encoder = encoding.createEncoder();
|
|
1744
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
1745
|
+
const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this);
|
|
1746
|
+
if (syncMessageType === syncProtocol.messageYjsSyncStep2) {
|
|
1747
|
+
// If we received SyncStep2, we're synced (and confirmed: we have
|
|
1748
|
+
// heard from a peer that had data for us)
|
|
1749
|
+
this._confirmed = true;
|
|
1750
|
+
this._noteResponse(true);
|
|
1751
|
+
this._markSynced();
|
|
1752
|
+
// Design E (phase 1d): a SyncStep2 carries its encoder's own pending
|
|
1753
|
+
// structs, so a responder missing a struct hands us the same hole.
|
|
1754
|
+
// Our response wait just ended - make sure something still asks.
|
|
1755
|
+
this._checkPendingAfterReply();
|
|
1756
|
+
// Someone else's SyncStep2 reply just arrived - our own pending
|
|
1757
|
+
// reply (if any) is now most likely redundant.
|
|
1758
|
+
this._cancelPendingSyncReply();
|
|
633
1759
|
}
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
// Decode JSON payload
|
|
641
|
-
const decoder = new TextDecoder();
|
|
642
|
-
const payloadStr = decoder.decode(payloadBytes);
|
|
643
|
-
const message = JSON.parse(payloadStr);
|
|
644
|
-
// Emit to pubsub channel
|
|
645
|
-
this.pubsub._handleMessage(topic, message);
|
|
646
|
-
}
|
|
647
|
-
catch (error) {
|
|
648
|
-
console.error('Error decoding pub/sub message:', error);
|
|
649
|
-
}
|
|
650
|
-
break;
|
|
1760
|
+
// Send reply if needed. Suppression only engages with genuine
|
|
1761
|
+
// redundancy (>=2 other known peers via awareness) - below that,
|
|
1762
|
+
// there's no "someone else" to rely on, so reply immediately
|
|
1763
|
+
// (still rate-limited via _sendSyncReply() as a hard backstop).
|
|
1764
|
+
if (encoding.length(encoder) > 1) {
|
|
1765
|
+
this._replyToSyncRequest(encoding.toUint8Array(encoder));
|
|
651
1766
|
}
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
1767
|
+
break;
|
|
1768
|
+
}
|
|
1769
|
+
case MESSAGE_AWARENESS: {
|
|
1770
|
+
const payload = decoding.readVarUint8Array(decoder);
|
|
1771
|
+
// Someone else's removal broadcast just arrived - cancel our own
|
|
1772
|
+
// suppressed removal for the same clientID(s), if pending (see
|
|
1773
|
+
// _scheduleAwarenessRemoval()). This MUST be checked here, at the
|
|
1774
|
+
// wire-message level, before calling applyAwarenessUpdate() below -
|
|
1775
|
+
// by the time this arrives we've very likely already independently
|
|
1776
|
+
// detected and applied the SAME removal ourselves (every peer's
|
|
1777
|
+
// 30s outdatedTimeout sweep fires near-simultaneously, well before
|
|
1778
|
+
// this message's network delay elapses - see
|
|
1779
|
+
// test/dummy/bench-awareness-removal-burst.ts), so
|
|
1780
|
+
// applyAwarenessUpdate() below will see a clock it already knows
|
|
1781
|
+
// and emit no 'update' event at all - mirrors exactly how the
|
|
1782
|
+
// MESSAGE_SYNC/MESSAGE_SYNC_VERIFIED case above cancels
|
|
1783
|
+
// _pendingSyncReply on seeing a SyncStep2 message TYPE arrive, not
|
|
1784
|
+
// on whether it changed anything locally.
|
|
1785
|
+
const scan = this._scanAwarenessPayload(payload);
|
|
1786
|
+
if (scan.removed.length > 0) {
|
|
1787
|
+
this._cancelPendingAwarenessRemovalIfOverlaps(scan.removed);
|
|
1788
|
+
}
|
|
1789
|
+
if (scan.coversUs)
|
|
1790
|
+
this._presenceCovered = true;
|
|
1791
|
+
// Round 5, item 3: with Trickle, settled peers rarely beacon, so a
|
|
1792
|
+
// joiner would learn them only from the few phase-winning beacons
|
|
1793
|
+
// per interval. The relayed presence table names everyone - ids
|
|
1794
|
+
// only, addresses stay beacon-learned (unicast needs a `from`).
|
|
1795
|
+
const heardAt = Date.now();
|
|
1796
|
+
for (const id of scan.present) {
|
|
1797
|
+
if (id !== this.doc.clientID)
|
|
1798
|
+
this._knownPeers.set(id, heardAt);
|
|
1799
|
+
}
|
|
1800
|
+
awarenessProtocol.applyAwarenessUpdate(this.awareness, payload, this);
|
|
1801
|
+
break;
|
|
1802
|
+
}
|
|
1803
|
+
case MESSAGE_PUBSUB: {
|
|
1804
|
+
// Read topic
|
|
1805
|
+
const topic = decoding.readVarString(decoder);
|
|
1806
|
+
// Read message payload
|
|
1807
|
+
const payloadBytes = decoding.readVarUint8Array(decoder);
|
|
1808
|
+
try {
|
|
1809
|
+
// Decode JSON payload
|
|
1810
|
+
const decoder = new TextDecoder();
|
|
1811
|
+
const payloadStr = decoder.decode(payloadBytes);
|
|
1812
|
+
const message = JSON.parse(payloadStr);
|
|
1813
|
+
// Emit to pubsub channel
|
|
1814
|
+
this.pubsub._handleMessage(topic, message);
|
|
1815
|
+
}
|
|
1816
|
+
catch (error) {
|
|
1817
|
+
console.error('Error decoding pub/sub message:', error);
|
|
1818
|
+
}
|
|
1819
|
+
break;
|
|
1820
|
+
}
|
|
1821
|
+
case MESSAGE_AWARENESS_APP: {
|
|
1822
|
+
// Deliberately none of the presence bookkeeping the core awareness
|
|
1823
|
+
// case does (_knownPeers, _presenceCovered, removal cancellation):
|
|
1824
|
+
// this channel is written by untrusted modules and must not be able
|
|
1825
|
+
// to influence the room's view of who is present.
|
|
1826
|
+
awarenessProtocol.applyAwarenessUpdate(this.appAwareness, decoding.readVarUint8Array(decoder), this);
|
|
1827
|
+
break;
|
|
1828
|
+
}
|
|
1829
|
+
case MESSAGE_PUBSUB_TARGETED: {
|
|
1830
|
+
const target = decoding.readVarString(decoder);
|
|
1831
|
+
const topic = decoding.readVarString(decoder);
|
|
1832
|
+
const payloadBytes = decoding.readVarUint8Array(decoder);
|
|
1833
|
+
// Drop messages aimed at someone else (broadcast-and-filter path).
|
|
1834
|
+
if (this._localId !== undefined && target !== this._localId) {
|
|
667
1835
|
break;
|
|
668
1836
|
}
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
//
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
1837
|
+
try {
|
|
1838
|
+
const message = JSON.parse(new TextDecoder().decode(payloadBytes));
|
|
1839
|
+
this.pubsub._handleMessage(topic, message);
|
|
1840
|
+
}
|
|
1841
|
+
catch (error) {
|
|
1842
|
+
console.error('Error decoding targeted pub/sub message:', error);
|
|
1843
|
+
}
|
|
1844
|
+
break;
|
|
1845
|
+
}
|
|
1846
|
+
case MESSAGE_SYNC_VERIFIED: {
|
|
1847
|
+
// Sync message with sequence number and hash verification
|
|
1848
|
+
// Read sequence number and clientID first
|
|
1849
|
+
const seqNum = decoding.readVarUint(decoder);
|
|
1850
|
+
const senderClientID = decoding.readVarUint(decoder);
|
|
1851
|
+
// Track for gap detection only — does NOT gate whether we apply
|
|
1852
|
+
// the update below (see _trackRemoteSeq() for why).
|
|
1853
|
+
this._trackRemoteSeq(senderClientID, seqNum);
|
|
1854
|
+
this._knownPeers.set(senderClientID, Date.now());
|
|
1855
|
+
if (from !== undefined)
|
|
1856
|
+
this._peerAddress.set(senderClientID, from);
|
|
1857
|
+
this._touchPeer(senderClientID);
|
|
1858
|
+
// Always apply the update. Yjs updates are idempotent/commutative,
|
|
1859
|
+
// so re-applying an already-seen update is a harmless no-op.
|
|
1860
|
+
// Under reordering, a merely-late (not actually duplicate) update
|
|
1861
|
+
// must still be applied here — the old "skip if seqNum <= last
|
|
1862
|
+
// seen" logic silently dropped such updates forever whenever a
|
|
1863
|
+
// later-numbered message happened to arrive first.
|
|
1864
|
+
// Peek at the sync sub-message's payload (SyncStep2/Update carry
|
|
1865
|
+
// an update as a varUint8Array right after the sub-type) without
|
|
1866
|
+
// consuming the decoder - _isLateUpdate() needs the bytes after
|
|
1867
|
+
// readSyncMessage() has applied them.
|
|
1868
|
+
const updateBytes = this._peekSyncUpdate(decoder);
|
|
1869
|
+
const encoder = encoding.createEncoder();
|
|
1870
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
1871
|
+
const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this);
|
|
1872
|
+
// Someone else's SyncStep2 reply just arrived - our own pending
|
|
1873
|
+
// reply (if any) is now most likely redundant. Mirrors the
|
|
1874
|
+
// MESSAGE_SYNC case: the reply encoded above is always a plain
|
|
1875
|
+
// MESSAGE_SYNC-typed message regardless of which message type
|
|
1876
|
+
// triggered it, so the same suppression scheme applies here too.
|
|
1877
|
+
if (syncMessageType === syncProtocol.messageYjsSyncStep2) {
|
|
1878
|
+
this._cancelPendingSyncReply();
|
|
1879
|
+
this._confirmed = true;
|
|
1880
|
+
this._checkPendingAfterReply();
|
|
1881
|
+
this._noteResponse(true);
|
|
1882
|
+
}
|
|
1883
|
+
// Read the expected hash from sender (signed integer)
|
|
1884
|
+
const expectedHash = decoding.readVarInt(decoder);
|
|
1885
|
+
// Compute our local hash after applying the update
|
|
1886
|
+
const localHash = computeDocHash(this.doc);
|
|
1887
|
+
// Verify hash match
|
|
1888
|
+
if (localHash !== expectedHash) {
|
|
1889
|
+
// If we already know this sender has a suspected reordering gap
|
|
1890
|
+
// (see _trackRemoteSeq()/_scheduleGapCheck()), a hash mismatch
|
|
1891
|
+
// right now is the *expected* transient state — we're missing a
|
|
1892
|
+
// piece that's very likely still in flight, not actually
|
|
1893
|
+
// diverged. Let the pending gap-check grace period resolve it
|
|
1894
|
+
// instead of also escalating the hash-mismatch backoff: under
|
|
1895
|
+
// heavy reordering this previously caused a burst of mismatches
|
|
1896
|
+
// to rack up the exponential backoff to its 10s cap within a
|
|
1897
|
+
// single edit burst, purely from timing, not real divergence.
|
|
1898
|
+
// A hash mismatch with NO pending gap (in-order, but still
|
|
1899
|
+
// wrong) is not explained by reordering and still escalates
|
|
1900
|
+
// normally below.
|
|
1901
|
+
const reorderingSuspected = this._gapCheckTimers.has(senderClientID);
|
|
1902
|
+
// A late update - one whose content we had already been past
|
|
1903
|
+
// when it arrived (its sender's clock in the update is below
|
|
1904
|
+
// ours, because a SyncStep2 or a reordered later update got here
|
|
1905
|
+
// first) - carries a hash of a state we have legitimately moved
|
|
1906
|
+
// beyond. Its mismatch says nothing about anything we lack; it
|
|
1907
|
+
// was the other half of the item-13 cascade (a resync's reply
|
|
1908
|
+
// fast-forwards a peer, then every in-flight keystroke behind it
|
|
1909
|
+
// mismatches). Cheap to detect from the update's own metadata.
|
|
1910
|
+
const lateUpdate = this._isLateUpdate(updateBytes);
|
|
1911
|
+
// Yjs's own verdict: if the update could not be fully integrated
|
|
1912
|
+
// because a causal dependency is missing, the struct store holds
|
|
1913
|
+
// it as pending. That is the ONE mismatch that is evidence of a
|
|
1914
|
+
// gap - and under jitter it is usually a reordering that the
|
|
1915
|
+
// next few ms resolve, so it gets the same grace a sequence gap
|
|
1916
|
+
// gets before a beacon goes out (_schedulePendingCheck). It also
|
|
1917
|
+
// covers the case the sequence anchor no longer does: since the
|
|
1918
|
+
// connect push carries no sequence number (MESSAGE_SYNC_PUSH), a
|
|
1919
|
+
// peer's first keystroke can be the first numbered message we see
|
|
1920
|
+
// from it, and a reordered first burst has no earlier number to
|
|
1921
|
+
// open a gap against.
|
|
1922
|
+
const pending = this.doc.store.pendingStructs !== null ||
|
|
1923
|
+
this.doc.store.pendingDs !== null;
|
|
1924
|
+
if (pending) {
|
|
1925
|
+
this._schedulePendingCheck();
|
|
733
1926
|
}
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
1927
|
+
else if (!reorderingSuspected && !lateUpdate) {
|
|
1928
|
+
// Push our full state AND request theirs (syncNow() does
|
|
1929
|
+
// both). A hash mismatch means the two peers have diverged -
|
|
1930
|
+
// one side may have edits the other lacks. Routed through the
|
|
1931
|
+
// shared coordinator so this doesn't stack an independent
|
|
1932
|
+
// timer on top of any corrupted-message/gap-confirmed resync
|
|
1933
|
+
// already pending.
|
|
1934
|
+
this._requestResync();
|
|
1935
|
+
// Logged with the shared attempt counter (kept as "#N" for
|
|
1936
|
+
// compatibility with existing tooling/benchmarks that grep
|
|
1937
|
+
// for this exact "Hash mismatch #" pattern) - it now reflects
|
|
1938
|
+
// the unified resync-attempt count rather than a
|
|
1939
|
+
// hash-mismatch-specific one, since the two escalation
|
|
1940
|
+
// counters were merged.
|
|
1941
|
+
console.warn(`[GenericProvider] Hash mismatch #${this._resyncAttemptCount} detected! Local: ${localHash}, Expected: ${expectedHash}`);
|
|
737
1942
|
}
|
|
738
|
-
break;
|
|
739
1943
|
}
|
|
740
|
-
|
|
741
|
-
|
|
1944
|
+
// If we received SyncStep2, we're synced (unless hash mismatched)
|
|
1945
|
+
if (syncMessageType === syncProtocol.messageYjsSyncStep2 &&
|
|
1946
|
+
localHash === expectedHash) {
|
|
1947
|
+
this._markSynced();
|
|
1948
|
+
}
|
|
1949
|
+
// Send reply if needed (as standard MESSAGE_SYNC). Suppression
|
|
1950
|
+
// only engages with genuine redundancy (>=2 other known peers via
|
|
1951
|
+
// awareness) - below that, reply immediately (still rate-limited
|
|
1952
|
+
// via _sendSyncReply() as a hard backstop). Matches the
|
|
1953
|
+
// MESSAGE_SYNC case's gate exactly; without this, a hash-mismatch
|
|
1954
|
+
// resync burst under packet loss bypassed suppression entirely,
|
|
1955
|
+
// since every peer answering a post-mismatch SyncStep1 replied
|
|
1956
|
+
// immediately via this path.
|
|
1957
|
+
if (encoding.length(encoder) > 1) {
|
|
1958
|
+
this._replyToSyncRequest(encoding.toUint8Array(encoder));
|
|
1959
|
+
}
|
|
1960
|
+
break;
|
|
742
1961
|
}
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
// This should only happen for logic errors, not corruption
|
|
746
|
-
// (corruption is caught by CRC32 check above)
|
|
747
|
-
console.error('[GenericProvider] Error handling message:', error);
|
|
1962
|
+
default:
|
|
1963
|
+
console.warn('Unknown message type:', messageType);
|
|
748
1964
|
}
|
|
749
1965
|
}
|
|
750
1966
|
/**
|
|
751
|
-
*
|
|
752
|
-
*
|
|
753
|
-
*
|
|
754
|
-
*
|
|
1967
|
+
* Handle a digest beacon (MESSAGE_SYNC_DIGEST). Reply rule (design doc
|
|
1968
|
+
* §3): SyncStep2 if the sender is behind us or its delete-set hash
|
|
1969
|
+
* differs from ours (the SyncStep2 always carries our full delete set, so
|
|
1970
|
+
* it also heals a lost delete on their side - and their beacon does the
|
|
1971
|
+
* same for us, symmetrically, within one interval); our own beacon as an
|
|
1972
|
+
* ack if the beacon is JOIN-flagged and states are equal; nothing
|
|
1973
|
+
* otherwise - which is what removes the ~5-12 empty replies per heartbeat
|
|
1974
|
+
* measured at N=50 in test/dummy/bench-idle-room.ts. "Sender is ahead of
|
|
1975
|
+
* us" triggers no reply: our own next beacon fetches it. Nothing here
|
|
1976
|
+
* removes a recovery path (the round-2 lesson in
|
|
1977
|
+
* 2026-09-04-resync-message-reduction-design.md's addendum), only
|
|
1978
|
+
* replies that carry no information.
|
|
1979
|
+
*
|
|
1980
|
+
* `synced`: a beacon we are not behind, with equal delete-set hash, is a
|
|
1981
|
+
* stronger statement than the empty SyncStep2 it replaces ("you lack
|
|
1982
|
+
* nothing I have"), so it marks us synced too - this is what keeps two
|
|
1983
|
+
* fresh peers, or a whole concurrent join burst, converging to `synced`
|
|
1984
|
+
* with no acks needing to survive the rate limiter.
|
|
755
1985
|
*/
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
if (
|
|
762
|
-
|
|
763
|
-
|
|
1986
|
+
_handleDigest(decoder, from) {
|
|
1987
|
+
decoding.readVarUint(decoder); // DIGEST_VERSION - append-only, nothing to branch on yet
|
|
1988
|
+
const flags = decoding.readVarUint(decoder);
|
|
1989
|
+
const senderClientID = decoding.readVarUint(decoder);
|
|
1990
|
+
this._knownPeers.set(senderClientID, Date.now());
|
|
1991
|
+
if (from !== undefined)
|
|
1992
|
+
this._peerAddress.set(senderClientID, from);
|
|
1993
|
+
this._touchPeer(senderClientID);
|
|
1994
|
+
const remoteSv = decoding.readVarUint8Array(decoder);
|
|
1995
|
+
const remoteDsHash = decoding.readVarUint(decoder);
|
|
1996
|
+
// Any trailing bytes belong to a newer version; ignored by design.
|
|
1997
|
+
const remote = Y.decodeStateVector(remoteSv);
|
|
1998
|
+
const local = Y.decodeStateVector(Y.encodeStateVector(this.doc));
|
|
1999
|
+
let senderBehind = false;
|
|
2000
|
+
for (const [client, clock] of local) {
|
|
2001
|
+
if ((remote.get(client) ?? 0) < clock) {
|
|
2002
|
+
senderBehind = true;
|
|
2003
|
+
break;
|
|
2004
|
+
}
|
|
764
2005
|
}
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
this.
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
//
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
2006
|
+
let weBehind = false;
|
|
2007
|
+
for (const [client, clock] of remote) {
|
|
2008
|
+
if ((local.get(client) ?? 0) < clock) {
|
|
2009
|
+
weBehind = true;
|
|
2010
|
+
break;
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
const dsEqual = remoteDsHash === this._deleteSetHash();
|
|
2014
|
+
const equal = !senderBehind && !weBehind && dsEqual;
|
|
2015
|
+
if (equal) {
|
|
2016
|
+
// Any peer holding exactly our state has confirmed it - see _confirmedSv.
|
|
2017
|
+
this._confirmedSv = remoteSv;
|
|
2018
|
+
this._confirmedDsHash = remoteDsHash;
|
|
2019
|
+
}
|
|
2020
|
+
if (flags & DIGEST_FLAG_ACK) {
|
|
2021
|
+
// Somebody confirmed the echoed state (see DIGEST_FLAG_ACK). If it is
|
|
2022
|
+
// ours, we're synced; if we were about to confirm the same state, we
|
|
2023
|
+
// no longer need to. Never a request: no reply, no presence.
|
|
2024
|
+
if (equal) {
|
|
2025
|
+
if (flags & DIGEST_FLAG_SETTLED) {
|
|
2026
|
+
this._confirmed = true;
|
|
2027
|
+
this._noteResponse(true);
|
|
2028
|
+
}
|
|
2029
|
+
else {
|
|
2030
|
+
this._equalUnsettledSeen = true;
|
|
2031
|
+
}
|
|
2032
|
+
this._markSynced();
|
|
2033
|
+
this._cancelPendingAck();
|
|
2034
|
+
}
|
|
2035
|
+
return;
|
|
2036
|
+
}
|
|
2037
|
+
// An equal-digest beacon from a settled peer's periodic tick has just
|
|
2038
|
+
// told the room (including whoever our pending ack was for) that this
|
|
2039
|
+
// state is confirmed. Our ack would say the same thing again. Task 3b
|
|
2040
|
+
// in the design doc: without this, acks were ~94% of a 50-peer join
|
|
2041
|
+
// burst's messages, throttled only by the rate limiter. An equal JOIN
|
|
2042
|
+
// beacon (another joiner in the same burst) asks for an ack too; our
|
|
2043
|
+
// pending ack - identical bytes, since it echoes that same state -
|
|
2044
|
+
// answers it as well, and _scheduleSyncReply() dedupes it below.
|
|
2045
|
+
if (equal && !(flags & (DIGEST_FLAG_JOIN | DIGEST_FLAG_CONFIRM))) {
|
|
2046
|
+
// A peer's periodic/resync beacon in our state: it makes our pending
|
|
2047
|
+
// ack redundant, and - if that peer is confirmed - it answers our
|
|
2048
|
+
// own outstanding join as well as any ack would. It also counts for
|
|
2049
|
+
// Trickle: the room has just compared itself against our digest.
|
|
2050
|
+
this._equalBeaconsHeard++;
|
|
2051
|
+
if (flags & DIGEST_FLAG_SETTLED) {
|
|
2052
|
+
this._confirmed = true;
|
|
2053
|
+
this._noteResponse(false);
|
|
2054
|
+
}
|
|
2055
|
+
else {
|
|
2056
|
+
this._equalUnsettledSeen = true;
|
|
2057
|
+
}
|
|
2058
|
+
this._cancelPendingAck();
|
|
2059
|
+
}
|
|
2060
|
+
if (senderBehind || !dsEqual) {
|
|
2061
|
+
const encoder = encoding.createEncoder();
|
|
2062
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
2063
|
+
syncProtocol.writeSyncStep2(encoder, this.doc, remoteSv);
|
|
2064
|
+
let reply = encoding.toUint8Array(encoder);
|
|
2065
|
+
if (!dsEqual) {
|
|
2066
|
+
// Round 5: a delete moves no clock, so the hash cannot say which of
|
|
2067
|
+
// us lacks a delete - and a SyncStep2 heals only its receiver. The
|
|
2068
|
+
// loser of a delete-only update was healed only by its OWN next
|
|
2069
|
+
// beacon, which idle backoff parks for up to 60 s (bench-idle-room
|
|
2070
|
+
// part b failed its 11 s cap at a 5 s interval). So the reply asks
|
|
2071
|
+
// back: our beacon rides in the same batch, and if the sender is
|
|
2072
|
+
// the one ahead it answers with its delete set. Terminates: a peer
|
|
2073
|
+
// healed by the SyncStep2 half sees an equal digest in the beacon
|
|
2074
|
+
// half and stays silent. (A behind-check timer on the mismatch was
|
|
2075
|
+
// tried first: every peer that overheard the loser's beacon armed
|
|
2076
|
+
// one and asked - ~2x the deliveries of a lossy edit burst on the
|
|
2077
|
+
// Matrix profile.)
|
|
2078
|
+
reply = this._encodeBatch([reply, this._encodeSyncStep1(0)]);
|
|
2079
|
+
}
|
|
2080
|
+
this._replyToSyncRequest(reply, false, remoteSv, senderClientID);
|
|
2081
|
+
}
|
|
2082
|
+
else if (flags & (DIGEST_FLAG_JOIN | DIGEST_FLAG_CONFIRM)) {
|
|
2083
|
+
this._replyToSyncRequest(this._encodeAck(remoteSv, remoteDsHash), true, null, senderClientID);
|
|
2084
|
+
}
|
|
2085
|
+
if (!weBehind && dsEqual) {
|
|
2086
|
+
this._markSynced();
|
|
2087
|
+
}
|
|
2088
|
+
if (weBehind) {
|
|
2089
|
+
// The sender has something we lack. Not a request yet - the update
|
|
2090
|
+
// may still be in flight (see _scheduleBehindCheck).
|
|
2091
|
+
this._scheduleBehindCheck(remoteSv);
|
|
2092
|
+
}
|
|
2093
|
+
if (flags & DIGEST_FLAG_JOIN && this.awareness.getLocalState() !== null) {
|
|
2094
|
+
// Presence on demand: the joiner asked. One broadcast per burst of
|
|
2095
|
+
// joiners (see _schedulePresenceResponse), never suppressed (each
|
|
2096
|
+
// responder's state is distinct). Skipped when we have no state to
|
|
2097
|
+
// announce.
|
|
2098
|
+
this._schedulePresenceResponse(senderClientID);
|
|
2099
|
+
}
|
|
2100
|
+
}
|
|
2101
|
+
/**
|
|
2102
|
+
* Answer a JOIN beacon's presence request once for all JOIN beacons that
|
|
2103
|
+
* arrive within `clamp(2 * minRTT, 100, 500)` ms of the first - long
|
|
2104
|
+
* enough to cover a join burst spread by latency, short enough that a
|
|
2105
|
+
* lone joiner sees the room's presence within a few round trips.
|
|
2106
|
+
*/
|
|
2107
|
+
_schedulePresenceResponse(requester) {
|
|
2108
|
+
this._presencePending.add(requester);
|
|
2109
|
+
if (this._presenceResponseTimer !== undefined)
|
|
2110
|
+
return;
|
|
2111
|
+
this._presenceCovered = false;
|
|
2112
|
+
const rtt = this._rttMinMs();
|
|
2113
|
+
// Phase 1e, relay path: one peer per 2 s bucket - the first-ranked
|
|
2114
|
+
// for a constant requester - relays the whole awareness table at once
|
|
2115
|
+
// (the way y-websocket's server does; clocks travel with the states,
|
|
2116
|
+
// a peer's own echoed state at an equal clock is ignored by
|
|
2117
|
+
// applyAwarenessUpdate). Everyone else waits the usual window, and
|
|
2118
|
+
// stays silent if that table carried their state (_presenceCovered).
|
|
2119
|
+
// Presence per late join: ~N deliveries instead of (N-1)^2 - it was
|
|
2120
|
+
// 82% of a late join into a 100-peer room (bench-join-census). Bytes
|
|
2121
|
+
// are unchanged (the table goes to everyone). Peers that cannot yet
|
|
2122
|
+
// tell (no RTT estimate on a slow link, so the table may arrive after
|
|
2123
|
+
// their window) fall back to the broadcast of their own state, as
|
|
2124
|
+
// before. The unicast path below is untouched.
|
|
2125
|
+
// Needs a view of the room: in a fresh burst the first JOIN arrives
|
|
2126
|
+
// before any peer is known, everyone would rank first and relay a
|
|
2127
|
+
// table each - the broadcast fallback is right for that case.
|
|
2128
|
+
const relayer = !this._canUnicast(requester) &&
|
|
2129
|
+
this._knownPeers.size >= 3 &&
|
|
2130
|
+
this._responderRank(0, 1) === 0;
|
|
2131
|
+
const delay = relayer ? 0 : Math.min(500, Math.max(100, rtt === null ? 0 : 2 * rtt));
|
|
2132
|
+
this._presenceResponseTimer = setTimeout(() => {
|
|
2133
|
+
this._presenceResponseTimer = undefined;
|
|
2134
|
+
const requesters = Array.from(this._presencePending);
|
|
2135
|
+
this._presencePending.clear();
|
|
2136
|
+
if (this._destroying || !this.transport.isConnected)
|
|
2137
|
+
return;
|
|
2138
|
+
if (this.awareness.getLocalState() === null)
|
|
2139
|
+
return;
|
|
2140
|
+
// Every joiner covered by this timer is addressable: one unicast
|
|
2141
|
+
// each ((N-1) deliveries per joiner room-wide) instead of one
|
|
2142
|
+
// broadcast ((N-1)^2). Otherwise the broadcast, as before.
|
|
2143
|
+
if (requesters.every((id) => this._canUnicast(id))) {
|
|
2144
|
+
const msg = this._encodeAwareness([this.doc.clientID]);
|
|
2145
|
+
for (const id of requesters)
|
|
2146
|
+
this._sendDirect(id, msg);
|
|
2147
|
+
}
|
|
2148
|
+
else if (relayer) {
|
|
2149
|
+
this._sendAwarenessNow(Array.from(this.awareness.getStates().keys()));
|
|
2150
|
+
}
|
|
2151
|
+
else if (!this._presenceCovered) {
|
|
2152
|
+
this._broadcastAwareness([this.doc.clientID]);
|
|
2153
|
+
}
|
|
2154
|
+
}, delay);
|
|
2155
|
+
}
|
|
2156
|
+
/**
|
|
2157
|
+
* Max random delay (ms) before replying to a SyncStep1 request, scaled by
|
|
2158
|
+
* a room-size signal already available (`this.awareness.getStates().size`
|
|
2159
|
+
* - the same signal read at the `>= 3` suppression gate). A fixed window
|
|
2160
|
+
* (the pre-fix behavior: always `_syncReplySuppressionMs`) doesn't scale
|
|
2161
|
+
* with room size, so a larger room has more independent repliers racing
|
|
2162
|
+
* to answer the same request within the same window - more of them lose
|
|
2163
|
+
* the race and get silently dropped by the `_sendSyncReply()` rate-limit
|
|
2164
|
+
* backstop instead of never sending in the first place. Measured in
|
|
2165
|
+
* test/dummy/bench-corruption-storm.ts: the SyncStep2/SyncStep1 ratio (
|
|
2166
|
+
* ideally ~1 if suppression alone were sufficient) grew from ~1.1-1.3 at
|
|
2167
|
+
* N=2 to ~4.5-5.9 at N=10 with the fixed 30ms window.
|
|
2168
|
+
*
|
|
2169
|
+
* `min(cap, base * log2(peerCount))` - log2 growth spreads replies over a
|
|
2170
|
+
* wider window as the room grows without the delay exploding at very high
|
|
2171
|
+
* N. Capped at 200ms: the slowest-profile round trip this project
|
|
2172
|
+
* benchmarks against (Matrix, ~350ms one-way) already tolerates hundreds
|
|
2173
|
+
* of ms of latency, so 200ms of extra requester-perceived delay stays
|
|
2174
|
+
* well inside that budget while still giving a 100-peer room roughly
|
|
2175
|
+
* 6-7x the base window instead of an unbounded one.
|
|
2176
|
+
*/
|
|
2177
|
+
/**
|
|
2178
|
+
* How many peers we believe are in the room: awareness states (includes
|
|
2179
|
+
* ourselves) or, if larger, the distinct beacon/update senders we have
|
|
2180
|
+
* heard plus ourselves. See `_knownPeers`.
|
|
2181
|
+
*/
|
|
2182
|
+
_peerCount() {
|
|
2183
|
+
return Math.max(this.awareness.getStates().size, this._knownPeers.size + 1);
|
|
2184
|
+
}
|
|
2185
|
+
/**
|
|
2186
|
+
* Resolves `_awarenessInterval` to a concrete ms value: the configured
|
|
2187
|
+
* fixed number, or (round 6, item 9) `max(transport hint ?? 100,
|
|
2188
|
+
* AWARENESS_AUTO_MS_PER_PEER * peerCount)` when set to `'auto'`.
|
|
2189
|
+
*/
|
|
2190
|
+
_effectiveAwarenessInterval() {
|
|
2191
|
+
if (this._awarenessInterval !== 'auto')
|
|
2192
|
+
return this._awarenessInterval;
|
|
2193
|
+
const hint = this.transport.preferredAwarenessMs ?? 100;
|
|
2194
|
+
return Math.max(hint, GenericProvider.AWARENESS_AUTO_MS_PER_PEER * this._peerCount());
|
|
2195
|
+
}
|
|
2196
|
+
_replySuppressionMaxDelay() {
|
|
2197
|
+
const peerCount = this._peerCount();
|
|
2198
|
+
const byRoomSize = Math.min(200, this._syncReplySuppressionMs * Math.log2(Math.max(2, peerCount)));
|
|
2199
|
+
// Phase 1b: the window must exceed the one-way latency or nobody
|
|
2200
|
+
// overhears anybody in time (see _rttSamples). 1.5x the smallest
|
|
2201
|
+
// observed round trip, capped at 2 s - on a 350 ms profile that is
|
|
2202
|
+
// ~1 s of extra requester-perceived delay in exchange for ~1 reply
|
|
2203
|
+
// instead of ~20.
|
|
2204
|
+
const rtt = this._rttMinMs();
|
|
2205
|
+
return rtt === null ? byRoomSize : Math.min(2000, Math.max(byRoomSize, 1.5 * rtt));
|
|
2206
|
+
}
|
|
2207
|
+
/**
|
|
2208
|
+
* Schedule a SyncStep2 reply after a short random delay instead of
|
|
2209
|
+
* sending immediately. If another peer's reply is overheard in the
|
|
2210
|
+
* meantime (`_cancelPendingSyncReply`), this reply is dropped as
|
|
2211
|
+
* redundant - the requester likely already got what it needed.
|
|
2212
|
+
*
|
|
2213
|
+
* A reply that is already pending when this is called answers a
|
|
2214
|
+
* *different* request (e.g. peer A's request, followed 5ms later by
|
|
2215
|
+
* peer B's) - it must not be silently overwritten by the new one. Flush
|
|
2216
|
+
* it immediately, then schedule the new reply fresh. The only sanctioned
|
|
2217
|
+
* ways a reply gets dropped are `_cancelPendingSyncReply()` (we overheard
|
|
2218
|
+
* someone else's SyncStep2 for the SAME request), `_cancelPendingAck()`,
|
|
2219
|
+
* and the identical-bytes case below.
|
|
2220
|
+
*
|
|
2221
|
+
* Identical-bytes case (Task 3c in the design doc): K peers with the same
|
|
2222
|
+
* state asking at once (K empty joiners in a burst) get K byte-identical
|
|
2223
|
+
* SyncStep2s from us - the same full document K times, one flushed
|
|
2224
|
+
* immediately per arriving request, each burning a rate-limit slot. If
|
|
2225
|
+
* the new reply's bytes equal the pending reply's bytes, the pending one
|
|
2226
|
+
* already answers this request too: keep it (same delay, same
|
|
2227
|
+
* suppression) and drop the new one. Measured in
|
|
2228
|
+
* test/dummy/bench-join-after-burst.ts.
|
|
2229
|
+
*/
|
|
2230
|
+
_scheduleSyncReply(reply, isAck = false, targetSv = null, requester = null) {
|
|
2231
|
+
if (this._pendingSyncReplyTimeoutId !== undefined && this._pendingSyncReply !== null) {
|
|
2232
|
+
if (bytesEqual(this._pendingSyncReply, reply)) {
|
|
2233
|
+
return; // identical answer already scheduled
|
|
2234
|
+
}
|
|
2235
|
+
if (targetSv !== null &&
|
|
2236
|
+
this._pendingSyncReplyTargetSv !== null &&
|
|
2237
|
+
bytesEqual(this._pendingSyncReplyTargetSv, targetSv)) {
|
|
2238
|
+
// Same question (same requester state), newer document: refresh
|
|
2239
|
+
// the answer, keep the timer. See _pendingSyncReplyTargetSv.
|
|
2240
|
+
this._pendingSyncReply = reply;
|
|
2241
|
+
return;
|
|
2242
|
+
}
|
|
2243
|
+
if (!isAck &&
|
|
2244
|
+
!this._pendingSyncReplyIsAck &&
|
|
2245
|
+
targetSv !== null &&
|
|
2246
|
+
this._pendingSyncReplyTargetSv !== null) {
|
|
2247
|
+
// Two requesters, both behind, different states: one SyncStep2
|
|
2248
|
+
// from the componentwise minimum of both state vectors contains
|
|
2249
|
+
// everything either of them lacks. Keep the pending reply's timer
|
|
2250
|
+
// and widen its content instead of flushing it. The flush (the
|
|
2251
|
+
// rule below, kept for acks and legacy SyncStep1s) sent an
|
|
2252
|
+
// unsuppressed broadcast for every second request that arrived
|
|
2253
|
+
// inside the suppression window; with the window at 1.5x RTT on a
|
|
2254
|
+
// 350 ms link and several peers behind after a lossy edit burst
|
|
2255
|
+
// that was ~4-5 broadcast replies per beacon (phase-1c results).
|
|
2256
|
+
const merged = minStateVector(this._pendingSyncReplyTargetSv, targetSv);
|
|
2257
|
+
const encoder = encoding.createEncoder();
|
|
2258
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
2259
|
+
syncProtocol.writeSyncStep2(encoder, this.doc, merged);
|
|
2260
|
+
this._pendingSyncReply = encoding.toUint8Array(encoder);
|
|
2261
|
+
this._pendingSyncReplyTargetSv = merged;
|
|
2262
|
+
return;
|
|
2263
|
+
}
|
|
2264
|
+
// Phase 1e: an ack and a SyncStep2 never flush each other. A room
|
|
2265
|
+
// whose JOIN waits expire together sends N CONFIRMs in the same
|
|
2266
|
+
// millisecond; while a lossy edit burst is still healing, some of
|
|
2267
|
+
// them find us equal (ack) and some behind (SyncStep2), and the flush
|
|
2268
|
+
// rule below turned every type change into an immediate broadcast -
|
|
2269
|
+
// ~650 replies in 200 ms at N=50, 5 % loss (probe timeline in the
|
|
2270
|
+
// phase-1e design doc). An ack adds nothing to a pending reply of
|
|
2271
|
+
// either kind (the requester's wait retries, or an equal peer's
|
|
2272
|
+
// SETTLED ack confirms it); a SyncStep2 replaces a pending ack.
|
|
2273
|
+
if (isAck)
|
|
2274
|
+
return;
|
|
2275
|
+
if (this._pendingSyncReplyIsAck) {
|
|
2276
|
+
clearTimeout(this._pendingSyncReplyTimeoutId);
|
|
2277
|
+
this._pendingSyncReplyTimeoutId = undefined;
|
|
2278
|
+
}
|
|
2279
|
+
}
|
|
2280
|
+
if (this._pendingSyncReplyTimeoutId !== undefined) {
|
|
2281
|
+
// Only a legacy plain SyncStep1 (no target state vector) still
|
|
2282
|
+
// flushes a pending SyncStep2.
|
|
2283
|
+
if (this._pendingSyncReply) {
|
|
2284
|
+
this._sendSyncReply(this._pendingSyncReply);
|
|
2285
|
+
}
|
|
2286
|
+
clearTimeout(this._pendingSyncReplyTimeoutId);
|
|
2287
|
+
this._pendingSyncReplyTimeoutId = undefined;
|
|
2288
|
+
}
|
|
2289
|
+
this._pendingSyncReply = reply;
|
|
2290
|
+
this._pendingSyncReplyIsAck = isAck;
|
|
2291
|
+
this._pendingSyncReplyTargetSv = targetSv;
|
|
2292
|
+
// Acks keep the uniform window: they carry no data, so their delay
|
|
2293
|
+
// costs nothing but a few ms on a joiner's `synced` flip, and in a
|
|
2294
|
+
// join burst one pending ack answers every equal JOIN that arrives
|
|
2295
|
+
// inside that window (identical bytes, deduped above). Ranked, rank 0
|
|
2296
|
+
// fired at once for every requester - measured: fresh-burst acks
|
|
2297
|
+
// 6,039 -> 15,147 at Gun N=100, join-after-burst Matrix 147 -> 686.
|
|
2298
|
+
const delay = isAck
|
|
2299
|
+
? Math.random() * this._replySuppressionMaxDelay()
|
|
2300
|
+
: this._replyDelay(requester);
|
|
2301
|
+
this._pendingSyncReplyTimeoutId = setTimeout(() => {
|
|
2302
|
+
this._pendingSyncReplyTimeoutId = undefined;
|
|
2303
|
+
if (this._pendingSyncReply) {
|
|
2304
|
+
this._sendSyncReply(this._pendingSyncReply);
|
|
2305
|
+
this._pendingSyncReply = null;
|
|
2306
|
+
this._pendingSyncReplyTargetSv = null;
|
|
2307
|
+
}
|
|
2308
|
+
}, delay);
|
|
2309
|
+
}
|
|
2310
|
+
/** Cancel a pending suppressed reply, if any. */
|
|
2311
|
+
_cancelPendingSyncReply() {
|
|
2312
|
+
if (this._pendingSyncReplyTimeoutId !== undefined) {
|
|
2313
|
+
clearTimeout(this._pendingSyncReplyTimeoutId);
|
|
2314
|
+
this._pendingSyncReplyTimeoutId = undefined;
|
|
2315
|
+
}
|
|
2316
|
+
this._pendingSyncReply = null;
|
|
2317
|
+
this._pendingSyncReplyIsAck = false;
|
|
2318
|
+
this._pendingSyncReplyTargetSv = null;
|
|
2319
|
+
}
|
|
2320
|
+
/**
|
|
2321
|
+
* Cancel a pending reply only if it is a digest ack - see
|
|
2322
|
+
* `_pendingSyncReplyIsAck`. Called from `_handleDigest()` on every
|
|
2323
|
+
* overheard beacon whose digest equals ours.
|
|
2324
|
+
*/
|
|
2325
|
+
_cancelPendingAck() {
|
|
2326
|
+
if (this._pendingSyncReplyIsAck) {
|
|
2327
|
+
this._cancelPendingSyncReply();
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
/**
|
|
2331
|
+
* Route a SyncStep2 (or digest-ack) reply through the redundancy
|
|
2332
|
+
* suppression when there's genuine redundancy (>= 2 other known peers via
|
|
2333
|
+
* awareness - below that there's no "someone else" to rely on), else send
|
|
2334
|
+
* immediately. Both paths are rate-limited by `_sendSyncReply()`. Shared
|
|
2335
|
+
* by the MESSAGE_SYNC, MESSAGE_SYNC_VERIFIED and MESSAGE_SYNC_DIGEST cases.
|
|
2336
|
+
*/
|
|
2337
|
+
_replyToSyncRequest(reply, isAck = false, targetSv = null, toClientID) {
|
|
2338
|
+
// A peer that knows it is incomplete does not answer. A SyncStep2 is
|
|
2339
|
+
// encoded from integrated structs only, so with structs (or a delete
|
|
2340
|
+
// set) still pending ours would be provably partial - and the
|
|
2341
|
+
// requester's response wait ends on the first SyncStep2 it gets, so a
|
|
2342
|
+
// partial answer strands it until its next trigger (phase-1c gates:
|
|
2343
|
+
// 5 s resync backoff, or a stall with syncInterval 0). In relay mode a
|
|
2344
|
+
// partial broadcast also cancels the complete replies other peers had
|
|
2345
|
+
// pending. Let them answer; the requester retries if nobody does, and
|
|
2346
|
+
// in unicast mode the rank bucket rotates the responders every 2 s.
|
|
2347
|
+
// An ack from us would likewise confirm a state we do not trust.
|
|
2348
|
+
if (this.doc.store.pendingStructs !== null ||
|
|
2349
|
+
this.doc.store.pendingDs !== null) {
|
|
2350
|
+
return;
|
|
2351
|
+
}
|
|
2352
|
+
// Unicast path (transport has sendTo and we know the requester's
|
|
2353
|
+
// address): nobody overhears a unicast, so the delay-and-cancel
|
|
2354
|
+
// suppression below cannot thin the replies. Instead each candidate
|
|
2355
|
+
// responder decides for itself whether it is one of ~3 that answer
|
|
2356
|
+
// (_selectedResponder), and answers at once - no suppression delay,
|
|
2357
|
+
// one delivery. The requester's response wait retries if all ~3 are
|
|
2358
|
+
// lost. Phase-1c design, item B.
|
|
2359
|
+
if (toClientID !== undefined && this._canUnicast(toClientID)) {
|
|
2360
|
+
if (!this._selectedResponder(toClientID))
|
|
2361
|
+
return;
|
|
2362
|
+
if (!this._tryReserveReplySlot())
|
|
2363
|
+
return;
|
|
2364
|
+
this._sendDirect(toClientID, reply);
|
|
2365
|
+
return;
|
|
2366
|
+
}
|
|
2367
|
+
// Acks ALWAYS take the delayed/suppressible path: they carry no data,
|
|
2368
|
+
// so the only cost of delaying one is a few ms on the joiner's `synced`
|
|
2369
|
+
// flip (measured before this rule: 37,240 of a 50-peer join burst's
|
|
2370
|
+
// 40,915 deliveries were immediate acks). Everything else goes through
|
|
2371
|
+
// suppression once there is someone else who could answer - counted
|
|
2372
|
+
// from beacon/update senders as well as awareness, see _peerCount().
|
|
2373
|
+
if (isAck || this._peerCount() >= 3) {
|
|
2374
|
+
this._scheduleSyncReply(reply, isAck, targetSv, toClientID ?? null);
|
|
2375
|
+
}
|
|
2376
|
+
else {
|
|
2377
|
+
this._sendSyncReply(reply);
|
|
2378
|
+
}
|
|
2379
|
+
}
|
|
2380
|
+
/** Whether a reply to `clientID` can go over Transport.sendTo. */
|
|
2381
|
+
_canUnicast(clientID) {
|
|
2382
|
+
return (typeof this.transport.sendTo === 'function' &&
|
|
2383
|
+
this._peerAddress.has(clientID));
|
|
2384
|
+
}
|
|
2385
|
+
/**
|
|
2386
|
+
* Responder self-selection for unicast replies: the three peers whose
|
|
2387
|
+
* hash for this requester ranks lowest among the peers we know answer
|
|
2388
|
+
* it. Every candidate ranks itself against the same known set, so the
|
|
2389
|
+
* sets agree wherever the views agree, and the peer that ranks first in
|
|
2390
|
+
* the true order always ranks first in its own view - the selection is
|
|
2391
|
+
* never empty. A 2 s time bucket in the hash rotates the ranking, so
|
|
2392
|
+
* three departed peers at the top only delay a reply until the
|
|
2393
|
+
* requester's next attempt. Everyone answers in rooms of four or fewer.
|
|
2394
|
+
* (A first cut chose each responder independently with probability 3/N;
|
|
2395
|
+
* ~5 % of requests then selected nobody and waited for the 1 s retry.)
|
|
2396
|
+
*/
|
|
2397
|
+
_selectedResponder(requester) {
|
|
2398
|
+
if (this._peerCount() < 4)
|
|
2399
|
+
return true;
|
|
2400
|
+
return this._responderRank(requester, 3) < 3;
|
|
2401
|
+
}
|
|
2402
|
+
/**
|
|
2403
|
+
* How many known peers rank below us for `requester` in the current 2 s
|
|
2404
|
+
* bucket (counting stops at `cap`). Shared by unicast self-selection
|
|
2405
|
+
* (rank < 3 answers) and, since phase 1e, the relay-mode reply delay
|
|
2406
|
+
* (rank r waits r slots, see _replyDelay()).
|
|
2407
|
+
*/
|
|
2408
|
+
_responderRank(requester, cap) {
|
|
2409
|
+
const bucket = Math.floor(Date.now() / 2000);
|
|
2410
|
+
const rank = (id) => (Math.imul(requester ^ bucket, 0x9e3779b1) ^ Math.imul(id, 0x85ebca6b)) >>> 0;
|
|
2411
|
+
const mine = rank(this.doc.clientID);
|
|
2412
|
+
let better = 0;
|
|
2413
|
+
for (const id of this._knownPeers.keys()) {
|
|
2414
|
+
if (id === requester || id === this.doc.clientID)
|
|
2415
|
+
continue;
|
|
2416
|
+
if (rank(id) < mine && ++better >= cap)
|
|
2417
|
+
break;
|
|
2418
|
+
}
|
|
2419
|
+
return better;
|
|
2420
|
+
}
|
|
2421
|
+
/**
|
|
2422
|
+
* Delay before a suppressible reply goes out (relay path). Phase 1e:
|
|
2423
|
+
* ranked, not uniform. A uniform draw from [0, W] lets ~N * L / W
|
|
2424
|
+
* repliers fire before the first reply is overheard (L = one-way
|
|
2425
|
+
* latency): 10-27 SyncStep2 sends per request at N=100 in
|
|
2426
|
+
* test/dummy/bench-join-census.ts, and the WebRTC join-burst cell's
|
|
2427
|
+
* 16-34k spread. With the responder rank (the same hash the unicast
|
|
2428
|
+
* self-selection uses) rank 0 answers at once and rank r waits r
|
|
2429
|
+
* windows (W = _replySuppressionMaxDelay(), 1.5x the minimum round
|
|
2430
|
+
* trip: with request arrival spread 2jL and reply flight L(1+j), rank 1
|
|
2431
|
+
* has overheard rank 0 iff the slot is >= L(1+3j), which 3L(1-j) covers
|
|
2432
|
+
* up to j~0.33). Ranks >= 8 add a random window on top so a room whose
|
|
2433
|
+
* first eight ranked peers are all gone does not answer in one
|
|
2434
|
+
* avalanche. Without an RTT sample or a requester id (legacy SyncStep1)
|
|
2435
|
+
* the uniform window stays.
|
|
2436
|
+
*/
|
|
2437
|
+
_replyDelay(requester) {
|
|
2438
|
+
const window = this._replySuppressionMaxDelay();
|
|
2439
|
+
const rtt = this._rttMinMs();
|
|
2440
|
+
if (requester === null || rtt === null)
|
|
2441
|
+
return Math.random() * window;
|
|
2442
|
+
// Half a window per rank (0.75 x the minimum round trip): a rank that
|
|
2443
|
+
// stays silent (pending structs during a lossy burst, phase 1d B)
|
|
2444
|
+
// costs the requester half a window, not a whole one - at a full
|
|
2445
|
+
// window the Matrix 5 % loss fan-out's median convergence doubled
|
|
2446
|
+
// (1.1 -> 2.2 s, worst 7.6 s); the price is an occasional second
|
|
2447
|
+
// reply where the jitter exceeds ~1/3 (probe numbers in the design doc).
|
|
2448
|
+
const slot = Math.max(this._syncReplySuppressionMs, 0.75 * rtt);
|
|
2449
|
+
const rank = this._responderRank(requester, 8);
|
|
2450
|
+
return rank * slot + (rank >= 8 ? Math.random() * window : 0);
|
|
2451
|
+
}
|
|
2452
|
+
/**
|
|
2453
|
+
* Send one already-encoded message to a single peer over
|
|
2454
|
+
* Transport.sendTo, with the same CRC32 wrapping and optional compression
|
|
2455
|
+
* as a broadcast. Not mirrored to BroadcastChannel (a same-browser tab
|
|
2456
|
+
* never appears as an addressable peer). Returns false if the peer's
|
|
2457
|
+
* address is unknown or the transport cannot unicast.
|
|
2458
|
+
*/
|
|
2459
|
+
_sendDirect(clientID, data) {
|
|
2460
|
+
const address = this._peerAddress.get(clientID);
|
|
2461
|
+
if (address === undefined || typeof this.transport.sendTo !== 'function') {
|
|
2462
|
+
return false;
|
|
2463
|
+
}
|
|
2464
|
+
if (!this.transport.isConnected)
|
|
2465
|
+
return false;
|
|
2466
|
+
this._sendToTransport(wrapMessageWithChecksum(data), address);
|
|
2467
|
+
return true;
|
|
2468
|
+
}
|
|
2469
|
+
/**
|
|
2470
|
+
* Re-check Yjs's pending-struct store after the gap grace period and
|
|
2471
|
+
* request a resync (a beacon, see _requestResync) only if something is
|
|
2472
|
+
* still missing. One timer; a check scheduled while one is pending is
|
|
2473
|
+
* absorbed. Cleared on disconnect/destroy.
|
|
2474
|
+
*/
|
|
2475
|
+
/**
|
|
2476
|
+
* A beacon (a peer's periodic tick, or its request) just showed its
|
|
2477
|
+
* sender ahead of us. Until phase 1d nothing happened with that: a peer
|
|
2478
|
+
* whose last update was lost (no later message to open a sequence gap
|
|
2479
|
+
* against), or whose request was answered by a responder that was
|
|
2480
|
+
* itself behind, waited for its OWN next periodic beacon - up to
|
|
2481
|
+
* syncInterval, up to idleBackoffMaxMs with idle backoff on. Now we
|
|
2482
|
+
* check again after a grace and, if still behind that state, ask through
|
|
2483
|
+
* the resync coordinator (coalesced, backed off, rate-limited).
|
|
2484
|
+
*
|
|
2485
|
+
* The grace is what keeps this quiet during typing: at Matrix latency
|
|
2486
|
+
* almost every receiver of a periodic beacon is "behind" by a keystroke
|
|
2487
|
+
* that is still in flight (jitter +-140 ms); max(gapGraceMs, 2 x minRTT)
|
|
2488
|
+
* later it has arrived and the check finds nothing to do. A lost
|
|
2489
|
+
* keystroke that opened a sequence gap is already being requested by the
|
|
2490
|
+
* gap check - the outstanding response wait tells us so, and we stay
|
|
2491
|
+
* quiet. One timer, the newest state vector: a later beacon that shows us
|
|
2492
|
+
* behind by more replaces the reference, the timer keeps running.
|
|
2493
|
+
*/
|
|
2494
|
+
_scheduleBehindCheck(remoteSv) {
|
|
2495
|
+
this._behindSv = remoteSv;
|
|
2496
|
+
if (this._behindCheckTimer !== undefined)
|
|
2497
|
+
return;
|
|
2498
|
+
const rtt = this._rttMinMs();
|
|
2499
|
+
const delay = Math.max(this._gapGraceMs, rtt === null ? 0 : 2 * rtt);
|
|
2500
|
+
this._behindCheckTimer = setTimeout(() => {
|
|
2501
|
+
this._behindCheckTimer = undefined;
|
|
2502
|
+
const sv = this._behindSv;
|
|
2503
|
+
this._behindSv = null;
|
|
2504
|
+
if (sv === null || this._destroying || !this.transport.isConnected)
|
|
2505
|
+
return;
|
|
2506
|
+
// A request of ours sent within the grace is still being answered.
|
|
2507
|
+
// An OLDER outstanding wait is not a reason to stay behind: a new
|
|
2508
|
+
// room's first peers keep their JOIN wait parked for seconds (three
|
|
2509
|
+
// retries, nobody SETTLED yet), and measured against it the check
|
|
2510
|
+
// never fired - bench-idle-backoff's recovery stayed at one
|
|
2511
|
+
// backed-off interval (1,846 ms) with this line reading
|
|
2512
|
+
// `_responseWaitTimer !== undefined`.
|
|
2513
|
+
if (this._requestSentAt > 0 && Date.now() - this._requestSentAt < delay)
|
|
2514
|
+
return;
|
|
2515
|
+
const remote = Y.decodeStateVector(sv);
|
|
2516
|
+
const local = Y.decodeStateVector(Y.encodeStateVector(this.doc));
|
|
2517
|
+
for (const [client, clock] of remote) {
|
|
2518
|
+
if ((local.get(client) ?? 0) < clock) {
|
|
2519
|
+
this._requestResync();
|
|
2520
|
+
return;
|
|
2521
|
+
}
|
|
2522
|
+
}
|
|
2523
|
+
}, delay);
|
|
2524
|
+
}
|
|
2525
|
+
/** Design E: after a reply or push, pending structs mean the sender had the same hole - arm the grace check. */
|
|
2526
|
+
_checkPendingAfterReply() {
|
|
2527
|
+
if (this.doc.store.pendingStructs !== null ||
|
|
2528
|
+
this.doc.store.pendingDs !== null) {
|
|
2529
|
+
this._schedulePendingCheck();
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
_schedulePendingCheck() {
|
|
2533
|
+
if (this._pendingCheckTimer !== undefined)
|
|
2534
|
+
return;
|
|
2535
|
+
this._pendingCheckTimer = setTimeout(() => {
|
|
2536
|
+
this._pendingCheckTimer = undefined;
|
|
2537
|
+
if (this._destroying || !this.transport.isConnected)
|
|
2538
|
+
return;
|
|
2539
|
+
// A request of ours may already be outstanding (JOIN or resync beacon
|
|
2540
|
+
// with its response wait running): whatever is pending is what that
|
|
2541
|
+
// request is fetching, and the response wait retries if it is lost.
|
|
2542
|
+
// Asking again here just raced the responders' suppression window
|
|
2543
|
+
// (measured: a joiner's keystrokes-before-content check fired at
|
|
2544
|
+
// 300 ms while the settled peers' replies were still delayed by a
|
|
2545
|
+
// window of up to ~300 ms, and every such re-beacon collected
|
|
2546
|
+
// another round of replies).
|
|
2547
|
+
// ... but only a request that went out within the last grace, whose
|
|
2548
|
+
// answer may still be on its way. Deferring to ANY outstanding
|
|
2549
|
+
// response wait (the rule until phase 1d) parked every pending check
|
|
2550
|
+
// behind a new room's JOIN wait - three retries with nobody SETTLED
|
|
2551
|
+
// yet, 19.6 s on a 700 ms link - so a peer that lost a keystroke in
|
|
2552
|
+
// such a room asked only once an overheard reply happened to clear
|
|
2553
|
+
// that wait (phase-1d design doc, "After Task 4", the fresh-budget
|
|
2554
|
+
// Matrix fan-out). Look again once the recent request is answered or
|
|
2555
|
+
// given up - dropping the check here left a joiner whose SyncStep2
|
|
2556
|
+
// predated the keystrokes it had received with nine pending updates
|
|
2557
|
+
// and no trigger (measured: 2 of 150 lossy 15-peer joins).
|
|
2558
|
+
const rtt = this._rttMinMs();
|
|
2559
|
+
const recent = Math.max(this._gapGraceMs, rtt === null ? 0 : 2 * rtt);
|
|
2560
|
+
if (this._requestSentAt > 0 && Date.now() - this._requestSentAt < recent) {
|
|
2561
|
+
this._schedulePendingCheck();
|
|
2562
|
+
return;
|
|
2563
|
+
}
|
|
2564
|
+
if (this.doc.store.pendingStructs !== null ||
|
|
2565
|
+
this.doc.store.pendingDs !== null) {
|
|
2566
|
+
this._requestResync();
|
|
2567
|
+
}
|
|
2568
|
+
}, this._gapGraceMs);
|
|
2569
|
+
}
|
|
2570
|
+
/**
|
|
2571
|
+
* Return the update payload of a SyncStep2/Update sync sub-message
|
|
2572
|
+
* without advancing `decoder` (null for SyncStep1 or malformed input).
|
|
2573
|
+
* y-protocols frames both as [subType varUint][update varUint8Array].
|
|
2574
|
+
*/
|
|
2575
|
+
_peekSyncUpdate(decoder) {
|
|
2576
|
+
const peek = decoding.clone(decoder);
|
|
2577
|
+
try {
|
|
2578
|
+
const subType = decoding.readVarUint(peek);
|
|
2579
|
+
if (subType === syncProtocol.messageYjsSyncStep1)
|
|
2580
|
+
return null;
|
|
2581
|
+
return decoding.readVarUint8Array(peek);
|
|
2582
|
+
}
|
|
2583
|
+
catch {
|
|
2584
|
+
return null;
|
|
2585
|
+
}
|
|
2586
|
+
}
|
|
2587
|
+
/**
|
|
2588
|
+
* Whether an update we just applied was already superseded here: every
|
|
2589
|
+
* client it touches ends at a clock we were at or beyond BEFORE this
|
|
2590
|
+
* update (i.e. it added nothing). Uses the update's own metadata
|
|
2591
|
+
* (`Y.parseUpdateMeta`), O(clients in the update).
|
|
2592
|
+
*/
|
|
2593
|
+
_isLateUpdate(updateBytes) {
|
|
2594
|
+
if (!updateBytes)
|
|
2595
|
+
return false;
|
|
2596
|
+
try {
|
|
2597
|
+
const { to } = Y.parseUpdateMeta(updateBytes);
|
|
2598
|
+
if (to.size === 0)
|
|
2599
|
+
return false;
|
|
2600
|
+
const local = Y.decodeStateVector(Y.encodeStateVector(this.doc));
|
|
2601
|
+
for (const [client, clock] of to) {
|
|
2602
|
+
// `to` is the exclusive end clock of the update's range for that
|
|
2603
|
+
// client; our state vector is exclusive too. Equal means the update
|
|
2604
|
+
// brought us exactly here (not late); greater means we were past it.
|
|
2605
|
+
if ((local.get(client) ?? 0) <= clock)
|
|
2606
|
+
return false;
|
|
2607
|
+
}
|
|
2608
|
+
return true;
|
|
2609
|
+
}
|
|
2610
|
+
catch {
|
|
2611
|
+
return false;
|
|
2612
|
+
}
|
|
2613
|
+
}
|
|
2614
|
+
/** Flip `synced` once and emit; idempotent. */
|
|
2615
|
+
_markSynced() {
|
|
2616
|
+
if (!this._synced) {
|
|
2617
|
+
this._synced = true;
|
|
2618
|
+
this.emit('synced', [true]);
|
|
2619
|
+
}
|
|
2620
|
+
}
|
|
2621
|
+
/**
|
|
2622
|
+
* Wait for a response to the JOIN or resync beacon we just sent. If
|
|
2623
|
+
* neither a SyncStep2 nor an equal ack/beacon arrives within 1s (then
|
|
2624
|
+
* 2s, 4s), ask again - with a CONFIRM beacon after a JOIN (so an equal
|
|
2625
|
+
* room acks), with a plain beacon after a resync (only peers ahead of us
|
|
2626
|
+
* need to answer; an equal room's silence is the correct answer and its
|
|
2627
|
+
* periodic beacons end the wait) - three times at most; after that the
|
|
2628
|
+
* periodic beacon is the fallback, as before. Requester-side retry is how the protocol
|
|
2629
|
+
* stays loss-tolerant now that reply suppression leaves ~1 reply per
|
|
2630
|
+
* request; N-fold redundant replies were the old (accidental) way.
|
|
2631
|
+
*/
|
|
2632
|
+
_armResponseWait(retryFlags) {
|
|
2633
|
+
if (this._responseWaitTimer !== undefined)
|
|
2634
|
+
return;
|
|
2635
|
+
this._responseSeen = false;
|
|
2636
|
+
if (this._responseWaitAttempts === 0)
|
|
2637
|
+
this._equalUnsettledSeen = false;
|
|
2638
|
+
this._responseWaitFlags = retryFlags;
|
|
2639
|
+
this._requestSentAt = Date.now();
|
|
2640
|
+
const rtt = this._rttMinMs();
|
|
2641
|
+
const delay = Math.max(1000, rtt === null ? 0 : 4 * rtt) *
|
|
2642
|
+
Math.pow(2, this._responseWaitAttempts);
|
|
2643
|
+
this._responseWaitTimer = setTimeout(() => {
|
|
2644
|
+
this._responseWaitTimer = undefined;
|
|
2645
|
+
// Phase 1e: a fresh room - equal but unsettled peers answered both
|
|
2646
|
+
// the JOIN and one CONFIRM retry, nobody settled did. Two rounds
|
|
2647
|
+
// instead of three; the first confirmed peers' acks then carry
|
|
2648
|
+
// SETTLED for everyone after them. Measured in
|
|
2649
|
+
// test/dummy/bench-join-census.ts (b): the third round was ~a third
|
|
2650
|
+
// of a fresh N=100 room's 92k-delivery join burst.
|
|
2651
|
+
const freshRoomDone = this._equalUnsettledSeen && this._responseWaitAttempts >= 1;
|
|
2652
|
+
if (this._responseSeen ||
|
|
2653
|
+
this._responseWaitAttempts >= 3 ||
|
|
2654
|
+
freshRoomDone ||
|
|
2655
|
+
!this.transport.isConnected ||
|
|
2656
|
+
this._destroying) {
|
|
2657
|
+
// Asked three times, nobody had more for us: we are the room's
|
|
2658
|
+
// state (or its first peer). Bootstraps DIGEST_FLAG_SETTLED in a
|
|
2659
|
+
// brand-new room so later joiners are confirmed by our acks.
|
|
2660
|
+
if (this._responseWaitAttempts >= 3 || freshRoomDone)
|
|
2661
|
+
this._confirmed = true;
|
|
2662
|
+
this._responseWaitAttempts = 0;
|
|
2663
|
+
return;
|
|
2664
|
+
}
|
|
2665
|
+
this._responseWaitAttempts++;
|
|
2666
|
+
this._sendSyncStep1(this._responseWaitFlags); // rate-limited; a dropped attempt is simply retried next round
|
|
2667
|
+
this._armResponseWait(this._responseWaitFlags);
|
|
2668
|
+
}, delay);
|
|
2669
|
+
}
|
|
2670
|
+
/**
|
|
2671
|
+
* A SyncStep2 or an equal ack/beacon arrived - whatever we asked for is
|
|
2672
|
+
* answered. `sample` = it was a direct reply (SyncStep2/ack), so its
|
|
2673
|
+
* timing is a round-trip sample; an equal periodic beacon from a settled
|
|
2674
|
+
* peer also ends the wait but says nothing about latency.
|
|
2675
|
+
*/
|
|
2676
|
+
_noteResponse(sample) {
|
|
2677
|
+
if (sample && this._requestSentAt > 0) {
|
|
2678
|
+
this._rttSamples.push(Date.now() - this._requestSentAt);
|
|
2679
|
+
if (this._rttSamples.length > 8)
|
|
2680
|
+
this._rttSamples.shift();
|
|
2681
|
+
}
|
|
2682
|
+
this._requestSentAt = 0;
|
|
2683
|
+
this._responseSeen = true;
|
|
2684
|
+
this._responseWaitAttempts = 0;
|
|
2685
|
+
if (this._responseWaitTimer !== undefined) {
|
|
2686
|
+
clearTimeout(this._responseWaitTimer);
|
|
2687
|
+
this._responseWaitTimer = undefined;
|
|
2688
|
+
}
|
|
2689
|
+
}
|
|
2690
|
+
/** Minimum of the recent round-trip samples, or null before the first reply. */
|
|
2691
|
+
_rttMinMs() {
|
|
2692
|
+
return this._rttSamples.length === 0 ? null : Math.min(...this._rttSamples);
|
|
2693
|
+
}
|
|
2694
|
+
/**
|
|
2695
|
+
* Delay a pure timeout-removal awareness broadcast and drop it if
|
|
2696
|
+
* another peer's broadcast of the SAME removal is overheard first (see
|
|
2697
|
+
* the `origin === this` branch in `_setupAwarenessSync()`'s handler,
|
|
2698
|
+
* which calls `_cancelPendingAwarenessRemovalIfOverlaps()`) - the exact
|
|
2699
|
+
* same NACK-style suppression `_scheduleSyncReply()` already applies to
|
|
2700
|
+
* SyncStep2 replies, reusing the same room-size-scaled delay
|
|
2701
|
+
* (`_replySuppressionMaxDelay()`).
|
|
2702
|
+
*
|
|
2703
|
+
* A pending removal already queued when this is called is for a
|
|
2704
|
+
* DIFFERENT departure (two peers timing out, or leaving, within the same
|
|
2705
|
+
* window): since round 5 the ids are merged into the pending set and its
|
|
2706
|
+
* timer kept - one broadcast carries both - instead of flushing the
|
|
2707
|
+
* first as an unsuppressed broadcast (with the long leave window below a
|
|
2708
|
+
* burst of departures would have flushed on every peer). An overheard
|
|
2709
|
+
* broadcast trims only the ids it covers from the pending set
|
|
2710
|
+
* (`_cancelPendingAwarenessRemovalIfOverlaps()`).
|
|
2711
|
+
*
|
|
2712
|
+
* Window: the reply-suppression window for timeouts (sweeps are spread
|
|
2713
|
+
* over seconds anyway); for leaves reported by the transport - all
|
|
2714
|
+
* survivors learn of them in the same millisecond - ten times that,
|
|
2715
|
+
* at least a second, so the first broadcast is overheard before the
|
|
2716
|
+
* rest fire. A departure is not urgent: every peer already dropped the
|
|
2717
|
+
* state locally.
|
|
2718
|
+
*/
|
|
2719
|
+
_scheduleAwarenessRemoval(clients, origin = 'timeout') {
|
|
2720
|
+
if (this._pendingAwarenessRemovalTimeoutId !== undefined) {
|
|
2721
|
+
const pending = this._pendingAwarenessRemoval ?? [];
|
|
2722
|
+
for (const id of clients)
|
|
2723
|
+
if (!pending.includes(id))
|
|
2724
|
+
pending.push(id);
|
|
2725
|
+
this._pendingAwarenessRemoval = pending;
|
|
2726
|
+
return;
|
|
2727
|
+
}
|
|
2728
|
+
this._pendingAwarenessRemoval = clients.slice();
|
|
2729
|
+
const window = this._replySuppressionMaxDelay();
|
|
2730
|
+
const delay = Math.random() * (origin === 'peer-left' ? Math.max(1000, 10 * window) : window);
|
|
2731
|
+
this._pendingAwarenessRemovalTimeoutId = setTimeout(() => {
|
|
2732
|
+
this._pendingAwarenessRemovalTimeoutId = undefined;
|
|
2733
|
+
if (this._pendingAwarenessRemoval) {
|
|
2734
|
+
this._broadcastAwareness(this._pendingAwarenessRemoval);
|
|
2735
|
+
this._pendingAwarenessRemoval = null;
|
|
2736
|
+
}
|
|
2737
|
+
}, delay);
|
|
2738
|
+
}
|
|
2739
|
+
/**
|
|
2740
|
+
* Peek at an awareness-update payload (still in
|
|
2741
|
+
* `awarenessProtocol.encodeAwarenessUpdate()`'s wire encoding) for
|
|
2742
|
+
* clientIDs whose state is `null` (a removal), without applying it.
|
|
2743
|
+
* y-protocols/awareness.js doesn't export a standalone decoder for this,
|
|
2744
|
+
* only `applyAwarenessUpdate()` (which also mutates state) and
|
|
2745
|
+
* `modifyAwarenessUpdate()` (which re-encodes) - so this mirrors the
|
|
2746
|
+
* format by hand: varUint length, then per entry
|
|
2747
|
+
* [varUint clientID][varUint clock][varString JSON state]. Used to cancel
|
|
2748
|
+
* a pending suppressed removal (see `_scheduleAwarenessRemoval()`) at the
|
|
2749
|
+
* wire-message level, before `applyAwarenessUpdate()` runs - see the
|
|
2750
|
+
* `MESSAGE_AWARENESS` case's comment for why timing matters here.
|
|
2751
|
+
*/
|
|
2752
|
+
_scanAwarenessPayload(payload) {
|
|
2753
|
+
const removed = [];
|
|
2754
|
+
const present = [];
|
|
2755
|
+
let coversUs = false;
|
|
2756
|
+
try {
|
|
2757
|
+
const d = decoding.createDecoder(payload);
|
|
2758
|
+
const len = decoding.readVarUint(d);
|
|
2759
|
+
const ourClock = this.awareness.meta.get(this.awareness.clientID)?.clock ?? 0;
|
|
2760
|
+
for (let i = 0; i < len; i++) {
|
|
2761
|
+
const clientID = decoding.readVarUint(d);
|
|
2762
|
+
const clock = decoding.readVarUint(d);
|
|
2763
|
+
const state = JSON.parse(decoding.readVarString(d));
|
|
2764
|
+
if (state === null)
|
|
2765
|
+
removed.push(clientID);
|
|
2766
|
+
else {
|
|
2767
|
+
present.push(clientID);
|
|
2768
|
+
if (clientID === this.awareness.clientID && clock >= ourClock)
|
|
2769
|
+
coversUs = true;
|
|
2770
|
+
}
|
|
2771
|
+
}
|
|
2772
|
+
}
|
|
2773
|
+
catch {
|
|
2774
|
+
// Malformed payload - let applyAwarenessUpdate() below be the one
|
|
2775
|
+
// that deals with it (or throws); suppression is a pure optimization,
|
|
2776
|
+
// never worth failing the actual message handling over.
|
|
2777
|
+
}
|
|
2778
|
+
return { removed, present, coversUs };
|
|
2779
|
+
}
|
|
2780
|
+
/**
|
|
2781
|
+
* Trim a pending suppressed removal broadcast by `removedClientIds` -
|
|
2782
|
+
* someone else already broadcast those departures; what they did not
|
|
2783
|
+
* cover stays queued.
|
|
2784
|
+
*/
|
|
2785
|
+
_cancelPendingAwarenessRemovalIfOverlaps(removedClientIds) {
|
|
2786
|
+
if (!this._pendingAwarenessRemoval)
|
|
2787
|
+
return;
|
|
2788
|
+
const rest = this._pendingAwarenessRemoval.filter((id) => !removedClientIds.includes(id));
|
|
2789
|
+
if (rest.length === this._pendingAwarenessRemoval.length)
|
|
2790
|
+
return;
|
|
2791
|
+
if (rest.length > 0) {
|
|
2792
|
+
this._pendingAwarenessRemoval = rest; // someone covered part of it; the rest stays queued
|
|
2793
|
+
return;
|
|
2794
|
+
}
|
|
2795
|
+
if (this._pendingAwarenessRemovalTimeoutId !== undefined) {
|
|
2796
|
+
clearTimeout(this._pendingAwarenessRemovalTimeoutId);
|
|
2797
|
+
this._pendingAwarenessRemovalTimeoutId = undefined;
|
|
2798
|
+
}
|
|
2799
|
+
this._pendingAwarenessRemoval = null;
|
|
2800
|
+
}
|
|
2801
|
+
/** Cancel a pending suppressed awareness-removal broadcast, if any. */
|
|
2802
|
+
_cancelPendingAwarenessRemoval() {
|
|
2803
|
+
if (this._pendingAwarenessRemovalTimeoutId !== undefined) {
|
|
2804
|
+
clearTimeout(this._pendingAwarenessRemovalTimeoutId);
|
|
2805
|
+
this._pendingAwarenessRemovalTimeoutId = undefined;
|
|
2806
|
+
}
|
|
2807
|
+
this._pendingAwarenessRemoval = null;
|
|
2808
|
+
}
|
|
2809
|
+
/**
|
|
2810
|
+
* Send a SyncStep2 reply, gated by the same shared per-peer budget as
|
|
2811
|
+
* SyncStep1 requests/syncNow() pushes (`_tryReserveSyncSlot()`).
|
|
2812
|
+
*
|
|
2813
|
+
* Previously SyncStep2 replies were completely unrated - the only
|
|
2814
|
+
* defense against redundant replies was the best-effort NACK-style
|
|
2815
|
+
* suppression in `_scheduleSyncReply()`/`_cancelPendingSyncReply()`,
|
|
2816
|
+
* which itself is just an ordinary broadcast message subject to the same
|
|
2817
|
+
* wire corruption as everything else. Under sustained corruption, more
|
|
2818
|
+
* competing repliers independently miss the "someone already answered"
|
|
2819
|
+
* signal as peer count grows, and none of that traffic was bounded.
|
|
2820
|
+
* Measured in test/dummy/bench-corruption-storm.ts: SyncStep2/SyncStep1
|
|
2821
|
+
* ratio grew from ~1.1-1.3 at N=2 to ~4.5-5.9 at N=10 (should stay near
|
|
2822
|
+
* 1 if suppression alone were sufficient). This is a hard backstop on
|
|
2823
|
+
* top of that suppression, not a replacement for it - a rate-limited
|
|
2824
|
+
* reply is dropped silently (no warn) since under normal, uncorrupted
|
|
2825
|
+
* operation this path is rarely exercised and logging every drop here
|
|
2826
|
+
* would itself become log spam exactly when things are already noisy.
|
|
2827
|
+
*/
|
|
2828
|
+
_sendSyncReply(reply) {
|
|
2829
|
+
if (!this._tryReserveReplySlot()) {
|
|
2830
|
+
return; // Rate limited - drop the reply silently
|
|
2831
|
+
}
|
|
2832
|
+
this._send(reply);
|
|
2833
|
+
}
|
|
2834
|
+
/**
|
|
2835
|
+
* Track a received sequence number for reordering-tolerant gap detection.
|
|
2836
|
+
* Does not gate whether the update gets applied — only decides whether a
|
|
2837
|
+
* gap looks suspicious enough to (eventually) request a resync.
|
|
2838
|
+
*/
|
|
2839
|
+
_trackRemoteSeq(senderClientID, seqNum) {
|
|
2840
|
+
let info = this._remoteSeqInfo.get(senderClientID);
|
|
2841
|
+
if (!info) {
|
|
2842
|
+
info = { highest: -1, seen: new Set() };
|
|
2843
|
+
this._remoteSeqInfo.set(senderClientID, info);
|
|
2844
|
+
}
|
|
2845
|
+
if (info.seen.has(seqNum)) {
|
|
2846
|
+
return; // genuine duplicate - nothing new to track
|
|
2847
|
+
}
|
|
2848
|
+
info.seen.add(seqNum);
|
|
2849
|
+
if (seqNum > info.highest) {
|
|
2850
|
+
if (info.highest >= 0 && seqNum > info.highest + 1) {
|
|
2851
|
+
this._scheduleGapCheck(senderClientID, info.highest + 1, seqNum - 1);
|
|
2852
|
+
}
|
|
2853
|
+
info.highest = seqNum;
|
|
2854
|
+
}
|
|
2855
|
+
// Bound memory: forget seqNums far behind the current high-water mark.
|
|
2856
|
+
const floor = info.highest - this._seqWindowSize;
|
|
2857
|
+
if (floor > 0) {
|
|
2858
|
+
for (const s of info.seen) {
|
|
2859
|
+
if (s < floor)
|
|
2860
|
+
info.seen.delete(s);
|
|
2861
|
+
}
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
/**
|
|
2865
|
+
* Re-check a suspected sequence gap after a short grace period instead of
|
|
2866
|
+
* requesting a resync immediately. Pure network reordering (a message
|
|
2867
|
+
* that's merely late, not lost) typically resolves itself within the
|
|
2868
|
+
* grace window, so this avoids the resync storms that immediate gap
|
|
2869
|
+
* detection caused under jitter. Real packet loss still gets caught —
|
|
2870
|
+
* just `_gapGraceMs` later — and the periodic sync interval / hash
|
|
2871
|
+
* verification remain as further safety nets regardless.
|
|
2872
|
+
*/
|
|
2873
|
+
_scheduleGapCheck(clientID, gapStart, gapEnd) {
|
|
2874
|
+
// Only one pending check per sender; a newly-opened gap while a check
|
|
2875
|
+
// is already scheduled will still be caught by periodic sync / hash
|
|
2876
|
+
// verification even if not by this specific check.
|
|
2877
|
+
if (this._gapCheckTimers.has(clientID))
|
|
2878
|
+
return;
|
|
2879
|
+
const timer = setTimeout(() => {
|
|
2880
|
+
this._gapCheckTimers.delete(clientID);
|
|
2881
|
+
const info = this._remoteSeqInfo.get(clientID);
|
|
2882
|
+
if (!info || this._destroying)
|
|
2883
|
+
return;
|
|
2884
|
+
let stillMissing = 0;
|
|
2885
|
+
for (let s = gapStart; s <= gapEnd; s++) {
|
|
2886
|
+
if (!info.seen.has(s))
|
|
2887
|
+
stillMissing++;
|
|
2888
|
+
}
|
|
2889
|
+
if (stillMissing > 0 && this.transport.isConnected) {
|
|
2890
|
+
console.warn(`[GenericProvider] Sequence gap confirmed from client ${clientID}: ` +
|
|
2891
|
+
`${stillMissing} message(s) still missing after ${this._gapGraceMs}ms grace period`);
|
|
2892
|
+
// Routed through the shared coordinator (previously called
|
|
2893
|
+
// _sendSyncStep1() directly with NO coalescing at all - the one
|
|
2894
|
+
// remaining gap that let this trigger steal rate-limit slots
|
|
2895
|
+
// independently of the hash-mismatch/corrupted-message triggers).
|
|
2896
|
+
this._requestResync();
|
|
2897
|
+
}
|
|
2898
|
+
}, this._gapGraceMs);
|
|
2899
|
+
this._gapCheckTimers.set(clientID, timer);
|
|
2900
|
+
}
|
|
2901
|
+
/**
|
|
2902
|
+
* Unified entry point for ALL resync triggers (hash mismatch, corrupted
|
|
2903
|
+
* message, confirmed sequence gap). Coalesces them behind a single
|
|
2904
|
+
* pending timer and a single shared escalation counter, so a burst of
|
|
2905
|
+
* triggers from different causes in a short window schedules exactly one
|
|
2906
|
+
* resync instead of three independent ones each able to draw on the
|
|
2907
|
+
* shared `_tryReserveSyncSlot()` budget on their own.
|
|
2908
|
+
*
|
|
2909
|
+
* Always resolves to `syncNow()` (push + pull) rather than distinguishing
|
|
2910
|
+
* a push-only/pull-only variant per trigger. `syncNow()`'s push half is
|
|
2911
|
+
* already a no-op when there's nothing to send (it only calls
|
|
2912
|
+
* `_sendUpdate()` when `update.length > 0`), so unifying on push+pull is
|
|
2913
|
+
* strictly simpler than threading a `push` flag through a *shared*
|
|
2914
|
+
* coordinator (where the "right" answer for an absorbed trigger is
|
|
2915
|
+
* ambiguous anyway - was it push-worthy or not?). It also closes a latent
|
|
2916
|
+
* gap where the corrupted-message and gap-confirmed triggers previously
|
|
2917
|
+
* called pull-only `_sendSyncStep1()` and could never deliver this peer's
|
|
2918
|
+
* own surplus edits made during a divergence window.
|
|
2919
|
+
*/
|
|
2920
|
+
_requestResync() {
|
|
2921
|
+
// Coalesced: if a resync is already pending (regardless of which
|
|
2922
|
+
// trigger scheduled it), this trigger is absorbed into it instead of
|
|
2923
|
+
// stacking another independent timer/broadcast. Escalation only
|
|
2924
|
+
// advances when we actually schedule a NEW timer below - incrementing
|
|
2925
|
+
// unconditionally here (once per absorbed trigger too) would let a
|
|
2926
|
+
// burst of many corrupted/mismatched messages while one resync is
|
|
2927
|
+
// already pending ratchet the counter straight to its cap, so the
|
|
2928
|
+
// *next* resync (after this one fires) always schedules at the max
|
|
2929
|
+
// backoff instead of escalating gradually.
|
|
2930
|
+
if (this._pendingResyncTimeoutId !== undefined) {
|
|
2931
|
+
return;
|
|
2932
|
+
}
|
|
2933
|
+
this._resyncAttemptCount++;
|
|
2934
|
+
const now = Date.now();
|
|
2935
|
+
// Reset the escalation counter if it's been stable for 10 seconds -
|
|
2936
|
+
// same quiet-period reset the old per-trigger counters used.
|
|
2937
|
+
if (now - this._lastResyncAttemptTime > 10000) {
|
|
2938
|
+
this._resyncAttemptCount = 1;
|
|
2939
|
+
}
|
|
2940
|
+
this._lastResyncAttemptTime = now;
|
|
2941
|
+
// Exponential backoff: 100ms, 500ms, 2.5s, then cap at 5s.
|
|
2942
|
+
const delay = Math.min(5000, 100 * Math.pow(5, Math.min(this._resyncAttemptCount - 1, 3)));
|
|
2943
|
+
console.warn(`[GenericProvider] Resync scheduled in ${delay}ms (attempt #${this._resyncAttemptCount})...`);
|
|
2944
|
+
this._pendingResyncTimeoutId = setTimeout(() => {
|
|
2945
|
+
this._pendingResyncTimeoutId = undefined;
|
|
2946
|
+
if (!this.transport.isConnected || this._destroying)
|
|
2947
|
+
return;
|
|
2948
|
+
// A resync trigger means "I may be missing something" - never "the
|
|
2949
|
+
// room is missing my data" (my updates travel on their own, and the
|
|
2950
|
+
// connect-time push covers offline edits). So: ask with a 12-byte
|
|
2951
|
+
// beacon; the peers ahead of me reply with exactly the diff (a
|
|
2952
|
+
// SyncStep2 against my state vector, suppressed as any reply), the
|
|
2953
|
+
// rest stay silent. Until phase 1b this pushed the WHOLE document to
|
|
2954
|
+
// the room on every trigger, and that push - a hashed update of my
|
|
2955
|
+
// state - failed the hash check at every peer ahead of me, which
|
|
2956
|
+
// scheduled a resync of its own: the cascade of research doc item 13
|
|
2957
|
+
// (198,990 deliveries for 10 keystrokes at N=100 on the Gun profile,
|
|
2958
|
+
// the rate limiter's ceiling). If the beacon or its reply is lost,
|
|
2959
|
+
// _armResponseWait() re-beacons (1s/2s/4s); the periodic beacon is
|
|
2960
|
+
// the fallback after that. A rate-limited attempt re-arms through
|
|
2961
|
+
// this same coordinator so a stranded peer keeps retrying.
|
|
2962
|
+
if (this._sendSyncStep1(0)) {
|
|
2963
|
+
this._armResponseWait(0);
|
|
2964
|
+
}
|
|
2965
|
+
else {
|
|
2966
|
+
this._requestResync();
|
|
2967
|
+
}
|
|
2968
|
+
}, delay);
|
|
2969
|
+
}
|
|
2970
|
+
/**
|
|
2971
|
+
* Reserve a slot in the sync rate limiter (max `_maxSyncRequestsPerWindow`
|
|
2972
|
+
* per `_syncRequestWindowMs`), recording the request if there's room.
|
|
2973
|
+
* Shared by `_sendSyncStep1()` and `syncNow()` so a burst of triggers from
|
|
2974
|
+
* different sources (periodic sync, hash-mismatch resyncs, gap-check
|
|
2975
|
+
* confirmations) draws from one combined budget instead of each having
|
|
2976
|
+
* its own uncapped or separately-capped allowance.
|
|
2977
|
+
*/
|
|
2978
|
+
_tryReserveSyncSlot() {
|
|
2979
|
+
return this._tryReserveSlot(this._syncRequestTimes);
|
|
2980
|
+
}
|
|
2981
|
+
/** Same limiter, separate budget, for SyncStep2 replies and acks. */
|
|
2982
|
+
_tryReserveReplySlot() {
|
|
2983
|
+
return this._tryReserveSlot(this._syncReplyTimes);
|
|
2984
|
+
}
|
|
2985
|
+
_tryReserveSlot(times) {
|
|
2986
|
+
const now = Date.now();
|
|
2987
|
+
// Drop entries outside the rolling window (in place: the arrays are
|
|
2988
|
+
// referenced from two fields)
|
|
2989
|
+
let keep = 0;
|
|
2990
|
+
for (const t of times) {
|
|
2991
|
+
if (now - t < this._syncRequestWindowMs)
|
|
2992
|
+
times[keep++] = t;
|
|
2993
|
+
}
|
|
2994
|
+
times.length = keep;
|
|
2995
|
+
if (times.length >= this._maxSyncRequestsPerWindow) {
|
|
2996
|
+
return false;
|
|
2997
|
+
}
|
|
2998
|
+
times.push(now);
|
|
2999
|
+
return true;
|
|
3000
|
+
}
|
|
3001
|
+
/**
|
|
3002
|
+
* Encode the digest beacon that replaces SyncStep1 (see
|
|
3003
|
+
* MESSAGE_SYNC_DIGEST). Still the one place every "request sync" path
|
|
3004
|
+
* goes through (connect()'s syncNow(), the periodic tick,
|
|
3005
|
+
* _requestResync()'s retry), so they all switched together.
|
|
3006
|
+
*/
|
|
3007
|
+
_encodeSyncStep1(flags = 0) {
|
|
3008
|
+
if (this._confirmed)
|
|
3009
|
+
flags |= DIGEST_FLAG_SETTLED;
|
|
3010
|
+
this._touchPeer(this.doc.clientID);
|
|
3011
|
+
const encoder = encoding.createEncoder();
|
|
3012
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC_DIGEST);
|
|
3013
|
+
encoding.writeVarUint(encoder, DIGEST_VERSION);
|
|
3014
|
+
encoding.writeVarUint(encoder, flags);
|
|
3015
|
+
encoding.writeVarUint(encoder, this.doc.clientID);
|
|
3016
|
+
encoding.writeVarUint8Array(encoder, Y.encodeStateVector(this.doc));
|
|
3017
|
+
encoding.writeVarUint(encoder, this._deleteSetHash());
|
|
3018
|
+
return encoding.toUint8Array(encoder);
|
|
3019
|
+
}
|
|
3020
|
+
/**
|
|
3021
|
+
* Encode an ack for a JOIN beacon: same framing as a beacon, DIGEST_FLAG_ACK
|
|
3022
|
+
* set, and the JOINER's state vector + delete-set hash echoed back instead
|
|
3023
|
+
* of ours (see DIGEST_FLAG_ACK for why it must never carry our own state).
|
|
3024
|
+
*/
|
|
3025
|
+
_encodeAck(ackedSv, ackedDsHash) {
|
|
3026
|
+
this._touchPeer(this.doc.clientID);
|
|
3027
|
+
const encoder = encoding.createEncoder();
|
|
3028
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC_DIGEST);
|
|
3029
|
+
encoding.writeVarUint(encoder, DIGEST_VERSION);
|
|
3030
|
+
encoding.writeVarUint(encoder, DIGEST_FLAG_ACK | (this._confirmed ? DIGEST_FLAG_SETTLED : 0));
|
|
3031
|
+
encoding.writeVarUint(encoder, this.doc.clientID);
|
|
3032
|
+
encoding.writeVarUint8Array(encoder, ackedSv);
|
|
3033
|
+
encoding.writeVarUint(encoder, ackedDsHash);
|
|
3034
|
+
return encoding.toUint8Array(encoder);
|
|
3035
|
+
}
|
|
3036
|
+
/**
|
|
3037
|
+
* Send the periodic digest beacon. Rate limited to prevent spam. Returns
|
|
3038
|
+
* whether it actually sent (false means rate-limited).
|
|
3039
|
+
*/
|
|
3040
|
+
_sendSyncStep1(flags = 0) {
|
|
3041
|
+
if (!this._tryReserveSyncSlot()) {
|
|
3042
|
+
console.warn(`[GenericProvider] Sync rate limit exceeded (${this._maxSyncRequestsPerWindow} requests per ${this._syncRequestWindowMs / 1000}s), throttling...`);
|
|
3043
|
+
return false; // Drop the request
|
|
3044
|
+
}
|
|
3045
|
+
this._send(this._encodeSyncStep1(flags));
|
|
3046
|
+
return true;
|
|
3047
|
+
}
|
|
3048
|
+
/**
|
|
3049
|
+
* Encode a document update, without sending it. Extracted from the old
|
|
3050
|
+
* `_sendUpdate()` so `_trySyncPushPull()` can fold the push half into a
|
|
3051
|
+
* batched wire send (see `_sendBatch()`); `_sendUpdate()` below is the
|
|
3052
|
+
* send-immediately form still used by every other update-emitting path
|
|
3053
|
+
* (the doc-update handler, batch-flush, disconnect/destroy flush) since
|
|
3054
|
+
* those aren't part of this batching effort's scope.
|
|
3055
|
+
*
|
|
3056
|
+
* NOTE: has a side effect (`_localSeqNum++`) - call exactly once per
|
|
3057
|
+
* logical update, same as before.
|
|
3058
|
+
*/
|
|
3059
|
+
_encodeUpdate(update) {
|
|
3060
|
+
const encoder = encoding.createEncoder();
|
|
3061
|
+
if (this._verifyUpdates) {
|
|
3062
|
+
this._touchPeer(this.doc.clientID);
|
|
3063
|
+
// Use verified sync protocol with sequence number and hash
|
|
3064
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC_VERIFIED);
|
|
3065
|
+
// Include sequence number and clientID for causal ordering
|
|
3066
|
+
encoding.writeVarUint(encoder, this._localSeqNum++);
|
|
785
3067
|
encoding.writeVarUint(encoder, this.doc.clientID);
|
|
786
3068
|
syncProtocol.writeUpdate(encoder, update);
|
|
787
3069
|
// Include document hash after applying this update (signed integer)
|
|
@@ -793,6 +3075,35 @@ export class GenericProvider extends Observable {
|
|
|
793
3075
|
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
794
3076
|
syncProtocol.writeUpdate(encoder, update);
|
|
795
3077
|
}
|
|
3078
|
+
return encoding.toUint8Array(encoder);
|
|
3079
|
+
}
|
|
3080
|
+
/**
|
|
3081
|
+
* Encode a full-state push (see MESSAGE_SYNC_PUSH): the document as one
|
|
3082
|
+
* update, deliberately without the hash and sequence number that
|
|
3083
|
+
* `_encodeUpdate()` adds to incremental updates.
|
|
3084
|
+
*/
|
|
3085
|
+
_encodePush(update) {
|
|
3086
|
+
const encoder = encoding.createEncoder();
|
|
3087
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC_PUSH);
|
|
3088
|
+
encoding.writeVarUint8Array(encoder, update);
|
|
3089
|
+
return encoding.toUint8Array(encoder);
|
|
3090
|
+
}
|
|
3091
|
+
/**
|
|
3092
|
+
* Send a document update to the transport, with whatever awareness change
|
|
3093
|
+
* the throttle is holding folded into the same wire message (round 5,
|
|
3094
|
+
* item 1). If verifyUpdates is enabled, the update carries a sequence
|
|
3095
|
+
* number and document hash for ordering and desync detection.
|
|
3096
|
+
*/
|
|
3097
|
+
_sendUpdate(update) {
|
|
3098
|
+
this._sendBatch([this._encodeUpdate(update), ...this._takePendingAwareness()]);
|
|
3099
|
+
}
|
|
3100
|
+
/**
|
|
3101
|
+
* Send awareness update to the transport.
|
|
3102
|
+
*/
|
|
3103
|
+
_sendAwarenessUpdate(changedClients) {
|
|
3104
|
+
const encoder = encoding.createEncoder();
|
|
3105
|
+
encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
|
|
3106
|
+
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, changedClients));
|
|
796
3107
|
this._send(encoding.toUint8Array(encoder));
|
|
797
3108
|
}
|
|
798
3109
|
/**
|
|
@@ -823,10 +3134,11 @@ export class GenericProvider extends Observable {
|
|
|
823
3134
|
}
|
|
824
3135
|
}
|
|
825
3136
|
/**
|
|
826
|
-
* Send a
|
|
827
|
-
*
|
|
828
|
-
*
|
|
829
|
-
*
|
|
3137
|
+
* Send a pub/sub message to a single target.
|
|
3138
|
+
*
|
|
3139
|
+
* With `Transport.sendTo` the frame is unicast to that peer; without it the
|
|
3140
|
+
* frame is broadcast with the target embedded and dropped on receipt by
|
|
3141
|
+
* every provider whose `localId` differs.
|
|
830
3142
|
*/
|
|
831
3143
|
_sendPubSubTo(target, topic, message) {
|
|
832
3144
|
if (!this.transport.isConnected) {
|
|
@@ -865,63 +3177,201 @@ export class GenericProvider extends Observable {
|
|
|
865
3177
|
* Throttled to prevent awareness updates from flooding document sync.
|
|
866
3178
|
* Multiple rapid updates are batched together.
|
|
867
3179
|
*/
|
|
868
|
-
_broadcastAwareness(clients
|
|
3180
|
+
_broadcastAwareness(clients) {
|
|
869
3181
|
if (clients.length === 0)
|
|
870
3182
|
return;
|
|
871
|
-
const
|
|
872
|
-
const pending = isApp
|
|
873
|
-
? this._pendingAppAwarenessClients
|
|
874
|
-
: this._pendingAwarenessClients;
|
|
3183
|
+
const interval = this._effectiveAwarenessInterval();
|
|
875
3184
|
// If throttling is disabled, send immediately
|
|
876
|
-
if (
|
|
877
|
-
this._sendAwarenessNow(clients
|
|
3185
|
+
if (interval <= 0) {
|
|
3186
|
+
this._sendAwarenessNow(clients);
|
|
878
3187
|
return;
|
|
879
3188
|
}
|
|
880
3189
|
// Add clients to pending set
|
|
881
3190
|
for (const client of clients) {
|
|
882
|
-
|
|
3191
|
+
this._pendingAwarenessClients.add(client);
|
|
883
3192
|
}
|
|
884
3193
|
// If we already have a scheduled broadcast, let it handle the batched clients
|
|
885
|
-
if (
|
|
886
|
-
undefined) {
|
|
3194
|
+
if (this._awarenessTimeoutId !== undefined) {
|
|
887
3195
|
return;
|
|
888
3196
|
}
|
|
889
3197
|
// Calculate delay - respect minimum interval since last broadcast
|
|
890
3198
|
const now = Date.now();
|
|
891
|
-
const
|
|
892
|
-
const delay = Math.max(0,
|
|
3199
|
+
const timeSinceLastBroadcast = now - this._lastAwarenessTime;
|
|
3200
|
+
const delay = Math.max(0, interval - timeSinceLastBroadcast);
|
|
893
3201
|
// Schedule the batched broadcast
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
3202
|
+
this._awarenessTimeoutId = setTimeout(() => {
|
|
3203
|
+
this._awarenessTimeoutId = undefined;
|
|
3204
|
+
this._lastAwarenessTime = Date.now();
|
|
3205
|
+
// Send all pending clients in one message - and a pending timed
|
|
3206
|
+
// update batch (`batchUpdates > 0`) with them, update first.
|
|
3207
|
+
const clientsToSend = Array.from(this._pendingAwarenessClients);
|
|
3208
|
+
this._pendingAwarenessClients.clear();
|
|
3209
|
+
if (clientsToSend.length > 0) {
|
|
3210
|
+
this._sendBatch([...this._takePendingUpdate(), this._encodeAwareness(clientsToSend)]);
|
|
902
3211
|
}
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
3212
|
+
}, delay);
|
|
3213
|
+
}
|
|
3214
|
+
/**
|
|
3215
|
+
* Register the app channel's update listener. Same echo suppression as the
|
|
3216
|
+
* core channel (never re-broadcast what came off the wire), but none of its
|
|
3217
|
+
* presence/removal handling - this channel has no bearing on who the room
|
|
3218
|
+
* thinks is present.
|
|
3219
|
+
*/
|
|
3220
|
+
_attachAppAwareness(aw) {
|
|
3221
|
+
if (this._appAwarenessUpdateHandler)
|
|
3222
|
+
return;
|
|
3223
|
+
this._appAwarenessUpdateHandler = ({ added, updated, removed, }, origin) => {
|
|
3224
|
+
if (origin === this)
|
|
3225
|
+
return;
|
|
3226
|
+
const changedClients = [...added, ...updated, ...removed];
|
|
3227
|
+
if (changedClients.length === 0)
|
|
3228
|
+
return;
|
|
3229
|
+
this._broadcastAppAwareness(changedClients);
|
|
3230
|
+
};
|
|
3231
|
+
aw.on('update', this._appAwarenessUpdateHandler);
|
|
3232
|
+
}
|
|
3233
|
+
/**
|
|
3234
|
+
* Broadcast app-channel awareness. Mirrors `_broadcastAwareness()`'s
|
|
3235
|
+
* throttle, against its own pending set and timer, and deliberately does
|
|
3236
|
+
* NOT piggyback on the core sync batch: module cursor churn must not pull
|
|
3237
|
+
* document or presence traffic onto its cadence (or vice versa).
|
|
3238
|
+
*/
|
|
3239
|
+
_broadcastAppAwareness(clients) {
|
|
3240
|
+
if (clients.length === 0)
|
|
3241
|
+
return;
|
|
3242
|
+
const interval = this._effectiveAwarenessInterval();
|
|
3243
|
+
if (interval <= 0) {
|
|
3244
|
+
this._sendAppAwarenessNow(clients);
|
|
3245
|
+
return;
|
|
3246
|
+
}
|
|
3247
|
+
for (const client of clients) {
|
|
3248
|
+
this._pendingAppAwarenessClients.add(client);
|
|
3249
|
+
}
|
|
3250
|
+
if (this._appAwarenessTimeoutId !== undefined) {
|
|
3251
|
+
return;
|
|
3252
|
+
}
|
|
3253
|
+
const delay = Math.max(0, interval - (Date.now() - this._lastAppAwarenessTime));
|
|
3254
|
+
this._appAwarenessTimeoutId = setTimeout(() => {
|
|
3255
|
+
this._appAwarenessTimeoutId = undefined;
|
|
3256
|
+
this._lastAppAwarenessTime = Date.now();
|
|
3257
|
+
const clientsToSend = Array.from(this._pendingAppAwarenessClients);
|
|
3258
|
+
this._pendingAppAwarenessClients.clear();
|
|
906
3259
|
if (clientsToSend.length > 0) {
|
|
907
|
-
this.
|
|
3260
|
+
this._sendAppAwarenessNow(clientsToSend);
|
|
908
3261
|
}
|
|
909
3262
|
}, delay);
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
3263
|
+
}
|
|
3264
|
+
/** Encode and send an app-channel awareness update immediately. */
|
|
3265
|
+
_sendAppAwarenessNow(clients) {
|
|
3266
|
+
const encoder = encoding.createEncoder();
|
|
3267
|
+
encoding.writeVarUint(encoder, MESSAGE_AWARENESS_APP);
|
|
3268
|
+
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.appAwareness, clients));
|
|
3269
|
+
this._send(encoding.toUint8Array(encoder));
|
|
914
3270
|
}
|
|
915
3271
|
/**
|
|
916
|
-
*
|
|
3272
|
+
* Round 5, item 1: the awareness change the throttle is holding rides
|
|
3273
|
+
* along with a wire message that is leaving anyway. Returns the encoded
|
|
3274
|
+
* awareness sub-message (or nothing) and commits the throttle state
|
|
3275
|
+
* exactly as the timer's own flush would. A piggybacked broadcast costs
|
|
3276
|
+
* no message, only its payload bytes, so it goes out early instead of as
|
|
3277
|
+
* its own message up to `_awarenessInterval` later. Measured in
|
|
3278
|
+
* test/dummy/bench-typing-census.ts: a keystroke in an editor binding is
|
|
3279
|
+
* a text insert plus a cursor update - two broadcasts per keystroke
|
|
3280
|
+
* before this, one after. Broadcast paths only: `_sendDirect` and the
|
|
3281
|
+
* BroadcastChannel-only publishes never call this.
|
|
3282
|
+
*/
|
|
3283
|
+
_takePendingAwareness() {
|
|
3284
|
+
if (this._awarenessTimeoutId === undefined)
|
|
3285
|
+
return [];
|
|
3286
|
+
clearTimeout(this._awarenessTimeoutId);
|
|
3287
|
+
this._awarenessTimeoutId = undefined;
|
|
3288
|
+
const clients = Array.from(this._pendingAwarenessClients);
|
|
3289
|
+
this._pendingAwarenessClients.clear();
|
|
3290
|
+
if (clients.length === 0)
|
|
3291
|
+
return [];
|
|
3292
|
+
this._lastAwarenessTime = Date.now();
|
|
3293
|
+
return [this._encodeAwareness(clients)];
|
|
3294
|
+
}
|
|
3295
|
+
/**
|
|
3296
|
+
* The counterpart for the timed batch: a `batchUpdates > 0` batch that is
|
|
3297
|
+
* still waiting rides along with an awareness flush (Matrix: both
|
|
3298
|
+
* default to 2 s, so a typist's cursor and text leave as one PUT).
|
|
917
3299
|
*/
|
|
918
|
-
|
|
919
|
-
|
|
3300
|
+
_takePendingUpdate() {
|
|
3301
|
+
if (this._batchTimeoutId === undefined || !this._pendingUpdate)
|
|
3302
|
+
return [];
|
|
3303
|
+
clearTimeout(this._batchTimeoutId);
|
|
3304
|
+
this._batchTimeoutId = undefined;
|
|
3305
|
+
this._flushScheduled = false;
|
|
3306
|
+
const update = this._pendingUpdate;
|
|
3307
|
+
this._pendingUpdate = null;
|
|
3308
|
+
return [this._encodeUpdate(update)];
|
|
3309
|
+
}
|
|
3310
|
+
/**
|
|
3311
|
+
* Encode an awareness update, without sending it. Extracted from the old
|
|
3312
|
+
* `_sendAwarenessNow()` so `_tryImmediateAwarenessMessage()` can fold it
|
|
3313
|
+
* into a batched wire send instead of always sending it as its own
|
|
3314
|
+
* message.
|
|
3315
|
+
*/
|
|
3316
|
+
_encodeAwareness(clients) {
|
|
920
3317
|
const encoder = encoding.createEncoder();
|
|
921
3318
|
encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
|
|
922
|
-
encoding.
|
|
923
|
-
encoding.
|
|
924
|
-
|
|
3319
|
+
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, clients));
|
|
3320
|
+
return encoding.toUint8Array(encoder);
|
|
3321
|
+
}
|
|
3322
|
+
/**
|
|
3323
|
+
* Send awareness update immediately without throttling.
|
|
3324
|
+
*/
|
|
3325
|
+
_sendAwarenessNow(clients) {
|
|
3326
|
+
this._send(this._encodeAwareness(clients));
|
|
3327
|
+
}
|
|
3328
|
+
/**
|
|
3329
|
+
* Attempt to build an awareness broadcast message for immediate
|
|
3330
|
+
* inclusion in the same wire send as a sync message a caller is about to
|
|
3331
|
+
* send anyway (see `_trySyncPushPull`'s `buildExtra` parameter), instead
|
|
3332
|
+
* of going through
|
|
3333
|
+
* `_broadcastAwareness()`'s independent debounce.
|
|
3334
|
+
*
|
|
3335
|
+
* Only returns non-null when the throttle would have let an immediate
|
|
3336
|
+
* send through anyway - i.e. no debounced broadcast is already pending
|
|
3337
|
+
* AND (throttling is disabled, or at least `_awarenessInterval` ms have
|
|
3338
|
+
* passed since the last broadcast) - so this never changes awareness
|
|
3339
|
+
* throttle semantics, only whether the resulting message travels as its
|
|
3340
|
+
* own wire send or bundled with a sync message that happens to be going
|
|
3341
|
+
* out "now" too.
|
|
3342
|
+
*
|
|
3343
|
+
* Mutates the same state `_broadcastAwareness()`'s own immediate-send
|
|
3344
|
+
* branches mutate (`_pendingAwarenessClients`, `_lastAwarenessTime`) -
|
|
3345
|
+
* once this returns non-null, the state is already committed as "sent
|
|
3346
|
+
* now", so the caller MUST actually send the returned message (bundled
|
|
3347
|
+
* or standalone) rather than discarding it.
|
|
3348
|
+
*/
|
|
3349
|
+
_tryImmediateAwarenessMessage(clients) {
|
|
3350
|
+
if (clients.length === 0)
|
|
3351
|
+
return null;
|
|
3352
|
+
// A debounced broadcast is already scheduled - let it handle these
|
|
3353
|
+
// clients via the normal pending-set path below, don't race it with an
|
|
3354
|
+
// immediate send here.
|
|
3355
|
+
if (this._awarenessTimeoutId !== undefined)
|
|
3356
|
+
return null;
|
|
3357
|
+
const interval = this._effectiveAwarenessInterval();
|
|
3358
|
+
if (interval > 0) {
|
|
3359
|
+
const timeSinceLastBroadcast = Date.now() - this._lastAwarenessTime;
|
|
3360
|
+
if (timeSinceLastBroadcast < interval)
|
|
3361
|
+
return null;
|
|
3362
|
+
}
|
|
3363
|
+
// Merge with anything already pending (normally empty here since no
|
|
3364
|
+
// timer is scheduled, but merge for safety) and commit to sending now -
|
|
3365
|
+
// mirrors exactly what _broadcastAwareness()'s own immediate
|
|
3366
|
+
// (_awarenessInterval <= 0) and debounced-timer-fired branches do.
|
|
3367
|
+
for (const c of clients)
|
|
3368
|
+
this._pendingAwarenessClients.add(c);
|
|
3369
|
+
const clientsToSend = Array.from(this._pendingAwarenessClients);
|
|
3370
|
+
this._pendingAwarenessClients.clear();
|
|
3371
|
+
if (clientsToSend.length === 0)
|
|
3372
|
+
return null;
|
|
3373
|
+
this._lastAwarenessTime = Date.now();
|
|
3374
|
+
return this._encodeAwareness(clientsToSend);
|
|
925
3375
|
}
|
|
926
3376
|
/**
|
|
927
3377
|
* Setup BroadcastChannel for cross-tab communication.
|
|
@@ -955,30 +3405,22 @@ export class GenericProvider extends Observable {
|
|
|
955
3405
|
const encoderSync = encoding.createEncoder();
|
|
956
3406
|
encoding.writeVarUint(encoderSync, MESSAGE_SYNC);
|
|
957
3407
|
syncProtocol.writeSyncStep1(encoderSync, this.doc);
|
|
958
|
-
|
|
3408
|
+
this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoderSync)));
|
|
959
3409
|
// Broadcast local state via BroadcastChannel (wrapped with CRC32)
|
|
960
3410
|
const encoderState = encoding.createEncoder();
|
|
961
3411
|
encoding.writeVarUint(encoderState, MESSAGE_SYNC);
|
|
962
3412
|
syncProtocol.writeSyncStep2(encoderState, this.doc);
|
|
963
|
-
|
|
3413
|
+
this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoderState)));
|
|
964
3414
|
// Broadcast local awareness state via BroadcastChannel (wrapped with CRC32)
|
|
965
3415
|
if (this.awareness.getLocalState() !== null) {
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
3416
|
+
const encoderAwareness = encoding.createEncoder();
|
|
3417
|
+
encoding.writeVarUint(encoderAwareness, MESSAGE_AWARENESS);
|
|
3418
|
+
encoding.writeVarUint8Array(encoderAwareness, awarenessProtocol.encodeAwarenessUpdate(this.awareness, [
|
|
3419
|
+
this.doc.clientID,
|
|
3420
|
+
]));
|
|
3421
|
+
this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoderAwareness)));
|
|
970
3422
|
}
|
|
971
3423
|
}
|
|
972
|
-
/**
|
|
973
|
-
* Encode and publish an awareness update for the local client to the BroadcastChannel.
|
|
974
|
-
*/
|
|
975
|
-
_publishAwarenessToBroadcastChannel(awareness, channel) {
|
|
976
|
-
const encoder = encoding.createEncoder();
|
|
977
|
-
encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
|
|
978
|
-
encoding.writeVarUint(encoder, channel);
|
|
979
|
-
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(awareness, [this.doc.clientID]));
|
|
980
|
-
bc.publish(this._bcChannel, wrapMessageWithChecksum(encoding.toUint8Array(encoder)), this);
|
|
981
|
-
}
|
|
982
3424
|
/**
|
|
983
3425
|
* Disconnect from BroadcastChannel and mark local client as offline.
|
|
984
3426
|
*/
|
|
@@ -987,21 +3429,71 @@ export class GenericProvider extends Observable {
|
|
|
987
3429
|
return;
|
|
988
3430
|
}
|
|
989
3431
|
// Broadcast awareness state with null (indicating disconnect) - wrapped with CRC32
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
const encoder = encoding.createEncoder();
|
|
995
|
-
encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
|
|
996
|
-
encoding.writeVarUint(encoder, channel);
|
|
997
|
-
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(awareness, [this.doc.clientID], new Map()));
|
|
998
|
-
bc.publish(this._bcChannel, wrapMessageWithChecksum(encoding.toUint8Array(encoder)), this);
|
|
999
|
-
}
|
|
3432
|
+
const encoder = encoding.createEncoder();
|
|
3433
|
+
encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
|
|
3434
|
+
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, [this.doc.clientID], new Map()));
|
|
3435
|
+
this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoder)));
|
|
1000
3436
|
// Unsubscribe from channel
|
|
1001
3437
|
bc.unsubscribe(this._bcChannel, this._bcSubscriber);
|
|
1002
3438
|
this._bcConnected = false;
|
|
1003
3439
|
this._bcSubscriber = undefined;
|
|
1004
3440
|
}
|
|
3441
|
+
/**
|
|
3442
|
+
* Send N already-encoded, already-typed sub-messages as ONE wire message
|
|
3443
|
+
* instead of N separate `transport.send()`/`bc.publish()` calls, when
|
|
3444
|
+
* there's more than one to send. Used at trigger points that
|
|
3445
|
+
* conceptually produce a single event but historically sent multiple
|
|
3446
|
+
* independent messages for it (sync push, sync pull, awareness) - see
|
|
3447
|
+
* `_trySyncPushPull()` (connect-time push + digest beacon + awareness in
|
|
3448
|
+
* one wire message).
|
|
3449
|
+
*
|
|
3450
|
+
* Design (see the task's framing requirements):
|
|
3451
|
+
* - Each sub-message is length-prefixed with `writeVarUint8Array`,
|
|
3452
|
+
* consistent with how this codebase already frames variable-length
|
|
3453
|
+
* payloads elsewhere (e.g. MESSAGE_AWARENESS). On receipt,
|
|
3454
|
+
* `_dispatchMessage()`'s `MESSAGE_BATCH` case unwraps and re-dispatches
|
|
3455
|
+
* each one through the EXACT SAME per-message-type logic used for a
|
|
3456
|
+
* top-level message - no parallel reimplementation.
|
|
3457
|
+
* - Sub-messages are NOT individually CRC32-wrapped here - the whole
|
|
3458
|
+
* batch envelope goes through the normal, single `_send()` pipeline
|
|
3459
|
+
* below, which wraps the WHOLE envelope in exactly one CRC32 checksum
|
|
3460
|
+
* (and, if `compressionThresholdBytes` is configured, one compression
|
|
3461
|
+
* pass) - built and computed exactly like any other outgoing message,
|
|
3462
|
+
* so this composes with the existing compression pipeline for free
|
|
3463
|
+
* rather than fighting it with a second, nested wrap/compress step.
|
|
3464
|
+
* The tradeoff: a single corrupted bit anywhere in a batched wire
|
|
3465
|
+
* message now invalidates every sub-message it carried, not just one -
|
|
3466
|
+
* per-sub-message CRC32s would avoid that, at the cost of ~4 extra
|
|
3467
|
+
* bytes per sub-message for a benefit that only matters under active
|
|
3468
|
+
* corruption. This tradeoff is exactly what
|
|
3469
|
+
* test/dummy/bench-corruption-storm.ts and bench-packet-loss.ts exist
|
|
3470
|
+
* to measure empirically, per this task's validation requirements,
|
|
3471
|
+
* rather than deciding it by design argument alone.
|
|
3472
|
+
* - BroadcastChannel (cross-tab) traffic is NOT specially batched beyond
|
|
3473
|
+
* whatever `_send()` already does per call - same-tab-group cross-tab
|
|
3474
|
+
* traffic is local/cheap, and `_send()` already only issues one
|
|
3475
|
+
* `bc.publish()` per call regardless, so a batch of N sub-messages
|
|
3476
|
+
* already becomes exactly one `bc.publish()` call for free once routed
|
|
3477
|
+
* through here - no separate BC-specific batching logic needed.
|
|
3478
|
+
*/
|
|
3479
|
+
_sendBatch(messages) {
|
|
3480
|
+
if (messages.length === 0)
|
|
3481
|
+
return;
|
|
3482
|
+
if (messages.length === 1) {
|
|
3483
|
+
this._send(messages[0]);
|
|
3484
|
+
return;
|
|
3485
|
+
}
|
|
3486
|
+
this._send(this._encodeBatch(messages));
|
|
3487
|
+
}
|
|
3488
|
+
/** The MESSAGE_BATCH envelope of `_sendBatch`, without sending it. */
|
|
3489
|
+
_encodeBatch(messages) {
|
|
3490
|
+
const encoder = encoding.createEncoder();
|
|
3491
|
+
encoding.writeVarUint(encoder, MESSAGE_BATCH);
|
|
3492
|
+
for (const message of messages) {
|
|
3493
|
+
encoding.writeVarUint8Array(encoder, message);
|
|
3494
|
+
}
|
|
3495
|
+
return encoding.toUint8Array(encoder);
|
|
3496
|
+
}
|
|
1005
3497
|
/**
|
|
1006
3498
|
* Send data through both BroadcastChannel (if connected) and transport.
|
|
1007
3499
|
* All messages are wrapped with CRC32 checksum for integrity verification.
|
|
@@ -1010,16 +3502,66 @@ export class GenericProvider extends Observable {
|
|
|
1010
3502
|
_send(data) {
|
|
1011
3503
|
// Wrap message with CRC32 checksum
|
|
1012
3504
|
const wrappedData = wrapMessageWithChecksum(data);
|
|
1013
|
-
// Send via BroadcastChannel to other tabs first
|
|
1014
|
-
if (this._bcConnected)
|
|
1015
|
-
|
|
1016
|
-
}
|
|
3505
|
+
// Send via BroadcastChannel to other tabs first.
|
|
3506
|
+
if (this._bcConnected)
|
|
3507
|
+
this._bcPublish(wrappedData);
|
|
1017
3508
|
// Send via network transport
|
|
1018
3509
|
if (!this.transport.isConnected) {
|
|
1019
3510
|
return;
|
|
1020
3511
|
}
|
|
3512
|
+
this._sendToTransport(wrappedData);
|
|
3513
|
+
}
|
|
3514
|
+
/**
|
|
3515
|
+
* Publish already-CRC32-wrapped bytes to the other tabs. BroadcastChannel
|
|
3516
|
+
* is same-process - never worth compressing - but when
|
|
3517
|
+
* compressionThresholdBytes is enabled every message still needs the
|
|
3518
|
+
* leading flag byte _handleIncomingMessage() expects regardless of
|
|
3519
|
+
* source, so this sends flag=0 in that case. The ONE place for every BC
|
|
3520
|
+
* publish: the connect-time burst in _setupBroadcastChannel() used to
|
|
3521
|
+
* publish without the flag, and with compression on (the transport
|
|
3522
|
+
* hints made that a default) the other tab read a CRC byte as the flag
|
|
3523
|
+
* and failed to inflate three messages per join (Nostr playground,
|
|
3524
|
+
* 2026-09-06).
|
|
3525
|
+
*/
|
|
3526
|
+
_bcPublish(wrappedData) {
|
|
3527
|
+
bc.publish(this._bcChannel, this._compressionThresholdBytes
|
|
3528
|
+
? prefixCompressionFlag(0, wrappedData)
|
|
3529
|
+
: wrappedData, this);
|
|
3530
|
+
}
|
|
3531
|
+
/**
|
|
3532
|
+
* Send already-CRC32-wrapped bytes to the network transport, compressing
|
|
3533
|
+
* first if compressionThresholdBytes is configured and this payload
|
|
3534
|
+
* clears it. See that option's doc comment for the size threshold
|
|
3535
|
+
* reasoning and the wire-format compatibility tradeoff of enabling it.
|
|
3536
|
+
*/
|
|
3537
|
+
_sendToTransport(wrappedData, to) {
|
|
3538
|
+
const threshold = this._compressionThresholdBytes;
|
|
3539
|
+
if (!threshold) {
|
|
3540
|
+
this._dispatchToTransport(wrappedData, to);
|
|
3541
|
+
return;
|
|
3542
|
+
}
|
|
3543
|
+
if (!COMPRESSION_AVAILABLE || wrappedData.length < threshold) {
|
|
3544
|
+
this._dispatchToTransport(prefixCompressionFlag(0, wrappedData), to);
|
|
3545
|
+
return;
|
|
3546
|
+
}
|
|
3547
|
+
compressDeflateRaw(wrappedData)
|
|
3548
|
+
.then((compressed) => {
|
|
3549
|
+
this._dispatchToTransport(prefixCompressionFlag(1, compressed), to);
|
|
3550
|
+
})
|
|
3551
|
+
.catch((error) => {
|
|
3552
|
+
console.error('[GenericProvider] Compression failed, sending uncompressed:', error);
|
|
3553
|
+
this._dispatchToTransport(prefixCompressionFlag(0, wrappedData), to);
|
|
3554
|
+
});
|
|
3555
|
+
}
|
|
3556
|
+
/**
|
|
3557
|
+
* Hand fully-framed bytes to transport.send() - or transport.sendTo() when
|
|
3558
|
+
* a peer address is given - tolerating a sync or async result.
|
|
3559
|
+
*/
|
|
3560
|
+
_dispatchToTransport(data, to) {
|
|
1021
3561
|
try {
|
|
1022
|
-
const result = this.transport.
|
|
3562
|
+
const result = to !== undefined && typeof this.transport.sendTo === 'function'
|
|
3563
|
+
? this.transport.sendTo(to, data)
|
|
3564
|
+
: this.transport.send(data);
|
|
1023
3565
|
// Handle async send
|
|
1024
3566
|
if (result instanceof Promise) {
|
|
1025
3567
|
result.catch((error) => {
|
|
@@ -1055,4 +3597,6 @@ export class GenericProvider extends Observable {
|
|
|
1055
3597
|
return this._localSeqNum;
|
|
1056
3598
|
}
|
|
1057
3599
|
}
|
|
3600
|
+
// ms of throttle added per known peer under `awarenessInterval: 'auto'`
|
|
3601
|
+
GenericProvider.AWARENESS_AUTO_MS_PER_PEER = 20;
|
|
1058
3602
|
//# sourceMappingURL=index.js.map
|