@edryslabs/genericprovider 1.0.4 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +129 -5
- package/dist/index.d.ts +903 -60
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2723 -428
- package/dist/index.js.map +1 -1
- package/dist/lib.d.ts +1 -0
- package/dist/lib.d.ts.map +1 -1
- package/dist/lib.js +3 -0
- package/dist/lib.js.map +1 -1
- package/dist/providers/ably/index.d.ts +203 -0
- package/dist/providers/ably/index.d.ts.map +1 -0
- package/dist/providers/ably/index.js +561 -0
- package/dist/providers/ably/index.js.map +1 -0
- package/dist/providers/chunking.d.ts +26 -0
- package/dist/providers/chunking.d.ts.map +1 -0
- package/dist/providers/chunking.js +52 -0
- package/dist/providers/chunking.js.map +1 -0
- package/dist/providers/dummy/index.d.ts +138 -2
- package/dist/providers/dummy/index.d.ts.map +1 -1
- package/dist/providers/dummy/index.js +266 -4
- package/dist/providers/dummy/index.js.map +1 -1
- package/dist/providers/gun/index.d.ts +11 -4
- package/dist/providers/gun/index.d.ts.map +1 -1
- package/dist/providers/gun/index.js +104 -22
- package/dist/providers/gun/index.js.map +1 -1
- package/dist/providers/indexeddb/index.d.ts +44 -20
- package/dist/providers/indexeddb/index.d.ts.map +1 -1
- package/dist/providers/indexeddb/index.js +85 -71
- package/dist/providers/indexeddb/index.js.map +1 -1
- package/dist/providers/matrix/index.d.ts +6 -1
- package/dist/providers/matrix/index.d.ts.map +1 -1
- package/dist/providers/matrix/index.js +44 -6
- package/dist/providers/matrix/index.js.map +1 -1
- package/dist/providers/nostr/index.d.ts +69 -5
- package/dist/providers/nostr/index.d.ts.map +1 -1
- package/dist/providers/nostr/index.js +204 -36
- package/dist/providers/nostr/index.js.map +1 -1
- package/dist/providers/peerjs/index.d.ts +11 -1
- package/dist/providers/peerjs/index.d.ts.map +1 -1
- package/dist/providers/peerjs/index.js +32 -2
- package/dist/providers/peerjs/index.js.map +1 -1
- package/dist/providers/pubnub/index.d.ts +35 -1
- package/dist/providers/pubnub/index.d.ts.map +1 -1
- package/dist/providers/pubnub/index.js +56 -35
- package/dist/providers/pubnub/index.js.map +1 -1
- package/dist/providers/simple-peer/index.d.ts +10 -15
- package/dist/providers/simple-peer/index.d.ts.map +1 -1
- package/dist/providers/simple-peer/index.js +81 -109
- package/dist/providers/simple-peer/index.js.map +1 -1
- package/dist/providers/supabase/index.d.ts +43 -1
- package/dist/providers/supabase/index.d.ts.map +1 -1
- package/dist/providers/supabase/index.js +211 -18
- package/dist/providers/supabase/index.js.map +1 -1
- package/dist/providers/trystero/index.d.ts +19 -1
- package/dist/providers/trystero/index.d.ts.map +1 -1
- package/dist/providers/trystero/index.js +37 -2
- package/dist/providers/trystero/index.js.map +1 -1
- package/dist/providers/websocket/index.d.ts +9 -1
- package/dist/providers/websocket/index.d.ts.map +1 -1
- package/dist/providers/websocket/index.js +7 -1
- package/dist/providers/websocket/index.js.map +1 -1
- package/dist/transport.d.ts +69 -16
- package/dist/transport.d.ts.map +1 -1
- package/package.json +35 -24
package/dist/index.js
CHANGED
|
@@ -10,10 +10,83 @@ const MESSAGE_SYNC = 0;
|
|
|
10
10
|
const MESSAGE_AWARENESS = 1;
|
|
11
11
|
const MESSAGE_PUBSUB = 2;
|
|
12
12
|
const MESSAGE_SYNC_VERIFIED = 3; // Sync message with hash verification
|
|
13
|
-
const
|
|
14
|
-
//
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
const MESSAGE_BATCH = 4; // Envelope for N independently-typed sub-messages sent as one wire message
|
|
14
|
+
// Digest beacon: replaces SyncStep1 on the wire. [version][flags][sender
|
|
15
|
+
// clientID][state vector][delete-set hash]. Receivers reply only when the
|
|
16
|
+
// sender is behind them or the delete-set hashes differ (SyncStep2, as
|
|
17
|
+
// before), or - on a JOIN-flagged beacon with equal state - with their own
|
|
18
|
+
// beacon as an ack; otherwise not at all. See
|
|
19
|
+
// docs/superpowers/specs/2026-09-05-digest-beacon-design.md and
|
|
20
|
+
// _handleDigest(). Versions are append-only: receivers read the fields
|
|
21
|
+
// they know and ignore trailing bytes.
|
|
22
|
+
const MESSAGE_SYNC_DIGEST = 5;
|
|
23
|
+
const DIGEST_VERSION = 1;
|
|
24
|
+
const DIGEST_FLAG_JOIN = 1; // bit 0: "I just joined - send me your presence and confirm my state"
|
|
25
|
+
// bit 1: this message is an ACK, not a request. It echoes the state vector
|
|
26
|
+
// and delete-set hash of the JOIN beacon it answers ("I hold exactly this
|
|
27
|
+
// state too"), never the sender's own state - so it is never read as a
|
|
28
|
+
// request and never collects a SyncStep2 from anyone. Receivers whose state
|
|
29
|
+
// equals the echoed digest mark themselves synced (the joiner it was for,
|
|
30
|
+
// and everyone else in that state); receivers holding a pending ack for the
|
|
31
|
+
// same digest drop it. See _handleDigest(). Before this flag existed the ack
|
|
32
|
+
// was the acker's own beacon; an acker that was itself behind the room (a
|
|
33
|
+
// joiner acking another joiner) made every peer ahead of it reply with a
|
|
34
|
+
// SyncStep2, and acks landing after an edit burst had started did the same
|
|
35
|
+
// (Task 3c in the design doc).
|
|
36
|
+
const DIGEST_FLAG_ACK = 2;
|
|
37
|
+
// bit 2: "ack me if our states are equal" WITHOUT the presence request of
|
|
38
|
+
// DIGEST_FLAG_JOIN. Sent only by the response-wait retry
|
|
39
|
+
// (_armResponseWait): a joiner that gets neither a SyncStep2 nor an ack
|
|
40
|
+
// within the wait has lost one message or the other and asks again - reply
|
|
41
|
+
// suppression deliberately leaves ~1 reply per request, so a single lost
|
|
42
|
+
// reply would otherwise strand the joiner's `synced` until the next
|
|
43
|
+
// periodic beacon (never, with syncInterval 0). Measured: the last joiner
|
|
44
|
+
// of a simultaneous 5-peer burst at 10% loss failed to reach `synced` in 1
|
|
45
|
+
// of 600 runs before this existed. Resync beacons do NOT request acks: in
|
|
46
|
+
// an equal room every peer would answer, and at high latency the
|
|
47
|
+
// suppression window cannot thin those replies (measured 4-5x more acks).
|
|
48
|
+
const DIGEST_FLAG_CONFIRM = 4;
|
|
49
|
+
// bit 3: the sender is CONFIRMED - it has received a SyncStep2, or an equal
|
|
50
|
+
// digest from a confirmed peer, or its own join asked three times and got
|
|
51
|
+
// nothing better (so it is the room). A joiner's response wait is satisfied
|
|
52
|
+
// only by data (SyncStep2) or by an equal digest carrying this bit: an
|
|
53
|
+
// equal ack from a fellow joiner says nothing about whether the room holds
|
|
54
|
+
// more than both of us, and treating it as an answer left a joiner whose
|
|
55
|
+
// SyncStep2 was lost with an empty document (measured: 1 of 150 lossy
|
|
56
|
+
// 15-peer joins; research doc item 13 follow-up in the phase-1b design).
|
|
57
|
+
const DIGEST_FLAG_SETTLED = 8;
|
|
58
|
+
// Full-state push (connect()/syncNow()): the whole document as one update,
|
|
59
|
+
// no hash, no sequence number, applied and nothing else. Before this type a
|
|
60
|
+
// push was a MESSAGE_SYNC_VERIFIED update whose hash was the PUSHER's state
|
|
61
|
+
// - every peer holding more data read that as divergence and scheduled a
|
|
62
|
+
// resync of its own (research doc item 12), the seed of the cascade in item
|
|
63
|
+
// 13. Not a SyncStep2 either: receiving one must not flip `synced` (an
|
|
64
|
+
// empty joiner's push says nothing about the room's content). See
|
|
65
|
+
// docs/superpowers/specs/2026-09-05-resync-cascade-design.md.
|
|
66
|
+
const MESSAGE_SYNC_PUSH = 6;
|
|
67
|
+
// Pub/sub message aimed at a single target: [target][topic][message]. Every
|
|
68
|
+
// provider whose `localId` differs drops it. On transports with sendTo it is
|
|
69
|
+
// unicast; elsewhere it is broadcast and filtered on receipt.
|
|
70
|
+
//
|
|
71
|
+
// NOTE: this was type 4 before the v1.5.0 merge, which upstream had taken for
|
|
72
|
+
// MESSAGE_BATCH. Renumbered to 7 (the first free slot) - the two are not wire
|
|
73
|
+
// compatible, so all peers of a room must run the same version, as the README
|
|
74
|
+
// already requires.
|
|
75
|
+
const MESSAGE_PUBSUB_TARGETED = 7;
|
|
76
|
+
// A second, independent awareness channel for application/module state
|
|
77
|
+
// (cursors, module presence), kept off MESSAGE_AWARENESS deliberately.
|
|
78
|
+
//
|
|
79
|
+
// Upstream's MESSAGE_AWARENESS receive path is now core presence machinery:
|
|
80
|
+
// it feeds _knownPeers, sets _presenceCovered, and cancels pending removal
|
|
81
|
+
// broadcasts. App awareness is written by untrusted third-party classroom
|
|
82
|
+
// modules, so routing it through that path would let module cursor traffic
|
|
83
|
+
// drive the room's peer bookkeeping - a module setting a state would mark
|
|
84
|
+
// phantom peers present and suppress real removals. A separate type keeps
|
|
85
|
+
// the two entirely disjoint: same throttle policy, no shared presence state.
|
|
86
|
+
//
|
|
87
|
+
// (Dev encoded this as a channel varint inside MESSAGE_AWARENESS, which was
|
|
88
|
+
// safe when that path did nothing but applyAwarenessUpdate. It no longer is.)
|
|
89
|
+
const MESSAGE_AWARENESS_APP = 8;
|
|
17
90
|
/**
|
|
18
91
|
* CRC32 lookup table for fast computation.
|
|
19
92
|
* Generated once and reused for all CRC calculations.
|
|
@@ -80,25 +153,105 @@ function unwrapAndVerifyMessage(wrapped) {
|
|
|
80
153
|
}
|
|
81
154
|
return message;
|
|
82
155
|
}
|
|
156
|
+
/** Byte-wise equality of two Uint8Arrays. */
|
|
157
|
+
function bytesEqual(a, b) {
|
|
158
|
+
if (a.length !== b.length)
|
|
159
|
+
return false;
|
|
160
|
+
for (let i = 0; i < a.length; i++) {
|
|
161
|
+
if (a[i] !== b[i])
|
|
162
|
+
return false;
|
|
163
|
+
}
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
83
166
|
/**
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
167
|
+
* Componentwise minimum of two encoded state vectors: the state a
|
|
168
|
+
* SyncStep2 must start from to serve both requesters. Clients missing from
|
|
169
|
+
* one side count as clock 0 and are omitted (omitted = 0 on the wire).
|
|
170
|
+
*/
|
|
171
|
+
function minStateVector(a, b) {
|
|
172
|
+
const ma = Y.decodeStateVector(a);
|
|
173
|
+
const mb = Y.decodeStateVector(b);
|
|
174
|
+
const out = new Map();
|
|
175
|
+
for (const [client, clock] of ma) {
|
|
176
|
+
const m = Math.min(clock, mb.get(client) ?? 0);
|
|
177
|
+
if (m > 0)
|
|
178
|
+
out.set(client, m);
|
|
179
|
+
}
|
|
180
|
+
return Y.encodeStateVector(out);
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Whether this runtime has the Compression Streams API (Node 18+, all
|
|
184
|
+
* evergreen browsers). Checked once at module load; compressionThresholdBytes
|
|
185
|
+
* falls back to sending uncompressed (still flag-byte-prefixed, flag=0) if
|
|
186
|
+
* this is false, rather than throwing.
|
|
187
|
+
*/
|
|
188
|
+
const COMPRESSION_AVAILABLE = typeof CompressionStream !== 'undefined' &&
|
|
189
|
+
typeof DecompressionStream !== 'undefined';
|
|
190
|
+
/** Drain a ReadableStream<Uint8Array> into a single concatenated Uint8Array. */
|
|
191
|
+
async function _readAllChunks(readable) {
|
|
192
|
+
const chunks = [];
|
|
193
|
+
const reader = readable.getReader();
|
|
194
|
+
for (;;) {
|
|
195
|
+
const { done, value } = await reader.read();
|
|
196
|
+
if (done)
|
|
197
|
+
break;
|
|
198
|
+
chunks.push(value);
|
|
199
|
+
}
|
|
200
|
+
const total = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
201
|
+
const out = new Uint8Array(total);
|
|
202
|
+
let offset = 0;
|
|
203
|
+
for (const chunk of chunks) {
|
|
204
|
+
out.set(chunk, offset);
|
|
205
|
+
offset += chunk.length;
|
|
206
|
+
}
|
|
207
|
+
return out;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Compress with deflate-raw (no gzip header/trailer - see
|
|
211
|
+
* compressionThresholdBytes's doc comment for why deflate-raw over gzip).
|
|
212
|
+
*/
|
|
213
|
+
async function compressDeflateRaw(data) {
|
|
214
|
+
const cs = new CompressionStream('deflate-raw');
|
|
215
|
+
const writer = cs.writable.getWriter();
|
|
216
|
+
// The writer promises reject too when the stream errors; the reader side
|
|
217
|
+
// already surfaces that error, and an unhandled rejection here crashes
|
|
218
|
+
// Node (seen with a corrupt deflate stream in the Nostr end-to-end test).
|
|
219
|
+
writer.write(data).catch(() => { });
|
|
220
|
+
writer.close().catch(() => { });
|
|
221
|
+
return _readAllChunks(cs.readable);
|
|
222
|
+
}
|
|
223
|
+
/** Inverse of compressDeflateRaw(). */
|
|
224
|
+
async function decompressDeflateRaw(data) {
|
|
225
|
+
const ds = new DecompressionStream('deflate-raw');
|
|
226
|
+
const writer = ds.writable.getWriter();
|
|
227
|
+
writer.write(data).catch(() => { });
|
|
228
|
+
writer.close().catch(() => { });
|
|
229
|
+
return _readAllChunks(ds.readable);
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Prepend a 1-byte compressed(1)/uncompressed(0) flag. Only used when
|
|
233
|
+
* compressionThresholdBytes is configured - see that option's doc comment
|
|
234
|
+
* for why this is a deliberate, opt-in wire-format change.
|
|
235
|
+
*/
|
|
236
|
+
function prefixCompressionFlag(flag, data) {
|
|
237
|
+
const out = new Uint8Array(1 + data.length);
|
|
238
|
+
out[0] = flag;
|
|
239
|
+
out.set(data, 1);
|
|
240
|
+
return out;
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Compute a cheap hash of document state for verification.
|
|
97
244
|
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
245
|
+
* Hashes the state VECTOR (each client's clock), not the full document
|
|
246
|
+
* content — O(number of distinct clients) instead of O(document content
|
|
247
|
+
* size). `Y.encodeStateVector()` writes entries sorted by clientID, so the
|
|
248
|
+
* result is deterministic regardless of the internal Map's iteration order.
|
|
249
|
+
* Two peers can only reach the same state vector by having applied the same
|
|
250
|
+
* set of operations, so this still catches real content divergence; it just
|
|
251
|
+
* no longer re-serializes the entire document on every single update.
|
|
252
|
+
* (CRC32 already guards against wire corruption, and sequence tracking
|
|
253
|
+
* guards against reordering/loss — this hash is the last line of defense
|
|
254
|
+
* against logical divergence between peers.)
|
|
102
255
|
*/
|
|
103
256
|
function computeDocHash(doc) {
|
|
104
257
|
const state = Y.encodeStateVector(doc);
|
|
@@ -108,6 +261,109 @@ function computeDocHash(doc) {
|
|
|
108
261
|
}
|
|
109
262
|
return hash;
|
|
110
263
|
}
|
|
264
|
+
/**
|
|
265
|
+
* Cheap, peer-deterministic hash of the document's delete set - the half of
|
|
266
|
+
* a Yjs document's identity that the state vector does NOT cover (yjs
|
|
267
|
+
* INTERNALS.md: "deletions are tracked in the DeleteSet, and do not update
|
|
268
|
+
* the state vector"). Two docs that differ only by a lost delete-only
|
|
269
|
+
* update have identical state vectors, so `computeDocHash` can never
|
|
270
|
+
* detect that divergence; this hash can, at heartbeat granularity (see
|
|
271
|
+
* `_encodeSyncStep1()` / `_handleDigest()`).
|
|
272
|
+
*
|
|
273
|
+
* Cost: `Y.createDeleteSetFromStructStore` walks every struct (Yjs keeps no
|
|
274
|
+
* incremental delete set), so this is O(items) - fine once per heartbeat
|
|
275
|
+
* (every empty SyncStep2 already did this exact walk inside
|
|
276
|
+
* `encodeStateAsUpdate`), NOT fine per update; hence the cache in
|
|
277
|
+
* `_deleteSetHash()`. Per-client runs come out already sorted and merged;
|
|
278
|
+
* the only per-peer non-determinism is `Map` insertion order, fixed by
|
|
279
|
+
* sorting client IDs before hashing.
|
|
280
|
+
*
|
|
281
|
+
* Exported for the property check in test/dummy/bench-idle-room.ts.
|
|
282
|
+
* @internal
|
|
283
|
+
*/
|
|
284
|
+
export function computeDeleteSetHash(doc) {
|
|
285
|
+
const ds = Y.createDeleteSetFromStructStore(doc.store);
|
|
286
|
+
const encoder = encoding.createEncoder();
|
|
287
|
+
const clients = Array.from(ds.clients.keys()).sort((x, y) => x - y);
|
|
288
|
+
for (const client of clients) {
|
|
289
|
+
encoding.writeVarUint(encoder, client);
|
|
290
|
+
for (const item of ds.clients.get(client)) {
|
|
291
|
+
encoding.writeVarUint(encoder, item.clock);
|
|
292
|
+
encoding.writeVarUint(encoder, item.len);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
return computeCRC32(encoding.toUint8Array(encoder));
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* The Yjs updates a CRC-wrapped frame (as handed to `Transport.send`)
|
|
299
|
+
* carries: the update of a MESSAGE_SYNC_VERIFIED or MESSAGE_SYNC
|
|
300
|
+
* Update/SyncStep2 message, the whole document of a MESSAGE_SYNC_PUSH,
|
|
301
|
+
* each such sub-message of a MESSAGE_BATCH - and nothing for awareness,
|
|
302
|
+
* pub/sub, digest beacons and SyncStep1 requests, which carry no document
|
|
303
|
+
* state. For persistence transports (`providers/indexeddb`, LiaScript's
|
|
304
|
+
* Dexie cache): store only what this returns, and only this. Storing whole
|
|
305
|
+
* frames and replaying them on the next load resurrects the previous
|
|
306
|
+
* session's clientID as a phantom peer (its presence, its beacons - which
|
|
307
|
+
* the provider then answers into the store, multiplying rows) and keeps
|
|
308
|
+
* ~10x the bytes (a keystroke's frame carries its cursor). A frame this
|
|
309
|
+
* cannot parse yields `[]`, never a partial read. Frames of a provider with
|
|
310
|
+
* `compressionThresholdBytes` set (a leading flag byte) are not supported.
|
|
311
|
+
* Round 7, item 6; measured in test/dummy/bench-persist-log.ts.
|
|
312
|
+
*/
|
|
313
|
+
export function extractDocUpdates(frame) {
|
|
314
|
+
const updates = [];
|
|
315
|
+
const walk = (message) => {
|
|
316
|
+
const decoder = decoding.createDecoder(message);
|
|
317
|
+
const type = decoding.readVarUint(decoder);
|
|
318
|
+
switch (type) {
|
|
319
|
+
case MESSAGE_BATCH:
|
|
320
|
+
while (decoding.hasContent(decoder))
|
|
321
|
+
walk(decoding.readVarUint8Array(decoder));
|
|
322
|
+
break;
|
|
323
|
+
case MESSAGE_SYNC_VERIFIED:
|
|
324
|
+
case MESSAGE_SYNC: {
|
|
325
|
+
if (type === MESSAGE_SYNC_VERIFIED) {
|
|
326
|
+
decoding.readVarUint(decoder); // sequence number
|
|
327
|
+
decoding.readVarUint(decoder); // sender clientID
|
|
328
|
+
}
|
|
329
|
+
const sub = decoding.readVarUint(decoder);
|
|
330
|
+
if (sub === syncProtocol.messageYjsSyncStep2 ||
|
|
331
|
+
sub === syncProtocol.messageYjsUpdate) {
|
|
332
|
+
updates.push(decoding.readVarUint8Array(decoder));
|
|
333
|
+
}
|
|
334
|
+
break;
|
|
335
|
+
}
|
|
336
|
+
case MESSAGE_SYNC_PUSH:
|
|
337
|
+
updates.push(decoding.readVarUint8Array(decoder));
|
|
338
|
+
break;
|
|
339
|
+
default:
|
|
340
|
+
break;
|
|
341
|
+
}
|
|
342
|
+
};
|
|
343
|
+
try {
|
|
344
|
+
if (frame.length > 4)
|
|
345
|
+
walk(frame.subarray(4));
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
return [];
|
|
349
|
+
}
|
|
350
|
+
return updates;
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* The counterpart of `extractDocUpdates()` for the load path of a
|
|
354
|
+
* persistence transport: wraps one (merged) update as a CRC-wrapped
|
|
355
|
+
* MESSAGE_SYNC SyncStep2 frame. Handed to the `onMessage` callback, the
|
|
356
|
+
* provider applies it as the answer to its own request - `synced` fires,
|
|
357
|
+
* nothing is sent back - exactly what a local copy is: the peer that had
|
|
358
|
+
* our document.
|
|
359
|
+
*/
|
|
360
|
+
export function frameDocUpdate(update) {
|
|
361
|
+
const encoder = encoding.createEncoder();
|
|
362
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
363
|
+
encoding.writeVarUint(encoder, syncProtocol.messageYjsSyncStep2);
|
|
364
|
+
encoding.writeVarUint8Array(encoder, update);
|
|
365
|
+
return wrapMessageWithChecksum(encoding.toUint8Array(encoder));
|
|
366
|
+
}
|
|
111
367
|
/**
|
|
112
368
|
* PubSub channel for real-time messaging alongside Yjs.
|
|
113
369
|
* Allows sending ephemeral messages that don't need CRDT properties.
|
|
@@ -207,6 +463,18 @@ export class PubSubChannel extends Observable {
|
|
|
207
463
|
* ```
|
|
208
464
|
*/
|
|
209
465
|
export class GenericProvider extends Observable {
|
|
466
|
+
get appAwareness() {
|
|
467
|
+
if (!this._appAwareness) {
|
|
468
|
+
// Created on first use, not in the constructor: every y-protocols
|
|
469
|
+
// Awareness starts its own setInterval that only destroy() clears, so
|
|
470
|
+
// an eagerly-built second instance cost one live timer per provider
|
|
471
|
+
// for the majority of consumers that never touch this channel
|
|
472
|
+
// (bench-reload-phantoms part 2 counts exactly that).
|
|
473
|
+
this._appAwareness = new awarenessProtocol.Awareness(this.doc);
|
|
474
|
+
this._attachAppAwareness(this._appAwareness);
|
|
475
|
+
}
|
|
476
|
+
return this._appAwareness;
|
|
477
|
+
}
|
|
210
478
|
/**
|
|
211
479
|
* Create a new generic provider.
|
|
212
480
|
*
|
|
@@ -219,6 +487,10 @@ export class GenericProvider extends Observable {
|
|
|
219
487
|
this._status = { state: 'disconnected' };
|
|
220
488
|
this._synced = false;
|
|
221
489
|
this._destroying = false;
|
|
490
|
+
this._lastActivityTime = Date.now();
|
|
491
|
+
this._lastPeriodicTickTime = Date.now();
|
|
492
|
+
this._equalBeaconsHeard = 0;
|
|
493
|
+
this._beaconForced = false;
|
|
222
494
|
// BroadcastChannel state for cross-tab sync
|
|
223
495
|
this._bcChannel = '';
|
|
224
496
|
this._bcConnected = false;
|
|
@@ -237,8 +509,15 @@ export class GenericProvider extends Observable {
|
|
|
237
509
|
// own. See _requestResync().
|
|
238
510
|
this._resyncAttemptCount = 0;
|
|
239
511
|
this._lastResyncAttemptTime = 0;
|
|
240
|
-
// Rate limiting for sync
|
|
512
|
+
// Rate limiting for sync traffic - two budgets since phase 1b: one for
|
|
513
|
+
// what we ask for (beacons, pushes, syncNow), one for what we owe
|
|
514
|
+
// (SyncStep2 replies, acks). With a single shared budget a join burst's
|
|
515
|
+
// replies spent the slots a peer needed for its own recovery (measured:
|
|
516
|
+
// joiners arriving within 10 s of a burst converged in 13 s once, and a
|
|
517
|
+
// 50-peer room's periodic beacons ran at a third of their rate for 10 s
|
|
518
|
+
// after every join burst). Same size, same window, independent.
|
|
241
519
|
this._syncRequestTimes = [];
|
|
520
|
+
this._syncReplyTimes = [];
|
|
242
521
|
// SyncStep2 reply suppression (NACK-suppression style): delay a reply to
|
|
243
522
|
// a SyncStep1 request briefly, and drop it if another peer's reply is
|
|
244
523
|
// overheard first - since every reply is broadcast to the whole room
|
|
@@ -248,6 +527,105 @@ export class GenericProvider extends Observable {
|
|
|
248
527
|
// peers there's no "someone else" to rely on, so replies go out
|
|
249
528
|
// immediately as before.
|
|
250
529
|
this._pendingSyncReply = null;
|
|
530
|
+
// Whether _pendingSyncReply is a digest-beacon ack (see _handleDigest())
|
|
531
|
+
// rather than a SyncStep2. An overheard beacon with a digest equal to ours
|
|
532
|
+
// makes a pending ACK redundant (the joiner it was for has received that
|
|
533
|
+
// same beacon and is synced by it) but says nothing about a pending
|
|
534
|
+
// SyncStep2, which carries data - so only acks are cancelled on that
|
|
535
|
+
// signal. Measured in test/dummy/bench-user-scaling.ts: without this, acks
|
|
536
|
+
// were ~94% of a 50-peer join burst's messages (Task 3b in the design doc).
|
|
537
|
+
this._pendingSyncReplyIsAck = false;
|
|
538
|
+
// The requester's state vector the pending SyncStep2 answers (null for
|
|
539
|
+
// acks and for replies to plain SyncStep1s). A later request with the
|
|
540
|
+
// same state vector is the same question: the pending reply's bytes are
|
|
541
|
+
// refreshed to the current document and the timer kept, instead of the
|
|
542
|
+
// old reply being flushed as "a different request" - measured: with one
|
|
543
|
+
// peer typing while K empty peers join, keystroke and JOIN-beacon
|
|
544
|
+
// arrivals interleave at random, every keystroke in between changed the
|
|
545
|
+
// reply bytes, and each settled peer flushed up to K replies (Task 7 in
|
|
546
|
+
// the phase-1b design doc).
|
|
547
|
+
this._pendingSyncReplyTargetSv = null;
|
|
548
|
+
this._responseWaitAttempts = 0;
|
|
549
|
+
this._responseSeen = false;
|
|
550
|
+
// Phase 1e: an equal ack or equal beacon from an UNSETTLED peer arrived
|
|
551
|
+
// during the current response wait. Not a response (a settled peer with
|
|
552
|
+
// content may still answer), but evidence that the room is a fresh one
|
|
553
|
+
// whose peers are all in our state - see _armResponseWait().
|
|
554
|
+
this._equalUnsettledSeen = false;
|
|
555
|
+
this._responseWaitFlags = 0; // flags for the retry beacon: CONFIRM after a JOIN, 0 after a resync
|
|
556
|
+
this._behindSv = null;
|
|
557
|
+
// Round-trip estimate from our own requests (JOIN/resync beacon -> first
|
|
558
|
+
// SyncStep2 or ack): the minimum of the last 8 samples, because a sample
|
|
559
|
+
// includes the responder's random suppression delay and the fastest
|
|
560
|
+
// reply had the least of it. Drives _replySuppressionMaxDelay() (a
|
|
561
|
+
// suppression window shorter than the one-way latency suppresses
|
|
562
|
+
// nothing: at 250-350 ms latency every peer ahead of a requester replied
|
|
563
|
+
// before any reply could be overheard, ~20 replies per request at N=100)
|
|
564
|
+
// and _armResponseWait()'s first delay (a fixed 1 s fired premature
|
|
565
|
+
// retries on the Matrix profile). See the phase-1b design doc, 1c.
|
|
566
|
+
this._rttSamples = [];
|
|
567
|
+
this._requestSentAt = 0;
|
|
568
|
+
// See DIGEST_FLAG_SETTLED. Reset on connect (a re-joining peer is a joiner).
|
|
569
|
+
this._confirmed = false;
|
|
570
|
+
// ClientIDs we have heard from (beacon and verified-update senders, and
|
|
571
|
+
// the ids a relayed presence table names), each with the time we last
|
|
572
|
+
// heard it. The reply-suppression gate ("is there someone else who could
|
|
573
|
+
// answer?") used awareness alone, and in a join burst the awareness
|
|
574
|
+
// messages trail the beacons - so the gate was still closed exactly when
|
|
575
|
+
// 49 requests arrived at once, and every one got an immediate reply
|
|
576
|
+
// (phase-1b design, item 3). Cleared on disconnect, pruned by the lease
|
|
577
|
+
// sweep after a lease of silence (round 7, item 3: on a relay transport a
|
|
578
|
+
// reload's old clientID never says goodbye, and every one of them counted
|
|
579
|
+
// in _peerCount() forever).
|
|
580
|
+
this._knownPeers = new Map();
|
|
581
|
+
// Transport address (the `from` of Transport.onMessage) per remote
|
|
582
|
+
// clientID, learned from beacons and verified updates; lets replies, acks
|
|
583
|
+
// and presence responses go to the requester alone when the transport
|
|
584
|
+
// has sendTo (phase-1c design, item B). Cleared on disconnect, pruned
|
|
585
|
+
// with _knownPeers.
|
|
586
|
+
this._peerAddress = new Map();
|
|
587
|
+
// Requesters whose JOIN presence request the pending presence-response
|
|
588
|
+
// timer covers (see _schedulePresenceResponse).
|
|
589
|
+
this._presencePending = new Set();
|
|
590
|
+
// Phase 1e: an awareness message carrying OUR state at our current clock
|
|
591
|
+
// arrived since the presence-response timer was armed - the room's
|
|
592
|
+
// relayer (see _schedulePresenceResponse) already told the joiner about
|
|
593
|
+
// us, our own response would repeat it.
|
|
594
|
+
this._presenceCovered = false;
|
|
595
|
+
// Same NACK-style suppression as _pendingSyncReply above, applied to
|
|
596
|
+
// awareness updates that are a pure timeout-triggered removal (see
|
|
597
|
+
// _scheduleAwarenessRemoval()) - every OTHER connected peer runs its own
|
|
598
|
+
// independent 30s outdatedTimeout sweep (y-protocols/awareness.js), so
|
|
599
|
+
// one peer going silent (crash/dirty drop, not a clean disconnect())
|
|
600
|
+
// causes an O(N) simultaneous "peer X is gone" broadcast burst without
|
|
601
|
+
// this. See docs/superpowers/specs/2026-09-04-sync-optimization-round-3-ideas.md
|
|
602
|
+
// item 7 and test/dummy/bench-awareness-removal-burst.ts.
|
|
603
|
+
this._pendingAwarenessRemoval = null;
|
|
604
|
+
// Peers whose channel opened since the debounce timer was armed (the
|
|
605
|
+
// `from`/sendTo address). Round 5, item 5: on a transport with sendTo
|
|
606
|
+
// each gets a plain beacon addressed to it - it answers with what we
|
|
607
|
+
// lack and its own JOIN beacon makes us answer it - instead of the
|
|
608
|
+
// full-state push + beacon broadcast to EVERY connection that
|
|
609
|
+
// `_syncNow(0)` did (the O(N^2) mesh-join burst CLAUDE.md warns about).
|
|
610
|
+
this._pendingPeerConnectIds = new Set();
|
|
611
|
+
// Round 5, item 5: the state vector and delete-set hash the room last
|
|
612
|
+
// confirmed as equal to ours (an equal digest from any peer, see
|
|
613
|
+
// _handleDigest). A reconnect's push then carries only what we produced
|
|
614
|
+
// since - our offline edits, the one thing a push exists for (round 2:
|
|
615
|
+
// one message must survive alone) - instead of the whole document, which
|
|
616
|
+
// on a chunking transport was several messages per reconnect. A room
|
|
617
|
+
// that has been replaced meanwhile shows up behind in its own JOIN
|
|
618
|
+
// beacons and is answered like any late joiner. Null until the first
|
|
619
|
+
// confirmation: the first connect still pushes everything.
|
|
620
|
+
this._confirmedSv = null;
|
|
621
|
+
this._confirmedDsHash = null;
|
|
622
|
+
// connect() is awaiting ConnectionConfig.waitFor: doc updates are the local load.
|
|
623
|
+
this._loading = false;
|
|
624
|
+
// Cached computeDeleteSetHash(doc); null = stale. Invalidated on every
|
|
625
|
+
// doc 'update' (deletes are content changes, so this is exact; inserts
|
|
626
|
+
// invalidate needlessly but cheaply). See computeDeleteSetHash's doc for
|
|
627
|
+
// why this must not be recomputed per update.
|
|
628
|
+
this._dsHashCache = null;
|
|
251
629
|
// Sequence numbers for causal ordering
|
|
252
630
|
this._localSeqNum = 0; // Our sequence number counter
|
|
253
631
|
// Per-sender sequence tracking for reordering-tolerant gap detection.
|
|
@@ -257,41 +635,77 @@ export class GenericProvider extends Observable {
|
|
|
257
635
|
// positives from mere network reordering. See _trackRemoteSeq().
|
|
258
636
|
this._remoteSeqInfo = new Map();
|
|
259
637
|
this._gapCheckTimers = new Map();
|
|
260
|
-
// Update batching/debouncing
|
|
261
|
-
|
|
638
|
+
// Update batching/debouncing. `_batchUpdates` 0 (the default) no longer
|
|
639
|
+
// means a synchronous send from inside the Y.Doc 'update' event: the
|
|
640
|
+
// update is merged into `_pendingUpdate` and flushed at the end of the
|
|
641
|
+
// current task (a microtask - no timer, no measurable delay), so that
|
|
642
|
+
// (a) the several transactions an editor binding can emit for one input
|
|
643
|
+
// event leave as one message and (b) the cursor awareness the binding
|
|
644
|
+
// sets right after the text change in the same task rides along in the
|
|
645
|
+
// same wire message (round 5, item 1 - see _flushPendingUpdate()).
|
|
646
|
+
// `_batchTimeoutId` is set only for the timed (`batchUpdates > 0`) flush;
|
|
647
|
+
// `_flushScheduled` covers both.
|
|
648
|
+
this._batchUpdates = 0; // milliseconds delay (0 = end of current task)
|
|
649
|
+
// Origins whose updates are never sent to the transport (local-only txns).
|
|
650
|
+
this._excludeOrigins = new Set();
|
|
651
|
+
// 'pull' never pushes local state unasked - for read-mostly replicas that
|
|
652
|
+
// must not write into the room.
|
|
653
|
+
this._syncMode = 'push-pull';
|
|
262
654
|
this._pendingUpdate = null;
|
|
655
|
+
this._flushScheduled = false;
|
|
263
656
|
// Awareness throttling - prevents awareness from flooding document sync
|
|
264
|
-
this._awarenessInterval = 100; // ms between awareness broadcasts
|
|
657
|
+
this._awarenessInterval = 100; // ms between awareness broadcasts, or 'auto' (round 6, item 9)
|
|
265
658
|
this._pendingAwarenessClients = new Set();
|
|
266
659
|
this._lastAwarenessTime = 0;
|
|
267
|
-
// Independent throttle state for the app awareness channel
|
|
660
|
+
// Independent throttle state for the app awareness channel, so module
|
|
661
|
+
// cursor churn never delays or coalesces with core presence.
|
|
268
662
|
this._pendingAppAwarenessClients = new Set();
|
|
269
663
|
this._lastAppAwarenessTime = 0;
|
|
270
|
-
// Origins whose updates are never sent to the transport (local-only txns).
|
|
271
|
-
this._excludeOrigins = new Set();
|
|
272
|
-
// Connect-time sync strategy: 'push-pull' (default) sends full local state
|
|
273
|
-
// then requests remote; 'pull' only requests remote state.
|
|
274
|
-
this._syncMode = 'push-pull';
|
|
275
664
|
this.doc = doc;
|
|
276
665
|
this.transport = transport;
|
|
277
666
|
this.pubsub = new PubSubChannel(this);
|
|
278
667
|
this.awareness = options.awareness || new awarenessProtocol.Awareness(doc);
|
|
279
|
-
|
|
280
|
-
|
|
668
|
+
// Only when supplied - otherwise the getter builds it on first use.
|
|
669
|
+
if (options.appAwareness) {
|
|
670
|
+
this._appAwareness = options.appAwareness;
|
|
671
|
+
}
|
|
281
672
|
this._syncInterval = options.syncInterval ?? 5000;
|
|
282
673
|
this._verifyUpdates = options.verifyUpdates ?? true;
|
|
283
674
|
this._batchUpdates =
|
|
284
675
|
options.batchUpdates ?? transport.preferredBatchMs ?? 0;
|
|
285
676
|
this._disableBc = options.disableBc ?? false;
|
|
286
|
-
this._awarenessInterval =
|
|
677
|
+
this._awarenessInterval =
|
|
678
|
+
options.awarenessInterval ?? transport.preferredAwarenessMs ?? 100;
|
|
287
679
|
this._excludeOrigins = new Set(options.excludeOrigins ?? []);
|
|
288
680
|
this._localId = options.localId;
|
|
289
681
|
this._syncMode = options.syncMode ?? 'push-pull';
|
|
290
682
|
this._maxSyncRequestsPerWindow = options.maxSyncRequestsPerWindow ?? 20;
|
|
291
683
|
this._syncRequestWindowMs = options.syncRequestWindowMs ?? 10000;
|
|
292
684
|
this._syncReplySuppressionMs = options.syncReplySuppressionMs ?? 30;
|
|
685
|
+
this._peerConnectDebounceMs = options.peerConnectDebounceMs ?? 50;
|
|
293
686
|
this._gapGraceMs = options.gapGraceMs ?? 300;
|
|
294
687
|
this._seqWindowSize = options.seqWindowSize ?? 64;
|
|
688
|
+
// Explicit 0 disables even when the transport hints a floor.
|
|
689
|
+
this._compressionThresholdBytes =
|
|
690
|
+
(options.compressionThresholdBytes ?? transport.preferredCompressMinBytes) ||
|
|
691
|
+
undefined;
|
|
692
|
+
this._idleBackoffEnabled = options.idleBackoffEnabled ?? true;
|
|
693
|
+
this._idleBackoffMaxMs = options.idleBackoffMaxMs ?? 60000;
|
|
694
|
+
this._trickleK = options.trickleK ?? 1;
|
|
695
|
+
this._ownsAwareness = !options.awareness;
|
|
696
|
+
this._awarenessTimeoutMs =
|
|
697
|
+
options.awarenessTimeoutMs ??
|
|
698
|
+
(typeof transport.onPeerDisconnect === 'function' ? 300000 : 30000);
|
|
699
|
+
// Silence y-protocols' own 3 s sweep on an awareness we created - the
|
|
700
|
+
// lease sweep that replaces it is armed in connect() and cleared in
|
|
701
|
+
// disconnect() (round 7, item 2: it used to start here and outlive
|
|
702
|
+
// disconnect(), keeping a dropped provider alive through its timer).
|
|
703
|
+
if (this._ownsAwareness) {
|
|
704
|
+
const aw = this.awareness;
|
|
705
|
+
if (aw._checkInterval !== undefined)
|
|
706
|
+
clearInterval(aw._checkInterval);
|
|
707
|
+
}
|
|
708
|
+
this._currentSyncIntervalMs = this._syncInterval;
|
|
295
709
|
this._setupDocumentSync();
|
|
296
710
|
this._setupAwarenessSync();
|
|
297
711
|
}
|
|
@@ -317,18 +731,21 @@ export class GenericProvider extends Observable {
|
|
|
317
731
|
try {
|
|
318
732
|
// Setup BroadcastChannel for cross-tab sync (if enabled and available)
|
|
319
733
|
this._setupBroadcastChannel(config);
|
|
320
|
-
//
|
|
321
|
-
|
|
322
|
-
//
|
|
323
|
-
|
|
324
|
-
|
|
734
|
+
// Register for incoming messages and new-peer notifications BEFORE
|
|
735
|
+
// connecting the transport. Some transports (e.g. PeerJS for a
|
|
736
|
+
// joining, non-coordinator peer) establish and fully open their first
|
|
737
|
+
// connection *inside* transport.connect() itself — so a peer-connect
|
|
738
|
+
// notification or an immediate reply from the other side can arrive
|
|
739
|
+
// before that promise resolves. Registering after the await left
|
|
740
|
+
// exactly that window uncovered: whatever arrived during it was
|
|
741
|
+
// silently dropped since neither callback was wired up yet.
|
|
742
|
+
this._unsubscribeTransport = this.transport.onMessage((data, from) => {
|
|
743
|
+
this._handleIncomingMessage(data, from);
|
|
325
744
|
});
|
|
326
|
-
// When a new WebRTC peer channel opens, immediately push our full state
|
|
327
|
-
// so peers that reconnected after offline edits receive our changes.
|
|
328
745
|
if (this.transport.onPeerConnect) {
|
|
329
|
-
const unsubPeer = this.transport.onPeerConnect((
|
|
746
|
+
const unsubPeer = this.transport.onPeerConnect((peerId) => {
|
|
330
747
|
if (!this._destroying)
|
|
331
|
-
this.
|
|
748
|
+
this._schedulePeerConnectSync(peerId);
|
|
332
749
|
});
|
|
333
750
|
const originalUnsub = this._unsubscribeTransport;
|
|
334
751
|
this._unsubscribeTransport = () => {
|
|
@@ -336,31 +753,125 @@ export class GenericProvider extends Observable {
|
|
|
336
753
|
unsubPeer();
|
|
337
754
|
};
|
|
338
755
|
}
|
|
756
|
+
if (this.transport.onPeerDisconnect) {
|
|
757
|
+
const unsubLeave = this.transport.onPeerDisconnect((peerId) => {
|
|
758
|
+
if (!this._destroying)
|
|
759
|
+
this._handlePeerLeave(peerId);
|
|
760
|
+
});
|
|
761
|
+
const originalUnsub = this._unsubscribeTransport;
|
|
762
|
+
this._unsubscribeTransport = () => {
|
|
763
|
+
originalUnsub?.();
|
|
764
|
+
unsubLeave();
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
// Connect the transport
|
|
768
|
+
await this.transport.connect(config);
|
|
339
769
|
this._setStatus({ state: 'connected' });
|
|
340
|
-
|
|
341
|
-
//
|
|
342
|
-
//
|
|
343
|
-
//
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
this._sendSyncStep1();
|
|
770
|
+
this._startAwarenessSweep();
|
|
771
|
+
// Seed the round-trip estimate from the transport's hint (see
|
|
772
|
+
// Transport.expectedRttMs); the minimum-of-8 rule lets real samples
|
|
773
|
+
// take over as soon as they arrive.
|
|
774
|
+
if (this.transport.expectedRttMs) {
|
|
775
|
+
this._rttSamples = [this.transport.expectedRttMs];
|
|
347
776
|
}
|
|
348
|
-
|
|
349
|
-
|
|
777
|
+
// Persistence first (round 5, item 7): let a local copy load before
|
|
778
|
+
// the first beacon says what we have. While it loads, the doc
|
|
779
|
+
// updates it produces are the load, not edits - they are not
|
|
780
|
+
// broadcast - and afterwards the loaded state counts as confirmed:
|
|
781
|
+
// no full-state push. The beacon reconciles: peers behind us (our
|
|
782
|
+
// offline edits) see themselves behind and ask; peers ahead of us
|
|
783
|
+
// answer. Measured in bench-reconnect-push part 2: a 50 KB copy that
|
|
784
|
+
// loaded 100 ms after connect cost the room a 50 KB SyncStep2, and a
|
|
785
|
+
// first version of waitFor that only delayed the beacon cost 100 KB
|
|
786
|
+
// (the load's broadcast plus the push). See ConnectionConfig.waitFor.
|
|
787
|
+
if (config.waitFor) {
|
|
788
|
+
this._loading = true;
|
|
789
|
+
try {
|
|
790
|
+
await config.waitFor;
|
|
791
|
+
}
|
|
792
|
+
catch {
|
|
793
|
+
// The local load failed; sync from the room as if there were none.
|
|
794
|
+
}
|
|
795
|
+
this._loading = false;
|
|
796
|
+
if (this._destroying || !this.transport.isConnected)
|
|
797
|
+
return;
|
|
798
|
+
this._confirmedSv = Y.encodeStateVector(this.doc);
|
|
799
|
+
this._confirmedDsHash = this._deleteSetHash();
|
|
350
800
|
}
|
|
351
|
-
//
|
|
352
|
-
|
|
353
|
-
this
|
|
801
|
+
// Send initial sync pushing our local state plus requesting remote state.
|
|
802
|
+
// syncNow() is used instead of _sendSyncStep1() so that any offline edits
|
|
803
|
+
// made before this connect() call are pushed to currently-connected peers
|
|
804
|
+
// (e.g. same-browser tabs via BroadcastChannel).
|
|
805
|
+
this.syncNow();
|
|
806
|
+
// (Local awareness goes out inside syncNow()'s batch, or via its
|
|
807
|
+
// throttled fallback - a second broadcast here was a duplicate 100ms
|
|
808
|
+
// later.)
|
|
809
|
+
// The app channel has no equivalent path into syncNow()'s batch, so
|
|
810
|
+
// its local state is announced here when there is any.
|
|
811
|
+
this._announceAppAwareness();
|
|
354
812
|
// Start periodic sync to handle packet loss
|
|
355
813
|
// Just request sync without sending full state (avoid redundant broadcasts)
|
|
356
814
|
// _sendSyncStep1() already checks the shared rate limiter internally
|
|
357
815
|
// and silently drops the request if it's exceeded.
|
|
816
|
+
//
|
|
817
|
+
// Uses a recursive setTimeout (re-jittered by ~20% each tick) rather
|
|
818
|
+
// than a plain setInterval so peers that connect() within a short
|
|
819
|
+
// window of each other - the common case: everyone joining a room
|
|
820
|
+
// near session start, or reconnecting together after a shared network
|
|
821
|
+
// blip - don't end up with near-synchronized periodic timers that all
|
|
822
|
+
// fire in the same few milliseconds every syncInterval. This doesn't
|
|
823
|
+
// reduce total periodic-sync traffic, only spreads it out so a room's
|
|
824
|
+
// background traffic is smooth instead of bursty.
|
|
358
825
|
if (this._syncInterval > 0) {
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
826
|
+
// Reset idle-backoff state on every (re)connect so a reconnect
|
|
827
|
+
// always starts its first tick at the base interval, never
|
|
828
|
+
// inheriting a backed-off value left over from a previous session.
|
|
829
|
+
this._currentSyncIntervalMs = this._syncInterval;
|
|
830
|
+
this._lastActivityTime = Date.now();
|
|
831
|
+
this._lastPeriodicTickTime = Date.now();
|
|
832
|
+
this._equalBeaconsHeard = 0;
|
|
833
|
+
this._beaconForced = false;
|
|
834
|
+
const scheduleNextPeriodicSync = (delayMs) => {
|
|
835
|
+
this._syncIntervalId = setTimeout(() => {
|
|
836
|
+
const tickTime = Date.now();
|
|
837
|
+
if (this._idleBackoffEnabled) {
|
|
838
|
+
// "Activity" = anything _markActivity() call sites observed
|
|
839
|
+
// (incoming message via transport or BroadcastChannel, local
|
|
840
|
+
// or remote doc change, local or remote awareness change)
|
|
841
|
+
// since the LAST tick fired - not since backoff started, so a
|
|
842
|
+
// single quiet tick after a burst of activity still resets to
|
|
843
|
+
// base rather than needing a full quiet cycle to catch up.
|
|
844
|
+
const hadActivitySinceLastTick = this._lastActivityTime > this._lastPeriodicTickTime;
|
|
845
|
+
this._currentSyncIntervalMs = hadActivitySinceLastTick
|
|
846
|
+
? this._syncInterval
|
|
847
|
+
: Math.min(this._idleBackoffMaxMs, this._currentSyncIntervalMs * 2);
|
|
848
|
+
}
|
|
849
|
+
this._lastPeriodicTickTime = tickTime;
|
|
850
|
+
if (this.transport.isConnected && !this._destroying) {
|
|
851
|
+
// Beacon only. Presence is no longer re-announced per tick on
|
|
852
|
+
// any transport: a joiner requests it via DIGEST_FLAG_JOIN
|
|
853
|
+
// (see _handleDigest()), and y-protocols/awareness renews the
|
|
854
|
+
// local state itself every outdatedTimeout/2 = 15s
|
|
855
|
+
// (awareness.js _checkInterval), which the awareness update
|
|
856
|
+
// handler broadcasts. Measured in
|
|
857
|
+
// test/dummy/bench-idle-room.ts: the per-tick re-announce was
|
|
858
|
+
// ~40% of an idle room's deliveries.
|
|
859
|
+
// Trickle (round 5, item 3): silent if enough equal beacons
|
|
860
|
+
// were overheard since the last tick - see _equalBeaconsHeard.
|
|
861
|
+
const suppressed = this._trickleK > 0 &&
|
|
862
|
+
!this._beaconForced &&
|
|
863
|
+
this._equalBeaconsHeard >= this._trickleK;
|
|
864
|
+
if (!suppressed)
|
|
865
|
+
this._sendSyncStep1();
|
|
866
|
+
this._equalBeaconsHeard = 0;
|
|
867
|
+
this._beaconForced = false;
|
|
868
|
+
}
|
|
869
|
+
if (!this._destroying)
|
|
870
|
+
scheduleNextPeriodicSync();
|
|
871
|
+
}, delayMs ?? this._jitteredSyncInterval());
|
|
872
|
+
};
|
|
873
|
+
this._periodicScheduler = scheduleNextPeriodicSync;
|
|
874
|
+
scheduleNextPeriodicSync();
|
|
364
875
|
}
|
|
365
876
|
}
|
|
366
877
|
catch (error) {
|
|
@@ -378,9 +889,15 @@ export class GenericProvider extends Observable {
|
|
|
378
889
|
disconnect() {
|
|
379
890
|
// Stop periodic sync
|
|
380
891
|
if (this._syncIntervalId !== undefined) {
|
|
381
|
-
|
|
892
|
+
clearTimeout(this._syncIntervalId);
|
|
382
893
|
this._syncIntervalId = undefined;
|
|
383
894
|
}
|
|
895
|
+
this._periodicScheduler = undefined;
|
|
896
|
+
// Stop the awareness lease sweep (armed in connect(); round 7, item 2).
|
|
897
|
+
if (this._awarenessSweepId !== undefined) {
|
|
898
|
+
clearTimeout(this._awarenessSweepId);
|
|
899
|
+
this._awarenessSweepId = undefined;
|
|
900
|
+
}
|
|
384
901
|
// Reset resync escalation tracking
|
|
385
902
|
this._resyncAttemptCount = 0;
|
|
386
903
|
this._lastResyncAttemptTime = 0;
|
|
@@ -391,26 +908,73 @@ export class GenericProvider extends Observable {
|
|
|
391
908
|
clearTimeout(this._pendingResyncTimeoutId);
|
|
392
909
|
this._pendingResyncTimeoutId = undefined;
|
|
393
910
|
}
|
|
911
|
+
// Stop any pending gap-check timers and forget per-sender sequence
|
|
912
|
+
// tracking. Without this, a gap-check timer armed before this
|
|
913
|
+
// disconnect() keeps running in the background and can fire
|
|
914
|
+
// _requestResync() after reconnect using sequence-number bookkeeping
|
|
915
|
+
// from the PREVIOUS connection - a spurious resync race disconnected
|
|
916
|
+
// from anything actually missing in the new session. Mirrors the
|
|
917
|
+
// equivalent cleanup in destroy().
|
|
918
|
+
for (const timer of this._gapCheckTimers.values()) {
|
|
919
|
+
clearTimeout(timer);
|
|
920
|
+
}
|
|
921
|
+
this._gapCheckTimers.clear();
|
|
922
|
+
this._remoteSeqInfo.clear();
|
|
923
|
+
// Cancel any pending debounced onPeerConnect sync - a reconnect gets a
|
|
924
|
+
// fresh burst of onPeerConnect events (if the transport supports it) and
|
|
925
|
+
// shouldn't fire a stale one left over from before this disconnect.
|
|
926
|
+
if (this._pendingPeerConnectSyncTimeoutId !== undefined) {
|
|
927
|
+
clearTimeout(this._pendingPeerConnectSyncTimeoutId);
|
|
928
|
+
this._pendingPeerConnectSyncTimeoutId = undefined;
|
|
929
|
+
}
|
|
930
|
+
this._pendingPeerConnectIds.clear();
|
|
394
931
|
// Reset the sync rate-limit budget. Without this, a reconnect inherits
|
|
395
932
|
// whatever budget was left over from before the disconnect - and since
|
|
396
933
|
// syncNow()'s full-state push now shares this same limiter (see
|
|
397
934
|
// _tryReserveSyncSlot()), a rate-limited reconnect could silently skip
|
|
398
935
|
// the very push that delivers edits made while offline.
|
|
399
936
|
this._syncRequestTimes = [];
|
|
937
|
+
this._syncReplyTimes = [];
|
|
938
|
+
this._knownPeers.clear();
|
|
939
|
+
this._peerAddress.clear();
|
|
940
|
+
this._presencePending.clear();
|
|
941
|
+
// A pending response-wait belongs to a request on the old connection.
|
|
942
|
+
if (this._responseWaitTimer !== undefined) {
|
|
943
|
+
clearTimeout(this._responseWaitTimer);
|
|
944
|
+
this._responseWaitTimer = undefined;
|
|
945
|
+
}
|
|
946
|
+
if (this._pendingCheckTimer !== undefined) {
|
|
947
|
+
clearTimeout(this._pendingCheckTimer);
|
|
948
|
+
this._pendingCheckTimer = undefined;
|
|
949
|
+
}
|
|
950
|
+
if (this._behindCheckTimer !== undefined) {
|
|
951
|
+
clearTimeout(this._behindCheckTimer);
|
|
952
|
+
this._behindCheckTimer = undefined;
|
|
953
|
+
}
|
|
954
|
+
this._behindSv = null;
|
|
955
|
+
if (this._presenceResponseTimer !== undefined) {
|
|
956
|
+
clearTimeout(this._presenceResponseTimer);
|
|
957
|
+
this._presenceResponseTimer = undefined;
|
|
958
|
+
}
|
|
959
|
+
this._responseWaitAttempts = 0;
|
|
960
|
+
this._responseSeen = false;
|
|
961
|
+
this._equalUnsettledSeen = false;
|
|
962
|
+
this._rttSamples = [];
|
|
963
|
+
this._requestSentAt = 0;
|
|
964
|
+
this._confirmed = false;
|
|
400
965
|
// Drop any pending suppressed sync reply - safe to simply discard (not
|
|
401
966
|
// flush/send like batched updates/awareness below), since a suppressed
|
|
402
967
|
// reply is by design redundant with whatever the room already has.
|
|
403
968
|
this._cancelPendingSyncReply();
|
|
404
|
-
//
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
}
|
|
969
|
+
// Same reasoning for a pending suppressed awareness-removal broadcast -
|
|
970
|
+
// every other surviving peer is independently running the same
|
|
971
|
+
// suppression for the same departure, so dropping ours on disconnect
|
|
972
|
+
// (rather than flushing it through a transport that's about to go
|
|
973
|
+
// down) is safe.
|
|
974
|
+
this._cancelPendingAwarenessRemoval();
|
|
975
|
+
// Flush any pending batched updates before disconnecting (sent only if
|
|
976
|
+
// the transport is still connected; dropped otherwise, as before)
|
|
977
|
+
this._flushPendingUpdate();
|
|
414
978
|
// Flush pending awareness updates before disconnecting
|
|
415
979
|
if (this._awarenessTimeoutId !== undefined) {
|
|
416
980
|
clearTimeout(this._awarenessTimeoutId);
|
|
@@ -423,13 +987,13 @@ export class GenericProvider extends Observable {
|
|
|
423
987
|
}
|
|
424
988
|
}
|
|
425
989
|
this._pendingAwarenessClients.clear();
|
|
426
|
-
//
|
|
990
|
+
// Same for the app channel's independent throttle.
|
|
427
991
|
if (this._appAwarenessTimeoutId !== undefined) {
|
|
428
992
|
clearTimeout(this._appAwarenessTimeoutId);
|
|
429
993
|
this._appAwarenessTimeoutId = undefined;
|
|
430
994
|
if (this._pendingAppAwarenessClients.size > 0 &&
|
|
431
995
|
this.transport.isConnected) {
|
|
432
|
-
this.
|
|
996
|
+
this._sendAppAwarenessNow(Array.from(this._pendingAppAwarenessClients));
|
|
433
997
|
}
|
|
434
998
|
}
|
|
435
999
|
this._pendingAppAwarenessClients.clear();
|
|
@@ -441,7 +1005,6 @@ export class GenericProvider extends Observable {
|
|
|
441
1005
|
}
|
|
442
1006
|
// Mark local client as offline in awareness
|
|
443
1007
|
awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'disconnect');
|
|
444
|
-
awarenessProtocol.removeAwarenessStates(this.appAwareness, [this.doc.clientID], 'disconnect');
|
|
445
1008
|
this.transport.disconnect();
|
|
446
1009
|
this._synced = false;
|
|
447
1010
|
this._setStatus({ state: 'disconnected' });
|
|
@@ -454,7 +1017,7 @@ export class GenericProvider extends Observable {
|
|
|
454
1017
|
this._destroying = true;
|
|
455
1018
|
// Stop periodic sync (disconnect() will also do this, but be explicit)
|
|
456
1019
|
if (this._syncIntervalId !== undefined) {
|
|
457
|
-
|
|
1020
|
+
clearTimeout(this._syncIntervalId);
|
|
458
1021
|
this._syncIntervalId = undefined;
|
|
459
1022
|
}
|
|
460
1023
|
// Stop any pending gap-check timers
|
|
@@ -465,38 +1028,42 @@ export class GenericProvider extends Observable {
|
|
|
465
1028
|
// Drop any pending suppressed sync reply (disconnect() will also do
|
|
466
1029
|
// this, but be explicit)
|
|
467
1030
|
this._cancelPendingSyncReply();
|
|
1031
|
+
// Same, for a pending suppressed awareness-removal broadcast
|
|
1032
|
+
// (disconnect() will also do this, but be explicit)
|
|
1033
|
+
this._cancelPendingAwarenessRemoval();
|
|
468
1034
|
// Flush any pending batched updates before destroying
|
|
469
|
-
|
|
470
|
-
clearTimeout(this._batchTimeoutId);
|
|
471
|
-
this._batchTimeoutId = undefined;
|
|
472
|
-
// Send pending update if transport is still connected
|
|
473
|
-
if (this._pendingUpdate && this.transport.isConnected) {
|
|
474
|
-
this._sendUpdate(this._pendingUpdate);
|
|
475
|
-
}
|
|
476
|
-
this._pendingUpdate = null;
|
|
477
|
-
}
|
|
1035
|
+
this._flushPendingUpdate();
|
|
478
1036
|
this.disconnect();
|
|
479
1037
|
// Remove document update listener
|
|
480
1038
|
if (this._updateHandler) {
|
|
481
1039
|
this.doc.off('update', this._updateHandler);
|
|
482
1040
|
this._updateHandler = undefined;
|
|
483
1041
|
}
|
|
484
|
-
// Remove awareness update
|
|
1042
|
+
// Remove awareness update listener
|
|
485
1043
|
if (this._awarenessUpdateHandler) {
|
|
486
|
-
this.awareness.off('update', this._awarenessUpdateHandler);
|
|
1044
|
+
this.awareness.off(this._ownsAwareness ? 'change' : 'update', this._awarenessUpdateHandler);
|
|
487
1045
|
this._awarenessUpdateHandler = undefined;
|
|
488
1046
|
}
|
|
489
1047
|
if (this._appAwarenessUpdateHandler) {
|
|
490
|
-
this.
|
|
1048
|
+
this._appAwareness?.off('update', this._appAwarenessUpdateHandler);
|
|
491
1049
|
this._appAwarenessUpdateHandler = undefined;
|
|
492
1050
|
}
|
|
1051
|
+
if (this._appAwarenessTimeoutId !== undefined) {
|
|
1052
|
+
clearTimeout(this._appAwarenessTimeoutId);
|
|
1053
|
+
this._appAwarenessTimeoutId = undefined;
|
|
1054
|
+
}
|
|
1055
|
+
if (this._awarenessSweepId !== undefined) {
|
|
1056
|
+
clearTimeout(this._awarenessSweepId);
|
|
1057
|
+
this._awarenessSweepId = undefined;
|
|
1058
|
+
}
|
|
493
1059
|
// Remove beforeunload handler
|
|
494
1060
|
if (this._beforeUnloadHandler && typeof window !== 'undefined') {
|
|
495
1061
|
window.removeEventListener('beforeunload', this._beforeUnloadHandler);
|
|
496
1062
|
this._beforeUnloadHandler = undefined;
|
|
497
1063
|
}
|
|
498
1064
|
this.awareness.destroy();
|
|
499
|
-
|
|
1065
|
+
// Only if one was ever built - never construct one just to destroy it.
|
|
1066
|
+
this._appAwareness?.destroy();
|
|
500
1067
|
super.destroy();
|
|
501
1068
|
}
|
|
502
1069
|
/**
|
|
@@ -524,14 +1091,29 @@ export class GenericProvider extends Observable {
|
|
|
524
1091
|
return this._synced;
|
|
525
1092
|
}
|
|
526
1093
|
/**
|
|
527
|
-
*
|
|
528
|
-
*
|
|
1094
|
+
* Push local state + request remote state, gated by the shared rate
|
|
1095
|
+
* limiter. Returns whether it actually reserved a slot and sent anything
|
|
1096
|
+
* - `false` means the caller was rate-limited right now. Extracted out of
|
|
1097
|
+
* `syncNow()` so `_requestResync()`'s scheduled retry (see below) can tell
|
|
1098
|
+
* the difference between "sent" and "silently skipped" and react to it,
|
|
1099
|
+
* instead of assuming a resync always succeeds once it fires.
|
|
1100
|
+
*
|
|
1101
|
+
* @param push - whether to also broadcast full local document state.
|
|
1102
|
+
* `_requestResync()`'s retry passes `false` (pull-only is enough for a
|
|
1103
|
+
* resync trigger - see its call site).
|
|
1104
|
+
* @param buildExtra - optional callback, invoked ONLY once a rate-limit
|
|
1105
|
+
* slot is actually reserved (so it never runs, and never mutates
|
|
1106
|
+
* whatever state it touches, on a call that ends up rate-limited),
|
|
1107
|
+
* returning additional already-encoded sub-messages to fold into the SAME
|
|
1108
|
+
* batched wire send as the push/pull messages below - e.g. an awareness
|
|
1109
|
+
* update that's ready to go out "now" anyway (see
|
|
1110
|
+
* `_tryImmediateAwarenessMessage()`). Pure wire-framing: whether a caller
|
|
1111
|
+
* passes this never changes whether/when the push+pull half itself sends,
|
|
1112
|
+
* only how many separate `transport.send()`/`bc.publish()` calls it costs.
|
|
1113
|
+
* @param flags - digest beacon flags: DIGEST_FLAG_JOIN from syncNow(), 0
|
|
1114
|
+
* from the peer-connect debounce and the resync retry (see _syncNow()).
|
|
529
1115
|
*/
|
|
530
|
-
|
|
531
|
-
if (!this.transport.isConnected) {
|
|
532
|
-
console.warn('Cannot sync: transport not connected');
|
|
533
|
-
return;
|
|
534
|
-
}
|
|
1116
|
+
_trySyncPushPull(push = true, buildExtra, flags = 0) {
|
|
535
1117
|
// Push (full document state) and pull (SyncStep1 request) share a
|
|
536
1118
|
// single rate-limit reservation. syncNow() is called from several
|
|
537
1119
|
// triggers that can all fire in a short window when many peers are
|
|
@@ -543,21 +1125,315 @@ export class GenericProvider extends Observable {
|
|
|
543
1125
|
// test/dummy/bench-user-scaling.ts: at 100 simulated users this drove
|
|
544
1126
|
// message counts to 20-200x the theoretical linear cost. See
|
|
545
1127
|
// docs/superpowers/specs/2026-07-26-dummy-benchmark-scaling-design.md.
|
|
546
|
-
if (this._tryReserveSyncSlot())
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
1128
|
+
if (!this._tryReserveSyncSlot())
|
|
1129
|
+
return false;
|
|
1130
|
+
const messages = [];
|
|
1131
|
+
// Send our document state to all peers - everything on the first
|
|
1132
|
+
// connect, only what we produced since the room last confirmed our
|
|
1133
|
+
// state afterwards (see _confirmedSv). This is what carries edits made
|
|
1134
|
+
// while offline.
|
|
1135
|
+
if (push) {
|
|
1136
|
+
const update = Y.encodeStateAsUpdate(this.doc, this._confirmedSv ?? undefined);
|
|
1137
|
+
let worthPushing = update.length > 0;
|
|
1138
|
+
if (worthPushing && this._confirmedSv !== null) {
|
|
1139
|
+
// A diff against a confirmed state is never byte-empty (it always
|
|
1140
|
+
// carries the delete set): push it only if it holds structs, or
|
|
1141
|
+
// deletes the room has not confirmed.
|
|
1142
|
+
try {
|
|
1143
|
+
worthPushing =
|
|
1144
|
+
Y.parseUpdateMeta(update).to.size > 0 ||
|
|
1145
|
+
this._deleteSetHash() !== this._confirmedDsHash;
|
|
1146
|
+
}
|
|
1147
|
+
catch {
|
|
1148
|
+
worthPushing = true;
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
if (worthPushing) {
|
|
1152
|
+
messages.push(this._encodePush(update));
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
// Send sync request to get updates from others
|
|
1156
|
+
messages.push(this._encodeSyncStep1(flags));
|
|
1157
|
+
if (buildExtra) {
|
|
1158
|
+
messages.push(...buildExtra());
|
|
1159
|
+
}
|
|
1160
|
+
// Batched into one wire message (MESSAGE_BATCH) instead of one
|
|
1161
|
+
// transport.send()/bc.publish() call per sub-message - see _sendBatch().
|
|
1162
|
+
this._sendBatch(messages);
|
|
1163
|
+
if (flags & DIGEST_FLAG_JOIN)
|
|
1164
|
+
this._armResponseWait(DIGEST_FLAG_CONFIRM);
|
|
1165
|
+
return true;
|
|
1166
|
+
}
|
|
1167
|
+
/**
|
|
1168
|
+
* Force an immediate sync with remote peers.
|
|
1169
|
+
* Useful after network interruptions or to manually trigger re-sync.
|
|
1170
|
+
* Sends the beacon with DIGEST_FLAG_JOIN: peers answer with their
|
|
1171
|
+
* presence and, if our state already matches theirs, with an ack beacon
|
|
1172
|
+
* so `synced` flips without a data round trip.
|
|
1173
|
+
*/
|
|
1174
|
+
syncNow() {
|
|
1175
|
+
this._syncNow(DIGEST_FLAG_JOIN);
|
|
1176
|
+
}
|
|
1177
|
+
/**
|
|
1178
|
+
* syncNow() body. `flags` = 0 for callers that must NOT request presence:
|
|
1179
|
+
* `_schedulePeerConnectSync()` (mesh transports already re-broadcast
|
|
1180
|
+
* presence to a newcomer via their own onPeerConnect -> syncNow()).
|
|
1181
|
+
*/
|
|
1182
|
+
_syncNow(flags) {
|
|
1183
|
+
if (!this.transport.isConnected) {
|
|
1184
|
+
console.warn('Cannot sync: transport not connected');
|
|
1185
|
+
return;
|
|
1186
|
+
}
|
|
1187
|
+
if (flags & DIGEST_FLAG_JOIN)
|
|
1188
|
+
this._confirmed = false; // a (re-)joiner until answered
|
|
1189
|
+
// Try to fold the awareness broadcast into the same wire send as the
|
|
1190
|
+
// sync push+pull below. _tryImmediateAwarenessMessage() only returns
|
|
1191
|
+
// non-null (and only mutates awareness-throttle state) when the
|
|
1192
|
+
// throttle would have let an immediate send through anyway - so this
|
|
1193
|
+
// never changes awareness throttle semantics, only whether it travels
|
|
1194
|
+
// as its own message or bundled with the sync message going out "now"
|
|
1195
|
+
// too. Built inside buildExtra so it's only even attempted once a sync
|
|
1196
|
+
// rate-limit slot is confirmed reserved (see _trySyncPushPull's doc).
|
|
1197
|
+
let awarenessBatched = false;
|
|
1198
|
+
const sent = this._trySyncPushPull(
|
|
1199
|
+
// 'pull': send the beacon but never the full-state push. A relay or
|
|
1200
|
+
// server holds authoritative state and we adopt it rather than pushing
|
|
1201
|
+
// a competing local copy on every (re)connect. Peers that are actually
|
|
1202
|
+
// behind us still ask, and we still answer - the beacon reconciles.
|
|
1203
|
+
this._syncMode !== 'pull', () => {
|
|
1204
|
+
const msg = this._tryImmediateAwarenessMessage([this.doc.clientID]);
|
|
1205
|
+
if (msg) {
|
|
1206
|
+
awarenessBatched = true;
|
|
1207
|
+
return [msg];
|
|
1208
|
+
}
|
|
1209
|
+
return [];
|
|
1210
|
+
}, flags);
|
|
1211
|
+
// Awareness broadcasting is independently throttled and explicitly NOT
|
|
1212
|
+
// gated by the sync rate limiter above - preserve that exactly: it
|
|
1213
|
+
// always ends up broadcast one way or another (batched above, or via
|
|
1214
|
+
// its own throttled path here), regardless of whether the sync half
|
|
1215
|
+
// above was rate-limited.
|
|
1216
|
+
if (!sent || !awarenessBatched) {
|
|
1217
|
+
this._broadcastAwareness([this.doc.clientID]);
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
/**
|
|
1221
|
+
* Compute the next periodic-sync delay, jittered by ~+/-20% around
|
|
1222
|
+
* `_currentSyncIntervalMs` (== `_syncInterval` unless `idleBackoffEnabled`
|
|
1223
|
+
* has backed it off - see that option's doc comment). Re-jittered fresh
|
|
1224
|
+
* each tick (not computed once per connect()) so a room's peers - which
|
|
1225
|
+
* commonly all connect() within a short window of each other - drift
|
|
1226
|
+
* apart over time instead of staying loosely synchronized. Extracted to
|
|
1227
|
+
* its own method purely so benchmarks can shadow it to compare against
|
|
1228
|
+
* the unjittered baseline.
|
|
1229
|
+
*/
|
|
1230
|
+
_jitteredSyncInterval() {
|
|
1231
|
+
const jitter = 1 + (Math.random() * 2 - 1) * 0.2; // +/-20%
|
|
1232
|
+
return this._currentSyncIntervalMs * jitter;
|
|
1233
|
+
}
|
|
1234
|
+
/**
|
|
1235
|
+
* Record that "activity" happened right now, for `idleBackoffEnabled`'s
|
|
1236
|
+
* benefit. Cheap (one timestamp write) and called unconditionally
|
|
1237
|
+
* regardless of whether idle backoff is enabled, so there's no behavioral
|
|
1238
|
+
* branch to keep in sync - the backoff decision in connect()'s periodic
|
|
1239
|
+
* tick is the only place that actually reads this.
|
|
1240
|
+
*
|
|
1241
|
+
* Call sites are deliberately NOT "any inbound wire message" - an earlier
|
|
1242
|
+
* version of this hooked `_handleIncomingMessage()` unconditionally, which
|
|
1243
|
+
* made the periodic tick's OWN SyncStep1 request and the SyncStep2 reply
|
|
1244
|
+
* answering it (empty payload - nothing to sync) each count as "activity",
|
|
1245
|
+
* permanently resetting the backoff on every single tick and making the
|
|
1246
|
+
* whole feature a no-op (caught by this bench script's own first run: ON
|
|
1247
|
+
* and OFF produced statistically indistinguishable message counts). Both
|
|
1248
|
+
* Yjs's `doc.emit('update', ...)` and y-protocols' `awareness.emit('update', ...)`
|
|
1249
|
+
* already only fire when something with actual content changed
|
|
1250
|
+
* (`hasContent`/non-empty added+updated+removed - confirmed by reading
|
|
1251
|
+
* yjs's `Transaction.js` and y-protocols' `awareness.js` directly), so
|
|
1252
|
+
* hooking THOSE instead is exactly "local or remote document/awareness
|
|
1253
|
+
* change" with no extra filtering needed - a no-op SyncStep2 reply, a
|
|
1254
|
+
* digest beacon, or a duplicate/no-change awareness re-announce (e.g. a
|
|
1255
|
+
* JOIN-triggered presence response that changed nothing) never reaches
|
|
1256
|
+
* these handlers. A corrupted (CRC32
|
|
1257
|
+
* mismatch) message is real evidence of wire activity that neither
|
|
1258
|
+
* handler would ever see (it's rejected before decoding) - see the
|
|
1259
|
+
* explicit call in `_processWrappedMessage()`'s corruption branch.
|
|
1260
|
+
*
|
|
1261
|
+
* Call sites: `_setupDocumentSync()`'s update handler (LOCAL document
|
|
1262
|
+
* edits only, since phase 1e) and `_processWrappedMessage()`'s
|
|
1263
|
+
* corrupted-message branch (wire noise, not silence).
|
|
1264
|
+
*/
|
|
1265
|
+
_markActivity() {
|
|
1266
|
+
this._lastActivityTime = Date.now();
|
|
1267
|
+
// A backed-off periodic timer is re-armed at the base interval NOW, not
|
|
1268
|
+
// at its next tick (which may be idleBackoffMaxMs away). Design D of
|
|
1269
|
+
// the phase-1d doc rests on it: the peer that just had activity beacons
|
|
1270
|
+
// within one base interval, and a peer that lost that activity's
|
|
1271
|
+
// message learns from that beacon that it is behind (design A) - its
|
|
1272
|
+
// own backed-off interval no longer bounds the recovery. Measured in
|
|
1273
|
+
// test/dummy/bench-idle-backoff.ts. At most once per idle stretch.
|
|
1274
|
+
if (this._idleBackoffEnabled &&
|
|
1275
|
+
this._currentSyncIntervalMs !== this._syncInterval &&
|
|
1276
|
+
this._syncIntervalId !== undefined &&
|
|
1277
|
+
this._periodicScheduler !== undefined) {
|
|
1278
|
+
clearTimeout(this._syncIntervalId);
|
|
1279
|
+
this._currentSyncIntervalMs = this._syncInterval;
|
|
1280
|
+
// Phase 1e: at a random point inside the base interval, not a full
|
|
1281
|
+
// one (the phase-1d note): the loser's recovery chain starts with
|
|
1282
|
+
// this beacon - so Trickle never suppresses it (round 5, item 3).
|
|
1283
|
+
this._beaconForced = true;
|
|
1284
|
+
this._periodicScheduler(Math.random() * this._syncInterval);
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
/**
|
|
1288
|
+
* The lease sweep: y-protocols' own (awareness.js `_checkInterval`: renew
|
|
1289
|
+
* at outdatedTimeout/2, remove at outdatedTimeout, every
|
|
1290
|
+
* outdatedTimeout/10) replaced by the same loop at `_awarenessTimeoutMs`,
|
|
1291
|
+
* the period jittered so a room that joined together does not renew in
|
|
1292
|
+
* one burst (measured: all 49 listeners of a 50-peer room renewed inside
|
|
1293
|
+
* the same 10 s window). The renew/remove half runs only on an awareness
|
|
1294
|
+
* we created (`_ownsAwareness`); the peer-table prune (round 7, item 3)
|
|
1295
|
+
* runs regardless. Armed by connect(), cleared by disconnect() - round 7,
|
|
1296
|
+
* item 2: started from the constructor it outlived disconnect(), ticking
|
|
1297
|
+
* ~20 times a minute and keeping the dropped provider reachable
|
|
1298
|
+
* (test/dummy/bench-reload-phantoms.ts, part 2).
|
|
1299
|
+
*/
|
|
1300
|
+
_startAwarenessSweep() {
|
|
1301
|
+
if (this._awarenessSweepId !== undefined)
|
|
1302
|
+
return;
|
|
1303
|
+
const lease = this._awarenessTimeoutMs;
|
|
1304
|
+
const arm = () => {
|
|
1305
|
+
this._awarenessSweepId = setTimeout(tick, (lease / 10) * (0.8 + Math.random() * 0.4));
|
|
1306
|
+
};
|
|
1307
|
+
const tick = () => {
|
|
1308
|
+
const now = Date.now();
|
|
1309
|
+
if (this._ownsAwareness) {
|
|
1310
|
+
const mine = this.awareness.meta.get(this.doc.clientID);
|
|
1311
|
+
if (this.awareness.getLocalState() !== null &&
|
|
1312
|
+
mine !== undefined &&
|
|
1313
|
+
lease / 2 <= now - mine.lastUpdated) {
|
|
1314
|
+
this.awareness.setLocalState(this.awareness.getLocalState()); // renew: bumps the clock
|
|
1315
|
+
this._broadcastAwareness([this.doc.clientID]); // 'change' does not fire for an equal state (item 8)
|
|
1316
|
+
}
|
|
1317
|
+
const remove = [];
|
|
1318
|
+
this.awareness.meta.forEach((meta, clientID) => {
|
|
1319
|
+
if (clientID !== this.doc.clientID &&
|
|
1320
|
+
lease <= now - meta.lastUpdated &&
|
|
1321
|
+
this.awareness.getStates().has(clientID)) {
|
|
1322
|
+
remove.push(clientID);
|
|
1323
|
+
}
|
|
1324
|
+
});
|
|
1325
|
+
if (remove.length > 0) {
|
|
1326
|
+
awarenessProtocol.removeAwarenessStates(this.awareness, remove, 'timeout');
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
// Round 7, item 3: forget peers not heard from for a lease. A live
|
|
1330
|
+
// peer is heard at least every lease/2 through its presence renewal
|
|
1331
|
+
// (every receiver scans it, above at MESSAGE_AWARENESS); a pruned
|
|
1332
|
+
// peer that speaks again is simply learned again. Measured before
|
|
1333
|
+
// this: 30 reloads in a 20-peer relay room left 49 known peers for
|
|
1334
|
+
// good, and every cursor moved at a 50-peer room's 'auto' interval
|
|
1335
|
+
// (test/dummy/bench-reload-phantoms.ts). awareness.meta is left to
|
|
1336
|
+
// y-protocols: it holds the clock a late message is checked against.
|
|
1337
|
+
for (const [id, heardAt] of this._knownPeers) {
|
|
1338
|
+
if (lease <= now - heardAt) {
|
|
1339
|
+
this._knownPeers.delete(id);
|
|
1340
|
+
this._peerAddress.delete(id);
|
|
1341
|
+
this._remoteSeqInfo.delete(id);
|
|
1342
|
+
}
|
|
552
1343
|
}
|
|
553
|
-
//
|
|
554
|
-
|
|
1344
|
+
// Re-arm only while connected: a disconnect() from inside a listener
|
|
1345
|
+
// above has just cleared the timer, and must stay cleared.
|
|
1346
|
+
if (!this._destroying && this._status.state === 'connected')
|
|
1347
|
+
arm();
|
|
1348
|
+
else
|
|
1349
|
+
this._awarenessSweepId = undefined;
|
|
1350
|
+
};
|
|
1351
|
+
arm();
|
|
1352
|
+
}
|
|
1353
|
+
/**
|
|
1354
|
+
* A digest, verified update or ack from `clientID` (or one we are about
|
|
1355
|
+
* to send, for our own id) is proof of presence: refresh the lease the
|
|
1356
|
+
* sweep above checks. Only for ids with a state - a departed peer's
|
|
1357
|
+
* `meta` entry survives its removal (y-protocols keeps it for the clock)
|
|
1358
|
+
* and must not be revived by a late message.
|
|
1359
|
+
*/
|
|
1360
|
+
_touchPeer(clientID) {
|
|
1361
|
+
const meta = this.awareness.meta.get(clientID);
|
|
1362
|
+
if (meta !== undefined && this.awareness.getStates().has(clientID)) {
|
|
1363
|
+
meta.lastUpdated = Date.now();
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
/**
|
|
1367
|
+
* Transport.onPeerDisconnect: the peer at `peerId` is gone. Forget its
|
|
1368
|
+
* address and id, drop its awareness state with origin 'peer-left': the
|
|
1369
|
+
* broadcast goes through the same suppression as a timeout removal, but
|
|
1370
|
+
* with a long window - every peer gets the leave signal in the same
|
|
1371
|
+
* millisecond, and at the reply window (~170 ms at N=50) 28 of 49
|
|
1372
|
+
* survivors broadcast before the first broadcast could be overheard
|
|
1373
|
+
* (bench-awareness-removal-burst, DUMMY_PEER_EVENTS=1). One broadcast
|
|
1374
|
+
* room-wide is still worth having: it corrects a joiner that received
|
|
1375
|
+
* this peer in a relayed presence table but had no channel to it yet.
|
|
1376
|
+
*/
|
|
1377
|
+
_handlePeerLeave(peerId) {
|
|
1378
|
+
const gone = [];
|
|
1379
|
+
for (const [clientID, address] of this._peerAddress) {
|
|
1380
|
+
if (address === peerId)
|
|
1381
|
+
gone.push(clientID);
|
|
1382
|
+
}
|
|
1383
|
+
for (const id of gone) {
|
|
1384
|
+
this._peerAddress.delete(id);
|
|
1385
|
+
this._knownPeers.delete(id);
|
|
1386
|
+
}
|
|
1387
|
+
const present = gone.filter((id) => this.awareness.getStates().has(id));
|
|
1388
|
+
if (present.length > 0) {
|
|
1389
|
+
awarenessProtocol.removeAwarenessStates(this.awareness, present, 'peer-left');
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
/** Cached delete-set hash - see computeDeleteSetHash(). */
|
|
1393
|
+
_deleteSetHash() {
|
|
1394
|
+
if (this._dsHashCache === null) {
|
|
1395
|
+
this._dsHashCache = computeDeleteSetHash(this.doc);
|
|
555
1396
|
}
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
1397
|
+
return this._dsHashCache;
|
|
1398
|
+
}
|
|
1399
|
+
/**
|
|
1400
|
+
* Debounce onPeerConnect-triggered syncNow() calls. A burst of connect
|
|
1401
|
+
* events within `_peerConnectDebounceMs` collapses into one call instead
|
|
1402
|
+
* of one per event - without this, N peers joining a mesh in a short
|
|
1403
|
+
* window each independently broadcast full state to everyone already
|
|
1404
|
+
* connected (O(N^2) traffic), since onPeerConnect fires once per
|
|
1405
|
+
* newly-opened peer connection with no coalescing of its own.
|
|
1406
|
+
*/
|
|
1407
|
+
_schedulePeerConnectSync(peerId) {
|
|
1408
|
+
if (peerId !== undefined)
|
|
1409
|
+
this._pendingPeerConnectIds.add(peerId);
|
|
1410
|
+
if (this._pendingPeerConnectSyncTimeoutId !== undefined)
|
|
1411
|
+
return;
|
|
1412
|
+
this._pendingPeerConnectSyncTimeoutId = setTimeout(() => {
|
|
1413
|
+
this._pendingPeerConnectSyncTimeoutId = undefined;
|
|
1414
|
+
const ids = Array.from(this._pendingPeerConnectIds);
|
|
1415
|
+
this._pendingPeerConnectIds.clear();
|
|
1416
|
+
if (!this.transport.isConnected || this._destroying)
|
|
1417
|
+
return;
|
|
1418
|
+
// A newly-joined mesh peer has no app-channel state for us yet: the
|
|
1419
|
+
// digest beacon below covers the document, and the core awareness
|
|
1420
|
+
// channel re-broadcasts on its own, but nothing replays app awareness.
|
|
1421
|
+
// Pre-merge this rode along as a second _broadcastAwareness() on the
|
|
1422
|
+
// APP channel here; the digest-beacon rewrite dropped it, which is why
|
|
1423
|
+
// cursors were invisible to whichever peer joined second.
|
|
1424
|
+
this._announceAppAwareness();
|
|
1425
|
+
if (typeof this.transport.sendTo === 'function' && ids.length > 0) {
|
|
1426
|
+
// Round 5, item 5: one plain beacon to each new peer, nothing to
|
|
1427
|
+
// the rest of the mesh. Not rate-limited as a request: it answers
|
|
1428
|
+
// a channel that just opened, and the peer's own JOIN beacon is
|
|
1429
|
+
// the fallback if it is lost.
|
|
1430
|
+
const beacon = wrapMessageWithChecksum(this._encodeSyncStep1(0));
|
|
1431
|
+
for (const id of ids)
|
|
1432
|
+
this._sendToTransport(beacon, id);
|
|
1433
|
+
return;
|
|
1434
|
+
}
|
|
1435
|
+
this._syncNow(0);
|
|
1436
|
+
}, this._peerConnectDebounceMs);
|
|
561
1437
|
}
|
|
562
1438
|
/**
|
|
563
1439
|
* Setup automatic document synchronization.
|
|
@@ -566,27 +1442,36 @@ export class GenericProvider extends Observable {
|
|
|
566
1442
|
*/
|
|
567
1443
|
_setupDocumentSync() {
|
|
568
1444
|
this._updateHandler = (update, origin) => {
|
|
569
|
-
|
|
570
|
-
//
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
//
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
1445
|
+
this._dsHashCache = null;
|
|
1446
|
+
this._equalBeaconsHeard = 0; // our digest changed - see the Trickle fields
|
|
1447
|
+
// Fires for BOTH local edits and remotely-applied updates (the latter
|
|
1448
|
+
// go through doc.transact with origin=this) - see _markActivity()'s
|
|
1449
|
+
// doc comment. Phase 1e: only a LOCAL edit counts as activity for
|
|
1450
|
+
// idle backoff - a listener has nothing a beacon would announce, and
|
|
1451
|
+
// the typist's base-interval beacon heals any listener that lost the
|
|
1452
|
+
// keystroke (phase 1d design A). Before this, one typist kept all N
|
|
1453
|
+
// peers at the base cadence: N*(N-1) deliveries per interval against
|
|
1454
|
+
// N-1 per keystroke.
|
|
1455
|
+
// Don't send updates that originated from this provider (received
|
|
1456
|
+
// from the wire) or from the local load connect() is waiting for.
|
|
1457
|
+
if (origin !== this && !this._loading) {
|
|
1458
|
+
// Local-only transaction origins (e.g. a rollback the peer must not
|
|
1459
|
+
// replicate) never reach the transport.
|
|
1460
|
+
if (this._excludeOrigins.has(origin))
|
|
1461
|
+
return;
|
|
1462
|
+
this._markActivity();
|
|
1463
|
+
// 'pull' replicas answer requests but never push unasked.
|
|
1464
|
+
if (this._syncMode === 'pull')
|
|
1465
|
+
return;
|
|
578
1466
|
this._batchUpdate(update);
|
|
579
1467
|
}
|
|
580
|
-
else {
|
|
581
|
-
// Immediate mode: send right away
|
|
582
|
-
this._sendUpdate(update);
|
|
583
|
-
}
|
|
584
1468
|
};
|
|
585
1469
|
this.doc.on('update', this._updateHandler);
|
|
586
1470
|
}
|
|
587
1471
|
/**
|
|
588
|
-
*
|
|
589
|
-
*
|
|
1472
|
+
* Merge a local update into the pending batch and schedule its flush:
|
|
1473
|
+
* after `batchUpdates` ms (debounced) when that is > 0, otherwise at the
|
|
1474
|
+
* end of the current task via queueMicrotask - see `_pendingUpdate`.
|
|
590
1475
|
*/
|
|
591
1476
|
_batchUpdate(update) {
|
|
592
1477
|
// Merge with pending update if exists
|
|
@@ -606,18 +1491,38 @@ export class GenericProvider extends Observable {
|
|
|
606
1491
|
else {
|
|
607
1492
|
this._pendingUpdate = update;
|
|
608
1493
|
}
|
|
609
|
-
|
|
1494
|
+
if (this._batchUpdates > 0) {
|
|
1495
|
+
// Debounce: restart the timer on every update.
|
|
1496
|
+
if (this._batchTimeoutId !== undefined)
|
|
1497
|
+
clearTimeout(this._batchTimeoutId);
|
|
1498
|
+
this._batchTimeoutId = setTimeout(() => {
|
|
1499
|
+
this._batchTimeoutId = undefined;
|
|
1500
|
+
this._flushPendingUpdate();
|
|
1501
|
+
}, this._batchUpdates);
|
|
1502
|
+
this._flushScheduled = true;
|
|
1503
|
+
return;
|
|
1504
|
+
}
|
|
1505
|
+
if (this._flushScheduled)
|
|
1506
|
+
return;
|
|
1507
|
+
this._flushScheduled = true;
|
|
1508
|
+
queueMicrotask(() => this._flushPendingUpdate());
|
|
1509
|
+
}
|
|
1510
|
+
/**
|
|
1511
|
+
* Send the pending update batch as one wire message, carrying any
|
|
1512
|
+
* awareness change the throttle is holding (see _takePendingAwareness).
|
|
1513
|
+
* Shared by the microtask flush, the timed flush, and the
|
|
1514
|
+
* disconnect()/destroy() flush.
|
|
1515
|
+
*/
|
|
1516
|
+
_flushPendingUpdate() {
|
|
1517
|
+
this._flushScheduled = false;
|
|
610
1518
|
if (this._batchTimeoutId !== undefined) {
|
|
611
1519
|
clearTimeout(this._batchTimeoutId);
|
|
612
|
-
}
|
|
613
|
-
// Set new timeout to send after delay
|
|
614
|
-
this._batchTimeoutId = setTimeout(() => {
|
|
615
|
-
if (this._pendingUpdate) {
|
|
616
|
-
this._sendUpdate(this._pendingUpdate);
|
|
617
|
-
this._pendingUpdate = null;
|
|
618
|
-
}
|
|
619
1520
|
this._batchTimeoutId = undefined;
|
|
620
|
-
}
|
|
1521
|
+
}
|
|
1522
|
+
const update = this._pendingUpdate;
|
|
1523
|
+
this._pendingUpdate = null;
|
|
1524
|
+
if (update && this.transport.isConnected)
|
|
1525
|
+
this._sendUpdate(update);
|
|
621
1526
|
}
|
|
622
1527
|
/**
|
|
623
1528
|
* Setup automatic awareness synchronization.
|
|
@@ -625,21 +1530,85 @@ export class GenericProvider extends Observable {
|
|
|
625
1530
|
*/
|
|
626
1531
|
_setupAwarenessSync() {
|
|
627
1532
|
this._awarenessUpdateHandler = ({ added, updated, removed, }, origin) => {
|
|
628
|
-
//
|
|
1533
|
+
// Not _markActivity(): awareness changes are not something a beacon
|
|
1534
|
+
// announces (phase 1e; see the idleBackoffEnabled option).
|
|
1535
|
+
// Broadcast awareness changes, UNLESS they came from remote (this
|
|
1536
|
+
// comment described the intent since the very first commit, but the
|
|
1537
|
+
// actual origin check was never implemented until now - confirmed by
|
|
1538
|
+
// `git log -p` on this handler). `origin === this` is exactly the
|
|
1539
|
+
// signature the MESSAGE_AWARENESS handler stamps on an update applied
|
|
1540
|
+
// from an incoming wire message (`applyAwarenessUpdate(this.awareness,
|
|
1541
|
+
// ..., this)`, below). Every transport this project targets is a
|
|
1542
|
+
// full-room relay (websocket/pubnub/gun/matrix/ably/supabase) or a
|
|
1543
|
+
// full mesh (peerjs/simple-peer/trystero - see CLAUDE.md), so the
|
|
1544
|
+
// sender's own broadcast already reached every other peer directly;
|
|
1545
|
+
// re-broadcasting it here on receipt is pure redundant traffic that
|
|
1546
|
+
// compounds across every OTHER receiver doing the same thing.
|
|
1547
|
+
// Verified with a throwaway probe: a single awareness field change in
|
|
1548
|
+
// an N-peer room cost N*(N-1) wire deliveries before this check (one
|
|
1549
|
+
// echo per receiver, each reaching N-1 peers) vs. N-1 after - e.g.
|
|
1550
|
+
// N=20: 380 -> 19.
|
|
1551
|
+
//
|
|
1552
|
+
// Carve-out: `applyAwarenessUpdate` has its own defense against a
|
|
1553
|
+
// remote peer incorrectly removing OUR OWN state (a stale/racy
|
|
1554
|
+
// timeout-removal from someone else's clock) - it bumps our clock
|
|
1555
|
+
// instead of deleting our state, but (a quirk of that function) still
|
|
1556
|
+
// reports it via `removed` including our own clientID. That specific
|
|
1557
|
+
// case must still be broadcast so the room's stale belief that we're
|
|
1558
|
+
// gone gets corrected promptly, instead of only self-healing on our
|
|
1559
|
+
// next unrelated state change/renewal (up to ~15s later).
|
|
1560
|
+
if (origin === this) {
|
|
1561
|
+
// An incoming removal might be the SAME departure we have a
|
|
1562
|
+
// suppressed broadcast queued for (see _scheduleAwarenessRemoval())
|
|
1563
|
+
// - someone else already told the room, drop ours.
|
|
1564
|
+
if (removed.length > 0) {
|
|
1565
|
+
this._cancelPendingAwarenessRemovalIfOverlaps(removed);
|
|
1566
|
+
}
|
|
1567
|
+
if (!removed.includes(this.awareness.clientID)) {
|
|
1568
|
+
return;
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
629
1571
|
const changedClients = added.concat(updated).concat(removed);
|
|
1572
|
+
// A pure timeout-triggered removal ('timeout' is the exact origin
|
|
1573
|
+
// string y-protocols/awareness.js's own _checkInterval passes to
|
|
1574
|
+
// removeAwarenessStates()) is redundant across the whole room: every
|
|
1575
|
+
// OTHER connected peer runs the identical 30s-timeout sweep
|
|
1576
|
+
// independently, so all of them detect and would broadcast the SAME
|
|
1577
|
+
// departure within the same ~3s tick - measured as O(N-1) broadcasts
|
|
1578
|
+
// / O((N-1)(N-2)) deliveries for ONE departure in
|
|
1579
|
+
// test/dummy/bench-awareness-removal-burst.ts (e.g. N=50: 2352
|
|
1580
|
+
// deliveries). Delay + drop-if-overheard, exactly mirroring
|
|
1581
|
+
// _scheduleSyncReply()/_cancelPendingSyncReply()'s suppression of
|
|
1582
|
+
// redundant SyncStep2 replies. Deliberately NOT applied to
|
|
1583
|
+
// 'disconnect'/'window unload' removals (a single broadcaster, not
|
|
1584
|
+
// redundant) or to added/updated clients (every sender's
|
|
1585
|
+
// cursor/presence data is meaningfully different and must never be
|
|
1586
|
+
// suppressed).
|
|
1587
|
+
if (origin === 'timeout' || origin === 'peer-left') {
|
|
1588
|
+
this._scheduleAwarenessRemoval(changedClients, origin);
|
|
1589
|
+
return;
|
|
1590
|
+
}
|
|
630
1591
|
this._broadcastAwareness(changedClients);
|
|
631
1592
|
};
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
1593
|
+
// Round 5, item 8: broadcast on 'change' (y-protocols filters updates
|
|
1594
|
+
// whose state deep-equals the previous one) rather than 'update'
|
|
1595
|
+
// (every setLocalState call) - an app that re-sets unchanged state no
|
|
1596
|
+
// longer costs a broadcast per call. The renewal is the one
|
|
1597
|
+
// equal-state update that must go out; the provider's own sweep sends
|
|
1598
|
+
// it explicitly (_startAwarenessSweep). With an app-supplied Awareness
|
|
1599
|
+
// y-protocols' own sweep renews through 'update' only, so that case
|
|
1600
|
+
// keeps listening on 'update'.
|
|
1601
|
+
this.awareness.on(this._ownsAwareness ? 'change' : 'update', this._awarenessUpdateHandler);
|
|
1602
|
+
// App channel: attached here only if one was supplied to the
|
|
1603
|
+
// constructor; otherwise the getter attaches on first use. Reading
|
|
1604
|
+
// `this.appAwareness` here would defeat the lazy construction.
|
|
1605
|
+
if (this._appAwareness) {
|
|
1606
|
+
this._attachAppAwareness(this._appAwareness);
|
|
1607
|
+
}
|
|
638
1608
|
// Cleanup: mark as offline and disconnect BC when page unloads
|
|
639
1609
|
if (typeof window !== 'undefined') {
|
|
640
1610
|
this._beforeUnloadHandler = () => {
|
|
641
1611
|
awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'window unload');
|
|
642
|
-
awarenessProtocol.removeAwarenessStates(this.appAwareness, [this.doc.clientID], 'window unload');
|
|
643
1612
|
// Disconnect BroadcastChannel to notify other tabs
|
|
644
1613
|
this._disconnectBroadcastChannel();
|
|
645
1614
|
};
|
|
@@ -647,17 +1616,81 @@ export class GenericProvider extends Observable {
|
|
|
647
1616
|
}
|
|
648
1617
|
}
|
|
649
1618
|
/**
|
|
650
|
-
* Handle incoming messages from the transport.
|
|
651
|
-
*
|
|
652
|
-
*
|
|
1619
|
+
* Handle incoming messages from the transport (or BroadcastChannel).
|
|
1620
|
+
*
|
|
1621
|
+
* When compressionThresholdBytes is disabled (the default), this is a
|
|
1622
|
+
* fully synchronous fast path, byte-for-byte the same behavior as before
|
|
1623
|
+
* that option existed: straight into _processWrappedMessage().
|
|
1624
|
+
*
|
|
1625
|
+
* When enabled, every message - from the network transport AND from
|
|
1626
|
+
* BroadcastChannel (see _send()) - carries a leading compressed(1)/
|
|
1627
|
+
* uncompressed(0) flag byte ahead of the usual CRC32 wrapper. Reading
|
|
1628
|
+
* that flag and, if set, decompressing is inherently async (the
|
|
1629
|
+
* Compression Streams API has no synchronous form), so this method
|
|
1630
|
+
* dispatches to a promise chain instead of processing inline in that
|
|
1631
|
+
* case. This means a large (compressed) message and a small (uncompressed
|
|
1632
|
+
* or below-threshold) message that arrive back-to-back can finish
|
|
1633
|
+
* processing out of arrival order - acceptable here because Yjs updates
|
|
1634
|
+
* are idempotent/commutative (see MESSAGE_SYNC_VERIFIED's handling below)
|
|
1635
|
+
* and because the compression threshold keeps this path almost entirely
|
|
1636
|
+
* to large, full-state syncs, not the per-keystroke incremental updates
|
|
1637
|
+
* that per-sender gap detection actually relies on ordering-sensitive
|
|
1638
|
+
* heuristics for.
|
|
1639
|
+
*/
|
|
1640
|
+
_handleIncomingMessage(data, from) {
|
|
1641
|
+
if (!this._compressionThresholdBytes) {
|
|
1642
|
+
this._processWrappedMessage(data, from);
|
|
1643
|
+
return;
|
|
1644
|
+
}
|
|
1645
|
+
if (data.length < 1) {
|
|
1646
|
+
console.warn('[GenericProvider] Dropping empty message (missing compression flag byte)');
|
|
1647
|
+
return;
|
|
1648
|
+
}
|
|
1649
|
+
const flag = data[0];
|
|
1650
|
+
const rest = data.subarray(1);
|
|
1651
|
+
if (flag === 0) {
|
|
1652
|
+
this._processWrappedMessage(rest, from);
|
|
1653
|
+
return;
|
|
1654
|
+
}
|
|
1655
|
+
if (!COMPRESSION_AVAILABLE) {
|
|
1656
|
+
console.warn('[GenericProvider] Received a compressed message but this runtime has no DecompressionStream - dropping it.');
|
|
1657
|
+
return;
|
|
1658
|
+
}
|
|
1659
|
+
decompressDeflateRaw(rest)
|
|
1660
|
+
.then((wrapped) => this._processWrappedMessage(wrapped, from))
|
|
1661
|
+
.catch((error) => {
|
|
1662
|
+
// Treat decompression failure the same as a CRC32 mismatch on the
|
|
1663
|
+
// uncompressed path: request a resync rather than silently dropping.
|
|
1664
|
+
console.warn('[GenericProvider] Failed to decompress incoming message, treating as corrupted:', error);
|
|
1665
|
+
this._requestResync();
|
|
1666
|
+
});
|
|
1667
|
+
}
|
|
1668
|
+
/**
|
|
1669
|
+
* Verify message integrity with CRC32 and decode. Corrupt messages are
|
|
1670
|
+
* rejected immediately without attempting to decode. Operates on bytes
|
|
1671
|
+
* that have already had any compression flag/decompression handled by
|
|
1672
|
+
* _handleIncomingMessage() - this is the pre-compression-feature
|
|
1673
|
+
* implementation, unchanged.
|
|
653
1674
|
*/
|
|
654
|
-
|
|
1675
|
+
_processWrappedMessage(data, from) {
|
|
655
1676
|
// Verify message integrity with CRC32 checksum
|
|
656
1677
|
const message = unwrapAndVerifyMessage(data);
|
|
657
1678
|
if (message === null) {
|
|
658
|
-
// Message is corrupted - reject it immediately
|
|
1679
|
+
// Message is corrupted - reject it immediately. Note: if this was a
|
|
1680
|
+
// MESSAGE_BATCH envelope, the CRC32 wrap covers the WHOLE batch, so a
|
|
1681
|
+
// single corrupted bit here loses every sub-message it contained, not
|
|
1682
|
+
// just one - a deliberate tradeoff of batching multiple logical
|
|
1683
|
+
// messages behind one wire message/one checksum. See _sendBatch()'s
|
|
1684
|
+
// doc comment for why this was chosen over per-sub-message checksums,
|
|
1685
|
+
// and this project's benchmark suite (bench-corruption-storm.ts,
|
|
1686
|
+
// bench-packet-loss.ts) for how that tradeoff was measured.
|
|
659
1687
|
console.warn(`[GenericProvider] 💥 Corrupted message rejected: CRC32 checksum mismatch. ` +
|
|
660
1688
|
`This is expected if data corruption simulation is enabled.`);
|
|
1689
|
+
// Wire noise, not silence - counts as activity for idleBackoffEnabled
|
|
1690
|
+
// even though nothing here reaches the doc/awareness update handlers
|
|
1691
|
+
// (the message is rejected before decoding). See _markActivity()'s
|
|
1692
|
+
// doc comment.
|
|
1693
|
+
this._markActivity();
|
|
661
1694
|
// Request re-sync to recover any lost data - routed through the
|
|
662
1695
|
// shared coordinator so this doesn't stack an independent timer on
|
|
663
1696
|
// top of any hash-mismatch/gap-confirmed resync already pending.
|
|
@@ -666,168 +1699,7 @@ export class GenericProvider extends Observable {
|
|
|
666
1699
|
}
|
|
667
1700
|
// Message integrity verified - safe to decode
|
|
668
1701
|
try {
|
|
669
|
-
|
|
670
|
-
const messageType = decoding.readVarUint(decoder);
|
|
671
|
-
switch (messageType) {
|
|
672
|
-
case MESSAGE_SYNC: {
|
|
673
|
-
const encoder = encoding.createEncoder();
|
|
674
|
-
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
675
|
-
const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this);
|
|
676
|
-
if (syncMessageType === syncProtocol.messageYjsSyncStep2) {
|
|
677
|
-
// If we received SyncStep2, we're synced
|
|
678
|
-
if (!this._synced) {
|
|
679
|
-
this._synced = true;
|
|
680
|
-
this.emit('synced', [true]);
|
|
681
|
-
}
|
|
682
|
-
// Someone else's SyncStep2 reply just arrived - our own pending
|
|
683
|
-
// reply (if any) is now most likely redundant.
|
|
684
|
-
this._cancelPendingSyncReply();
|
|
685
|
-
}
|
|
686
|
-
// Send reply if needed. Suppression only engages with genuine
|
|
687
|
-
// redundancy (>=2 other known peers via awareness) - below that,
|
|
688
|
-
// there's no "someone else" to rely on, so reply immediately
|
|
689
|
-
// (still rate-limited via _sendSyncReply() as a hard backstop).
|
|
690
|
-
if (encoding.length(encoder) > 1) {
|
|
691
|
-
if (this.awareness.getStates().size >= 3) {
|
|
692
|
-
this._scheduleSyncReply(encoding.toUint8Array(encoder));
|
|
693
|
-
}
|
|
694
|
-
else {
|
|
695
|
-
this._sendSyncReply(encoding.toUint8Array(encoder));
|
|
696
|
-
}
|
|
697
|
-
}
|
|
698
|
-
break;
|
|
699
|
-
}
|
|
700
|
-
case MESSAGE_AWARENESS: {
|
|
701
|
-
const channel = decoding.readVarUint(decoder);
|
|
702
|
-
awarenessProtocol.applyAwarenessUpdate(channel === AWARENESS_CHANNEL_APP
|
|
703
|
-
? this.appAwareness
|
|
704
|
-
: this.awareness, decoding.readVarUint8Array(decoder), this);
|
|
705
|
-
break;
|
|
706
|
-
}
|
|
707
|
-
case MESSAGE_PUBSUB: {
|
|
708
|
-
// Read topic
|
|
709
|
-
const topic = decoding.readVarString(decoder);
|
|
710
|
-
// Read message payload
|
|
711
|
-
const payloadBytes = decoding.readVarUint8Array(decoder);
|
|
712
|
-
try {
|
|
713
|
-
// Decode JSON payload
|
|
714
|
-
const decoder = new TextDecoder();
|
|
715
|
-
const payloadStr = decoder.decode(payloadBytes);
|
|
716
|
-
const message = JSON.parse(payloadStr);
|
|
717
|
-
// Emit to pubsub channel
|
|
718
|
-
this.pubsub._handleMessage(topic, message);
|
|
719
|
-
}
|
|
720
|
-
catch (error) {
|
|
721
|
-
console.error('Error decoding pub/sub message:', error);
|
|
722
|
-
}
|
|
723
|
-
break;
|
|
724
|
-
}
|
|
725
|
-
case MESSAGE_PUBSUB_TARGETED: {
|
|
726
|
-
const target = decoding.readVarString(decoder);
|
|
727
|
-
const topic = decoding.readVarString(decoder);
|
|
728
|
-
const payloadBytes = decoding.readVarUint8Array(decoder);
|
|
729
|
-
// Drop messages aimed at someone else (broadcast-and-filter path).
|
|
730
|
-
if (this._localId !== undefined && target !== this._localId) {
|
|
731
|
-
break;
|
|
732
|
-
}
|
|
733
|
-
try {
|
|
734
|
-
const message = JSON.parse(new TextDecoder().decode(payloadBytes));
|
|
735
|
-
this.pubsub._handleMessage(topic, message);
|
|
736
|
-
}
|
|
737
|
-
catch (error) {
|
|
738
|
-
console.error('Error decoding targeted pub/sub message:', error);
|
|
739
|
-
}
|
|
740
|
-
break;
|
|
741
|
-
}
|
|
742
|
-
case MESSAGE_SYNC_VERIFIED: {
|
|
743
|
-
// Sync message with sequence number and hash verification
|
|
744
|
-
// Read sequence number and clientID first
|
|
745
|
-
const seqNum = decoding.readVarUint(decoder);
|
|
746
|
-
const senderClientID = decoding.readVarUint(decoder);
|
|
747
|
-
// Track for gap detection only — does NOT gate whether we apply
|
|
748
|
-
// the update below (see _trackRemoteSeq() for why).
|
|
749
|
-
this._trackRemoteSeq(senderClientID, seqNum);
|
|
750
|
-
// Always apply the update. Yjs updates are idempotent/commutative,
|
|
751
|
-
// so re-applying an already-seen update is a harmless no-op.
|
|
752
|
-
// Under reordering, a merely-late (not actually duplicate) update
|
|
753
|
-
// must still be applied here — the old "skip if seqNum <= last
|
|
754
|
-
// seen" logic silently dropped such updates forever whenever a
|
|
755
|
-
// later-numbered message happened to arrive first.
|
|
756
|
-
const encoder = encoding.createEncoder();
|
|
757
|
-
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
758
|
-
const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this);
|
|
759
|
-
// Someone else's SyncStep2 reply just arrived - our own pending
|
|
760
|
-
// reply (if any) is now most likely redundant. Mirrors the
|
|
761
|
-
// MESSAGE_SYNC case: the reply encoded above is always a plain
|
|
762
|
-
// MESSAGE_SYNC-typed message regardless of which message type
|
|
763
|
-
// triggered it, so the same suppression scheme applies here too.
|
|
764
|
-
if (syncMessageType === syncProtocol.messageYjsSyncStep2) {
|
|
765
|
-
this._cancelPendingSyncReply();
|
|
766
|
-
}
|
|
767
|
-
// Read the expected hash from sender (signed integer)
|
|
768
|
-
const expectedHash = decoding.readVarInt(decoder);
|
|
769
|
-
// Compute our local hash after applying the update
|
|
770
|
-
const localHash = computeDocHash(this.doc);
|
|
771
|
-
// Verify hash match
|
|
772
|
-
if (localHash !== expectedHash) {
|
|
773
|
-
// If we already know this sender has a suspected reordering gap
|
|
774
|
-
// (see _trackRemoteSeq()/_scheduleGapCheck()), a hash mismatch
|
|
775
|
-
// right now is the *expected* transient state — we're missing a
|
|
776
|
-
// piece that's very likely still in flight, not actually
|
|
777
|
-
// diverged. Let the pending gap-check grace period resolve it
|
|
778
|
-
// instead of also escalating the hash-mismatch backoff: under
|
|
779
|
-
// heavy reordering this previously caused a burst of mismatches
|
|
780
|
-
// to rack up the exponential backoff to its 10s cap within a
|
|
781
|
-
// single edit burst, purely from timing, not real divergence.
|
|
782
|
-
// A hash mismatch with NO pending gap (in-order, but still
|
|
783
|
-
// wrong) is not explained by reordering and still escalates
|
|
784
|
-
// normally below.
|
|
785
|
-
const reorderingSuspected = this._gapCheckTimers.has(senderClientID);
|
|
786
|
-
if (!reorderingSuspected) {
|
|
787
|
-
// Push our full state AND request theirs (syncNow() does
|
|
788
|
-
// both). A hash mismatch means the two peers have diverged -
|
|
789
|
-
// one side may have edits the other lacks. Routed through the
|
|
790
|
-
// shared coordinator so this doesn't stack an independent
|
|
791
|
-
// timer on top of any corrupted-message/gap-confirmed resync
|
|
792
|
-
// already pending.
|
|
793
|
-
this._requestResync();
|
|
794
|
-
// Logged with the shared attempt counter (kept as "#N" for
|
|
795
|
-
// compatibility with existing tooling/benchmarks that grep
|
|
796
|
-
// for this exact "Hash mismatch #" pattern) - it now reflects
|
|
797
|
-
// the unified resync-attempt count rather than a
|
|
798
|
-
// hash-mismatch-specific one, since the two escalation
|
|
799
|
-
// counters were merged.
|
|
800
|
-
console.warn(`[GenericProvider] Hash mismatch #${this._resyncAttemptCount} detected! Local: ${localHash}, Expected: ${expectedHash}`);
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
// If we received SyncStep2, we're synced (unless hash mismatched)
|
|
804
|
-
if (syncMessageType === syncProtocol.messageYjsSyncStep2 &&
|
|
805
|
-
!this._synced &&
|
|
806
|
-
localHash === expectedHash) {
|
|
807
|
-
this._synced = true;
|
|
808
|
-
this.emit('synced', [true]);
|
|
809
|
-
}
|
|
810
|
-
// Send reply if needed (as standard MESSAGE_SYNC). Suppression
|
|
811
|
-
// only engages with genuine redundancy (>=2 other known peers via
|
|
812
|
-
// awareness) - below that, reply immediately (still rate-limited
|
|
813
|
-
// via _sendSyncReply() as a hard backstop). Matches the
|
|
814
|
-
// MESSAGE_SYNC case's gate exactly; without this, a hash-mismatch
|
|
815
|
-
// resync burst under packet loss bypassed suppression entirely,
|
|
816
|
-
// since every peer answering a post-mismatch SyncStep1 replied
|
|
817
|
-
// immediately via this path.
|
|
818
|
-
if (encoding.length(encoder) > 1) {
|
|
819
|
-
if (this.awareness.getStates().size >= 3) {
|
|
820
|
-
this._scheduleSyncReply(encoding.toUint8Array(encoder));
|
|
821
|
-
}
|
|
822
|
-
else {
|
|
823
|
-
this._sendSyncReply(encoding.toUint8Array(encoder));
|
|
824
|
-
}
|
|
825
|
-
}
|
|
826
|
-
break;
|
|
827
|
-
}
|
|
828
|
-
default:
|
|
829
|
-
console.warn('Unknown message type:', messageType);
|
|
830
|
-
}
|
|
1702
|
+
this._dispatchMessage(message, from);
|
|
831
1703
|
}
|
|
832
1704
|
catch (error) {
|
|
833
1705
|
// This should only happen for logic errors, not corruption
|
|
@@ -836,36 +1708,618 @@ export class GenericProvider extends Observable {
|
|
|
836
1708
|
}
|
|
837
1709
|
}
|
|
838
1710
|
/**
|
|
839
|
-
*
|
|
840
|
-
*
|
|
841
|
-
*
|
|
842
|
-
*
|
|
843
|
-
*
|
|
844
|
-
*
|
|
845
|
-
*
|
|
846
|
-
*
|
|
847
|
-
*
|
|
848
|
-
* sanctioned way a reply gets dropped is `_cancelPendingSyncReply()`,
|
|
849
|
-
* because we overheard someone else's SyncStep2 for the SAME request.
|
|
1711
|
+
* Decode and act on one already-integrity-verified, already-decompressed
|
|
1712
|
+
* message. Split out of `_processWrappedMessage()` so `MESSAGE_BATCH`
|
|
1713
|
+
* (see `_sendBatch()`) can recurse into this for each sub-message it
|
|
1714
|
+
* unwraps, running the EXACT SAME per-message-type logic used for a
|
|
1715
|
+
* top-level message rather than a parallel reimplementation. A thrown
|
|
1716
|
+
* error partway through a batch's sub-messages aborts the REST of that
|
|
1717
|
+
* batch (propagates up to `_processWrappedMessage()`'s catch) - same as
|
|
1718
|
+
* a logic error aborting a single top-level message today, just now
|
|
1719
|
+
* scoped to "the rest of this batch" instead of "this one message".
|
|
850
1720
|
*/
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
1721
|
+
_dispatchMessage(message, from) {
|
|
1722
|
+
const decoder = decoding.createDecoder(message);
|
|
1723
|
+
const messageType = decoding.readVarUint(decoder);
|
|
1724
|
+
switch (messageType) {
|
|
1725
|
+
case MESSAGE_BATCH: {
|
|
1726
|
+
// Payload is N length-prefixed sub-messages (writeVarUint8Array
|
|
1727
|
+
// per sub-message, mirroring MESSAGE_AWARENESS's own framing).
|
|
1728
|
+
// Each was NOT individually CRC32-wrapped - see _sendBatch()'s doc
|
|
1729
|
+
// comment - so just decode and dispatch each one directly.
|
|
1730
|
+
while (decoding.hasContent(decoder)) {
|
|
1731
|
+
const subMessage = decoding.readVarUint8Array(decoder);
|
|
1732
|
+
this._dispatchMessage(subMessage, from);
|
|
1733
|
+
}
|
|
1734
|
+
break;
|
|
855
1735
|
}
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
this._pendingSyncReply = reply;
|
|
860
|
-
const delay = Math.random() * this._syncReplySuppressionMs;
|
|
861
|
-
this._pendingSyncReplyTimeoutId = setTimeout(() => {
|
|
862
|
-
this._pendingSyncReplyTimeoutId = undefined;
|
|
863
|
-
if (this._pendingSyncReply) {
|
|
864
|
-
this._sendSyncReply(this._pendingSyncReply);
|
|
865
|
-
this._pendingSyncReply = null;
|
|
1736
|
+
case MESSAGE_SYNC_DIGEST: {
|
|
1737
|
+
this._handleDigest(decoder, from);
|
|
1738
|
+
break;
|
|
866
1739
|
}
|
|
867
|
-
|
|
868
|
-
|
|
1740
|
+
case MESSAGE_SYNC_PUSH: {
|
|
1741
|
+
// Somebody's whole document: apply it, nothing else (see the
|
|
1742
|
+
// constant's comment for why no hash check and no synced flip).
|
|
1743
|
+
Y.applyUpdate(this.doc, decoding.readVarUint8Array(decoder), this);
|
|
1744
|
+
this._checkPendingAfterReply();
|
|
1745
|
+
break;
|
|
1746
|
+
}
|
|
1747
|
+
case MESSAGE_SYNC: {
|
|
1748
|
+
const encoder = encoding.createEncoder();
|
|
1749
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
1750
|
+
const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this);
|
|
1751
|
+
if (syncMessageType === syncProtocol.messageYjsSyncStep2) {
|
|
1752
|
+
// If we received SyncStep2, we're synced (and confirmed: we have
|
|
1753
|
+
// heard from a peer that had data for us)
|
|
1754
|
+
this._confirmed = true;
|
|
1755
|
+
this._noteResponse(true);
|
|
1756
|
+
this._markSynced();
|
|
1757
|
+
// Design E (phase 1d): a SyncStep2 carries its encoder's own pending
|
|
1758
|
+
// structs, so a responder missing a struct hands us the same hole.
|
|
1759
|
+
// Our response wait just ended - make sure something still asks.
|
|
1760
|
+
this._checkPendingAfterReply();
|
|
1761
|
+
// Someone else's SyncStep2 reply just arrived - our own pending
|
|
1762
|
+
// reply (if any) is now most likely redundant.
|
|
1763
|
+
this._cancelPendingSyncReply();
|
|
1764
|
+
}
|
|
1765
|
+
// Send reply if needed. Suppression only engages with genuine
|
|
1766
|
+
// redundancy (>=2 other known peers via awareness) - below that,
|
|
1767
|
+
// there's no "someone else" to rely on, so reply immediately
|
|
1768
|
+
// (still rate-limited via _sendSyncReply() as a hard backstop).
|
|
1769
|
+
if (encoding.length(encoder) > 1) {
|
|
1770
|
+
this._replyToSyncRequest(encoding.toUint8Array(encoder));
|
|
1771
|
+
}
|
|
1772
|
+
break;
|
|
1773
|
+
}
|
|
1774
|
+
case MESSAGE_AWARENESS: {
|
|
1775
|
+
const payload = decoding.readVarUint8Array(decoder);
|
|
1776
|
+
// Someone else's removal broadcast just arrived - cancel our own
|
|
1777
|
+
// suppressed removal for the same clientID(s), if pending (see
|
|
1778
|
+
// _scheduleAwarenessRemoval()). This MUST be checked here, at the
|
|
1779
|
+
// wire-message level, before calling applyAwarenessUpdate() below -
|
|
1780
|
+
// by the time this arrives we've very likely already independently
|
|
1781
|
+
// detected and applied the SAME removal ourselves (every peer's
|
|
1782
|
+
// 30s outdatedTimeout sweep fires near-simultaneously, well before
|
|
1783
|
+
// this message's network delay elapses - see
|
|
1784
|
+
// test/dummy/bench-awareness-removal-burst.ts), so
|
|
1785
|
+
// applyAwarenessUpdate() below will see a clock it already knows
|
|
1786
|
+
// and emit no 'update' event at all - mirrors exactly how the
|
|
1787
|
+
// MESSAGE_SYNC/MESSAGE_SYNC_VERIFIED case above cancels
|
|
1788
|
+
// _pendingSyncReply on seeing a SyncStep2 message TYPE arrive, not
|
|
1789
|
+
// on whether it changed anything locally.
|
|
1790
|
+
const scan = this._scanAwarenessPayload(payload);
|
|
1791
|
+
if (scan.removed.length > 0) {
|
|
1792
|
+
this._cancelPendingAwarenessRemovalIfOverlaps(scan.removed);
|
|
1793
|
+
}
|
|
1794
|
+
if (scan.coversUs)
|
|
1795
|
+
this._presenceCovered = true;
|
|
1796
|
+
// Round 5, item 3: with Trickle, settled peers rarely beacon, so a
|
|
1797
|
+
// joiner would learn them only from the few phase-winning beacons
|
|
1798
|
+
// per interval. The relayed presence table names everyone - ids
|
|
1799
|
+
// only, addresses stay beacon-learned (unicast needs a `from`).
|
|
1800
|
+
const heardAt = Date.now();
|
|
1801
|
+
for (const id of scan.present) {
|
|
1802
|
+
if (id !== this.doc.clientID)
|
|
1803
|
+
this._knownPeers.set(id, heardAt);
|
|
1804
|
+
}
|
|
1805
|
+
awarenessProtocol.applyAwarenessUpdate(this.awareness, payload, this);
|
|
1806
|
+
break;
|
|
1807
|
+
}
|
|
1808
|
+
case MESSAGE_PUBSUB: {
|
|
1809
|
+
// Read topic
|
|
1810
|
+
const topic = decoding.readVarString(decoder);
|
|
1811
|
+
// Read message payload
|
|
1812
|
+
const payloadBytes = decoding.readVarUint8Array(decoder);
|
|
1813
|
+
try {
|
|
1814
|
+
// Decode JSON payload
|
|
1815
|
+
const decoder = new TextDecoder();
|
|
1816
|
+
const payloadStr = decoder.decode(payloadBytes);
|
|
1817
|
+
const message = JSON.parse(payloadStr);
|
|
1818
|
+
// Emit to pubsub channel
|
|
1819
|
+
this.pubsub._handleMessage(topic, message);
|
|
1820
|
+
}
|
|
1821
|
+
catch (error) {
|
|
1822
|
+
console.error('Error decoding pub/sub message:', error);
|
|
1823
|
+
}
|
|
1824
|
+
break;
|
|
1825
|
+
}
|
|
1826
|
+
case MESSAGE_AWARENESS_APP: {
|
|
1827
|
+
// Deliberately none of the presence bookkeeping the core awareness
|
|
1828
|
+
// case does (_knownPeers, _presenceCovered, removal cancellation):
|
|
1829
|
+
// this channel is written by untrusted modules and must not be able
|
|
1830
|
+
// to influence the room's view of who is present.
|
|
1831
|
+
awarenessProtocol.applyAwarenessUpdate(this.appAwareness, decoding.readVarUint8Array(decoder), this);
|
|
1832
|
+
break;
|
|
1833
|
+
}
|
|
1834
|
+
case MESSAGE_PUBSUB_TARGETED: {
|
|
1835
|
+
const target = decoding.readVarString(decoder);
|
|
1836
|
+
const topic = decoding.readVarString(decoder);
|
|
1837
|
+
const payloadBytes = decoding.readVarUint8Array(decoder);
|
|
1838
|
+
// Drop messages aimed at someone else (broadcast-and-filter path).
|
|
1839
|
+
if (this._localId !== undefined && target !== this._localId) {
|
|
1840
|
+
break;
|
|
1841
|
+
}
|
|
1842
|
+
try {
|
|
1843
|
+
const message = JSON.parse(new TextDecoder().decode(payloadBytes));
|
|
1844
|
+
this.pubsub._handleMessage(topic, message);
|
|
1845
|
+
}
|
|
1846
|
+
catch (error) {
|
|
1847
|
+
console.error('Error decoding targeted pub/sub message:', error);
|
|
1848
|
+
}
|
|
1849
|
+
break;
|
|
1850
|
+
}
|
|
1851
|
+
case MESSAGE_SYNC_VERIFIED: {
|
|
1852
|
+
// Sync message with sequence number and hash verification
|
|
1853
|
+
// Read sequence number and clientID first
|
|
1854
|
+
const seqNum = decoding.readVarUint(decoder);
|
|
1855
|
+
const senderClientID = decoding.readVarUint(decoder);
|
|
1856
|
+
// Track for gap detection only — does NOT gate whether we apply
|
|
1857
|
+
// the update below (see _trackRemoteSeq() for why).
|
|
1858
|
+
this._trackRemoteSeq(senderClientID, seqNum);
|
|
1859
|
+
this._knownPeers.set(senderClientID, Date.now());
|
|
1860
|
+
if (from !== undefined)
|
|
1861
|
+
this._peerAddress.set(senderClientID, from);
|
|
1862
|
+
this._touchPeer(senderClientID);
|
|
1863
|
+
// Always apply the update. Yjs updates are idempotent/commutative,
|
|
1864
|
+
// so re-applying an already-seen update is a harmless no-op.
|
|
1865
|
+
// Under reordering, a merely-late (not actually duplicate) update
|
|
1866
|
+
// must still be applied here — the old "skip if seqNum <= last
|
|
1867
|
+
// seen" logic silently dropped such updates forever whenever a
|
|
1868
|
+
// later-numbered message happened to arrive first.
|
|
1869
|
+
// Peek at the sync sub-message's payload (SyncStep2/Update carry
|
|
1870
|
+
// an update as a varUint8Array right after the sub-type) without
|
|
1871
|
+
// consuming the decoder - _isLateUpdate() needs the bytes after
|
|
1872
|
+
// readSyncMessage() has applied them.
|
|
1873
|
+
const updateBytes = this._peekSyncUpdate(decoder);
|
|
1874
|
+
const encoder = encoding.createEncoder();
|
|
1875
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
1876
|
+
const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, this.doc, this);
|
|
1877
|
+
// Someone else's SyncStep2 reply just arrived - our own pending
|
|
1878
|
+
// reply (if any) is now most likely redundant. Mirrors the
|
|
1879
|
+
// MESSAGE_SYNC case: the reply encoded above is always a plain
|
|
1880
|
+
// MESSAGE_SYNC-typed message regardless of which message type
|
|
1881
|
+
// triggered it, so the same suppression scheme applies here too.
|
|
1882
|
+
if (syncMessageType === syncProtocol.messageYjsSyncStep2) {
|
|
1883
|
+
this._cancelPendingSyncReply();
|
|
1884
|
+
this._confirmed = true;
|
|
1885
|
+
this._checkPendingAfterReply();
|
|
1886
|
+
this._noteResponse(true);
|
|
1887
|
+
}
|
|
1888
|
+
// Read the expected hash from sender (signed integer)
|
|
1889
|
+
const expectedHash = decoding.readVarInt(decoder);
|
|
1890
|
+
// Compute our local hash after applying the update
|
|
1891
|
+
const localHash = computeDocHash(this.doc);
|
|
1892
|
+
// Verify hash match
|
|
1893
|
+
if (localHash !== expectedHash) {
|
|
1894
|
+
// If we already know this sender has a suspected reordering gap
|
|
1895
|
+
// (see _trackRemoteSeq()/_scheduleGapCheck()), a hash mismatch
|
|
1896
|
+
// right now is the *expected* transient state — we're missing a
|
|
1897
|
+
// piece that's very likely still in flight, not actually
|
|
1898
|
+
// diverged. Let the pending gap-check grace period resolve it
|
|
1899
|
+
// instead of also escalating the hash-mismatch backoff: under
|
|
1900
|
+
// heavy reordering this previously caused a burst of mismatches
|
|
1901
|
+
// to rack up the exponential backoff to its 10s cap within a
|
|
1902
|
+
// single edit burst, purely from timing, not real divergence.
|
|
1903
|
+
// A hash mismatch with NO pending gap (in-order, but still
|
|
1904
|
+
// wrong) is not explained by reordering and still escalates
|
|
1905
|
+
// normally below.
|
|
1906
|
+
const reorderingSuspected = this._gapCheckTimers.has(senderClientID);
|
|
1907
|
+
// A late update - one whose content we had already been past
|
|
1908
|
+
// when it arrived (its sender's clock in the update is below
|
|
1909
|
+
// ours, because a SyncStep2 or a reordered later update got here
|
|
1910
|
+
// first) - carries a hash of a state we have legitimately moved
|
|
1911
|
+
// beyond. Its mismatch says nothing about anything we lack; it
|
|
1912
|
+
// was the other half of the item-13 cascade (a resync's reply
|
|
1913
|
+
// fast-forwards a peer, then every in-flight keystroke behind it
|
|
1914
|
+
// mismatches). Cheap to detect from the update's own metadata.
|
|
1915
|
+
const lateUpdate = this._isLateUpdate(updateBytes);
|
|
1916
|
+
// Yjs's own verdict: if the update could not be fully integrated
|
|
1917
|
+
// because a causal dependency is missing, the struct store holds
|
|
1918
|
+
// it as pending. That is the ONE mismatch that is evidence of a
|
|
1919
|
+
// gap - and under jitter it is usually a reordering that the
|
|
1920
|
+
// next few ms resolve, so it gets the same grace a sequence gap
|
|
1921
|
+
// gets before a beacon goes out (_schedulePendingCheck). It also
|
|
1922
|
+
// covers the case the sequence anchor no longer does: since the
|
|
1923
|
+
// connect push carries no sequence number (MESSAGE_SYNC_PUSH), a
|
|
1924
|
+
// peer's first keystroke can be the first numbered message we see
|
|
1925
|
+
// from it, and a reordered first burst has no earlier number to
|
|
1926
|
+
// open a gap against.
|
|
1927
|
+
const pending = this.doc.store.pendingStructs !== null ||
|
|
1928
|
+
this.doc.store.pendingDs !== null;
|
|
1929
|
+
if (pending) {
|
|
1930
|
+
this._schedulePendingCheck();
|
|
1931
|
+
}
|
|
1932
|
+
else if (!reorderingSuspected && !lateUpdate) {
|
|
1933
|
+
// Push our full state AND request theirs (syncNow() does
|
|
1934
|
+
// both). A hash mismatch means the two peers have diverged -
|
|
1935
|
+
// one side may have edits the other lacks. Routed through the
|
|
1936
|
+
// shared coordinator so this doesn't stack an independent
|
|
1937
|
+
// timer on top of any corrupted-message/gap-confirmed resync
|
|
1938
|
+
// already pending.
|
|
1939
|
+
this._requestResync();
|
|
1940
|
+
// Logged with the shared attempt counter (kept as "#N" for
|
|
1941
|
+
// compatibility with existing tooling/benchmarks that grep
|
|
1942
|
+
// for this exact "Hash mismatch #" pattern) - it now reflects
|
|
1943
|
+
// the unified resync-attempt count rather than a
|
|
1944
|
+
// hash-mismatch-specific one, since the two escalation
|
|
1945
|
+
// counters were merged.
|
|
1946
|
+
console.warn(`[GenericProvider] Hash mismatch #${this._resyncAttemptCount} detected! Local: ${localHash}, Expected: ${expectedHash}`);
|
|
1947
|
+
}
|
|
1948
|
+
}
|
|
1949
|
+
// If we received SyncStep2, we're synced (unless hash mismatched)
|
|
1950
|
+
if (syncMessageType === syncProtocol.messageYjsSyncStep2 &&
|
|
1951
|
+
localHash === expectedHash) {
|
|
1952
|
+
this._markSynced();
|
|
1953
|
+
}
|
|
1954
|
+
// Send reply if needed (as standard MESSAGE_SYNC). Suppression
|
|
1955
|
+
// only engages with genuine redundancy (>=2 other known peers via
|
|
1956
|
+
// awareness) - below that, reply immediately (still rate-limited
|
|
1957
|
+
// via _sendSyncReply() as a hard backstop). Matches the
|
|
1958
|
+
// MESSAGE_SYNC case's gate exactly; without this, a hash-mismatch
|
|
1959
|
+
// resync burst under packet loss bypassed suppression entirely,
|
|
1960
|
+
// since every peer answering a post-mismatch SyncStep1 replied
|
|
1961
|
+
// immediately via this path.
|
|
1962
|
+
if (encoding.length(encoder) > 1) {
|
|
1963
|
+
this._replyToSyncRequest(encoding.toUint8Array(encoder));
|
|
1964
|
+
}
|
|
1965
|
+
break;
|
|
1966
|
+
}
|
|
1967
|
+
default:
|
|
1968
|
+
console.warn('Unknown message type:', messageType);
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
/**
|
|
1972
|
+
* Handle a digest beacon (MESSAGE_SYNC_DIGEST). Reply rule (design doc
|
|
1973
|
+
* §3): SyncStep2 if the sender is behind us or its delete-set hash
|
|
1974
|
+
* differs from ours (the SyncStep2 always carries our full delete set, so
|
|
1975
|
+
* it also heals a lost delete on their side - and their beacon does the
|
|
1976
|
+
* same for us, symmetrically, within one interval); our own beacon as an
|
|
1977
|
+
* ack if the beacon is JOIN-flagged and states are equal; nothing
|
|
1978
|
+
* otherwise - which is what removes the ~5-12 empty replies per heartbeat
|
|
1979
|
+
* measured at N=50 in test/dummy/bench-idle-room.ts. "Sender is ahead of
|
|
1980
|
+
* us" triggers no reply: our own next beacon fetches it. Nothing here
|
|
1981
|
+
* removes a recovery path (the round-2 lesson in
|
|
1982
|
+
* 2026-09-04-resync-message-reduction-design.md's addendum), only
|
|
1983
|
+
* replies that carry no information.
|
|
1984
|
+
*
|
|
1985
|
+
* `synced`: a beacon we are not behind, with equal delete-set hash, is a
|
|
1986
|
+
* stronger statement than the empty SyncStep2 it replaces ("you lack
|
|
1987
|
+
* nothing I have"), so it marks us synced too - this is what keeps two
|
|
1988
|
+
* fresh peers, or a whole concurrent join burst, converging to `synced`
|
|
1989
|
+
* with no acks needing to survive the rate limiter.
|
|
1990
|
+
*/
|
|
1991
|
+
_handleDigest(decoder, from) {
|
|
1992
|
+
decoding.readVarUint(decoder); // DIGEST_VERSION - append-only, nothing to branch on yet
|
|
1993
|
+
const flags = decoding.readVarUint(decoder);
|
|
1994
|
+
const senderClientID = decoding.readVarUint(decoder);
|
|
1995
|
+
this._knownPeers.set(senderClientID, Date.now());
|
|
1996
|
+
if (from !== undefined)
|
|
1997
|
+
this._peerAddress.set(senderClientID, from);
|
|
1998
|
+
this._touchPeer(senderClientID);
|
|
1999
|
+
const remoteSv = decoding.readVarUint8Array(decoder);
|
|
2000
|
+
const remoteDsHash = decoding.readVarUint(decoder);
|
|
2001
|
+
// Any trailing bytes belong to a newer version; ignored by design.
|
|
2002
|
+
const remote = Y.decodeStateVector(remoteSv);
|
|
2003
|
+
const local = Y.decodeStateVector(Y.encodeStateVector(this.doc));
|
|
2004
|
+
let senderBehind = false;
|
|
2005
|
+
for (const [client, clock] of local) {
|
|
2006
|
+
if ((remote.get(client) ?? 0) < clock) {
|
|
2007
|
+
senderBehind = true;
|
|
2008
|
+
break;
|
|
2009
|
+
}
|
|
2010
|
+
}
|
|
2011
|
+
let weBehind = false;
|
|
2012
|
+
for (const [client, clock] of remote) {
|
|
2013
|
+
if ((local.get(client) ?? 0) < clock) {
|
|
2014
|
+
weBehind = true;
|
|
2015
|
+
break;
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
const dsEqual = remoteDsHash === this._deleteSetHash();
|
|
2019
|
+
const equal = !senderBehind && !weBehind && dsEqual;
|
|
2020
|
+
if (equal) {
|
|
2021
|
+
// Any peer holding exactly our state has confirmed it - see _confirmedSv.
|
|
2022
|
+
this._confirmedSv = remoteSv;
|
|
2023
|
+
this._confirmedDsHash = remoteDsHash;
|
|
2024
|
+
}
|
|
2025
|
+
if (flags & DIGEST_FLAG_ACK) {
|
|
2026
|
+
// Somebody confirmed the echoed state (see DIGEST_FLAG_ACK). If it is
|
|
2027
|
+
// ours, we're synced; if we were about to confirm the same state, we
|
|
2028
|
+
// no longer need to. Never a request: no reply, no presence.
|
|
2029
|
+
if (equal) {
|
|
2030
|
+
if (flags & DIGEST_FLAG_SETTLED) {
|
|
2031
|
+
this._confirmed = true;
|
|
2032
|
+
this._noteResponse(true);
|
|
2033
|
+
}
|
|
2034
|
+
else {
|
|
2035
|
+
this._equalUnsettledSeen = true;
|
|
2036
|
+
}
|
|
2037
|
+
this._markSynced();
|
|
2038
|
+
this._cancelPendingAck();
|
|
2039
|
+
}
|
|
2040
|
+
return;
|
|
2041
|
+
}
|
|
2042
|
+
// An equal-digest beacon from a settled peer's periodic tick has just
|
|
2043
|
+
// told the room (including whoever our pending ack was for) that this
|
|
2044
|
+
// state is confirmed. Our ack would say the same thing again. Task 3b
|
|
2045
|
+
// in the design doc: without this, acks were ~94% of a 50-peer join
|
|
2046
|
+
// burst's messages, throttled only by the rate limiter. An equal JOIN
|
|
2047
|
+
// beacon (another joiner in the same burst) asks for an ack too; our
|
|
2048
|
+
// pending ack - identical bytes, since it echoes that same state -
|
|
2049
|
+
// answers it as well, and _scheduleSyncReply() dedupes it below.
|
|
2050
|
+
if (equal && !(flags & (DIGEST_FLAG_JOIN | DIGEST_FLAG_CONFIRM))) {
|
|
2051
|
+
// A peer's periodic/resync beacon in our state: it makes our pending
|
|
2052
|
+
// ack redundant, and - if that peer is confirmed - it answers our
|
|
2053
|
+
// own outstanding join as well as any ack would. It also counts for
|
|
2054
|
+
// Trickle: the room has just compared itself against our digest.
|
|
2055
|
+
this._equalBeaconsHeard++;
|
|
2056
|
+
if (flags & DIGEST_FLAG_SETTLED) {
|
|
2057
|
+
this._confirmed = true;
|
|
2058
|
+
this._noteResponse(false);
|
|
2059
|
+
}
|
|
2060
|
+
else {
|
|
2061
|
+
this._equalUnsettledSeen = true;
|
|
2062
|
+
}
|
|
2063
|
+
this._cancelPendingAck();
|
|
2064
|
+
}
|
|
2065
|
+
if (senderBehind || !dsEqual) {
|
|
2066
|
+
const encoder = encoding.createEncoder();
|
|
2067
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
2068
|
+
syncProtocol.writeSyncStep2(encoder, this.doc, remoteSv);
|
|
2069
|
+
let reply = encoding.toUint8Array(encoder);
|
|
2070
|
+
if (!dsEqual) {
|
|
2071
|
+
// Round 5: a delete moves no clock, so the hash cannot say which of
|
|
2072
|
+
// us lacks a delete - and a SyncStep2 heals only its receiver. The
|
|
2073
|
+
// loser of a delete-only update was healed only by its OWN next
|
|
2074
|
+
// beacon, which idle backoff parks for up to 60 s (bench-idle-room
|
|
2075
|
+
// part b failed its 11 s cap at a 5 s interval). So the reply asks
|
|
2076
|
+
// back: our beacon rides in the same batch, and if the sender is
|
|
2077
|
+
// the one ahead it answers with its delete set. Terminates: a peer
|
|
2078
|
+
// healed by the SyncStep2 half sees an equal digest in the beacon
|
|
2079
|
+
// half and stays silent. (A behind-check timer on the mismatch was
|
|
2080
|
+
// tried first: every peer that overheard the loser's beacon armed
|
|
2081
|
+
// one and asked - ~2x the deliveries of a lossy edit burst on the
|
|
2082
|
+
// Matrix profile.)
|
|
2083
|
+
reply = this._encodeBatch([reply, this._encodeSyncStep1(0)]);
|
|
2084
|
+
}
|
|
2085
|
+
this._replyToSyncRequest(reply, false, remoteSv, senderClientID);
|
|
2086
|
+
}
|
|
2087
|
+
else if (flags & (DIGEST_FLAG_JOIN | DIGEST_FLAG_CONFIRM)) {
|
|
2088
|
+
this._replyToSyncRequest(this._encodeAck(remoteSv, remoteDsHash), true, null, senderClientID);
|
|
2089
|
+
}
|
|
2090
|
+
if (!weBehind && dsEqual) {
|
|
2091
|
+
this._markSynced();
|
|
2092
|
+
}
|
|
2093
|
+
if (weBehind) {
|
|
2094
|
+
// The sender has something we lack. Not a request yet - the update
|
|
2095
|
+
// may still be in flight (see _scheduleBehindCheck).
|
|
2096
|
+
this._scheduleBehindCheck(remoteSv);
|
|
2097
|
+
}
|
|
2098
|
+
if (flags & DIGEST_FLAG_JOIN && this.awareness.getLocalState() !== null) {
|
|
2099
|
+
// Presence on demand: the joiner asked. One broadcast per burst of
|
|
2100
|
+
// joiners (see _schedulePresenceResponse), never suppressed (each
|
|
2101
|
+
// responder's state is distinct). Skipped when we have no state to
|
|
2102
|
+
// announce.
|
|
2103
|
+
this._schedulePresenceResponse(senderClientID);
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
/**
|
|
2107
|
+
* Answer a JOIN beacon's presence request once for all JOIN beacons that
|
|
2108
|
+
* arrive within `clamp(2 * minRTT, 100, 500)` ms of the first - long
|
|
2109
|
+
* enough to cover a join burst spread by latency, short enough that a
|
|
2110
|
+
* lone joiner sees the room's presence within a few round trips.
|
|
2111
|
+
*/
|
|
2112
|
+
_schedulePresenceResponse(requester) {
|
|
2113
|
+
this._presencePending.add(requester);
|
|
2114
|
+
if (this._presenceResponseTimer !== undefined)
|
|
2115
|
+
return;
|
|
2116
|
+
this._presenceCovered = false;
|
|
2117
|
+
const rtt = this._rttMinMs();
|
|
2118
|
+
// Phase 1e, relay path: one peer per 2 s bucket - the first-ranked
|
|
2119
|
+
// for a constant requester - relays the whole awareness table at once
|
|
2120
|
+
// (the way y-websocket's server does; clocks travel with the states,
|
|
2121
|
+
// a peer's own echoed state at an equal clock is ignored by
|
|
2122
|
+
// applyAwarenessUpdate). Everyone else waits the usual window, and
|
|
2123
|
+
// stays silent if that table carried their state (_presenceCovered).
|
|
2124
|
+
// Presence per late join: ~N deliveries instead of (N-1)^2 - it was
|
|
2125
|
+
// 82% of a late join into a 100-peer room (bench-join-census). Bytes
|
|
2126
|
+
// are unchanged (the table goes to everyone). Peers that cannot yet
|
|
2127
|
+
// tell (no RTT estimate on a slow link, so the table may arrive after
|
|
2128
|
+
// their window) fall back to the broadcast of their own state, as
|
|
2129
|
+
// before. The unicast path below is untouched.
|
|
2130
|
+
// Needs a view of the room: in a fresh burst the first JOIN arrives
|
|
2131
|
+
// before any peer is known, everyone would rank first and relay a
|
|
2132
|
+
// table each - the broadcast fallback is right for that case.
|
|
2133
|
+
const relayer = !this._canUnicast(requester) &&
|
|
2134
|
+
this._knownPeers.size >= 3 &&
|
|
2135
|
+
this._responderRank(0, 1) === 0;
|
|
2136
|
+
const delay = relayer ? 0 : Math.min(500, Math.max(100, rtt === null ? 0 : 2 * rtt));
|
|
2137
|
+
this._presenceResponseTimer = setTimeout(() => {
|
|
2138
|
+
this._presenceResponseTimer = undefined;
|
|
2139
|
+
const requesters = Array.from(this._presencePending);
|
|
2140
|
+
this._presencePending.clear();
|
|
2141
|
+
if (this._destroying || !this.transport.isConnected)
|
|
2142
|
+
return;
|
|
2143
|
+
// The app channel has no presence machinery of its own (deliberately -
|
|
2144
|
+
// it must not drive _knownPeers), so it has nothing that would tell a
|
|
2145
|
+
// joiner about existing cursors. A JOIN is the one signal that someone
|
|
2146
|
+
// needs our state, so answer it on both channels. Sent on its own wire
|
|
2147
|
+
// type and its own throttle; the relay/unicast fan-out optimisations
|
|
2148
|
+
// below apply to core presence only, since the app table is typically
|
|
2149
|
+
// a few cursors rather than the room's full census.
|
|
2150
|
+
this._announceAppAwareness();
|
|
2151
|
+
if (this.awareness.getLocalState() === null)
|
|
2152
|
+
return;
|
|
2153
|
+
// Every joiner covered by this timer is addressable: one unicast
|
|
2154
|
+
// each ((N-1) deliveries per joiner room-wide) instead of one
|
|
2155
|
+
// broadcast ((N-1)^2). Otherwise the broadcast, as before.
|
|
2156
|
+
if (requesters.every((id) => this._canUnicast(id))) {
|
|
2157
|
+
const msg = this._encodeAwareness([this.doc.clientID]);
|
|
2158
|
+
for (const id of requesters)
|
|
2159
|
+
this._sendDirect(id, msg);
|
|
2160
|
+
}
|
|
2161
|
+
else if (relayer) {
|
|
2162
|
+
this._sendAwarenessNow(Array.from(this.awareness.getStates().keys()));
|
|
2163
|
+
}
|
|
2164
|
+
else if (!this._presenceCovered) {
|
|
2165
|
+
this._broadcastAwareness([this.doc.clientID]);
|
|
2166
|
+
}
|
|
2167
|
+
}, delay);
|
|
2168
|
+
}
|
|
2169
|
+
/**
|
|
2170
|
+
* Max random delay (ms) before replying to a SyncStep1 request, scaled by
|
|
2171
|
+
* a room-size signal already available (`this.awareness.getStates().size`
|
|
2172
|
+
* - the same signal read at the `>= 3` suppression gate). A fixed window
|
|
2173
|
+
* (the pre-fix behavior: always `_syncReplySuppressionMs`) doesn't scale
|
|
2174
|
+
* with room size, so a larger room has more independent repliers racing
|
|
2175
|
+
* to answer the same request within the same window - more of them lose
|
|
2176
|
+
* the race and get silently dropped by the `_sendSyncReply()` rate-limit
|
|
2177
|
+
* backstop instead of never sending in the first place. Measured in
|
|
2178
|
+
* test/dummy/bench-corruption-storm.ts: the SyncStep2/SyncStep1 ratio (
|
|
2179
|
+
* ideally ~1 if suppression alone were sufficient) grew from ~1.1-1.3 at
|
|
2180
|
+
* N=2 to ~4.5-5.9 at N=10 with the fixed 30ms window.
|
|
2181
|
+
*
|
|
2182
|
+
* `min(cap, base * log2(peerCount))` - log2 growth spreads replies over a
|
|
2183
|
+
* wider window as the room grows without the delay exploding at very high
|
|
2184
|
+
* N. Capped at 200ms: the slowest-profile round trip this project
|
|
2185
|
+
* benchmarks against (Matrix, ~350ms one-way) already tolerates hundreds
|
|
2186
|
+
* of ms of latency, so 200ms of extra requester-perceived delay stays
|
|
2187
|
+
* well inside that budget while still giving a 100-peer room roughly
|
|
2188
|
+
* 6-7x the base window instead of an unbounded one.
|
|
2189
|
+
*/
|
|
2190
|
+
/**
|
|
2191
|
+
* How many peers we believe are in the room: awareness states (includes
|
|
2192
|
+
* ourselves) or, if larger, the distinct beacon/update senders we have
|
|
2193
|
+
* heard plus ourselves. See `_knownPeers`.
|
|
2194
|
+
*/
|
|
2195
|
+
_peerCount() {
|
|
2196
|
+
return Math.max(this.awareness.getStates().size, this._knownPeers.size + 1);
|
|
2197
|
+
}
|
|
2198
|
+
/**
|
|
2199
|
+
* Resolves `_awarenessInterval` to a concrete ms value: the configured
|
|
2200
|
+
* fixed number, or (round 6, item 9) `max(transport hint ?? 100,
|
|
2201
|
+
* AWARENESS_AUTO_MS_PER_PEER * peerCount)` when set to `'auto'`.
|
|
2202
|
+
*/
|
|
2203
|
+
_effectiveAwarenessInterval() {
|
|
2204
|
+
if (this._awarenessInterval !== 'auto')
|
|
2205
|
+
return this._awarenessInterval;
|
|
2206
|
+
const hint = this.transport.preferredAwarenessMs ?? 100;
|
|
2207
|
+
return Math.max(hint, GenericProvider.AWARENESS_AUTO_MS_PER_PEER * this._peerCount());
|
|
2208
|
+
}
|
|
2209
|
+
_replySuppressionMaxDelay() {
|
|
2210
|
+
const peerCount = this._peerCount();
|
|
2211
|
+
const byRoomSize = Math.min(200, this._syncReplySuppressionMs * Math.log2(Math.max(2, peerCount)));
|
|
2212
|
+
// Phase 1b: the window must exceed the one-way latency or nobody
|
|
2213
|
+
// overhears anybody in time (see _rttSamples). 1.5x the smallest
|
|
2214
|
+
// observed round trip, capped at 2 s - on a 350 ms profile that is
|
|
2215
|
+
// ~1 s of extra requester-perceived delay in exchange for ~1 reply
|
|
2216
|
+
// instead of ~20.
|
|
2217
|
+
const rtt = this._rttMinMs();
|
|
2218
|
+
return rtt === null ? byRoomSize : Math.min(2000, Math.max(byRoomSize, 1.5 * rtt));
|
|
2219
|
+
}
|
|
2220
|
+
/**
|
|
2221
|
+
* Schedule a SyncStep2 reply after a short random delay instead of
|
|
2222
|
+
* sending immediately. If another peer's reply is overheard in the
|
|
2223
|
+
* meantime (`_cancelPendingSyncReply`), this reply is dropped as
|
|
2224
|
+
* redundant - the requester likely already got what it needed.
|
|
2225
|
+
*
|
|
2226
|
+
* A reply that is already pending when this is called answers a
|
|
2227
|
+
* *different* request (e.g. peer A's request, followed 5ms later by
|
|
2228
|
+
* peer B's) - it must not be silently overwritten by the new one. Flush
|
|
2229
|
+
* it immediately, then schedule the new reply fresh. The only sanctioned
|
|
2230
|
+
* ways a reply gets dropped are `_cancelPendingSyncReply()` (we overheard
|
|
2231
|
+
* someone else's SyncStep2 for the SAME request), `_cancelPendingAck()`,
|
|
2232
|
+
* and the identical-bytes case below.
|
|
2233
|
+
*
|
|
2234
|
+
* Identical-bytes case (Task 3c in the design doc): K peers with the same
|
|
2235
|
+
* state asking at once (K empty joiners in a burst) get K byte-identical
|
|
2236
|
+
* SyncStep2s from us - the same full document K times, one flushed
|
|
2237
|
+
* immediately per arriving request, each burning a rate-limit slot. If
|
|
2238
|
+
* the new reply's bytes equal the pending reply's bytes, the pending one
|
|
2239
|
+
* already answers this request too: keep it (same delay, same
|
|
2240
|
+
* suppression) and drop the new one. Measured in
|
|
2241
|
+
* test/dummy/bench-join-after-burst.ts.
|
|
2242
|
+
*/
|
|
2243
|
+
_scheduleSyncReply(reply, isAck = false, targetSv = null, requester = null) {
|
|
2244
|
+
if (this._pendingSyncReplyTimeoutId !== undefined && this._pendingSyncReply !== null) {
|
|
2245
|
+
if (bytesEqual(this._pendingSyncReply, reply)) {
|
|
2246
|
+
return; // identical answer already scheduled
|
|
2247
|
+
}
|
|
2248
|
+
if (targetSv !== null &&
|
|
2249
|
+
this._pendingSyncReplyTargetSv !== null &&
|
|
2250
|
+
bytesEqual(this._pendingSyncReplyTargetSv, targetSv)) {
|
|
2251
|
+
// Same question (same requester state), newer document: refresh
|
|
2252
|
+
// the answer, keep the timer. See _pendingSyncReplyTargetSv.
|
|
2253
|
+
this._pendingSyncReply = reply;
|
|
2254
|
+
return;
|
|
2255
|
+
}
|
|
2256
|
+
if (!isAck &&
|
|
2257
|
+
!this._pendingSyncReplyIsAck &&
|
|
2258
|
+
targetSv !== null &&
|
|
2259
|
+
this._pendingSyncReplyTargetSv !== null) {
|
|
2260
|
+
// Two requesters, both behind, different states: one SyncStep2
|
|
2261
|
+
// from the componentwise minimum of both state vectors contains
|
|
2262
|
+
// everything either of them lacks. Keep the pending reply's timer
|
|
2263
|
+
// and widen its content instead of flushing it. The flush (the
|
|
2264
|
+
// rule below, kept for acks and legacy SyncStep1s) sent an
|
|
2265
|
+
// unsuppressed broadcast for every second request that arrived
|
|
2266
|
+
// inside the suppression window; with the window at 1.5x RTT on a
|
|
2267
|
+
// 350 ms link and several peers behind after a lossy edit burst
|
|
2268
|
+
// that was ~4-5 broadcast replies per beacon (phase-1c results).
|
|
2269
|
+
const merged = minStateVector(this._pendingSyncReplyTargetSv, targetSv);
|
|
2270
|
+
const encoder = encoding.createEncoder();
|
|
2271
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
2272
|
+
syncProtocol.writeSyncStep2(encoder, this.doc, merged);
|
|
2273
|
+
this._pendingSyncReply = encoding.toUint8Array(encoder);
|
|
2274
|
+
this._pendingSyncReplyTargetSv = merged;
|
|
2275
|
+
return;
|
|
2276
|
+
}
|
|
2277
|
+
// Phase 1e: an ack and a SyncStep2 never flush each other. A room
|
|
2278
|
+
// whose JOIN waits expire together sends N CONFIRMs in the same
|
|
2279
|
+
// millisecond; while a lossy edit burst is still healing, some of
|
|
2280
|
+
// them find us equal (ack) and some behind (SyncStep2), and the flush
|
|
2281
|
+
// rule below turned every type change into an immediate broadcast -
|
|
2282
|
+
// ~650 replies in 200 ms at N=50, 5 % loss (probe timeline in the
|
|
2283
|
+
// phase-1e design doc). An ack adds nothing to a pending reply of
|
|
2284
|
+
// either kind (the requester's wait retries, or an equal peer's
|
|
2285
|
+
// SETTLED ack confirms it); a SyncStep2 replaces a pending ack.
|
|
2286
|
+
if (isAck)
|
|
2287
|
+
return;
|
|
2288
|
+
if (this._pendingSyncReplyIsAck) {
|
|
2289
|
+
clearTimeout(this._pendingSyncReplyTimeoutId);
|
|
2290
|
+
this._pendingSyncReplyTimeoutId = undefined;
|
|
2291
|
+
}
|
|
2292
|
+
}
|
|
2293
|
+
if (this._pendingSyncReplyTimeoutId !== undefined) {
|
|
2294
|
+
// Only a legacy plain SyncStep1 (no target state vector) still
|
|
2295
|
+
// flushes a pending SyncStep2.
|
|
2296
|
+
if (this._pendingSyncReply) {
|
|
2297
|
+
this._sendSyncReply(this._pendingSyncReply);
|
|
2298
|
+
}
|
|
2299
|
+
clearTimeout(this._pendingSyncReplyTimeoutId);
|
|
2300
|
+
this._pendingSyncReplyTimeoutId = undefined;
|
|
2301
|
+
}
|
|
2302
|
+
this._pendingSyncReply = reply;
|
|
2303
|
+
this._pendingSyncReplyIsAck = isAck;
|
|
2304
|
+
this._pendingSyncReplyTargetSv = targetSv;
|
|
2305
|
+
// Acks keep the uniform window: they carry no data, so their delay
|
|
2306
|
+
// costs nothing but a few ms on a joiner's `synced` flip, and in a
|
|
2307
|
+
// join burst one pending ack answers every equal JOIN that arrives
|
|
2308
|
+
// inside that window (identical bytes, deduped above). Ranked, rank 0
|
|
2309
|
+
// fired at once for every requester - measured: fresh-burst acks
|
|
2310
|
+
// 6,039 -> 15,147 at Gun N=100, join-after-burst Matrix 147 -> 686.
|
|
2311
|
+
const delay = isAck
|
|
2312
|
+
? Math.random() * this._replySuppressionMaxDelay()
|
|
2313
|
+
: this._replyDelay(requester);
|
|
2314
|
+
this._pendingSyncReplyTimeoutId = setTimeout(() => {
|
|
2315
|
+
this._pendingSyncReplyTimeoutId = undefined;
|
|
2316
|
+
if (this._pendingSyncReply) {
|
|
2317
|
+
this._sendSyncReply(this._pendingSyncReply);
|
|
2318
|
+
this._pendingSyncReply = null;
|
|
2319
|
+
this._pendingSyncReplyTargetSv = null;
|
|
2320
|
+
}
|
|
2321
|
+
}, delay);
|
|
2322
|
+
}
|
|
869
2323
|
/** Cancel a pending suppressed reply, if any. */
|
|
870
2324
|
_cancelPendingSyncReply() {
|
|
871
2325
|
if (this._pendingSyncReplyTimeoutId !== undefined) {
|
|
@@ -873,6 +2327,497 @@ export class GenericProvider extends Observable {
|
|
|
873
2327
|
this._pendingSyncReplyTimeoutId = undefined;
|
|
874
2328
|
}
|
|
875
2329
|
this._pendingSyncReply = null;
|
|
2330
|
+
this._pendingSyncReplyIsAck = false;
|
|
2331
|
+
this._pendingSyncReplyTargetSv = null;
|
|
2332
|
+
}
|
|
2333
|
+
/**
|
|
2334
|
+
* Cancel a pending reply only if it is a digest ack - see
|
|
2335
|
+
* `_pendingSyncReplyIsAck`. Called from `_handleDigest()` on every
|
|
2336
|
+
* overheard beacon whose digest equals ours.
|
|
2337
|
+
*/
|
|
2338
|
+
_cancelPendingAck() {
|
|
2339
|
+
if (this._pendingSyncReplyIsAck) {
|
|
2340
|
+
this._cancelPendingSyncReply();
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
/**
|
|
2344
|
+
* Route a SyncStep2 (or digest-ack) reply through the redundancy
|
|
2345
|
+
* suppression when there's genuine redundancy (>= 2 other known peers via
|
|
2346
|
+
* awareness - below that there's no "someone else" to rely on), else send
|
|
2347
|
+
* immediately. Both paths are rate-limited by `_sendSyncReply()`. Shared
|
|
2348
|
+
* by the MESSAGE_SYNC, MESSAGE_SYNC_VERIFIED and MESSAGE_SYNC_DIGEST cases.
|
|
2349
|
+
*/
|
|
2350
|
+
_replyToSyncRequest(reply, isAck = false, targetSv = null, toClientID) {
|
|
2351
|
+
// A peer that knows it is incomplete does not answer. A SyncStep2 is
|
|
2352
|
+
// encoded from integrated structs only, so with structs (or a delete
|
|
2353
|
+
// set) still pending ours would be provably partial - and the
|
|
2354
|
+
// requester's response wait ends on the first SyncStep2 it gets, so a
|
|
2355
|
+
// partial answer strands it until its next trigger (phase-1c gates:
|
|
2356
|
+
// 5 s resync backoff, or a stall with syncInterval 0). In relay mode a
|
|
2357
|
+
// partial broadcast also cancels the complete replies other peers had
|
|
2358
|
+
// pending. Let them answer; the requester retries if nobody does, and
|
|
2359
|
+
// in unicast mode the rank bucket rotates the responders every 2 s.
|
|
2360
|
+
// An ack from us would likewise confirm a state we do not trust.
|
|
2361
|
+
if (this.doc.store.pendingStructs !== null ||
|
|
2362
|
+
this.doc.store.pendingDs !== null) {
|
|
2363
|
+
return;
|
|
2364
|
+
}
|
|
2365
|
+
// Unicast path (transport has sendTo and we know the requester's
|
|
2366
|
+
// address): nobody overhears a unicast, so the delay-and-cancel
|
|
2367
|
+
// suppression below cannot thin the replies. Instead each candidate
|
|
2368
|
+
// responder decides for itself whether it is one of ~3 that answer
|
|
2369
|
+
// (_selectedResponder), and answers at once - no suppression delay,
|
|
2370
|
+
// one delivery. The requester's response wait retries if all ~3 are
|
|
2371
|
+
// lost. Phase-1c design, item B.
|
|
2372
|
+
if (toClientID !== undefined && this._canUnicast(toClientID)) {
|
|
2373
|
+
if (!this._selectedResponder(toClientID))
|
|
2374
|
+
return;
|
|
2375
|
+
if (!this._tryReserveReplySlot())
|
|
2376
|
+
return;
|
|
2377
|
+
this._sendDirect(toClientID, reply);
|
|
2378
|
+
return;
|
|
2379
|
+
}
|
|
2380
|
+
// Acks ALWAYS take the delayed/suppressible path: they carry no data,
|
|
2381
|
+
// so the only cost of delaying one is a few ms on the joiner's `synced`
|
|
2382
|
+
// flip (measured before this rule: 37,240 of a 50-peer join burst's
|
|
2383
|
+
// 40,915 deliveries were immediate acks). Everything else goes through
|
|
2384
|
+
// suppression once there is someone else who could answer - counted
|
|
2385
|
+
// from beacon/update senders as well as awareness, see _peerCount().
|
|
2386
|
+
if (isAck || this._peerCount() >= 3) {
|
|
2387
|
+
this._scheduleSyncReply(reply, isAck, targetSv, toClientID ?? null);
|
|
2388
|
+
}
|
|
2389
|
+
else {
|
|
2390
|
+
this._sendSyncReply(reply);
|
|
2391
|
+
}
|
|
2392
|
+
}
|
|
2393
|
+
/** Whether a reply to `clientID` can go over Transport.sendTo. */
|
|
2394
|
+
_canUnicast(clientID) {
|
|
2395
|
+
return (typeof this.transport.sendTo === 'function' &&
|
|
2396
|
+
this._peerAddress.has(clientID));
|
|
2397
|
+
}
|
|
2398
|
+
/**
|
|
2399
|
+
* Responder self-selection for unicast replies: the three peers whose
|
|
2400
|
+
* hash for this requester ranks lowest among the peers we know answer
|
|
2401
|
+
* it. Every candidate ranks itself against the same known set, so the
|
|
2402
|
+
* sets agree wherever the views agree, and the peer that ranks first in
|
|
2403
|
+
* the true order always ranks first in its own view - the selection is
|
|
2404
|
+
* never empty. A 2 s time bucket in the hash rotates the ranking, so
|
|
2405
|
+
* three departed peers at the top only delay a reply until the
|
|
2406
|
+
* requester's next attempt. Everyone answers in rooms of four or fewer.
|
|
2407
|
+
* (A first cut chose each responder independently with probability 3/N;
|
|
2408
|
+
* ~5 % of requests then selected nobody and waited for the 1 s retry.)
|
|
2409
|
+
*/
|
|
2410
|
+
_selectedResponder(requester) {
|
|
2411
|
+
if (this._peerCount() < 4)
|
|
2412
|
+
return true;
|
|
2413
|
+
return this._responderRank(requester, 3) < 3;
|
|
2414
|
+
}
|
|
2415
|
+
/**
|
|
2416
|
+
* How many known peers rank below us for `requester` in the current 2 s
|
|
2417
|
+
* bucket (counting stops at `cap`). Shared by unicast self-selection
|
|
2418
|
+
* (rank < 3 answers) and, since phase 1e, the relay-mode reply delay
|
|
2419
|
+
* (rank r waits r slots, see _replyDelay()).
|
|
2420
|
+
*/
|
|
2421
|
+
_responderRank(requester, cap) {
|
|
2422
|
+
const bucket = Math.floor(Date.now() / 2000);
|
|
2423
|
+
const rank = (id) => (Math.imul(requester ^ bucket, 0x9e3779b1) ^ Math.imul(id, 0x85ebca6b)) >>> 0;
|
|
2424
|
+
const mine = rank(this.doc.clientID);
|
|
2425
|
+
let better = 0;
|
|
2426
|
+
for (const id of this._knownPeers.keys()) {
|
|
2427
|
+
if (id === requester || id === this.doc.clientID)
|
|
2428
|
+
continue;
|
|
2429
|
+
if (rank(id) < mine && ++better >= cap)
|
|
2430
|
+
break;
|
|
2431
|
+
}
|
|
2432
|
+
return better;
|
|
2433
|
+
}
|
|
2434
|
+
/**
|
|
2435
|
+
* Delay before a suppressible reply goes out (relay path). Phase 1e:
|
|
2436
|
+
* ranked, not uniform. A uniform draw from [0, W] lets ~N * L / W
|
|
2437
|
+
* repliers fire before the first reply is overheard (L = one-way
|
|
2438
|
+
* latency): 10-27 SyncStep2 sends per request at N=100 in
|
|
2439
|
+
* test/dummy/bench-join-census.ts, and the WebRTC join-burst cell's
|
|
2440
|
+
* 16-34k spread. With the responder rank (the same hash the unicast
|
|
2441
|
+
* self-selection uses) rank 0 answers at once and rank r waits r
|
|
2442
|
+
* windows (W = _replySuppressionMaxDelay(), 1.5x the minimum round
|
|
2443
|
+
* trip: with request arrival spread 2jL and reply flight L(1+j), rank 1
|
|
2444
|
+
* has overheard rank 0 iff the slot is >= L(1+3j), which 3L(1-j) covers
|
|
2445
|
+
* up to j~0.33). Ranks >= 8 add a random window on top so a room whose
|
|
2446
|
+
* first eight ranked peers are all gone does not answer in one
|
|
2447
|
+
* avalanche. Without an RTT sample or a requester id (legacy SyncStep1)
|
|
2448
|
+
* the uniform window stays.
|
|
2449
|
+
*/
|
|
2450
|
+
_replyDelay(requester) {
|
|
2451
|
+
const window = this._replySuppressionMaxDelay();
|
|
2452
|
+
const rtt = this._rttMinMs();
|
|
2453
|
+
if (requester === null || rtt === null)
|
|
2454
|
+
return Math.random() * window;
|
|
2455
|
+
// Half a window per rank (0.75 x the minimum round trip): a rank that
|
|
2456
|
+
// stays silent (pending structs during a lossy burst, phase 1d B)
|
|
2457
|
+
// costs the requester half a window, not a whole one - at a full
|
|
2458
|
+
// window the Matrix 5 % loss fan-out's median convergence doubled
|
|
2459
|
+
// (1.1 -> 2.2 s, worst 7.6 s); the price is an occasional second
|
|
2460
|
+
// reply where the jitter exceeds ~1/3 (probe numbers in the design doc).
|
|
2461
|
+
const slot = Math.max(this._syncReplySuppressionMs, 0.75 * rtt);
|
|
2462
|
+
const rank = this._responderRank(requester, 8);
|
|
2463
|
+
return rank * slot + (rank >= 8 ? Math.random() * window : 0);
|
|
2464
|
+
}
|
|
2465
|
+
/**
|
|
2466
|
+
* Send one already-encoded message to a single peer over
|
|
2467
|
+
* Transport.sendTo, with the same CRC32 wrapping and optional compression
|
|
2468
|
+
* as a broadcast. Not mirrored to BroadcastChannel (a same-browser tab
|
|
2469
|
+
* never appears as an addressable peer). Returns false if the peer's
|
|
2470
|
+
* address is unknown or the transport cannot unicast.
|
|
2471
|
+
*/
|
|
2472
|
+
_sendDirect(clientID, data) {
|
|
2473
|
+
const address = this._peerAddress.get(clientID);
|
|
2474
|
+
if (address === undefined || typeof this.transport.sendTo !== 'function') {
|
|
2475
|
+
return false;
|
|
2476
|
+
}
|
|
2477
|
+
if (!this.transport.isConnected)
|
|
2478
|
+
return false;
|
|
2479
|
+
this._sendToTransport(wrapMessageWithChecksum(data), address);
|
|
2480
|
+
return true;
|
|
2481
|
+
}
|
|
2482
|
+
/**
|
|
2483
|
+
* Re-check Yjs's pending-struct store after the gap grace period and
|
|
2484
|
+
* request a resync (a beacon, see _requestResync) only if something is
|
|
2485
|
+
* still missing. One timer; a check scheduled while one is pending is
|
|
2486
|
+
* absorbed. Cleared on disconnect/destroy.
|
|
2487
|
+
*/
|
|
2488
|
+
/**
|
|
2489
|
+
* A beacon (a peer's periodic tick, or its request) just showed its
|
|
2490
|
+
* sender ahead of us. Until phase 1d nothing happened with that: a peer
|
|
2491
|
+
* whose last update was lost (no later message to open a sequence gap
|
|
2492
|
+
* against), or whose request was answered by a responder that was
|
|
2493
|
+
* itself behind, waited for its OWN next periodic beacon - up to
|
|
2494
|
+
* syncInterval, up to idleBackoffMaxMs with idle backoff on. Now we
|
|
2495
|
+
* check again after a grace and, if still behind that state, ask through
|
|
2496
|
+
* the resync coordinator (coalesced, backed off, rate-limited).
|
|
2497
|
+
*
|
|
2498
|
+
* The grace is what keeps this quiet during typing: at Matrix latency
|
|
2499
|
+
* almost every receiver of a periodic beacon is "behind" by a keystroke
|
|
2500
|
+
* that is still in flight (jitter +-140 ms); max(gapGraceMs, 2 x minRTT)
|
|
2501
|
+
* later it has arrived and the check finds nothing to do. A lost
|
|
2502
|
+
* keystroke that opened a sequence gap is already being requested by the
|
|
2503
|
+
* gap check - the outstanding response wait tells us so, and we stay
|
|
2504
|
+
* quiet. One timer, the newest state vector: a later beacon that shows us
|
|
2505
|
+
* behind by more replaces the reference, the timer keeps running.
|
|
2506
|
+
*/
|
|
2507
|
+
_scheduleBehindCheck(remoteSv) {
|
|
2508
|
+
this._behindSv = remoteSv;
|
|
2509
|
+
if (this._behindCheckTimer !== undefined)
|
|
2510
|
+
return;
|
|
2511
|
+
const rtt = this._rttMinMs();
|
|
2512
|
+
const delay = Math.max(this._gapGraceMs, rtt === null ? 0 : 2 * rtt);
|
|
2513
|
+
this._behindCheckTimer = setTimeout(() => {
|
|
2514
|
+
this._behindCheckTimer = undefined;
|
|
2515
|
+
const sv = this._behindSv;
|
|
2516
|
+
this._behindSv = null;
|
|
2517
|
+
if (sv === null || this._destroying || !this.transport.isConnected)
|
|
2518
|
+
return;
|
|
2519
|
+
// A request of ours sent within the grace is still being answered.
|
|
2520
|
+
// An OLDER outstanding wait is not a reason to stay behind: a new
|
|
2521
|
+
// room's first peers keep their JOIN wait parked for seconds (three
|
|
2522
|
+
// retries, nobody SETTLED yet), and measured against it the check
|
|
2523
|
+
// never fired - bench-idle-backoff's recovery stayed at one
|
|
2524
|
+
// backed-off interval (1,846 ms) with this line reading
|
|
2525
|
+
// `_responseWaitTimer !== undefined`.
|
|
2526
|
+
if (this._requestSentAt > 0 && Date.now() - this._requestSentAt < delay)
|
|
2527
|
+
return;
|
|
2528
|
+
const remote = Y.decodeStateVector(sv);
|
|
2529
|
+
const local = Y.decodeStateVector(Y.encodeStateVector(this.doc));
|
|
2530
|
+
for (const [client, clock] of remote) {
|
|
2531
|
+
if ((local.get(client) ?? 0) < clock) {
|
|
2532
|
+
this._requestResync();
|
|
2533
|
+
return;
|
|
2534
|
+
}
|
|
2535
|
+
}
|
|
2536
|
+
}, delay);
|
|
2537
|
+
}
|
|
2538
|
+
/** Design E: after a reply or push, pending structs mean the sender had the same hole - arm the grace check. */
|
|
2539
|
+
_checkPendingAfterReply() {
|
|
2540
|
+
if (this.doc.store.pendingStructs !== null ||
|
|
2541
|
+
this.doc.store.pendingDs !== null) {
|
|
2542
|
+
this._schedulePendingCheck();
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
_schedulePendingCheck() {
|
|
2546
|
+
if (this._pendingCheckTimer !== undefined)
|
|
2547
|
+
return;
|
|
2548
|
+
this._pendingCheckTimer = setTimeout(() => {
|
|
2549
|
+
this._pendingCheckTimer = undefined;
|
|
2550
|
+
if (this._destroying || !this.transport.isConnected)
|
|
2551
|
+
return;
|
|
2552
|
+
// A request of ours may already be outstanding (JOIN or resync beacon
|
|
2553
|
+
// with its response wait running): whatever is pending is what that
|
|
2554
|
+
// request is fetching, and the response wait retries if it is lost.
|
|
2555
|
+
// Asking again here just raced the responders' suppression window
|
|
2556
|
+
// (measured: a joiner's keystrokes-before-content check fired at
|
|
2557
|
+
// 300 ms while the settled peers' replies were still delayed by a
|
|
2558
|
+
// window of up to ~300 ms, and every such re-beacon collected
|
|
2559
|
+
// another round of replies).
|
|
2560
|
+
// ... but only a request that went out within the last grace, whose
|
|
2561
|
+
// answer may still be on its way. Deferring to ANY outstanding
|
|
2562
|
+
// response wait (the rule until phase 1d) parked every pending check
|
|
2563
|
+
// behind a new room's JOIN wait - three retries with nobody SETTLED
|
|
2564
|
+
// yet, 19.6 s on a 700 ms link - so a peer that lost a keystroke in
|
|
2565
|
+
// such a room asked only once an overheard reply happened to clear
|
|
2566
|
+
// that wait (phase-1d design doc, "After Task 4", the fresh-budget
|
|
2567
|
+
// Matrix fan-out). Look again once the recent request is answered or
|
|
2568
|
+
// given up - dropping the check here left a joiner whose SyncStep2
|
|
2569
|
+
// predated the keystrokes it had received with nine pending updates
|
|
2570
|
+
// and no trigger (measured: 2 of 150 lossy 15-peer joins).
|
|
2571
|
+
const rtt = this._rttMinMs();
|
|
2572
|
+
const recent = Math.max(this._gapGraceMs, rtt === null ? 0 : 2 * rtt);
|
|
2573
|
+
if (this._requestSentAt > 0 && Date.now() - this._requestSentAt < recent) {
|
|
2574
|
+
this._schedulePendingCheck();
|
|
2575
|
+
return;
|
|
2576
|
+
}
|
|
2577
|
+
if (this.doc.store.pendingStructs !== null ||
|
|
2578
|
+
this.doc.store.pendingDs !== null) {
|
|
2579
|
+
this._requestResync();
|
|
2580
|
+
}
|
|
2581
|
+
}, this._gapGraceMs);
|
|
2582
|
+
}
|
|
2583
|
+
/**
|
|
2584
|
+
* Return the update payload of a SyncStep2/Update sync sub-message
|
|
2585
|
+
* without advancing `decoder` (null for SyncStep1 or malformed input).
|
|
2586
|
+
* y-protocols frames both as [subType varUint][update varUint8Array].
|
|
2587
|
+
*/
|
|
2588
|
+
_peekSyncUpdate(decoder) {
|
|
2589
|
+
const peek = decoding.clone(decoder);
|
|
2590
|
+
try {
|
|
2591
|
+
const subType = decoding.readVarUint(peek);
|
|
2592
|
+
if (subType === syncProtocol.messageYjsSyncStep1)
|
|
2593
|
+
return null;
|
|
2594
|
+
return decoding.readVarUint8Array(peek);
|
|
2595
|
+
}
|
|
2596
|
+
catch {
|
|
2597
|
+
return null;
|
|
2598
|
+
}
|
|
2599
|
+
}
|
|
2600
|
+
/**
|
|
2601
|
+
* Whether an update we just applied was already superseded here: every
|
|
2602
|
+
* client it touches ends at a clock we were at or beyond BEFORE this
|
|
2603
|
+
* update (i.e. it added nothing). Uses the update's own metadata
|
|
2604
|
+
* (`Y.parseUpdateMeta`), O(clients in the update).
|
|
2605
|
+
*/
|
|
2606
|
+
_isLateUpdate(updateBytes) {
|
|
2607
|
+
if (!updateBytes)
|
|
2608
|
+
return false;
|
|
2609
|
+
try {
|
|
2610
|
+
const { to } = Y.parseUpdateMeta(updateBytes);
|
|
2611
|
+
if (to.size === 0)
|
|
2612
|
+
return false;
|
|
2613
|
+
const local = Y.decodeStateVector(Y.encodeStateVector(this.doc));
|
|
2614
|
+
for (const [client, clock] of to) {
|
|
2615
|
+
// `to` is the exclusive end clock of the update's range for that
|
|
2616
|
+
// client; our state vector is exclusive too. Equal means the update
|
|
2617
|
+
// brought us exactly here (not late); greater means we were past it.
|
|
2618
|
+
if ((local.get(client) ?? 0) <= clock)
|
|
2619
|
+
return false;
|
|
2620
|
+
}
|
|
2621
|
+
return true;
|
|
2622
|
+
}
|
|
2623
|
+
catch {
|
|
2624
|
+
return false;
|
|
2625
|
+
}
|
|
2626
|
+
}
|
|
2627
|
+
/** Flip `synced` once and emit; idempotent. */
|
|
2628
|
+
_markSynced() {
|
|
2629
|
+
if (!this._synced) {
|
|
2630
|
+
this._synced = true;
|
|
2631
|
+
this.emit('synced', [true]);
|
|
2632
|
+
}
|
|
2633
|
+
}
|
|
2634
|
+
/**
|
|
2635
|
+
* Wait for a response to the JOIN or resync beacon we just sent. If
|
|
2636
|
+
* neither a SyncStep2 nor an equal ack/beacon arrives within 1s (then
|
|
2637
|
+
* 2s, 4s), ask again - with a CONFIRM beacon after a JOIN (so an equal
|
|
2638
|
+
* room acks), with a plain beacon after a resync (only peers ahead of us
|
|
2639
|
+
* need to answer; an equal room's silence is the correct answer and its
|
|
2640
|
+
* periodic beacons end the wait) - three times at most; after that the
|
|
2641
|
+
* periodic beacon is the fallback, as before. Requester-side retry is how the protocol
|
|
2642
|
+
* stays loss-tolerant now that reply suppression leaves ~1 reply per
|
|
2643
|
+
* request; N-fold redundant replies were the old (accidental) way.
|
|
2644
|
+
*/
|
|
2645
|
+
_armResponseWait(retryFlags) {
|
|
2646
|
+
if (this._responseWaitTimer !== undefined)
|
|
2647
|
+
return;
|
|
2648
|
+
this._responseSeen = false;
|
|
2649
|
+
if (this._responseWaitAttempts === 0)
|
|
2650
|
+
this._equalUnsettledSeen = false;
|
|
2651
|
+
this._responseWaitFlags = retryFlags;
|
|
2652
|
+
this._requestSentAt = Date.now();
|
|
2653
|
+
const rtt = this._rttMinMs();
|
|
2654
|
+
const delay = Math.max(1000, rtt === null ? 0 : 4 * rtt) *
|
|
2655
|
+
Math.pow(2, this._responseWaitAttempts);
|
|
2656
|
+
this._responseWaitTimer = setTimeout(() => {
|
|
2657
|
+
this._responseWaitTimer = undefined;
|
|
2658
|
+
// Phase 1e: a fresh room - equal but unsettled peers answered both
|
|
2659
|
+
// the JOIN and one CONFIRM retry, nobody settled did. Two rounds
|
|
2660
|
+
// instead of three; the first confirmed peers' acks then carry
|
|
2661
|
+
// SETTLED for everyone after them. Measured in
|
|
2662
|
+
// test/dummy/bench-join-census.ts (b): the third round was ~a third
|
|
2663
|
+
// of a fresh N=100 room's 92k-delivery join burst.
|
|
2664
|
+
const freshRoomDone = this._equalUnsettledSeen && this._responseWaitAttempts >= 1;
|
|
2665
|
+
if (this._responseSeen ||
|
|
2666
|
+
this._responseWaitAttempts >= 3 ||
|
|
2667
|
+
freshRoomDone ||
|
|
2668
|
+
!this.transport.isConnected ||
|
|
2669
|
+
this._destroying) {
|
|
2670
|
+
// Asked three times, nobody had more for us: we are the room's
|
|
2671
|
+
// state (or its first peer). Bootstraps DIGEST_FLAG_SETTLED in a
|
|
2672
|
+
// brand-new room so later joiners are confirmed by our acks.
|
|
2673
|
+
if (this._responseWaitAttempts >= 3 || freshRoomDone)
|
|
2674
|
+
this._confirmed = true;
|
|
2675
|
+
this._responseWaitAttempts = 0;
|
|
2676
|
+
return;
|
|
2677
|
+
}
|
|
2678
|
+
this._responseWaitAttempts++;
|
|
2679
|
+
this._sendSyncStep1(this._responseWaitFlags); // rate-limited; a dropped attempt is simply retried next round
|
|
2680
|
+
this._armResponseWait(this._responseWaitFlags);
|
|
2681
|
+
}, delay);
|
|
2682
|
+
}
|
|
2683
|
+
/**
|
|
2684
|
+
* A SyncStep2 or an equal ack/beacon arrived - whatever we asked for is
|
|
2685
|
+
* answered. `sample` = it was a direct reply (SyncStep2/ack), so its
|
|
2686
|
+
* timing is a round-trip sample; an equal periodic beacon from a settled
|
|
2687
|
+
* peer also ends the wait but says nothing about latency.
|
|
2688
|
+
*/
|
|
2689
|
+
_noteResponse(sample) {
|
|
2690
|
+
if (sample && this._requestSentAt > 0) {
|
|
2691
|
+
this._rttSamples.push(Date.now() - this._requestSentAt);
|
|
2692
|
+
if (this._rttSamples.length > 8)
|
|
2693
|
+
this._rttSamples.shift();
|
|
2694
|
+
}
|
|
2695
|
+
this._requestSentAt = 0;
|
|
2696
|
+
this._responseSeen = true;
|
|
2697
|
+
this._responseWaitAttempts = 0;
|
|
2698
|
+
if (this._responseWaitTimer !== undefined) {
|
|
2699
|
+
clearTimeout(this._responseWaitTimer);
|
|
2700
|
+
this._responseWaitTimer = undefined;
|
|
2701
|
+
}
|
|
2702
|
+
}
|
|
2703
|
+
/** Minimum of the recent round-trip samples, or null before the first reply. */
|
|
2704
|
+
_rttMinMs() {
|
|
2705
|
+
return this._rttSamples.length === 0 ? null : Math.min(...this._rttSamples);
|
|
2706
|
+
}
|
|
2707
|
+
/**
|
|
2708
|
+
* Delay a pure timeout-removal awareness broadcast and drop it if
|
|
2709
|
+
* another peer's broadcast of the SAME removal is overheard first (see
|
|
2710
|
+
* the `origin === this` branch in `_setupAwarenessSync()`'s handler,
|
|
2711
|
+
* which calls `_cancelPendingAwarenessRemovalIfOverlaps()`) - the exact
|
|
2712
|
+
* same NACK-style suppression `_scheduleSyncReply()` already applies to
|
|
2713
|
+
* SyncStep2 replies, reusing the same room-size-scaled delay
|
|
2714
|
+
* (`_replySuppressionMaxDelay()`).
|
|
2715
|
+
*
|
|
2716
|
+
* A pending removal already queued when this is called is for a
|
|
2717
|
+
* DIFFERENT departure (two peers timing out, or leaving, within the same
|
|
2718
|
+
* window): since round 5 the ids are merged into the pending set and its
|
|
2719
|
+
* timer kept - one broadcast carries both - instead of flushing the
|
|
2720
|
+
* first as an unsuppressed broadcast (with the long leave window below a
|
|
2721
|
+
* burst of departures would have flushed on every peer). An overheard
|
|
2722
|
+
* broadcast trims only the ids it covers from the pending set
|
|
2723
|
+
* (`_cancelPendingAwarenessRemovalIfOverlaps()`).
|
|
2724
|
+
*
|
|
2725
|
+
* Window: the reply-suppression window for timeouts (sweeps are spread
|
|
2726
|
+
* over seconds anyway); for leaves reported by the transport - all
|
|
2727
|
+
* survivors learn of them in the same millisecond - ten times that,
|
|
2728
|
+
* at least a second, so the first broadcast is overheard before the
|
|
2729
|
+
* rest fire. A departure is not urgent: every peer already dropped the
|
|
2730
|
+
* state locally.
|
|
2731
|
+
*/
|
|
2732
|
+
_scheduleAwarenessRemoval(clients, origin = 'timeout') {
|
|
2733
|
+
if (this._pendingAwarenessRemovalTimeoutId !== undefined) {
|
|
2734
|
+
const pending = this._pendingAwarenessRemoval ?? [];
|
|
2735
|
+
for (const id of clients)
|
|
2736
|
+
if (!pending.includes(id))
|
|
2737
|
+
pending.push(id);
|
|
2738
|
+
this._pendingAwarenessRemoval = pending;
|
|
2739
|
+
return;
|
|
2740
|
+
}
|
|
2741
|
+
this._pendingAwarenessRemoval = clients.slice();
|
|
2742
|
+
const window = this._replySuppressionMaxDelay();
|
|
2743
|
+
const delay = Math.random() * (origin === 'peer-left' ? Math.max(1000, 10 * window) : window);
|
|
2744
|
+
this._pendingAwarenessRemovalTimeoutId = setTimeout(() => {
|
|
2745
|
+
this._pendingAwarenessRemovalTimeoutId = undefined;
|
|
2746
|
+
if (this._pendingAwarenessRemoval) {
|
|
2747
|
+
this._broadcastAwareness(this._pendingAwarenessRemoval);
|
|
2748
|
+
this._pendingAwarenessRemoval = null;
|
|
2749
|
+
}
|
|
2750
|
+
}, delay);
|
|
2751
|
+
}
|
|
2752
|
+
/**
|
|
2753
|
+
* Peek at an awareness-update payload (still in
|
|
2754
|
+
* `awarenessProtocol.encodeAwarenessUpdate()`'s wire encoding) for
|
|
2755
|
+
* clientIDs whose state is `null` (a removal), without applying it.
|
|
2756
|
+
* y-protocols/awareness.js doesn't export a standalone decoder for this,
|
|
2757
|
+
* only `applyAwarenessUpdate()` (which also mutates state) and
|
|
2758
|
+
* `modifyAwarenessUpdate()` (which re-encodes) - so this mirrors the
|
|
2759
|
+
* format by hand: varUint length, then per entry
|
|
2760
|
+
* [varUint clientID][varUint clock][varString JSON state]. Used to cancel
|
|
2761
|
+
* a pending suppressed removal (see `_scheduleAwarenessRemoval()`) at the
|
|
2762
|
+
* wire-message level, before `applyAwarenessUpdate()` runs - see the
|
|
2763
|
+
* `MESSAGE_AWARENESS` case's comment for why timing matters here.
|
|
2764
|
+
*/
|
|
2765
|
+
_scanAwarenessPayload(payload) {
|
|
2766
|
+
const removed = [];
|
|
2767
|
+
const present = [];
|
|
2768
|
+
let coversUs = false;
|
|
2769
|
+
try {
|
|
2770
|
+
const d = decoding.createDecoder(payload);
|
|
2771
|
+
const len = decoding.readVarUint(d);
|
|
2772
|
+
const ourClock = this.awareness.meta.get(this.awareness.clientID)?.clock ?? 0;
|
|
2773
|
+
for (let i = 0; i < len; i++) {
|
|
2774
|
+
const clientID = decoding.readVarUint(d);
|
|
2775
|
+
const clock = decoding.readVarUint(d);
|
|
2776
|
+
const state = JSON.parse(decoding.readVarString(d));
|
|
2777
|
+
if (state === null)
|
|
2778
|
+
removed.push(clientID);
|
|
2779
|
+
else {
|
|
2780
|
+
present.push(clientID);
|
|
2781
|
+
if (clientID === this.awareness.clientID && clock >= ourClock)
|
|
2782
|
+
coversUs = true;
|
|
2783
|
+
}
|
|
2784
|
+
}
|
|
2785
|
+
}
|
|
2786
|
+
catch {
|
|
2787
|
+
// Malformed payload - let applyAwarenessUpdate() below be the one
|
|
2788
|
+
// that deals with it (or throws); suppression is a pure optimization,
|
|
2789
|
+
// never worth failing the actual message handling over.
|
|
2790
|
+
}
|
|
2791
|
+
return { removed, present, coversUs };
|
|
2792
|
+
}
|
|
2793
|
+
/**
|
|
2794
|
+
* Trim a pending suppressed removal broadcast by `removedClientIds` -
|
|
2795
|
+
* someone else already broadcast those departures; what they did not
|
|
2796
|
+
* cover stays queued.
|
|
2797
|
+
*/
|
|
2798
|
+
_cancelPendingAwarenessRemovalIfOverlaps(removedClientIds) {
|
|
2799
|
+
if (!this._pendingAwarenessRemoval)
|
|
2800
|
+
return;
|
|
2801
|
+
const rest = this._pendingAwarenessRemoval.filter((id) => !removedClientIds.includes(id));
|
|
2802
|
+
if (rest.length === this._pendingAwarenessRemoval.length)
|
|
2803
|
+
return;
|
|
2804
|
+
if (rest.length > 0) {
|
|
2805
|
+
this._pendingAwarenessRemoval = rest; // someone covered part of it; the rest stays queued
|
|
2806
|
+
return;
|
|
2807
|
+
}
|
|
2808
|
+
if (this._pendingAwarenessRemovalTimeoutId !== undefined) {
|
|
2809
|
+
clearTimeout(this._pendingAwarenessRemovalTimeoutId);
|
|
2810
|
+
this._pendingAwarenessRemovalTimeoutId = undefined;
|
|
2811
|
+
}
|
|
2812
|
+
this._pendingAwarenessRemoval = null;
|
|
2813
|
+
}
|
|
2814
|
+
/** Cancel a pending suppressed awareness-removal broadcast, if any. */
|
|
2815
|
+
_cancelPendingAwarenessRemoval() {
|
|
2816
|
+
if (this._pendingAwarenessRemovalTimeoutId !== undefined) {
|
|
2817
|
+
clearTimeout(this._pendingAwarenessRemovalTimeoutId);
|
|
2818
|
+
this._pendingAwarenessRemovalTimeoutId = undefined;
|
|
2819
|
+
}
|
|
2820
|
+
this._pendingAwarenessRemoval = null;
|
|
876
2821
|
}
|
|
877
2822
|
/**
|
|
878
2823
|
* Send a SyncStep2 reply, gated by the same shared per-peer budget as
|
|
@@ -894,7 +2839,7 @@ export class GenericProvider extends Observable {
|
|
|
894
2839
|
* would itself become log spam exactly when things are already noisy.
|
|
895
2840
|
*/
|
|
896
2841
|
_sendSyncReply(reply) {
|
|
897
|
-
if (!this.
|
|
2842
|
+
if (!this._tryReserveReplySlot()) {
|
|
898
2843
|
return; // Rate limited - drop the reply silently
|
|
899
2844
|
}
|
|
900
2845
|
this._send(reply);
|
|
@@ -1011,8 +2956,27 @@ export class GenericProvider extends Observable {
|
|
|
1011
2956
|
console.warn(`[GenericProvider] Resync scheduled in ${delay}ms (attempt #${this._resyncAttemptCount})...`);
|
|
1012
2957
|
this._pendingResyncTimeoutId = setTimeout(() => {
|
|
1013
2958
|
this._pendingResyncTimeoutId = undefined;
|
|
1014
|
-
if (this.transport.isConnected
|
|
1015
|
-
|
|
2959
|
+
if (!this.transport.isConnected || this._destroying)
|
|
2960
|
+
return;
|
|
2961
|
+
// A resync trigger means "I may be missing something" - never "the
|
|
2962
|
+
// room is missing my data" (my updates travel on their own, and the
|
|
2963
|
+
// connect-time push covers offline edits). So: ask with a 12-byte
|
|
2964
|
+
// beacon; the peers ahead of me reply with exactly the diff (a
|
|
2965
|
+
// SyncStep2 against my state vector, suppressed as any reply), the
|
|
2966
|
+
// rest stay silent. Until phase 1b this pushed the WHOLE document to
|
|
2967
|
+
// the room on every trigger, and that push - a hashed update of my
|
|
2968
|
+
// state - failed the hash check at every peer ahead of me, which
|
|
2969
|
+
// scheduled a resync of its own: the cascade of research doc item 13
|
|
2970
|
+
// (198,990 deliveries for 10 keystrokes at N=100 on the Gun profile,
|
|
2971
|
+
// the rate limiter's ceiling). If the beacon or its reply is lost,
|
|
2972
|
+
// _armResponseWait() re-beacons (1s/2s/4s); the periodic beacon is
|
|
2973
|
+
// the fallback after that. A rate-limited attempt re-arms through
|
|
2974
|
+
// this same coordinator so a stranded peer keeps retrying.
|
|
2975
|
+
if (this._sendSyncStep1(0)) {
|
|
2976
|
+
this._armResponseWait(0);
|
|
2977
|
+
}
|
|
2978
|
+
else {
|
|
2979
|
+
this._requestResync();
|
|
1016
2980
|
}
|
|
1017
2981
|
}, delay);
|
|
1018
2982
|
}
|
|
@@ -1025,48 +2989,90 @@ export class GenericProvider extends Observable {
|
|
|
1025
2989
|
* its own uncapped or separately-capped allowance.
|
|
1026
2990
|
*/
|
|
1027
2991
|
_tryReserveSyncSlot() {
|
|
2992
|
+
return this._tryReserveSlot(this._syncRequestTimes);
|
|
2993
|
+
}
|
|
2994
|
+
/** Same limiter, separate budget, for SyncStep2 replies and acks. */
|
|
2995
|
+
_tryReserveReplySlot() {
|
|
2996
|
+
return this._tryReserveSlot(this._syncReplyTimes);
|
|
2997
|
+
}
|
|
2998
|
+
_tryReserveSlot(times) {
|
|
1028
2999
|
const now = Date.now();
|
|
1029
|
-
//
|
|
1030
|
-
|
|
1031
|
-
|
|
3000
|
+
// Drop entries outside the rolling window (in place: the arrays are
|
|
3001
|
+
// referenced from two fields)
|
|
3002
|
+
let keep = 0;
|
|
3003
|
+
for (const t of times) {
|
|
3004
|
+
if (now - t < this._syncRequestWindowMs)
|
|
3005
|
+
times[keep++] = t;
|
|
3006
|
+
}
|
|
3007
|
+
times.length = keep;
|
|
3008
|
+
if (times.length >= this._maxSyncRequestsPerWindow) {
|
|
1032
3009
|
return false;
|
|
1033
3010
|
}
|
|
1034
|
-
|
|
3011
|
+
times.push(now);
|
|
1035
3012
|
return true;
|
|
1036
3013
|
}
|
|
1037
3014
|
/**
|
|
1038
|
-
* Encode
|
|
1039
|
-
*
|
|
1040
|
-
*
|
|
3015
|
+
* Encode the digest beacon that replaces SyncStep1 (see
|
|
3016
|
+
* MESSAGE_SYNC_DIGEST). Still the one place every "request sync" path
|
|
3017
|
+
* goes through (connect()'s syncNow(), the periodic tick,
|
|
3018
|
+
* _requestResync()'s retry), so they all switched together.
|
|
1041
3019
|
*/
|
|
1042
|
-
|
|
3020
|
+
_encodeSyncStep1(flags = 0) {
|
|
3021
|
+
if (this._confirmed)
|
|
3022
|
+
flags |= DIGEST_FLAG_SETTLED;
|
|
3023
|
+
this._touchPeer(this.doc.clientID);
|
|
1043
3024
|
const encoder = encoding.createEncoder();
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
encoding.writeVarUint(encoder,
|
|
1047
|
-
|
|
1048
|
-
|
|
3025
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC_DIGEST);
|
|
3026
|
+
encoding.writeVarUint(encoder, DIGEST_VERSION);
|
|
3027
|
+
encoding.writeVarUint(encoder, flags);
|
|
3028
|
+
encoding.writeVarUint(encoder, this.doc.clientID);
|
|
3029
|
+
encoding.writeVarUint8Array(encoder, Y.encodeStateVector(this.doc));
|
|
3030
|
+
encoding.writeVarUint(encoder, this._deleteSetHash());
|
|
3031
|
+
return encoding.toUint8Array(encoder);
|
|
3032
|
+
}
|
|
3033
|
+
/**
|
|
3034
|
+
* Encode an ack for a JOIN beacon: same framing as a beacon, DIGEST_FLAG_ACK
|
|
3035
|
+
* set, and the JOINER's state vector + delete-set hash echoed back instead
|
|
3036
|
+
* of ours (see DIGEST_FLAG_ACK for why it must never carry our own state).
|
|
3037
|
+
*/
|
|
3038
|
+
_encodeAck(ackedSv, ackedDsHash) {
|
|
3039
|
+
this._touchPeer(this.doc.clientID);
|
|
3040
|
+
const encoder = encoding.createEncoder();
|
|
3041
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC_DIGEST);
|
|
3042
|
+
encoding.writeVarUint(encoder, DIGEST_VERSION);
|
|
3043
|
+
encoding.writeVarUint(encoder, DIGEST_FLAG_ACK | (this._confirmed ? DIGEST_FLAG_SETTLED : 0));
|
|
3044
|
+
encoding.writeVarUint(encoder, this.doc.clientID);
|
|
3045
|
+
encoding.writeVarUint8Array(encoder, ackedSv);
|
|
3046
|
+
encoding.writeVarUint(encoder, ackedDsHash);
|
|
3047
|
+
return encoding.toUint8Array(encoder);
|
|
1049
3048
|
}
|
|
1050
3049
|
/**
|
|
1051
|
-
* Send
|
|
1052
|
-
*
|
|
1053
|
-
* Note: SyncStep1 is just a request and doesn't include hash verification.
|
|
1054
|
-
* Rate limited to prevent spam.
|
|
3050
|
+
* Send the periodic digest beacon. Rate limited to prevent spam. Returns
|
|
3051
|
+
* whether it actually sent (false means rate-limited).
|
|
1055
3052
|
*/
|
|
1056
|
-
_sendSyncStep1() {
|
|
3053
|
+
_sendSyncStep1(flags = 0) {
|
|
1057
3054
|
if (!this._tryReserveSyncSlot()) {
|
|
1058
3055
|
console.warn(`[GenericProvider] Sync rate limit exceeded (${this._maxSyncRequestsPerWindow} requests per ${this._syncRequestWindowMs / 1000}s), throttling...`);
|
|
1059
|
-
return; // Drop the request
|
|
3056
|
+
return false; // Drop the request
|
|
1060
3057
|
}
|
|
1061
|
-
this.
|
|
3058
|
+
this._send(this._encodeSyncStep1(flags));
|
|
3059
|
+
return true;
|
|
1062
3060
|
}
|
|
1063
3061
|
/**
|
|
1064
|
-
*
|
|
1065
|
-
*
|
|
3062
|
+
* Encode a document update, without sending it. Extracted from the old
|
|
3063
|
+
* `_sendUpdate()` so `_trySyncPushPull()` can fold the push half into a
|
|
3064
|
+
* batched wire send (see `_sendBatch()`); `_sendUpdate()` below is the
|
|
3065
|
+
* send-immediately form still used by every other update-emitting path
|
|
3066
|
+
* (the doc-update handler, batch-flush, disconnect/destroy flush) since
|
|
3067
|
+
* those aren't part of this batching effort's scope.
|
|
3068
|
+
*
|
|
3069
|
+
* NOTE: has a side effect (`_localSeqNum++`) - call exactly once per
|
|
3070
|
+
* logical update, same as before.
|
|
1066
3071
|
*/
|
|
1067
|
-
|
|
3072
|
+
_encodeUpdate(update) {
|
|
1068
3073
|
const encoder = encoding.createEncoder();
|
|
1069
3074
|
if (this._verifyUpdates) {
|
|
3075
|
+
this._touchPeer(this.doc.clientID);
|
|
1070
3076
|
// Use verified sync protocol with sequence number and hash
|
|
1071
3077
|
encoding.writeVarUint(encoder, MESSAGE_SYNC_VERIFIED);
|
|
1072
3078
|
// Include sequence number and clientID for causal ordering
|
|
@@ -1082,6 +3088,35 @@ export class GenericProvider extends Observable {
|
|
|
1082
3088
|
encoding.writeVarUint(encoder, MESSAGE_SYNC);
|
|
1083
3089
|
syncProtocol.writeUpdate(encoder, update);
|
|
1084
3090
|
}
|
|
3091
|
+
return encoding.toUint8Array(encoder);
|
|
3092
|
+
}
|
|
3093
|
+
/**
|
|
3094
|
+
* Encode a full-state push (see MESSAGE_SYNC_PUSH): the document as one
|
|
3095
|
+
* update, deliberately without the hash and sequence number that
|
|
3096
|
+
* `_encodeUpdate()` adds to incremental updates.
|
|
3097
|
+
*/
|
|
3098
|
+
_encodePush(update) {
|
|
3099
|
+
const encoder = encoding.createEncoder();
|
|
3100
|
+
encoding.writeVarUint(encoder, MESSAGE_SYNC_PUSH);
|
|
3101
|
+
encoding.writeVarUint8Array(encoder, update);
|
|
3102
|
+
return encoding.toUint8Array(encoder);
|
|
3103
|
+
}
|
|
3104
|
+
/**
|
|
3105
|
+
* Send a document update to the transport, with whatever awareness change
|
|
3106
|
+
* the throttle is holding folded into the same wire message (round 5,
|
|
3107
|
+
* item 1). If verifyUpdates is enabled, the update carries a sequence
|
|
3108
|
+
* number and document hash for ordering and desync detection.
|
|
3109
|
+
*/
|
|
3110
|
+
_sendUpdate(update) {
|
|
3111
|
+
this._sendBatch([this._encodeUpdate(update), ...this._takePendingAwareness()]);
|
|
3112
|
+
}
|
|
3113
|
+
/**
|
|
3114
|
+
* Send awareness update to the transport.
|
|
3115
|
+
*/
|
|
3116
|
+
_sendAwarenessUpdate(changedClients) {
|
|
3117
|
+
const encoder = encoding.createEncoder();
|
|
3118
|
+
encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
|
|
3119
|
+
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, changedClients));
|
|
1085
3120
|
this._send(encoding.toUint8Array(encoder));
|
|
1086
3121
|
}
|
|
1087
3122
|
/**
|
|
@@ -1112,10 +3147,11 @@ export class GenericProvider extends Observable {
|
|
|
1112
3147
|
}
|
|
1113
3148
|
}
|
|
1114
3149
|
/**
|
|
1115
|
-
* Send a
|
|
1116
|
-
*
|
|
1117
|
-
*
|
|
1118
|
-
*
|
|
3150
|
+
* Send a pub/sub message to a single target.
|
|
3151
|
+
*
|
|
3152
|
+
* With `Transport.sendTo` the frame is unicast to that peer; without it the
|
|
3153
|
+
* frame is broadcast with the target embedded and dropped on receipt by
|
|
3154
|
+
* every provider whose `localId` differs.
|
|
1119
3155
|
*/
|
|
1120
3156
|
_sendPubSubTo(target, topic, message) {
|
|
1121
3157
|
if (!this.transport.isConnected) {
|
|
@@ -1154,63 +3190,215 @@ export class GenericProvider extends Observable {
|
|
|
1154
3190
|
* Throttled to prevent awareness updates from flooding document sync.
|
|
1155
3191
|
* Multiple rapid updates are batched together.
|
|
1156
3192
|
*/
|
|
1157
|
-
_broadcastAwareness(clients
|
|
3193
|
+
_broadcastAwareness(clients) {
|
|
1158
3194
|
if (clients.length === 0)
|
|
1159
3195
|
return;
|
|
1160
|
-
const
|
|
1161
|
-
const pending = isApp
|
|
1162
|
-
? this._pendingAppAwarenessClients
|
|
1163
|
-
: this._pendingAwarenessClients;
|
|
3196
|
+
const interval = this._effectiveAwarenessInterval();
|
|
1164
3197
|
// If throttling is disabled, send immediately
|
|
1165
|
-
if (
|
|
1166
|
-
this._sendAwarenessNow(clients
|
|
3198
|
+
if (interval <= 0) {
|
|
3199
|
+
this._sendAwarenessNow(clients);
|
|
1167
3200
|
return;
|
|
1168
3201
|
}
|
|
1169
3202
|
// Add clients to pending set
|
|
1170
3203
|
for (const client of clients) {
|
|
1171
|
-
|
|
3204
|
+
this._pendingAwarenessClients.add(client);
|
|
1172
3205
|
}
|
|
1173
3206
|
// If we already have a scheduled broadcast, let it handle the batched clients
|
|
1174
|
-
if (
|
|
1175
|
-
undefined) {
|
|
3207
|
+
if (this._awarenessTimeoutId !== undefined) {
|
|
1176
3208
|
return;
|
|
1177
3209
|
}
|
|
1178
3210
|
// Calculate delay - respect minimum interval since last broadcast
|
|
1179
3211
|
const now = Date.now();
|
|
1180
|
-
const
|
|
1181
|
-
const delay = Math.max(0,
|
|
3212
|
+
const timeSinceLastBroadcast = now - this._lastAwarenessTime;
|
|
3213
|
+
const delay = Math.max(0, interval - timeSinceLastBroadcast);
|
|
1182
3214
|
// Schedule the batched broadcast
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
3215
|
+
this._awarenessTimeoutId = setTimeout(() => {
|
|
3216
|
+
this._awarenessTimeoutId = undefined;
|
|
3217
|
+
this._lastAwarenessTime = Date.now();
|
|
3218
|
+
// Send all pending clients in one message - and a pending timed
|
|
3219
|
+
// update batch (`batchUpdates > 0`) with them, update first.
|
|
3220
|
+
const clientsToSend = Array.from(this._pendingAwarenessClients);
|
|
3221
|
+
this._pendingAwarenessClients.clear();
|
|
3222
|
+
if (clientsToSend.length > 0) {
|
|
3223
|
+
this._sendBatch([...this._takePendingUpdate(), this._encodeAwareness(clientsToSend)]);
|
|
1191
3224
|
}
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
3225
|
+
}, delay);
|
|
3226
|
+
}
|
|
3227
|
+
/**
|
|
3228
|
+
* Register the app channel's update listener. Same echo suppression as the
|
|
3229
|
+
* core channel (never re-broadcast what came off the wire), but none of its
|
|
3230
|
+
* presence/removal handling - this channel has no bearing on who the room
|
|
3231
|
+
* thinks is present.
|
|
3232
|
+
*/
|
|
3233
|
+
/**
|
|
3234
|
+
* Announce local app-channel state, if the channel is in use and has any.
|
|
3235
|
+
*
|
|
3236
|
+
* Reads the private field, never the getter: an announce must never be the
|
|
3237
|
+
* thing that constructs the instance (that would re-introduce the eager
|
|
3238
|
+
* timer the lazy getter exists to avoid). A provider whose modules never
|
|
3239
|
+
* touch the channel therefore announces nothing, which is correct - there
|
|
3240
|
+
* is no local state to announce.
|
|
3241
|
+
*/
|
|
3242
|
+
_announceAppAwareness() {
|
|
3243
|
+
if (this._appAwareness?.getLocalState() != null) {
|
|
3244
|
+
this._broadcastAppAwareness([this.doc.clientID]);
|
|
3245
|
+
}
|
|
3246
|
+
}
|
|
3247
|
+
_attachAppAwareness(aw) {
|
|
3248
|
+
if (this._appAwarenessUpdateHandler)
|
|
3249
|
+
return;
|
|
3250
|
+
this._appAwarenessUpdateHandler = ({ added, updated, removed, }, origin) => {
|
|
3251
|
+
if (origin === this)
|
|
3252
|
+
return;
|
|
3253
|
+
const changedClients = [...added, ...updated, ...removed];
|
|
3254
|
+
if (changedClients.length === 0)
|
|
3255
|
+
return;
|
|
3256
|
+
this._broadcastAppAwareness(changedClients);
|
|
3257
|
+
};
|
|
3258
|
+
aw.on('update', this._appAwarenessUpdateHandler);
|
|
3259
|
+
}
|
|
3260
|
+
/**
|
|
3261
|
+
* Broadcast app-channel awareness. Mirrors `_broadcastAwareness()`'s
|
|
3262
|
+
* throttle, against its own pending set and timer, and deliberately does
|
|
3263
|
+
* NOT piggyback on the core sync batch: module cursor churn must not pull
|
|
3264
|
+
* document or presence traffic onto its cadence (or vice versa).
|
|
3265
|
+
*/
|
|
3266
|
+
_broadcastAppAwareness(clients) {
|
|
3267
|
+
if (clients.length === 0)
|
|
3268
|
+
return;
|
|
3269
|
+
const interval = this._effectiveAwarenessInterval();
|
|
3270
|
+
if (interval <= 0) {
|
|
3271
|
+
this._sendAppAwarenessNow(clients);
|
|
3272
|
+
return;
|
|
3273
|
+
}
|
|
3274
|
+
for (const client of clients) {
|
|
3275
|
+
this._pendingAppAwarenessClients.add(client);
|
|
3276
|
+
}
|
|
3277
|
+
if (this._appAwarenessTimeoutId !== undefined) {
|
|
3278
|
+
return;
|
|
3279
|
+
}
|
|
3280
|
+
const delay = Math.max(0, interval - (Date.now() - this._lastAppAwarenessTime));
|
|
3281
|
+
this._appAwarenessTimeoutId = setTimeout(() => {
|
|
3282
|
+
this._appAwarenessTimeoutId = undefined;
|
|
3283
|
+
this._lastAppAwarenessTime = Date.now();
|
|
3284
|
+
const clientsToSend = Array.from(this._pendingAppAwarenessClients);
|
|
3285
|
+
this._pendingAppAwarenessClients.clear();
|
|
1195
3286
|
if (clientsToSend.length > 0) {
|
|
1196
|
-
this.
|
|
3287
|
+
this._sendAppAwarenessNow(clientsToSend);
|
|
1197
3288
|
}
|
|
1198
3289
|
}, delay);
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
3290
|
+
}
|
|
3291
|
+
/** Encode and send an app-channel awareness update immediately. */
|
|
3292
|
+
_sendAppAwarenessNow(clients) {
|
|
3293
|
+
const encoder = encoding.createEncoder();
|
|
3294
|
+
encoding.writeVarUint(encoder, MESSAGE_AWARENESS_APP);
|
|
3295
|
+
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.appAwareness, clients));
|
|
3296
|
+
this._send(encoding.toUint8Array(encoder));
|
|
1203
3297
|
}
|
|
1204
3298
|
/**
|
|
1205
|
-
*
|
|
3299
|
+
* Round 5, item 1: the awareness change the throttle is holding rides
|
|
3300
|
+
* along with a wire message that is leaving anyway. Returns the encoded
|
|
3301
|
+
* awareness sub-message (or nothing) and commits the throttle state
|
|
3302
|
+
* exactly as the timer's own flush would. A piggybacked broadcast costs
|
|
3303
|
+
* no message, only its payload bytes, so it goes out early instead of as
|
|
3304
|
+
* its own message up to `_awarenessInterval` later. Measured in
|
|
3305
|
+
* test/dummy/bench-typing-census.ts: a keystroke in an editor binding is
|
|
3306
|
+
* a text insert plus a cursor update - two broadcasts per keystroke
|
|
3307
|
+
* before this, one after. Broadcast paths only: `_sendDirect` and the
|
|
3308
|
+
* BroadcastChannel-only publishes never call this.
|
|
1206
3309
|
*/
|
|
1207
|
-
|
|
1208
|
-
|
|
3310
|
+
_takePendingAwareness() {
|
|
3311
|
+
if (this._awarenessTimeoutId === undefined)
|
|
3312
|
+
return [];
|
|
3313
|
+
clearTimeout(this._awarenessTimeoutId);
|
|
3314
|
+
this._awarenessTimeoutId = undefined;
|
|
3315
|
+
const clients = Array.from(this._pendingAwarenessClients);
|
|
3316
|
+
this._pendingAwarenessClients.clear();
|
|
3317
|
+
if (clients.length === 0)
|
|
3318
|
+
return [];
|
|
3319
|
+
this._lastAwarenessTime = Date.now();
|
|
3320
|
+
return [this._encodeAwareness(clients)];
|
|
3321
|
+
}
|
|
3322
|
+
/**
|
|
3323
|
+
* The counterpart for the timed batch: a `batchUpdates > 0` batch that is
|
|
3324
|
+
* still waiting rides along with an awareness flush (Matrix: both
|
|
3325
|
+
* default to 2 s, so a typist's cursor and text leave as one PUT).
|
|
3326
|
+
*/
|
|
3327
|
+
_takePendingUpdate() {
|
|
3328
|
+
if (this._batchTimeoutId === undefined || !this._pendingUpdate)
|
|
3329
|
+
return [];
|
|
3330
|
+
clearTimeout(this._batchTimeoutId);
|
|
3331
|
+
this._batchTimeoutId = undefined;
|
|
3332
|
+
this._flushScheduled = false;
|
|
3333
|
+
const update = this._pendingUpdate;
|
|
3334
|
+
this._pendingUpdate = null;
|
|
3335
|
+
return [this._encodeUpdate(update)];
|
|
3336
|
+
}
|
|
3337
|
+
/**
|
|
3338
|
+
* Encode an awareness update, without sending it. Extracted from the old
|
|
3339
|
+
* `_sendAwarenessNow()` so `_tryImmediateAwarenessMessage()` can fold it
|
|
3340
|
+
* into a batched wire send instead of always sending it as its own
|
|
3341
|
+
* message.
|
|
3342
|
+
*/
|
|
3343
|
+
_encodeAwareness(clients) {
|
|
1209
3344
|
const encoder = encoding.createEncoder();
|
|
1210
3345
|
encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
|
|
1211
|
-
encoding.
|
|
1212
|
-
encoding.
|
|
1213
|
-
|
|
3346
|
+
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, clients));
|
|
3347
|
+
return encoding.toUint8Array(encoder);
|
|
3348
|
+
}
|
|
3349
|
+
/**
|
|
3350
|
+
* Send awareness update immediately without throttling.
|
|
3351
|
+
*/
|
|
3352
|
+
_sendAwarenessNow(clients) {
|
|
3353
|
+
this._send(this._encodeAwareness(clients));
|
|
3354
|
+
}
|
|
3355
|
+
/**
|
|
3356
|
+
* Attempt to build an awareness broadcast message for immediate
|
|
3357
|
+
* inclusion in the same wire send as a sync message a caller is about to
|
|
3358
|
+
* send anyway (see `_trySyncPushPull`'s `buildExtra` parameter), instead
|
|
3359
|
+
* of going through
|
|
3360
|
+
* `_broadcastAwareness()`'s independent debounce.
|
|
3361
|
+
*
|
|
3362
|
+
* Only returns non-null when the throttle would have let an immediate
|
|
3363
|
+
* send through anyway - i.e. no debounced broadcast is already pending
|
|
3364
|
+
* AND (throttling is disabled, or at least `_awarenessInterval` ms have
|
|
3365
|
+
* passed since the last broadcast) - so this never changes awareness
|
|
3366
|
+
* throttle semantics, only whether the resulting message travels as its
|
|
3367
|
+
* own wire send or bundled with a sync message that happens to be going
|
|
3368
|
+
* out "now" too.
|
|
3369
|
+
*
|
|
3370
|
+
* Mutates the same state `_broadcastAwareness()`'s own immediate-send
|
|
3371
|
+
* branches mutate (`_pendingAwarenessClients`, `_lastAwarenessTime`) -
|
|
3372
|
+
* once this returns non-null, the state is already committed as "sent
|
|
3373
|
+
* now", so the caller MUST actually send the returned message (bundled
|
|
3374
|
+
* or standalone) rather than discarding it.
|
|
3375
|
+
*/
|
|
3376
|
+
_tryImmediateAwarenessMessage(clients) {
|
|
3377
|
+
if (clients.length === 0)
|
|
3378
|
+
return null;
|
|
3379
|
+
// A debounced broadcast is already scheduled - let it handle these
|
|
3380
|
+
// clients via the normal pending-set path below, don't race it with an
|
|
3381
|
+
// immediate send here.
|
|
3382
|
+
if (this._awarenessTimeoutId !== undefined)
|
|
3383
|
+
return null;
|
|
3384
|
+
const interval = this._effectiveAwarenessInterval();
|
|
3385
|
+
if (interval > 0) {
|
|
3386
|
+
const timeSinceLastBroadcast = Date.now() - this._lastAwarenessTime;
|
|
3387
|
+
if (timeSinceLastBroadcast < interval)
|
|
3388
|
+
return null;
|
|
3389
|
+
}
|
|
3390
|
+
// Merge with anything already pending (normally empty here since no
|
|
3391
|
+
// timer is scheduled, but merge for safety) and commit to sending now -
|
|
3392
|
+
// mirrors exactly what _broadcastAwareness()'s own immediate
|
|
3393
|
+
// (_awarenessInterval <= 0) and debounced-timer-fired branches do.
|
|
3394
|
+
for (const c of clients)
|
|
3395
|
+
this._pendingAwarenessClients.add(c);
|
|
3396
|
+
const clientsToSend = Array.from(this._pendingAwarenessClients);
|
|
3397
|
+
this._pendingAwarenessClients.clear();
|
|
3398
|
+
if (clientsToSend.length === 0)
|
|
3399
|
+
return null;
|
|
3400
|
+
this._lastAwarenessTime = Date.now();
|
|
3401
|
+
return this._encodeAwareness(clientsToSend);
|
|
1214
3402
|
}
|
|
1215
3403
|
/**
|
|
1216
3404
|
* Setup BroadcastChannel for cross-tab communication.
|
|
@@ -1244,30 +3432,35 @@ export class GenericProvider extends Observable {
|
|
|
1244
3432
|
const encoderSync = encoding.createEncoder();
|
|
1245
3433
|
encoding.writeVarUint(encoderSync, MESSAGE_SYNC);
|
|
1246
3434
|
syncProtocol.writeSyncStep1(encoderSync, this.doc);
|
|
1247
|
-
|
|
3435
|
+
this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoderSync)));
|
|
1248
3436
|
// Broadcast local state via BroadcastChannel (wrapped with CRC32)
|
|
1249
3437
|
const encoderState = encoding.createEncoder();
|
|
1250
3438
|
encoding.writeVarUint(encoderState, MESSAGE_SYNC);
|
|
1251
3439
|
syncProtocol.writeSyncStep2(encoderState, this.doc);
|
|
1252
|
-
|
|
3440
|
+
this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoderState)));
|
|
1253
3441
|
// Broadcast local awareness state via BroadcastChannel (wrapped with CRC32)
|
|
1254
3442
|
if (this.awareness.getLocalState() !== null) {
|
|
1255
|
-
|
|
3443
|
+
const encoderAwareness = encoding.createEncoder();
|
|
3444
|
+
encoding.writeVarUint(encoderAwareness, MESSAGE_AWARENESS);
|
|
3445
|
+
encoding.writeVarUint8Array(encoderAwareness, awarenessProtocol.encodeAwarenessUpdate(this.awareness, [
|
|
3446
|
+
this.doc.clientID,
|
|
3447
|
+
]));
|
|
3448
|
+
this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoderAwareness)));
|
|
1256
3449
|
}
|
|
1257
|
-
|
|
1258
|
-
|
|
3450
|
+
// Same for the app channel: ongoing changes reach other tabs for free
|
|
3451
|
+
// (_sendAppAwarenessNow goes through _send, which publishes to BC), but
|
|
3452
|
+
// the connect-time announce does not - so without this a second tab saw
|
|
3453
|
+
// the first tab's presence and never its cursors. Guarded on the private
|
|
3454
|
+
// field, like every other announce: never construct the instance here.
|
|
3455
|
+
if (this._appAwareness?.getLocalState() != null) {
|
|
3456
|
+
const encoderApp = encoding.createEncoder();
|
|
3457
|
+
encoding.writeVarUint(encoderApp, MESSAGE_AWARENESS_APP);
|
|
3458
|
+
encoding.writeVarUint8Array(encoderApp, awarenessProtocol.encodeAwarenessUpdate(this._appAwareness, [
|
|
3459
|
+
this.doc.clientID,
|
|
3460
|
+
]));
|
|
3461
|
+
this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoderApp)));
|
|
1259
3462
|
}
|
|
1260
3463
|
}
|
|
1261
|
-
/**
|
|
1262
|
-
* Encode and publish an awareness update for the local client to the BroadcastChannel.
|
|
1263
|
-
*/
|
|
1264
|
-
_publishAwarenessToBroadcastChannel(awareness, channel) {
|
|
1265
|
-
const encoder = encoding.createEncoder();
|
|
1266
|
-
encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
|
|
1267
|
-
encoding.writeVarUint(encoder, channel);
|
|
1268
|
-
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(awareness, [this.doc.clientID]));
|
|
1269
|
-
bc.publish(this._bcChannel, wrapMessageWithChecksum(encoding.toUint8Array(encoder)), this);
|
|
1270
|
-
}
|
|
1271
3464
|
/**
|
|
1272
3465
|
* Disconnect from BroadcastChannel and mark local client as offline.
|
|
1273
3466
|
*/
|
|
@@ -1276,21 +3469,71 @@ export class GenericProvider extends Observable {
|
|
|
1276
3469
|
return;
|
|
1277
3470
|
}
|
|
1278
3471
|
// Broadcast awareness state with null (indicating disconnect) - wrapped with CRC32
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
const encoder = encoding.createEncoder();
|
|
1284
|
-
encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
|
|
1285
|
-
encoding.writeVarUint(encoder, channel);
|
|
1286
|
-
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(awareness, [this.doc.clientID], new Map()));
|
|
1287
|
-
bc.publish(this._bcChannel, wrapMessageWithChecksum(encoding.toUint8Array(encoder)), this);
|
|
1288
|
-
}
|
|
3472
|
+
const encoder = encoding.createEncoder();
|
|
3473
|
+
encoding.writeVarUint(encoder, MESSAGE_AWARENESS);
|
|
3474
|
+
encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(this.awareness, [this.doc.clientID], new Map()));
|
|
3475
|
+
this._bcPublish(wrapMessageWithChecksum(encoding.toUint8Array(encoder)));
|
|
1289
3476
|
// Unsubscribe from channel
|
|
1290
3477
|
bc.unsubscribe(this._bcChannel, this._bcSubscriber);
|
|
1291
3478
|
this._bcConnected = false;
|
|
1292
3479
|
this._bcSubscriber = undefined;
|
|
1293
3480
|
}
|
|
3481
|
+
/**
|
|
3482
|
+
* Send N already-encoded, already-typed sub-messages as ONE wire message
|
|
3483
|
+
* instead of N separate `transport.send()`/`bc.publish()` calls, when
|
|
3484
|
+
* there's more than one to send. Used at trigger points that
|
|
3485
|
+
* conceptually produce a single event but historically sent multiple
|
|
3486
|
+
* independent messages for it (sync push, sync pull, awareness) - see
|
|
3487
|
+
* `_trySyncPushPull()` (connect-time push + digest beacon + awareness in
|
|
3488
|
+
* one wire message).
|
|
3489
|
+
*
|
|
3490
|
+
* Design (see the task's framing requirements):
|
|
3491
|
+
* - Each sub-message is length-prefixed with `writeVarUint8Array`,
|
|
3492
|
+
* consistent with how this codebase already frames variable-length
|
|
3493
|
+
* payloads elsewhere (e.g. MESSAGE_AWARENESS). On receipt,
|
|
3494
|
+
* `_dispatchMessage()`'s `MESSAGE_BATCH` case unwraps and re-dispatches
|
|
3495
|
+
* each one through the EXACT SAME per-message-type logic used for a
|
|
3496
|
+
* top-level message - no parallel reimplementation.
|
|
3497
|
+
* - Sub-messages are NOT individually CRC32-wrapped here - the whole
|
|
3498
|
+
* batch envelope goes through the normal, single `_send()` pipeline
|
|
3499
|
+
* below, which wraps the WHOLE envelope in exactly one CRC32 checksum
|
|
3500
|
+
* (and, if `compressionThresholdBytes` is configured, one compression
|
|
3501
|
+
* pass) - built and computed exactly like any other outgoing message,
|
|
3502
|
+
* so this composes with the existing compression pipeline for free
|
|
3503
|
+
* rather than fighting it with a second, nested wrap/compress step.
|
|
3504
|
+
* The tradeoff: a single corrupted bit anywhere in a batched wire
|
|
3505
|
+
* message now invalidates every sub-message it carried, not just one -
|
|
3506
|
+
* per-sub-message CRC32s would avoid that, at the cost of ~4 extra
|
|
3507
|
+
* bytes per sub-message for a benefit that only matters under active
|
|
3508
|
+
* corruption. This tradeoff is exactly what
|
|
3509
|
+
* test/dummy/bench-corruption-storm.ts and bench-packet-loss.ts exist
|
|
3510
|
+
* to measure empirically, per this task's validation requirements,
|
|
3511
|
+
* rather than deciding it by design argument alone.
|
|
3512
|
+
* - BroadcastChannel (cross-tab) traffic is NOT specially batched beyond
|
|
3513
|
+
* whatever `_send()` already does per call - same-tab-group cross-tab
|
|
3514
|
+
* traffic is local/cheap, and `_send()` already only issues one
|
|
3515
|
+
* `bc.publish()` per call regardless, so a batch of N sub-messages
|
|
3516
|
+
* already becomes exactly one `bc.publish()` call for free once routed
|
|
3517
|
+
* through here - no separate BC-specific batching logic needed.
|
|
3518
|
+
*/
|
|
3519
|
+
_sendBatch(messages) {
|
|
3520
|
+
if (messages.length === 0)
|
|
3521
|
+
return;
|
|
3522
|
+
if (messages.length === 1) {
|
|
3523
|
+
this._send(messages[0]);
|
|
3524
|
+
return;
|
|
3525
|
+
}
|
|
3526
|
+
this._send(this._encodeBatch(messages));
|
|
3527
|
+
}
|
|
3528
|
+
/** The MESSAGE_BATCH envelope of `_sendBatch`, without sending it. */
|
|
3529
|
+
_encodeBatch(messages) {
|
|
3530
|
+
const encoder = encoding.createEncoder();
|
|
3531
|
+
encoding.writeVarUint(encoder, MESSAGE_BATCH);
|
|
3532
|
+
for (const message of messages) {
|
|
3533
|
+
encoding.writeVarUint8Array(encoder, message);
|
|
3534
|
+
}
|
|
3535
|
+
return encoding.toUint8Array(encoder);
|
|
3536
|
+
}
|
|
1294
3537
|
/**
|
|
1295
3538
|
* Send data through both BroadcastChannel (if connected) and transport.
|
|
1296
3539
|
* All messages are wrapped with CRC32 checksum for integrity verification.
|
|
@@ -1299,16 +3542,66 @@ export class GenericProvider extends Observable {
|
|
|
1299
3542
|
_send(data) {
|
|
1300
3543
|
// Wrap message with CRC32 checksum
|
|
1301
3544
|
const wrappedData = wrapMessageWithChecksum(data);
|
|
1302
|
-
// Send via BroadcastChannel to other tabs first
|
|
1303
|
-
if (this._bcConnected)
|
|
1304
|
-
|
|
1305
|
-
}
|
|
3545
|
+
// Send via BroadcastChannel to other tabs first.
|
|
3546
|
+
if (this._bcConnected)
|
|
3547
|
+
this._bcPublish(wrappedData);
|
|
1306
3548
|
// Send via network transport
|
|
1307
3549
|
if (!this.transport.isConnected) {
|
|
1308
3550
|
return;
|
|
1309
3551
|
}
|
|
3552
|
+
this._sendToTransport(wrappedData);
|
|
3553
|
+
}
|
|
3554
|
+
/**
|
|
3555
|
+
* Publish already-CRC32-wrapped bytes to the other tabs. BroadcastChannel
|
|
3556
|
+
* is same-process - never worth compressing - but when
|
|
3557
|
+
* compressionThresholdBytes is enabled every message still needs the
|
|
3558
|
+
* leading flag byte _handleIncomingMessage() expects regardless of
|
|
3559
|
+
* source, so this sends flag=0 in that case. The ONE place for every BC
|
|
3560
|
+
* publish: the connect-time burst in _setupBroadcastChannel() used to
|
|
3561
|
+
* publish without the flag, and with compression on (the transport
|
|
3562
|
+
* hints made that a default) the other tab read a CRC byte as the flag
|
|
3563
|
+
* and failed to inflate three messages per join (Nostr playground,
|
|
3564
|
+
* 2026-09-06).
|
|
3565
|
+
*/
|
|
3566
|
+
_bcPublish(wrappedData) {
|
|
3567
|
+
bc.publish(this._bcChannel, this._compressionThresholdBytes
|
|
3568
|
+
? prefixCompressionFlag(0, wrappedData)
|
|
3569
|
+
: wrappedData, this);
|
|
3570
|
+
}
|
|
3571
|
+
/**
|
|
3572
|
+
* Send already-CRC32-wrapped bytes to the network transport, compressing
|
|
3573
|
+
* first if compressionThresholdBytes is configured and this payload
|
|
3574
|
+
* clears it. See that option's doc comment for the size threshold
|
|
3575
|
+
* reasoning and the wire-format compatibility tradeoff of enabling it.
|
|
3576
|
+
*/
|
|
3577
|
+
_sendToTransport(wrappedData, to) {
|
|
3578
|
+
const threshold = this._compressionThresholdBytes;
|
|
3579
|
+
if (!threshold) {
|
|
3580
|
+
this._dispatchToTransport(wrappedData, to);
|
|
3581
|
+
return;
|
|
3582
|
+
}
|
|
3583
|
+
if (!COMPRESSION_AVAILABLE || wrappedData.length < threshold) {
|
|
3584
|
+
this._dispatchToTransport(prefixCompressionFlag(0, wrappedData), to);
|
|
3585
|
+
return;
|
|
3586
|
+
}
|
|
3587
|
+
compressDeflateRaw(wrappedData)
|
|
3588
|
+
.then((compressed) => {
|
|
3589
|
+
this._dispatchToTransport(prefixCompressionFlag(1, compressed), to);
|
|
3590
|
+
})
|
|
3591
|
+
.catch((error) => {
|
|
3592
|
+
console.error('[GenericProvider] Compression failed, sending uncompressed:', error);
|
|
3593
|
+
this._dispatchToTransport(prefixCompressionFlag(0, wrappedData), to);
|
|
3594
|
+
});
|
|
3595
|
+
}
|
|
3596
|
+
/**
|
|
3597
|
+
* Hand fully-framed bytes to transport.send() - or transport.sendTo() when
|
|
3598
|
+
* a peer address is given - tolerating a sync or async result.
|
|
3599
|
+
*/
|
|
3600
|
+
_dispatchToTransport(data, to) {
|
|
1310
3601
|
try {
|
|
1311
|
-
const result = this.transport.
|
|
3602
|
+
const result = to !== undefined && typeof this.transport.sendTo === 'function'
|
|
3603
|
+
? this.transport.sendTo(to, data)
|
|
3604
|
+
: this.transport.send(data);
|
|
1312
3605
|
// Handle async send
|
|
1313
3606
|
if (result instanceof Promise) {
|
|
1314
3607
|
result.catch((error) => {
|
|
@@ -1344,4 +3637,6 @@ export class GenericProvider extends Observable {
|
|
|
1344
3637
|
return this._localSeqNum;
|
|
1345
3638
|
}
|
|
1346
3639
|
}
|
|
3640
|
+
// ms of throttle added per known peer under `awarenessInterval: 'auto'`
|
|
3641
|
+
GenericProvider.AWARENESS_AUTO_MS_PER_PEER = 20;
|
|
1347
3642
|
//# sourceMappingURL=index.js.map
|