@tldraw/sync-core 5.5.0-next.f9308d324e1c → 5.6.0-canary.5bfc37740468

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist-cjs/index.d.ts +8 -4
  2. package/dist-cjs/index.js +1 -1
  3. package/dist-cjs/lib/ClientWebSocketAdapter.js +46 -18
  4. package/dist-cjs/lib/ClientWebSocketAdapter.js.map +2 -2
  5. package/dist-cjs/lib/InMemorySyncStorage.js +18 -18
  6. package/dist-cjs/lib/InMemorySyncStorage.js.map +2 -2
  7. package/dist-cjs/lib/SQLiteSyncStorage.js +15 -7
  8. package/dist-cjs/lib/SQLiteSyncStorage.js.map +2 -2
  9. package/dist-cjs/lib/TLSocketRoom.js +9 -8
  10. package/dist-cjs/lib/TLSocketRoom.js.map +2 -2
  11. package/dist-cjs/lib/TLSyncClient.js +8 -1
  12. package/dist-cjs/lib/TLSyncClient.js.map +2 -2
  13. package/dist-cjs/lib/TLSyncRoom.js +82 -50
  14. package/dist-cjs/lib/TLSyncRoom.js.map +2 -2
  15. package/dist-cjs/lib/TLSyncStorage.js +2 -2
  16. package/dist-cjs/lib/TLSyncStorage.js.map +2 -2
  17. package/dist-cjs/lib/chunk.js +35 -3
  18. package/dist-cjs/lib/chunk.js.map +2 -2
  19. package/dist-cjs/lib/diff.js +15 -7
  20. package/dist-cjs/lib/diff.js.map +2 -2
  21. package/dist-esm/index.d.mts +8 -4
  22. package/dist-esm/index.mjs +1 -1
  23. package/dist-esm/lib/ClientWebSocketAdapter.mjs +47 -19
  24. package/dist-esm/lib/ClientWebSocketAdapter.mjs.map +2 -2
  25. package/dist-esm/lib/InMemorySyncStorage.mjs +18 -18
  26. package/dist-esm/lib/InMemorySyncStorage.mjs.map +2 -2
  27. package/dist-esm/lib/SQLiteSyncStorage.mjs +15 -7
  28. package/dist-esm/lib/SQLiteSyncStorage.mjs.map +2 -2
  29. package/dist-esm/lib/TLSocketRoom.mjs +10 -9
  30. package/dist-esm/lib/TLSocketRoom.mjs.map +2 -2
  31. package/dist-esm/lib/TLSyncClient.mjs +8 -1
  32. package/dist-esm/lib/TLSyncClient.mjs.map +2 -2
  33. package/dist-esm/lib/TLSyncRoom.mjs +82 -50
  34. package/dist-esm/lib/TLSyncRoom.mjs.map +2 -2
  35. package/dist-esm/lib/TLSyncStorage.mjs +2 -2
  36. package/dist-esm/lib/TLSyncStorage.mjs.map +2 -2
  37. package/dist-esm/lib/chunk.mjs +35 -3
  38. package/dist-esm/lib/chunk.mjs.map +2 -2
  39. package/dist-esm/lib/diff.mjs +16 -8
  40. package/dist-esm/lib/diff.mjs.map +2 -2
  41. package/package.json +22 -18
  42. package/src/lib/ClientWebSocketAdapter.test.ts +221 -16
  43. package/src/lib/ClientWebSocketAdapter.ts +62 -23
  44. package/src/lib/InMemorySyncStorage.test.ts +21 -0
  45. package/src/lib/InMemorySyncStorage.ts +22 -24
  46. package/src/lib/SQLiteSyncStorage.test.ts +60 -1
  47. package/src/lib/SQLiteSyncStorage.ts +24 -16
  48. package/src/lib/TLSocketRoom.ts +16 -9
  49. package/src/lib/TLSyncClient.test.ts +61 -0
  50. package/src/lib/TLSyncClient.ts +11 -0
  51. package/src/lib/TLSyncRoom.ts +122 -54
  52. package/src/lib/TLSyncStorage.ts +7 -2
  53. package/src/lib/chunk.test.ts +88 -1
  54. package/src/lib/chunk.ts +66 -3
  55. package/src/lib/diff.test.ts +91 -1
  56. package/src/lib/diff.ts +30 -9
  57. package/src/test/TLSocketRoom.test.ts +63 -0
  58. package/src/test/TLSyncRoom.test.ts +300 -0
  59. package/src/test/objectStore.test.ts +35 -0
  60. package/src/test/upgradeDowngrade.test.ts +21 -0
@@ -16,6 +16,7 @@ import {
16
16
  objectMapEntriesIterable,
17
17
  Result,
18
18
  throttle,
19
+ type ThrottledFunction,
19
20
  } from '@tldraw/utils'
20
21
  import { createNanoEvents } from 'nanoevents'
21
22
  import {
@@ -88,6 +89,29 @@ export const DATA_MESSAGE_DEBOUNCE_INTERVAL = 1000 / 60
88
89
 
89
90
  const timeSince = (time: number) => Date.now() - time
90
91
 
92
+ // WebSocket close frames cap the reason at 123 UTF-8 bytes; a longer one makes `close()` throw
93
+ const MAX_CLOSE_REASON_BYTES = 123
94
+ const closeReasonEncoder = new TextEncoder()
95
+
96
+ const truncatedSuffix = (droppedBytes: number) => `... (+${droppedBytes} bytes)`
97
+
98
+ function truncateCloseReason(reason: string) {
99
+ const totalBytes = closeReasonEncoder.encode(reason).length
100
+ if (totalBytes <= MAX_CLOSE_REASON_BYTES) return reason
101
+ // the dropped count can't have more digits than the total, so reserving room for it always fits
102
+ const budget = MAX_CLOSE_REASON_BYTES - truncatedSuffix(totalBytes).length
103
+ let out = ''
104
+ let outBytes = 0
105
+ // step by code point so a multi-byte character is never cut in half
106
+ for (const char of reason) {
107
+ const charBytes = closeReasonEncoder.encode(char).length
108
+ if (outBytes + charBytes > budget) break
109
+ out += char
110
+ outBytes += charBytes
111
+ }
112
+ return out + truncatedSuffix(totalBytes - outBytes)
113
+ }
114
+
91
115
  /**
92
116
  * Snapshot of a room's complete state that can be persisted and restored.
93
117
  * Contains all documents, tombstones, and metadata needed to reconstruct the room.
@@ -217,7 +241,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
217
241
 
218
242
  private pruneTimer: ReturnType<typeof setTimeout> | null = null
219
243
 
220
- pruneSessions = throttle(() => {
244
+ pruneSessions: ThrottledFunction<() => void> = throttle(() => {
221
245
  if (this.pruneTimer) {
222
246
  clearTimeout(this.pruneTimer)
223
247
  this.pruneTimer = null
@@ -258,7 +282,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
258
282
  }, 1000)
259
283
 
260
284
  private scheduleFollowUpPrune() {
261
- if (this.pruneTimer) return
285
+ // don't leave a stray timer on a room the host has already torn down
286
+ if (this._isClosed || this.pruneTimer) return
262
287
  this.pruneTimer = setTimeout(this.pruneSessions, SESSION_REMOVAL_WAIT_TIME + 100)
263
288
  }
264
289
 
@@ -273,11 +298,27 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
273
298
  * and stops background processes.
274
299
  */
275
300
  close() {
301
+ this._isClosed = true
276
302
  this.disposables.forEach((d) => d())
277
303
  this.sessions.forEach((session) => {
278
- session.socket.close()
304
+ this.clearDebounceTimer(session)
305
+ try {
306
+ session.socket.close()
307
+ } catch {
308
+ // noop, one bad socket must not leave the rest open
309
+ }
279
310
  })
280
- this._isClosed = true
311
+ // forgetting the sessions is what makes late socket close/error events no-ops, so
312
+ // nothing can emit session_removed / room_became_empty on a closed room
313
+ this.sessions.clear()
314
+ }
315
+
316
+ private clearDebounceTimer(session: RoomSession<R, SessionMeta>) {
317
+ if (session.state === RoomSessionState.Connected && session.debounceTimer !== null) {
318
+ clearTimeout(session.debounceTimer)
319
+ session.debounceTimer = null
320
+ session.outstandingDataMessages = []
321
+ }
281
322
  }
282
323
 
283
324
  /**
@@ -519,6 +560,12 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
519
560
  // place, so sockets that defer serialization don't see an emptied array
520
561
  const data = session.outstandingDataMessages
521
562
  session.outstandingDataMessages = []
563
+ if (!session.socket.isOpen) {
564
+ // same as the immediate-send path: send() into a closed socket throws on
565
+ // some runtimes (Cloudflare), and here that would be from inside a timer
566
+ this.cancelSession(sessionId)
567
+ return
568
+ }
522
569
  session.socket.sendMessage({ type: 'data', data })
523
570
  }
524
571
  }
@@ -532,10 +579,12 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
532
579
  }
533
580
 
534
581
  this.sessions.delete(sessionId)
582
+ this.clearDebounceTimer(session)
535
583
 
536
584
  try {
537
585
  if (fatalReason) {
538
- session.socket.close(TLSyncErrorCloseEventCode, fatalReason)
586
+ // the session is already gone, so a throwing close() would leave its socket open
587
+ session.socket.close(TLSyncErrorCloseEventCode, truncateCloseReason(fatalReason))
539
588
  } else {
540
589
  session.socket.close()
541
590
  }
@@ -570,6 +619,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
570
619
  return
571
620
  }
572
621
 
622
+ this.clearDebounceTimer(session)
573
623
  this.sessions.set(sessionId, {
574
624
  state: RoomSessionState.AwaitingRemoval,
575
625
  sessionId,
@@ -607,9 +657,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
607
657
  networkDiff?: NetworkDiff<R> | null,
608
658
  sourceSessionId?: string
609
659
  ) {
610
- // Pre-compute network diff if not provided
611
- const unmigrated = networkDiff ?? toNetworkDiff(diff)
612
- if (!unmigrated) return this
660
+ // Computed once and shared by every session that needs no down-migration; the push path
661
+ // hands us the diff it already computed (in legacy append mode when needed), and re-deriving
662
+ // it per session would both repeat the diffing work and lose that legacy handling.
663
+ const legacyAppendMode = !this.getCanEmitStringAppend()
664
+ const unmigrated = networkDiff ?? toNetworkDiff(diff, legacyAppendMode)
613
665
 
614
666
  this.sessions.forEach((session) => {
615
667
  if (session.state !== RoomSessionState.Connected) return
@@ -623,7 +675,9 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
623
675
  session.sessionId,
624
676
  session.serializedSchema,
625
677
  session.requiresDownMigrations,
626
- diff
678
+ diff,
679
+ unmigrated,
680
+ legacyAppendMode
627
681
  )
628
682
  if (!diffResult.ok) return
629
683
 
@@ -692,7 +746,13 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
692
746
  objectAccess?: TLObjectStoreAccess
693
747
  }) {
694
748
  const { sessionId, socket, meta, isReadonly, objectAccess } = opts
749
+ // a connect racing close() would otherwise create a session nothing ever prunes
750
+ if (this._isClosed) {
751
+ socket.close()
752
+ return this
753
+ }
695
754
  const existing = this.sessions.get(sessionId)
755
+ if (existing) this.clearDebounceTimer(existing)
696
756
  this.sessions.set(sessionId, {
697
757
  state: RoomSessionState.AwaitingConnectMessage,
698
758
  sessionId,
@@ -741,6 +801,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
741
801
  supportsStringAppend,
742
802
  } = opts
743
803
 
804
+ if (this._isClosed) {
805
+ socket.close()
806
+ return
807
+ }
808
+
744
809
  const migrations = this.schema.getMigrationsSince(serializedSchema)
745
810
  const requiresDownMigrations = migrations.ok ? migrations.value.length > 0 : false
746
811
 
@@ -761,9 +826,22 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
761
826
  supportsStringAppend,
762
827
  })
763
828
 
829
+ // Restore presence before the schema checks: other clients still hold it from before the
830
+ // socket slept, and a rejection only broadcasts its removal if it's in the store
764
831
  if (presenceRecord && presenceId) {
765
832
  this.presenceStore.set(presenceId, presenceRecord as R)
766
833
  }
834
+
835
+ // The server may have changed builds while the socket slept, so re-run the handshake's
836
+ // schema checks (HS3): a schema we can no longer reconcile must not be served raw diffs.
837
+ if (!migrations.ok) {
838
+ this.rejectSession(sessionId, this.getVersionMismatchReason(serializedSchema))
839
+ return
840
+ }
841
+ if (migrations.value.some((m) => m.scope !== 'record' || !m.down)) {
842
+ this.rejectSession(sessionId, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD)
843
+ return
844
+ }
767
845
  }
768
846
 
769
847
  /**
@@ -798,6 +876,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
798
876
  * @param requiresDownMigrations - Whether the client needs down migrations
799
877
  * @param diff - The TLSyncForwardDiff containing full records to migrate
800
878
  * @param unmigrated - Optional pre-computed NetworkDiff for when no migration is needed
879
+ * @param legacyAppendMode - Emit string appends as puts (SES5); defaults to the room-wide state
801
880
  * @returns A NetworkDiff with migrated records, or a migration failure
802
881
  */
803
882
  private migrateDiffOrRejectSession(
@@ -805,10 +884,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
805
884
  serializedSchema: SerializedSchema,
806
885
  requiresDownMigrations: boolean,
807
886
  diff: TLSyncForwardDiff<R>,
808
- unmigrated?: NetworkDiff<R>
887
+ unmigrated?: NetworkDiff<R>,
888
+ legacyAppendMode = !this.getCanEmitStringAppend()
809
889
  ): Result<NetworkDiff<R>, MigrationFailureReason> {
810
890
  if (!requiresDownMigrations) {
811
- return Result.ok(unmigrated ?? toNetworkDiff(diff) ?? {})
891
+ return Result.ok(unmigrated ?? toNetworkDiff(diff, legacyAppendMode))
812
892
  }
813
893
 
814
894
  const result: NetworkDiff<R> = {}
@@ -828,7 +908,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
828
908
  this.rejectSession(sessionId, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD)
829
909
  return Result.err(toResult.reason)
830
910
  }
831
- const patch = diffRecord(fromResult.value, toResult.value)
911
+ const patch = diffRecord(fromResult.value, toResult.value, legacyAppendMode)
832
912
  if (patch) {
833
913
  result[id] = [RecordOpType.Patch, patch]
834
914
  }
@@ -905,7 +985,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
905
985
  * Sends appropriate error messages before closing the connection.
906
986
  *
907
987
  * @param sessionId - The session to reject
908
- * @param fatalReason - The reason for rejection (optional)
988
+ * @param fatalReason - The reason for rejection (optional). WebSocket close reasons are capped
989
+ * at 123 UTF-8 bytes, so a longer reason is truncated and ends with `... (+N bytes)`.
909
990
  * @example
910
991
  * ```ts
911
992
  * // Reject due to version mismatch
@@ -962,6 +1043,10 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
962
1043
 
963
1044
  private forceAllReconnect() {
964
1045
  for (const session of this.sessions.values()) {
1046
+ // Only connected clients hold state at a clock we can no longer diff from. A session
1047
+ // mid-handshake gets hydrated from its own lastServerClock in the same transaction, so
1048
+ // removing it would close its socket and then re-add it as Connected (resurrected).
1049
+ if (session.state !== RoomSessionState.Connected) continue
965
1050
  this.removeSession(session.sessionId)
966
1051
  }
967
1052
  }
@@ -991,7 +1076,13 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
991
1076
  return TLSyncErrorCloseEventReason.SERVER_TOO_OLD
992
1077
  }
993
1078
 
994
- if (theirSchema.schemaVersion === 2 && ourSchema.schemaVersion === 2) {
1079
+ if (
1080
+ theirSchema.schemaVersion === 2 &&
1081
+ ourSchema.schemaVersion === 2 &&
1082
+ // a malformed client schema must reject the session, not throw
1083
+ typeof theirSchema.sequences === 'object' &&
1084
+ theirSchema.sequences !== null
1085
+ ) {
995
1086
  for (const [sequenceId, theirVersion] of Object.entries(theirSchema.sequences)) {
996
1087
  const ourVersion = ourSchema.sequences[sequenceId]
997
1088
  if (ourVersion === undefined || theirVersion > ourVersion) {
@@ -1055,7 +1146,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
1055
1146
 
1056
1147
  const requiresDownMigrations = migrations.value.length > 0
1057
1148
 
1058
- const connect = async (msg: Extract<TLSocketServerSentEvent<R>, { type: 'connect' }>) => {
1149
+ const connect = (msg: Extract<TLSocketServerSentEvent<R>, { type: 'connect' }>) => {
1059
1150
  this.sessions.set(session.sessionId, {
1060
1151
  state: RoomSessionState.Connected,
1061
1152
  sessionId: session.sessionId,
@@ -1196,7 +1287,6 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
1196
1287
  }
1197
1288
  let { value: state } = res
1198
1289
 
1199
- // Get the existing document, if any
1200
1290
  const doc =
1201
1291
  prevDoc !== undefined ? (prevDoc ?? undefined) : (storage.get(id) as R | undefined)
1202
1292
 
@@ -1212,7 +1302,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
1212
1302
  // If there's an existing document, replace it with the new state
1213
1303
  // but propagate a diff rather than the entire value
1214
1304
  const recordType = assertExists(getOwnProperty(this.schema.types, doc.typeName))
1215
- const diff = diffAndValidateRecord(doc, state, recordType)
1305
+ const diff = diffAndValidateRecord(doc, state, recordType, legacyAppendMode)
1216
1306
  if (diff) {
1217
1307
  storage.set(id, state)
1218
1308
  propagateOp(changes, id, [RecordOpType.Patch, diff], doc, state)
@@ -1330,6 +1420,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
1330
1420
  !session ||
1331
1421
  (this.objectTypes.has(typeName) ? session.objectAccess !== 'read' : !session.isReadonly)
1332
1422
 
1423
+ // what authorizers see of the pushing session, shared by every authorized op in this push
1424
+ const authSession = session
1425
+ ? { sessionId: session.sessionId, isReadonly: session.isReadonly, meta: session.meta }
1426
+ : null
1427
+
1333
1428
  if (message.diff) {
1334
1429
  // The push request was for the document scope.
1335
1430
  for (const [id, op] of objectMapEntriesIterable(message.diff!)) {
@@ -1380,26 +1475,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
1380
1475
  authorize = (prevRec, next) => {
1381
1476
  const result = authorizePut(
1382
1477
  prevRec
1383
- ? {
1384
- session: {
1385
- sessionId: session.sessionId,
1386
- isReadonly: session.isReadonly,
1387
- meta: session.meta,
1388
- },
1389
- type: 'update',
1390
- prev: prevRec,
1391
- next,
1392
- }
1393
- : {
1394
- session: {
1395
- sessionId: session.sessionId,
1396
- isReadonly: session.isReadonly,
1397
- meta: session.meta,
1398
- },
1399
- type: 'create',
1400
- prev: null,
1401
- next,
1402
- }
1478
+ ? { session: authSession!, type: 'update', prev: prevRec, next }
1479
+ : { session: authSession!, type: 'create', prev: null, next }
1403
1480
  )
1404
1481
  if (!result) {
1405
1482
  this.log?.warn?.(
@@ -1431,11 +1508,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
1431
1508
  const authorize = authorizePatch
1432
1509
  ? (prev: R, next: R) => {
1433
1510
  const result = authorizePatch({
1434
- session: {
1435
- sessionId: session.sessionId,
1436
- isReadonly: session.isReadonly,
1437
- meta: session.meta,
1438
- },
1511
+ session: authSession!,
1439
1512
  type: 'update',
1440
1513
  prev,
1441
1514
  next,
@@ -1469,16 +1542,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
1469
1542
  const authorizeRemove = session && this.authorizerFor(doc.typeName)
1470
1543
  if (
1471
1544
  authorizeRemove &&
1472
- !authorizeRemove({
1473
- session: {
1474
- sessionId: session.sessionId,
1475
- isReadonly: session.isReadonly,
1476
- meta: session.meta,
1477
- },
1478
- type: 'delete',
1479
- prev: doc,
1480
- next: null,
1481
- })
1545
+ !authorizeRemove({ session: authSession!, type: 'delete', prev: doc, next: null })
1482
1546
  ) {
1483
1547
  this.log?.warn?.(
1484
1548
  'authorizer vetoed delete',
@@ -1510,7 +1574,10 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
1510
1574
  let pushResult: TLSocketServerSentEvent<R> | undefined
1511
1575
  if (changes && session) {
1512
1576
  // txn did not apply verbatim so we should broadcast the actual changes
1513
- result.docChanges.diffs = { networkDiff: toNetworkDiff(changes) ?? {}, diff: changes }
1577
+ result.docChanges.diffs = {
1578
+ networkDiff: toNetworkDiff(changes, legacyAppendMode),
1579
+ diff: changes,
1580
+ }
1514
1581
  }
1515
1582
 
1516
1583
  if (isEqual(result.docChanges.diffs?.networkDiff, message.diff)) {
@@ -1535,7 +1602,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
1535
1602
  session.serializedSchema,
1536
1603
  session.requiresDownMigrations,
1537
1604
  result.docChanges.diffs.diff,
1538
- result.docChanges.diffs.networkDiff
1605
+ result.docChanges.diffs.networkDiff,
1606
+ legacyAppendMode
1539
1607
  )
1540
1608
  if (diff.ok) {
1541
1609
  pushResult = {
@@ -146,13 +146,18 @@ export interface TLSyncForwardDiff<R extends UnknownRecord> {
146
146
  }
147
147
 
148
148
  /**
149
+ * @param legacyAppendMode - When true, string appends are emitted as puts (see `diffRecord`); the
150
+ * room sets this whenever a connected client predates string-append support.
149
151
  * @internal
150
152
  */
151
- export function toNetworkDiff<R extends UnknownRecord>(diff: TLSyncForwardDiff<R>): NetworkDiff<R> {
153
+ export function toNetworkDiff<R extends UnknownRecord>(
154
+ diff: TLSyncForwardDiff<R>,
155
+ legacyAppendMode = false
156
+ ): NetworkDiff<R> {
152
157
  const networkDiff: NetworkDiff<R> = {}
153
158
  for (const [id, put] of objectMapEntriesIterable(diff.puts)) {
154
159
  if (Array.isArray(put)) {
155
- const patch = diffRecord(put[0], put[1])
160
+ const patch = diffRecord(put[0], put[1], legacyAppendMode)
156
161
  if (patch) {
157
162
  networkDiff[id] = [RecordOpType.Patch, patch]
158
163
  }
@@ -1,5 +1,11 @@
1
1
  import { assert } from 'tldraw'
2
- import { JsonChunkAssembler, chunk } from './chunk'
2
+ import {
3
+ JsonChunkAssembler,
4
+ MAX_ASSEMBLED_MESSAGE_CHARS,
5
+ MAX_CHUNK_COUNT,
6
+ MessageTooLargeError,
7
+ chunk,
8
+ } from './chunk'
3
9
 
4
10
  describe('chunk (CH1–CH3)', () => {
5
11
  describe('size boundary (CH1)', () => {
@@ -134,6 +140,40 @@ describe('chunk (CH1–CH3)', () => {
134
140
  `)
135
141
  })
136
142
  })
143
+
144
+ describe('surrogate pairs (CH9)', () => {
145
+ it('[CH9] never splits a surrogate pair across two chunks', () => {
146
+ // WebSocket.send converts each chunk to a USVString, so a lone surrogate would arrive
147
+ // as U+FFFD and the reassembled message would carry corrupted text
148
+ const emoji = '\u{1F600}'
149
+ const isHigh = (code: number) => code >= 0xd800 && code <= 0xdbff
150
+ const isLow = (code: number) => code >= 0xdc00 && code <= 0xdfff
151
+ const msgs = [
152
+ ...Array.from(
153
+ { length: 8 },
154
+ (_, padding) => 'x'.repeat(padding) + emoji + 'y'.repeat(6) + emoji
155
+ ),
156
+ // adjacent pairs: after a nudge the next boundary lands on another pair
157
+ 'x' + emoji.repeat(3),
158
+ ]
159
+ for (const msg of msgs) {
160
+ for (const maxSize of [3, 4, 5, 6, 7, 8]) {
161
+ const chunks = chunk(msg, maxSize)
162
+ const bodies = chunks.map((c) => c.slice(c.indexOf('_') + 1))
163
+ expect(bodies.join('')).toBe(msg)
164
+ for (const [i, body] of bodies.entries()) {
165
+ expect(body.length).toBeGreaterThan(0)
166
+ expect(isHigh(body.charCodeAt(body.length - 1))).toBe(false)
167
+ expect(isLow(body.charCodeAt(0))).toBe(false)
168
+ // keeping a pair whole may cost one character over the CH3 bound, never more
169
+ expect(chunks[i].length).toBeLessThanOrEqual(
170
+ Math.max(maxSize, chunks[i].indexOf('_') + 2) + 1
171
+ )
172
+ }
173
+ }
174
+ }
175
+ })
176
+ })
137
177
  })
138
178
 
139
179
  const testObject = {} as any
@@ -369,4 +409,51 @@ describe('JsonChunkAssembler (CH4–CH8)', () => {
369
409
  })
370
410
  })
371
411
  })
412
+
413
+ describe('assembly bounds (CH10)', () => {
414
+ it('[CH10] rejects a first chunk declaring more than MAX_CHUNK_COUNT chunks', () => {
415
+ const unchunker = new JsonChunkAssembler()
416
+
417
+ const result = unchunker.handleMessage(`${MAX_CHUNK_COUNT}_{"a":`)
418
+ expect(result).toMatchObject({
419
+ error: expect.objectContaining({ message: `Too many chunks: ${MAX_CHUNK_COUNT + 1}` }),
420
+ })
421
+ expect(unchunker.state).toBe('idle')
422
+ })
423
+
424
+ it('[CH10] accepts a declared count at the limit', () => {
425
+ const unchunker = new JsonChunkAssembler()
426
+
427
+ expect(unchunker.handleMessage(`${MAX_CHUNK_COUNT - 1}_{"a":`)).toBeNull()
428
+ expect(unchunker.state).not.toBe('idle')
429
+ })
430
+
431
+ it('[CH10] rejects an assembly once its accumulated bodies pass MAX_ASSEMBLED_MESSAGE_CHARS', () => {
432
+ const unchunker = new JsonChunkAssembler()
433
+ // A chunk stream that never sends its final chunk. Before CH10 this accumulated
434
+ // without limit for as long as the sender kept sending.
435
+ const body = 'x'.repeat(1024 * 1024)
436
+ const chunksNeeded = Math.ceil(MAX_ASSEMBLED_MESSAGE_CHARS / body.length) + 1
437
+
438
+ let result = null
439
+ for (let i = 0; i < chunksNeeded; i++) {
440
+ result = unchunker.handleMessage(`${chunksNeeded - i}_${body}`)
441
+ if (result) break
442
+ }
443
+
444
+ expect(result).toMatchObject({ error: expect.any(MessageTooLargeError) })
445
+ expect(unchunker.state).toBe('idle')
446
+ })
447
+
448
+ it('[CH10] leaves the assembler usable after rejecting an oversized assembly', () => {
449
+ const unchunker = new JsonChunkAssembler()
450
+ const body = 'x'.repeat(1024 * 1024)
451
+ const chunksNeeded = Math.ceil(MAX_ASSEMBLED_MESSAGE_CHARS / body.length) + 1
452
+ for (let i = 0; i < chunksNeeded; i++) {
453
+ if (unchunker.handleMessage(`${chunksNeeded - i}_${body}`)) break
454
+ }
455
+
456
+ expect(unchunker.handleMessage('{"test": true}')).toMatchObject({ data: { test: true } })
457
+ })
458
+ })
372
459
  })
package/src/lib/chunk.ts CHANGED
@@ -8,6 +8,35 @@ const MAX_BYTES_PER_CHAR = 4
8
8
  // in the (admittedly impossible) worst case, the max size is 1/4 of a megabyte
9
9
  const MAX_SAFE_MESSAGE_SIZE = MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES / MAX_BYTES_PER_CHAR
10
10
 
11
+ // JsonChunkAssembler holds every chunk of a message until the last one arrives, and the sender
12
+ // decides how many that is — the count is just a prefix on the first chunk. Nothing in the
13
+ // protocol obliges a sender to ever send the last chunk, so without a ceiling a single connection
14
+ // can make the receiver hold an arbitrarily large buffer for as long as the socket stays open.
15
+ //
16
+ // The cap counts UTF-16 code units, which is what a JS string costs in memory: 16M of them is the
17
+ // 32MB of retained string data a room can afford to hold per session and stay inside a Durable
18
+ // Object's 128MB instance limit with room for the document itself. It is far above anything
19
+ // chunk() produces for a real document push. The character cap is the real bound; the chunk-count
20
+ // cap only rejects an absurd declared count on arrival instead of waiting for the characters to
21
+ // accumulate, and is loose enough not to constrain a sender that chunks more finely than the
22
+ // default.
23
+ export const MAX_ASSEMBLED_MESSAGE_CHARS = 16 * 1024 * 1024
24
+ export const MAX_CHUNK_COUNT = 100_000
25
+
26
+ /**
27
+ * Thrown when a message would exceed MAX_ASSEMBLED_MESSAGE_CHARS, on either side of the wire.
28
+ * Distinct from the other assembly errors because it is fatal: retrying sends the same
29
+ * oversized message, so the session is rejected rather than reset.
30
+ *
31
+ * @internal
32
+ */
33
+ export class MessageTooLargeError extends Error {
34
+ constructor(chars: number) {
35
+ super(`Message too large: ${chars} characters, max ${MAX_ASSEMBLED_MESSAGE_CHARS}`)
36
+ this.name = 'MessageTooLargeError'
37
+ }
38
+ }
39
+
11
40
  /**
12
41
  * Splits a string into smaller chunks suitable for transmission over WebSockets.
13
42
  * This function ensures messages don't exceed size limits imposed by platforms like Cloudflare Workers (1MB max).
@@ -38,7 +67,18 @@ export function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
38
67
  let offset = msg.length
39
68
  while (offset > 0) {
40
69
  const prefix = `${chunkNumber}_`
41
- const chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1)
70
+ let chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1)
71
+ // Never split a surrogate pair across chunks: WebSocket.send converts each chunk to a
72
+ // USVString, turning a lone surrogate into U+FFFD, so the reassembled JSON silently
73
+ // carries corrupted text. Shrink by one so the pair moves whole into the preceding
74
+ // chunk; a single-character chunk grows by one instead (one over the CH3 bound, see CH9).
75
+ if (
76
+ chunkSize < offset &&
77
+ isLowSurrogate(msg.charCodeAt(offset - chunkSize)) &&
78
+ isHighSurrogate(msg.charCodeAt(offset - chunkSize - 1))
79
+ ) {
80
+ chunkSize += chunkSize > 1 ? -1 : 1
81
+ }
42
82
  chunks.unshift(prefix + msg.slice(offset - chunkSize, offset))
43
83
  offset -= chunkSize
44
84
  chunkNumber++
@@ -47,6 +87,14 @@ export function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
47
87
  }
48
88
  }
49
89
 
90
+ function isHighSurrogate(code: number) {
91
+ return code >= 0xd800 && code <= 0xdbff
92
+ }
93
+
94
+ function isLowSurrogate(code: number) {
95
+ return code >= 0xdc00 && code <= 0xdfff
96
+ }
97
+
50
98
  // The 's' flag (dotAll) makes '.' match any character including line terminators
51
99
  // like U+2028 and U+2029, which are commonly introduced via copy/paste from Word
52
100
  const chunkRe = /^(\d+)_(.*)$/s
@@ -81,6 +129,7 @@ export class JsonChunkAssembler {
81
129
  | {
82
130
  chunksReceived: string[]
83
131
  totalChunks: number
132
+ charsReceived: number
84
133
  } = 'idle'
85
134
 
86
135
  /**
@@ -91,7 +140,8 @@ export class JsonChunkAssembler {
91
140
  * @param msg - The message to process, either JSON or chunk format
92
141
  * @returns Result object with data/stringified on success, error object on failure, or null for incomplete chunks
93
142
  * - `\{ data: object, stringified: string \}` - Successfully parsed complete message
94
- * - `\{ error: Error \}` - Parse error or invalid chunk sequence
143
+ * - `\{ error: Error \}` - Parse error, invalid chunk sequence, or a message exceeding
144
+ * MAX_ASSEMBLED_MESSAGE_CHARS / MAX_CHUNK_COUNT
95
145
  * - `null` - Chunk received but more chunks expected
96
146
  *
97
147
  * @example
@@ -125,17 +175,30 @@ export class JsonChunkAssembler {
125
175
  const data = match[2]
126
176
 
127
177
  if (this.state === 'idle') {
178
+ const totalChunks = numChunksRemaining + 1
179
+ if (totalChunks > MAX_CHUNK_COUNT) {
180
+ // Nothing has been buffered yet, so the state is already idle.
181
+ return { error: new Error(`Too many chunks: ${totalChunks}`) }
182
+ }
128
183
  this.state = {
129
184
  chunksReceived: [data],
130
- totalChunks: numChunksRemaining + 1,
185
+ totalChunks,
186
+ charsReceived: data.length,
131
187
  }
132
188
  } else {
133
189
  this.state.chunksReceived.push(data)
190
+ this.state.charsReceived += data.length
134
191
  if (numChunksRemaining !== this.state.totalChunks - this.state.chunksReceived.length) {
135
192
  this.state = 'idle'
136
193
  return { error: new Error(`Chunks received in wrong order`) }
137
194
  }
138
195
  }
196
+ if (this.state.charsReceived > MAX_ASSEMBLED_MESSAGE_CHARS) {
197
+ // Drop what was buffered rather than keeping it until the socket closes.
198
+ const charsReceived = this.state.charsReceived
199
+ this.state = 'idle'
200
+ return { error: new MessageTooLargeError(charsReceived) }
201
+ }
139
202
  if (this.state.chunksReceived.length === this.state.totalChunks) {
140
203
  try {
141
204
  const stringified = this.state.chunksReceived.join('')