@tldraw/sync-core 5.5.0-next.f9308d324e1c → 5.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/dist-cjs/index.d.ts +7 -2
  2. package/dist-cjs/index.js +1 -1
  3. package/dist-cjs/lib/ClientWebSocketAdapter.js +46 -18
  4. package/dist-cjs/lib/ClientWebSocketAdapter.js.map +2 -2
  5. package/dist-cjs/lib/InMemorySyncStorage.js +18 -18
  6. package/dist-cjs/lib/InMemorySyncStorage.js.map +2 -2
  7. package/dist-cjs/lib/SQLiteSyncStorage.js +15 -7
  8. package/dist-cjs/lib/SQLiteSyncStorage.js.map +2 -2
  9. package/dist-cjs/lib/TLSocketRoom.js +9 -8
  10. package/dist-cjs/lib/TLSocketRoom.js.map +2 -2
  11. package/dist-cjs/lib/TLSyncClient.js +3 -1
  12. package/dist-cjs/lib/TLSyncClient.js.map +2 -2
  13. package/dist-cjs/lib/TLSyncRoom.js +52 -6
  14. package/dist-cjs/lib/TLSyncRoom.js.map +2 -2
  15. package/dist-cjs/lib/chunk.js +35 -3
  16. package/dist-cjs/lib/chunk.js.map +2 -2
  17. package/dist-esm/index.d.mts +7 -2
  18. package/dist-esm/index.mjs +1 -1
  19. package/dist-esm/lib/ClientWebSocketAdapter.mjs +47 -19
  20. package/dist-esm/lib/ClientWebSocketAdapter.mjs.map +2 -2
  21. package/dist-esm/lib/InMemorySyncStorage.mjs +18 -18
  22. package/dist-esm/lib/InMemorySyncStorage.mjs.map +2 -2
  23. package/dist-esm/lib/SQLiteSyncStorage.mjs +15 -7
  24. package/dist-esm/lib/SQLiteSyncStorage.mjs.map +2 -2
  25. package/dist-esm/lib/TLSocketRoom.mjs +10 -9
  26. package/dist-esm/lib/TLSocketRoom.mjs.map +2 -2
  27. package/dist-esm/lib/TLSyncClient.mjs +3 -1
  28. package/dist-esm/lib/TLSyncClient.mjs.map +2 -2
  29. package/dist-esm/lib/TLSyncRoom.mjs +52 -6
  30. package/dist-esm/lib/TLSyncRoom.mjs.map +2 -2
  31. package/dist-esm/lib/chunk.mjs +35 -3
  32. package/dist-esm/lib/chunk.mjs.map +2 -2
  33. package/package.json +11 -7
  34. package/src/lib/ClientWebSocketAdapter.test.ts +221 -16
  35. package/src/lib/ClientWebSocketAdapter.ts +62 -23
  36. package/src/lib/InMemorySyncStorage.test.ts +21 -0
  37. package/src/lib/InMemorySyncStorage.ts +20 -22
  38. package/src/lib/SQLiteSyncStorage.test.ts +60 -1
  39. package/src/lib/SQLiteSyncStorage.ts +22 -14
  40. package/src/lib/TLSocketRoom.ts +16 -9
  41. package/src/lib/TLSyncClient.ts +2 -0
  42. package/src/lib/TLSyncRoom.ts +71 -6
  43. package/src/lib/chunk.test.ts +88 -1
  44. package/src/lib/chunk.ts +66 -3
  45. package/src/test/TLSocketRoom.test.ts +63 -0
  46. package/src/test/TLSyncRoom.test.ts +176 -0
  47. package/src/test/objectStore.test.ts +35 -0
@@ -1,7 +1,7 @@
1
1
  import { DatabaseSync } from 'node:sqlite'
2
2
  import { DocumentRecordType, PageRecordType, TLDOCUMENT_ID, TLRecord } from '@tldraw/tlschema'
3
3
  import { ZERO_INDEX_KEY } from '@tldraw/utils'
4
- import { describe, expect, it } from 'vitest'
4
+ import { describe, expect, it, vi } from 'vitest'
5
5
  import {
6
6
  contractRecords,
7
7
  contractSchema,
@@ -9,6 +9,7 @@ import {
9
9
  makePage,
10
10
  registerStorageContractTests,
11
11
  } from '../test/storageContractSuite'
12
+ import { MAX_TOMBSTONES } from './InMemorySyncStorage'
12
13
  import { NodeSqliteWrapper } from './NodeSqliteWrapper'
13
14
  import { SQLiteSyncStorage } from './SQLiteSyncStorage'
14
15
 
@@ -263,4 +264,62 @@ describe('SQLiteSyncStorage', () => {
263
264
  expect(row?.migrationVersion).toBe(3)
264
265
  })
265
266
  })
267
+
268
+ describe('tombstone pruning robustness', () => {
269
+ const makeTombstones = (count: number) => {
270
+ const tombstones: Record<string, number> = {}
271
+ for (let i = 0; i < count; i++) tombstones[`shape:doc${i}`] = i + 1
272
+ return tombstones
273
+ }
274
+ function makeOverfullStorage() {
275
+ const db = new DatabaseSync(':memory:')
276
+ const storage = new SQLiteSyncStorage<TLRecord>({
277
+ sql: new NodeSqliteWrapper(db),
278
+ snapshot: makeContractSnapshot(contractRecords, {
279
+ tombstones: makeTombstones(MAX_TOMBSTONES + 500),
280
+ documentClock: MAX_TOMBSTONES + 501,
281
+ tombstoneHistoryStartsAtClock: 0,
282
+ }),
283
+ })
284
+ return { db, storage, prune: (storage as any).pruneTombstones as { (): void; flush(): void } }
285
+ }
286
+
287
+ it('[SS18] a prune that fires after the database was closed does not throw', () => {
288
+ const { db, storage, prune } = makeOverfullStorage()
289
+ const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
290
+ try {
291
+ // a delete schedules the throttled prune; the host then closes everything
292
+ storage.transaction((txn) => txn.delete(contractRecords[1].id))
293
+ db.close()
294
+ // the throttle timer fires later, from outside any caller that could catch it
295
+ expect(() => prune.flush()).not.toThrow()
296
+ expect(consoleSpy).toHaveBeenCalledWith('Failed to prune tombstones', expect.anything())
297
+ } finally {
298
+ consoleSpy.mockRestore()
299
+ }
300
+ })
301
+
302
+ it('[SS18] a prune whose delete fails leaves the history clock where it was', () => {
303
+ const { storage, prune } = makeOverfullStorage()
304
+ const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
305
+ try {
306
+ const stmts = (storage as any).stmts
307
+ const originalRun = stmts.deleteTombstonesBefore.run
308
+ stmts.deleteTombstonesBefore.run = () => {
309
+ throw new Error('SQLITE_BUSY: database is locked')
310
+ }
311
+ prune()
312
+ expect(() => prune.flush()).not.toThrow()
313
+ stmts.deleteTombstonesBefore.run = originalRun
314
+
315
+ // neither half of the prune applied: the history start was not advanced past
316
+ // tombstones that still exist
317
+ const snapshot = storage.getSnapshot()
318
+ expect(snapshot.tombstoneHistoryStartsAtClock).toBe(0)
319
+ expect(Object.keys(snapshot.tombstones!).length).toBe(MAX_TOMBSTONES + 500)
320
+ } finally {
321
+ consoleSpy.mockRestore()
322
+ }
323
+ })
324
+ })
266
325
  })
@@ -394,7 +394,6 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
394
394
  const documentClock = snapshot.documentClock ?? snapshot.clock ?? 0
395
395
  const tombstoneHistoryStartsAtClock = snapshot.tombstoneHistoryStartsAtClock ?? documentClock
396
396
 
397
- // Clear existing data
398
397
  this.sql.exec(`
399
398
  DELETE FROM ${documentsTable};
400
399
  DELETE FROM ${objectsTable};
@@ -410,14 +409,12 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
410
409
  table.insert.run(doc.state.id, encodeState(doc.state), doc.lastChangedClock)
411
410
  }
412
411
 
413
- // Insert tombstones
414
412
  if (snapshot.tombstones) {
415
413
  for (const [id, clock] of objectMapEntries(snapshot.tombstones)) {
416
414
  this.stmts.insertTombstone.run(id, clock)
417
415
  }
418
416
  }
419
417
 
420
- // Insert metadata row
421
418
  this.stmts.updateMetadata.run(
422
419
  documentClock,
423
420
  tombstoneHistoryStartsAtClock,
@@ -513,18 +510,29 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
513
510
  /** @internal */
514
511
  pruneTombstones = throttle(
515
512
  () => {
516
- const tombstoneCount = this.stmts.countTombstones.all()[0].count as number
517
- if (tombstoneCount > MAX_TOMBSTONES) {
518
- // Get all tombstones sorted by clock ascending (oldest first)
519
- const tombstones = this.stmts.iterateTombstones.all()
520
-
521
- const result = computeTombstonePruning({ tombstones, documentClock: this.getClock() })
522
- if (result) {
523
- this.stmts.setTombstoneHistoryStartsAtClock.run(result.newTombstoneHistoryStartsAtClock)
524
- // Delete all tombstones with clock < newTombstoneHistoryStartsAtClock in one operation.
525
- // This works because computeTombstonePruning ensures we never split a clock value.
526
- this.stmts.deleteTombstonesBefore.run(result.newTombstoneHistoryStartsAtClock)
513
+ // Runs from a timer, so a host that closed the database right after the last delete
514
+ // (e.g. `room.close(); db.close()` in onSessionRemoved) would otherwise get an uncaught
515
+ // throw from here; pruning is best-effort and simply runs again on the next delete.
516
+ try {
517
+ const tombstoneCount = this.stmts.countTombstones.all()[0].count as number
518
+ if (tombstoneCount > MAX_TOMBSTONES) {
519
+ // Get all tombstones sorted by clock ascending (oldest first)
520
+ const tombstones = this.stmts.iterateTombstones.all()
521
+
522
+ const result = computeTombstonePruning({ tombstones, documentClock: this.getClock() })
523
+ if (result) {
524
+ this.sql.transaction(() => {
525
+ this.stmts.setTombstoneHistoryStartsAtClock.run(
526
+ result.newTombstoneHistoryStartsAtClock
527
+ )
528
+ // Delete all tombstones with clock < newTombstoneHistoryStartsAtClock in one operation.
529
+ // This works because computeTombstonePruning ensures we never split a clock value.
530
+ this.stmts.deleteTombstonesBefore.run(result.newTombstoneHistoryStartsAtClock)
531
+ })
532
+ }
527
533
  }
534
+ } catch (e) {
535
+ console.error('Failed to prune tombstones', e)
528
536
  }
529
537
  },
530
538
  1000,
@@ -1,7 +1,7 @@
1
1
  import type { SerializedSchema, StoreSchema, UnknownRecord } from '@tldraw/store'
2
2
  import { createTLSchema, TLInstancePresence, TLStoreSnapshot } from '@tldraw/tlschema'
3
3
  import { getOwnProperty, hasOwnProperty, isEqual, structuredClone } from '@tldraw/utils'
4
- import { JsonChunkAssembler } from './chunk'
4
+ import { JsonChunkAssembler, MessageTooLargeError } from './chunk'
5
5
  import { DEFAULT_INITIAL_SNAPSHOT, InMemorySyncStorage } from './InMemorySyncStorage'
6
6
  import { TLObjectStoreAccess, TLSocketServerSentEvent } from './protocol'
7
7
  import { RoomSessionState } from './RoomSession'
@@ -273,13 +273,16 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
273
273
  })
274
274
  )
275
275
  }
276
+ // The default logger belongs to both layers: TLSyncRoom is where authorizer throws are
277
+ // logged, and a host that passes no `log` would otherwise never see them.
278
+ this.log = 'log' in opts ? opts.log : { error: console.error }
276
279
  this.room = new TLSyncRoom<R, SessionMeta>({
277
280
  onPresenceChange: opts.onPresenceChange,
278
281
  onCommittedChanges: opts.onCommittedChanges,
279
282
  objectTypes: opts.objectTypes,
280
283
  authorizeRecord: opts.authorizeRecord,
281
284
  schema: opts.schema ?? (createTLSchema() as any),
282
- log: opts.log,
285
+ log: this.log,
283
286
  storage,
284
287
  clientTimeout: opts.clientTimeout,
285
288
  })
@@ -295,7 +298,6 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
295
298
  })
296
299
  }
297
300
  })
298
- this.log = 'log' in opts ? opts.log : { error: console.error }
299
301
  }
300
302
 
301
303
  /**
@@ -475,6 +477,11 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
475
477
  })
476
478
  this.room.pruneSessions()
477
479
  this.scheduleDebouncedSnapshot(sessionId)
480
+ } else if (res.error instanceof MessageTooLargeError) {
481
+ this.log?.error?.('Error assembling message', res.error)
482
+ // Resetting the connection would have the client reconnect and re-send the same
483
+ // oversized message, so reject the session instead and let the client surface it.
484
+ this.room.rejectSession(sessionId, TLSyncErrorCloseEventReason.MESSAGE_TOO_LARGE)
478
485
  } else {
479
486
  this.log?.error?.('Error assembling message', res.error)
480
487
  // close the socket to reset the connection
@@ -854,12 +861,11 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
854
861
  if (this.isClosed()) {
855
862
  throw new Error('Cannot update store on a closed room')
856
863
  }
864
+ // read through the transaction rather than getSnapshot(): the document snapshot excludes the
865
+ // object-store lane, which would make object-lane records invisible (and undeletable) here
866
+ const records = this.storage.transaction((txn) => Object.fromEntries(txn.entries())).result
857
867
  // eslint-disable-next-line @typescript-eslint/no-deprecated
858
- const ctx = new StoreUpdateContext<R>(
859
- // eslint-disable-next-line @typescript-eslint/no-deprecated
860
- Object.fromEntries(this.getCurrentSnapshot().documents.map((d) => [d.state.id, d.state])),
861
- this.room.schema
862
- )
868
+ const ctx = new StoreUpdateContext<R>(records, this.room.schema)
863
869
  try {
864
870
  await updater(ctx)
865
871
  } finally {
@@ -908,7 +914,8 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
908
914
  * The client will attempt to reconnect automatically unless a fatal reason is provided.
909
915
  *
910
916
  * @param sessionId - Session identifier to remove
911
- * @param fatalReason - Optional fatal error reason that prevents reconnection
917
+ * @param fatalReason - Optional fatal error reason that prevents reconnection. WebSocket close
918
+ * reasons are capped at 123 UTF-8 bytes, so a longer reason is truncated and ends with `... (+N bytes)`.
912
919
  *
913
920
  * @example
914
921
  * ```ts
@@ -122,6 +122,8 @@ export const TLSyncErrorCloseEventReason = {
122
122
  RATE_LIMITED: 'RATE_LIMITED',
123
123
  /** Room has reached maximum capacity */
124
124
  ROOM_FULL: 'ROOM_FULL',
125
+ /** A single message exceeded the size the server will assemble */
126
+ MESSAGE_TOO_LARGE: 'MESSAGE_TOO_LARGE',
125
127
  } as const
126
128
 
127
129
  /**
@@ -88,6 +88,29 @@ export const DATA_MESSAGE_DEBOUNCE_INTERVAL = 1000 / 60
88
88
 
89
89
  const timeSince = (time: number) => Date.now() - time
90
90
 
91
+ // WebSocket close frames cap the reason at 123 UTF-8 bytes; a longer one makes `close()` throw
92
+ const MAX_CLOSE_REASON_BYTES = 123
93
+ const closeReasonEncoder = new TextEncoder()
94
+
95
+ const truncatedSuffix = (droppedBytes: number) => `... (+${droppedBytes} bytes)`
96
+
97
+ function truncateCloseReason(reason: string) {
98
+ const totalBytes = closeReasonEncoder.encode(reason).length
99
+ if (totalBytes <= MAX_CLOSE_REASON_BYTES) return reason
100
+ // the dropped count can't have more digits than the total, so reserving room for it always fits
101
+ const budget = MAX_CLOSE_REASON_BYTES - truncatedSuffix(totalBytes).length
102
+ let out = ''
103
+ let outBytes = 0
104
+ // step by code point so a multi-byte character is never cut in half
105
+ for (const char of reason) {
106
+ const charBytes = closeReasonEncoder.encode(char).length
107
+ if (outBytes + charBytes > budget) break
108
+ out += char
109
+ outBytes += charBytes
110
+ }
111
+ return out + truncatedSuffix(totalBytes - outBytes)
112
+ }
113
+
91
114
  /**
92
115
  * Snapshot of a room's complete state that can be persisted and restored.
93
116
  * Contains all documents, tombstones, and metadata needed to reconstruct the room.
@@ -258,7 +281,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
258
281
  }, 1000)
259
282
 
260
283
  private scheduleFollowUpPrune() {
261
- if (this.pruneTimer) return
284
+ // don't leave a stray timer on a room the host has already torn down
285
+ if (this._isClosed || this.pruneTimer) return
262
286
  this.pruneTimer = setTimeout(this.pruneSessions, SESSION_REMOVAL_WAIT_TIME + 100)
263
287
  }
264
288
 
@@ -273,11 +297,27 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
273
297
  * and stops background processes.
274
298
  */
275
299
  close() {
300
+ this._isClosed = true
276
301
  this.disposables.forEach((d) => d())
277
302
  this.sessions.forEach((session) => {
278
- session.socket.close()
303
+ this.clearDebounceTimer(session)
304
+ try {
305
+ session.socket.close()
306
+ } catch {
307
+ // noop, one bad socket must not leave the rest open
308
+ }
279
309
  })
280
- this._isClosed = true
310
+ // forgetting the sessions is what makes late socket close/error events no-ops, so
311
+ // nothing can emit session_removed / room_became_empty on a closed room
312
+ this.sessions.clear()
313
+ }
314
+
315
+ private clearDebounceTimer(session: RoomSession<R, SessionMeta>) {
316
+ if (session.state === RoomSessionState.Connected && session.debounceTimer !== null) {
317
+ clearTimeout(session.debounceTimer)
318
+ session.debounceTimer = null
319
+ session.outstandingDataMessages = []
320
+ }
281
321
  }
282
322
 
283
323
  /**
@@ -519,6 +559,12 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
519
559
  // place, so sockets that defer serialization don't see an emptied array
520
560
  const data = session.outstandingDataMessages
521
561
  session.outstandingDataMessages = []
562
+ if (!session.socket.isOpen) {
563
+ // same as the immediate-send path: send() into a closed socket throws on
564
+ // some runtimes (Cloudflare), and here that would be from inside a timer
565
+ this.cancelSession(sessionId)
566
+ return
567
+ }
522
568
  session.socket.sendMessage({ type: 'data', data })
523
569
  }
524
570
  }
@@ -532,10 +578,12 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
532
578
  }
533
579
 
534
580
  this.sessions.delete(sessionId)
581
+ this.clearDebounceTimer(session)
535
582
 
536
583
  try {
537
584
  if (fatalReason) {
538
- session.socket.close(TLSyncErrorCloseEventCode, fatalReason)
585
+ // the session is already gone, so a throwing close() would leave its socket open
586
+ session.socket.close(TLSyncErrorCloseEventCode, truncateCloseReason(fatalReason))
539
587
  } else {
540
588
  session.socket.close()
541
589
  }
@@ -570,6 +618,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
570
618
  return
571
619
  }
572
620
 
621
+ this.clearDebounceTimer(session)
573
622
  this.sessions.set(sessionId, {
574
623
  state: RoomSessionState.AwaitingRemoval,
575
624
  sessionId,
@@ -692,7 +741,13 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
692
741
  objectAccess?: TLObjectStoreAccess
693
742
  }) {
694
743
  const { sessionId, socket, meta, isReadonly, objectAccess } = opts
744
+ // a connect racing close() would otherwise create a session nothing ever prunes
745
+ if (this._isClosed) {
746
+ socket.close()
747
+ return this
748
+ }
695
749
  const existing = this.sessions.get(sessionId)
750
+ if (existing) this.clearDebounceTimer(existing)
696
751
  this.sessions.set(sessionId, {
697
752
  state: RoomSessionState.AwaitingConnectMessage,
698
753
  sessionId,
@@ -741,6 +796,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
741
796
  supportsStringAppend,
742
797
  } = opts
743
798
 
799
+ if (this._isClosed) {
800
+ socket.close()
801
+ return
802
+ }
803
+
744
804
  const migrations = this.schema.getMigrationsSince(serializedSchema)
745
805
  const requiresDownMigrations = migrations.ok ? migrations.value.length > 0 : false
746
806
 
@@ -905,7 +965,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
905
965
  * Sends appropriate error messages before closing the connection.
906
966
  *
907
967
  * @param sessionId - The session to reject
908
- * @param fatalReason - The reason for rejection (optional)
968
+ * @param fatalReason - The reason for rejection (optional). WebSocket close reasons are capped
969
+ * at 123 UTF-8 bytes, so a longer reason is truncated and ends with `... (+N bytes)`.
909
970
  * @example
910
971
  * ```ts
911
972
  * // Reject due to version mismatch
@@ -962,6 +1023,10 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
962
1023
 
963
1024
  private forceAllReconnect() {
964
1025
  for (const session of this.sessions.values()) {
1026
+ // Only connected clients hold state at a clock we can no longer diff from. A session
1027
+ // mid-handshake gets hydrated from its own lastServerClock in the same transaction, so
1028
+ // removing it would close its socket and then re-add it as Connected (resurrected).
1029
+ if (session.state !== RoomSessionState.Connected) continue
965
1030
  this.removeSession(session.sessionId)
966
1031
  }
967
1032
  }
@@ -1055,7 +1120,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
1055
1120
 
1056
1121
  const requiresDownMigrations = migrations.value.length > 0
1057
1122
 
1058
- const connect = async (msg: Extract<TLSocketServerSentEvent<R>, { type: 'connect' }>) => {
1123
+ const connect = (msg: Extract<TLSocketServerSentEvent<R>, { type: 'connect' }>) => {
1059
1124
  this.sessions.set(session.sessionId, {
1060
1125
  state: RoomSessionState.Connected,
1061
1126
  sessionId: session.sessionId,
@@ -1,5 +1,11 @@
1
1
  import { assert } from 'tldraw'
2
- import { JsonChunkAssembler, chunk } from './chunk'
2
+ import {
3
+ JsonChunkAssembler,
4
+ MAX_ASSEMBLED_MESSAGE_CHARS,
5
+ MAX_CHUNK_COUNT,
6
+ MessageTooLargeError,
7
+ chunk,
8
+ } from './chunk'
3
9
 
4
10
  describe('chunk (CH1–CH3)', () => {
5
11
  describe('size boundary (CH1)', () => {
@@ -134,6 +140,40 @@ describe('chunk (CH1–CH3)', () => {
134
140
  `)
135
141
  })
136
142
  })
143
+
144
+ describe('surrogate pairs (CH9)', () => {
145
+ it('[CH9] never splits a surrogate pair across two chunks', () => {
146
+ // WebSocket.send converts each chunk to a USVString, so a lone surrogate would arrive
147
+ // as U+FFFD and the reassembled message would carry corrupted text
148
+ const emoji = '\u{1F600}'
149
+ const isHigh = (code: number) => code >= 0xd800 && code <= 0xdbff
150
+ const isLow = (code: number) => code >= 0xdc00 && code <= 0xdfff
151
+ const msgs = [
152
+ ...Array.from(
153
+ { length: 8 },
154
+ (_, padding) => 'x'.repeat(padding) + emoji + 'y'.repeat(6) + emoji
155
+ ),
156
+ // adjacent pairs: after a nudge the next boundary lands on another pair
157
+ 'x' + emoji.repeat(3),
158
+ ]
159
+ for (const msg of msgs) {
160
+ for (const maxSize of [3, 4, 5, 6, 7, 8]) {
161
+ const chunks = chunk(msg, maxSize)
162
+ const bodies = chunks.map((c) => c.slice(c.indexOf('_') + 1))
163
+ expect(bodies.join('')).toBe(msg)
164
+ for (const [i, body] of bodies.entries()) {
165
+ expect(body.length).toBeGreaterThan(0)
166
+ expect(isHigh(body.charCodeAt(body.length - 1))).toBe(false)
167
+ expect(isLow(body.charCodeAt(0))).toBe(false)
168
+ // keeping a pair whole may cost one character over the CH3 bound, never more
169
+ expect(chunks[i].length).toBeLessThanOrEqual(
170
+ Math.max(maxSize, chunks[i].indexOf('_') + 2) + 1
171
+ )
172
+ }
173
+ }
174
+ }
175
+ })
176
+ })
137
177
  })
138
178
 
139
179
  const testObject = {} as any
@@ -369,4 +409,51 @@ describe('JsonChunkAssembler (CH4–CH8)', () => {
369
409
  })
370
410
  })
371
411
  })
412
+
413
+ describe('assembly bounds (CH10)', () => {
414
+ it('[CH10] rejects a first chunk declaring more than MAX_CHUNK_COUNT chunks', () => {
415
+ const unchunker = new JsonChunkAssembler()
416
+
417
+ const result = unchunker.handleMessage(`${MAX_CHUNK_COUNT}_{"a":`)
418
+ expect(result).toMatchObject({
419
+ error: expect.objectContaining({ message: `Too many chunks: ${MAX_CHUNK_COUNT + 1}` }),
420
+ })
421
+ expect(unchunker.state).toBe('idle')
422
+ })
423
+
424
+ it('[CH10] accepts a declared count at the limit', () => {
425
+ const unchunker = new JsonChunkAssembler()
426
+
427
+ expect(unchunker.handleMessage(`${MAX_CHUNK_COUNT - 1}_{"a":`)).toBeNull()
428
+ expect(unchunker.state).not.toBe('idle')
429
+ })
430
+
431
+ it('[CH10] rejects an assembly once its accumulated bodies pass MAX_ASSEMBLED_MESSAGE_CHARS', () => {
432
+ const unchunker = new JsonChunkAssembler()
433
+ // A chunk stream that never sends its final chunk. Before CH10 this accumulated
434
+ // without limit for as long as the sender kept sending.
435
+ const body = 'x'.repeat(1024 * 1024)
436
+ const chunksNeeded = Math.ceil(MAX_ASSEMBLED_MESSAGE_CHARS / body.length) + 1
437
+
438
+ let result = null
439
+ for (let i = 0; i < chunksNeeded; i++) {
440
+ result = unchunker.handleMessage(`${chunksNeeded - i}_${body}`)
441
+ if (result) break
442
+ }
443
+
444
+ expect(result).toMatchObject({ error: expect.any(MessageTooLargeError) })
445
+ expect(unchunker.state).toBe('idle')
446
+ })
447
+
448
+ it('[CH10] leaves the assembler usable after rejecting an oversized assembly', () => {
449
+ const unchunker = new JsonChunkAssembler()
450
+ const body = 'x'.repeat(1024 * 1024)
451
+ const chunksNeeded = Math.ceil(MAX_ASSEMBLED_MESSAGE_CHARS / body.length) + 1
452
+ for (let i = 0; i < chunksNeeded; i++) {
453
+ if (unchunker.handleMessage(`${chunksNeeded - i}_${body}`)) break
454
+ }
455
+
456
+ expect(unchunker.handleMessage('{"test": true}')).toMatchObject({ data: { test: true } })
457
+ })
458
+ })
372
459
  })
package/src/lib/chunk.ts CHANGED
@@ -8,6 +8,35 @@ const MAX_BYTES_PER_CHAR = 4
8
8
  // in the (admittedly impossible) worst case, the max size is 1/4 of a megabyte
9
9
  const MAX_SAFE_MESSAGE_SIZE = MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES / MAX_BYTES_PER_CHAR
10
10
 
11
+ // JsonChunkAssembler holds every chunk of a message until the last one arrives, and the sender
12
+ // decides how many that is — the count is just a prefix on the first chunk. Nothing in the
13
+ // protocol obliges a sender to ever send the last chunk, so without a ceiling a single connection
14
+ // can make the receiver hold an arbitrarily large buffer for as long as the socket stays open.
15
+ //
16
+ // The cap counts UTF-16 code units, which is what a JS string costs in memory: 16M of them is the
17
+ // 32MB of retained string data a room can afford to hold per session and stay inside a Durable
18
+ // Object's 128MB instance limit with room for the document itself. It is far above anything
19
+ // chunk() produces for a real document push. The character cap is the real bound; the chunk-count
20
+ // cap only rejects an absurd declared count on arrival instead of waiting for the characters to
21
+ // accumulate, and is loose enough not to constrain a sender that chunks more finely than the
22
+ // default.
23
+ export const MAX_ASSEMBLED_MESSAGE_CHARS = 16 * 1024 * 1024
24
+ export const MAX_CHUNK_COUNT = 100_000
25
+
26
+ /**
27
+ * Thrown when a message would exceed MAX_ASSEMBLED_MESSAGE_CHARS, on either side of the wire.
28
+ * Distinct from the other assembly errors because it is fatal: retrying sends the same
29
+ * oversized message, so the session is rejected rather than reset.
30
+ *
31
+ * @internal
32
+ */
33
+ export class MessageTooLargeError extends Error {
34
+ constructor(chars: number) {
35
+ super(`Message too large: ${chars} characters, max ${MAX_ASSEMBLED_MESSAGE_CHARS}`)
36
+ this.name = 'MessageTooLargeError'
37
+ }
38
+ }
39
+
11
40
  /**
12
41
  * Splits a string into smaller chunks suitable for transmission over WebSockets.
13
42
  * This function ensures messages don't exceed size limits imposed by platforms like Cloudflare Workers (1MB max).
@@ -38,7 +67,18 @@ export function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
38
67
  let offset = msg.length
39
68
  while (offset > 0) {
40
69
  const prefix = `${chunkNumber}_`
41
- const chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1)
70
+ let chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1)
71
+ // Never split a surrogate pair across chunks: WebSocket.send converts each chunk to a
72
+ // USVString, turning a lone surrogate into U+FFFD, so the reassembled JSON silently
73
+ // carries corrupted text. Shrink by one so the pair moves whole into the preceding
74
+ // chunk; a single-character chunk grows by one instead (one over the CH3 bound, see CH9).
75
+ if (
76
+ chunkSize < offset &&
77
+ isLowSurrogate(msg.charCodeAt(offset - chunkSize)) &&
78
+ isHighSurrogate(msg.charCodeAt(offset - chunkSize - 1))
79
+ ) {
80
+ chunkSize += chunkSize > 1 ? -1 : 1
81
+ }
42
82
  chunks.unshift(prefix + msg.slice(offset - chunkSize, offset))
43
83
  offset -= chunkSize
44
84
  chunkNumber++
@@ -47,6 +87,14 @@ export function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
47
87
  }
48
88
  }
49
89
 
90
+ function isHighSurrogate(code: number) {
91
+ return code >= 0xd800 && code <= 0xdbff
92
+ }
93
+
94
+ function isLowSurrogate(code: number) {
95
+ return code >= 0xdc00 && code <= 0xdfff
96
+ }
97
+
50
98
  // The 's' flag (dotAll) makes '.' match any character including line terminators
51
99
  // like U+2028 and U+2029, which are commonly introduced via copy/paste from Word
52
100
  const chunkRe = /^(\d+)_(.*)$/s
@@ -81,6 +129,7 @@ export class JsonChunkAssembler {
81
129
  | {
82
130
  chunksReceived: string[]
83
131
  totalChunks: number
132
+ charsReceived: number
84
133
  } = 'idle'
85
134
 
86
135
  /**
@@ -91,7 +140,8 @@ export class JsonChunkAssembler {
91
140
  * @param msg - The message to process, either JSON or chunk format
92
141
  * @returns Result object with data/stringified on success, error object on failure, or null for incomplete chunks
93
142
  * - `\{ data: object, stringified: string \}` - Successfully parsed complete message
94
- * - `\{ error: Error \}` - Parse error or invalid chunk sequence
143
+ * - `\{ error: Error \}` - Parse error, invalid chunk sequence, or a message exceeding
144
+ * MAX_ASSEMBLED_MESSAGE_CHARS / MAX_CHUNK_COUNT
95
145
  * - `null` - Chunk received but more chunks expected
96
146
  *
97
147
  * @example
@@ -125,17 +175,30 @@ export class JsonChunkAssembler {
125
175
  const data = match[2]
126
176
 
127
177
  if (this.state === 'idle') {
178
+ const totalChunks = numChunksRemaining + 1
179
+ if (totalChunks > MAX_CHUNK_COUNT) {
180
+ // Nothing has been buffered yet, so the state is already idle.
181
+ return { error: new Error(`Too many chunks: ${totalChunks}`) }
182
+ }
128
183
  this.state = {
129
184
  chunksReceived: [data],
130
- totalChunks: numChunksRemaining + 1,
185
+ totalChunks,
186
+ charsReceived: data.length,
131
187
  }
132
188
  } else {
133
189
  this.state.chunksReceived.push(data)
190
+ this.state.charsReceived += data.length
134
191
  if (numChunksRemaining !== this.state.totalChunks - this.state.chunksReceived.length) {
135
192
  this.state = 'idle'
136
193
  return { error: new Error(`Chunks received in wrong order`) }
137
194
  }
138
195
  }
196
+ if (this.state.charsReceived > MAX_ASSEMBLED_MESSAGE_CHARS) {
197
+ // Drop what was buffered rather than keeping it until the socket closes.
198
+ const charsReceived = this.state.charsReceived
199
+ this.state = 'idle'
200
+ return { error: new MessageTooLargeError(charsReceived) }
201
+ }
139
202
  if (this.state.chunksReceived.length === this.state.totalChunks) {
140
203
  try {
141
204
  const stringified = this.state.chunksReceived.join('')