@tldraw/sync-core 5.5.0-next.f9308d324e1c → 5.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-cjs/index.d.ts +7 -2
- package/dist-cjs/index.js +1 -1
- package/dist-cjs/lib/ClientWebSocketAdapter.js +46 -18
- package/dist-cjs/lib/ClientWebSocketAdapter.js.map +2 -2
- package/dist-cjs/lib/InMemorySyncStorage.js +18 -18
- package/dist-cjs/lib/InMemorySyncStorage.js.map +2 -2
- package/dist-cjs/lib/SQLiteSyncStorage.js +15 -7
- package/dist-cjs/lib/SQLiteSyncStorage.js.map +2 -2
- package/dist-cjs/lib/TLSocketRoom.js +9 -8
- package/dist-cjs/lib/TLSocketRoom.js.map +2 -2
- package/dist-cjs/lib/TLSyncClient.js +3 -1
- package/dist-cjs/lib/TLSyncClient.js.map +2 -2
- package/dist-cjs/lib/TLSyncRoom.js +52 -6
- package/dist-cjs/lib/TLSyncRoom.js.map +2 -2
- package/dist-cjs/lib/chunk.js +35 -3
- package/dist-cjs/lib/chunk.js.map +2 -2
- package/dist-esm/index.d.mts +7 -2
- package/dist-esm/index.mjs +1 -1
- package/dist-esm/lib/ClientWebSocketAdapter.mjs +47 -19
- package/dist-esm/lib/ClientWebSocketAdapter.mjs.map +2 -2
- package/dist-esm/lib/InMemorySyncStorage.mjs +18 -18
- package/dist-esm/lib/InMemorySyncStorage.mjs.map +2 -2
- package/dist-esm/lib/SQLiteSyncStorage.mjs +15 -7
- package/dist-esm/lib/SQLiteSyncStorage.mjs.map +2 -2
- package/dist-esm/lib/TLSocketRoom.mjs +10 -9
- package/dist-esm/lib/TLSocketRoom.mjs.map +2 -2
- package/dist-esm/lib/TLSyncClient.mjs +3 -1
- package/dist-esm/lib/TLSyncClient.mjs.map +2 -2
- package/dist-esm/lib/TLSyncRoom.mjs +52 -6
- package/dist-esm/lib/TLSyncRoom.mjs.map +2 -2
- package/dist-esm/lib/chunk.mjs +35 -3
- package/dist-esm/lib/chunk.mjs.map +2 -2
- package/package.json +11 -7
- package/src/lib/ClientWebSocketAdapter.test.ts +221 -16
- package/src/lib/ClientWebSocketAdapter.ts +62 -23
- package/src/lib/InMemorySyncStorage.test.ts +21 -0
- package/src/lib/InMemorySyncStorage.ts +20 -22
- package/src/lib/SQLiteSyncStorage.test.ts +60 -1
- package/src/lib/SQLiteSyncStorage.ts +22 -14
- package/src/lib/TLSocketRoom.ts +16 -9
- package/src/lib/TLSyncClient.ts +2 -0
- package/src/lib/TLSyncRoom.ts +71 -6
- package/src/lib/chunk.test.ts +88 -1
- package/src/lib/chunk.ts +66 -3
- package/src/test/TLSocketRoom.test.ts +63 -0
- package/src/test/TLSyncRoom.test.ts +176 -0
- package/src/test/objectStore.test.ts +35 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite'
|
|
2
2
|
import { DocumentRecordType, PageRecordType, TLDOCUMENT_ID, TLRecord } from '@tldraw/tlschema'
|
|
3
3
|
import { ZERO_INDEX_KEY } from '@tldraw/utils'
|
|
4
|
-
import { describe, expect, it } from 'vitest'
|
|
4
|
+
import { describe, expect, it, vi } from 'vitest'
|
|
5
5
|
import {
|
|
6
6
|
contractRecords,
|
|
7
7
|
contractSchema,
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
makePage,
|
|
10
10
|
registerStorageContractTests,
|
|
11
11
|
} from '../test/storageContractSuite'
|
|
12
|
+
import { MAX_TOMBSTONES } from './InMemorySyncStorage'
|
|
12
13
|
import { NodeSqliteWrapper } from './NodeSqliteWrapper'
|
|
13
14
|
import { SQLiteSyncStorage } from './SQLiteSyncStorage'
|
|
14
15
|
|
|
@@ -263,4 +264,62 @@ describe('SQLiteSyncStorage', () => {
|
|
|
263
264
|
expect(row?.migrationVersion).toBe(3)
|
|
264
265
|
})
|
|
265
266
|
})
|
|
267
|
+
|
|
268
|
+
describe('tombstone pruning robustness', () => {
|
|
269
|
+
const makeTombstones = (count: number) => {
|
|
270
|
+
const tombstones: Record<string, number> = {}
|
|
271
|
+
for (let i = 0; i < count; i++) tombstones[`shape:doc${i}`] = i + 1
|
|
272
|
+
return tombstones
|
|
273
|
+
}
|
|
274
|
+
function makeOverfullStorage() {
|
|
275
|
+
const db = new DatabaseSync(':memory:')
|
|
276
|
+
const storage = new SQLiteSyncStorage<TLRecord>({
|
|
277
|
+
sql: new NodeSqliteWrapper(db),
|
|
278
|
+
snapshot: makeContractSnapshot(contractRecords, {
|
|
279
|
+
tombstones: makeTombstones(MAX_TOMBSTONES + 500),
|
|
280
|
+
documentClock: MAX_TOMBSTONES + 501,
|
|
281
|
+
tombstoneHistoryStartsAtClock: 0,
|
|
282
|
+
}),
|
|
283
|
+
})
|
|
284
|
+
return { db, storage, prune: (storage as any).pruneTombstones as { (): void; flush(): void } }
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
it('[SS18] a prune that fires after the database was closed does not throw', () => {
|
|
288
|
+
const { db, storage, prune } = makeOverfullStorage()
|
|
289
|
+
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
|
290
|
+
try {
|
|
291
|
+
// a delete schedules the throttled prune; the host then closes everything
|
|
292
|
+
storage.transaction((txn) => txn.delete(contractRecords[1].id))
|
|
293
|
+
db.close()
|
|
294
|
+
// the throttle timer fires later, from outside any caller that could catch it
|
|
295
|
+
expect(() => prune.flush()).not.toThrow()
|
|
296
|
+
expect(consoleSpy).toHaveBeenCalledWith('Failed to prune tombstones', expect.anything())
|
|
297
|
+
} finally {
|
|
298
|
+
consoleSpy.mockRestore()
|
|
299
|
+
}
|
|
300
|
+
})
|
|
301
|
+
|
|
302
|
+
it('[SS18] a prune whose delete fails leaves the history clock where it was', () => {
|
|
303
|
+
const { storage, prune } = makeOverfullStorage()
|
|
304
|
+
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
|
305
|
+
try {
|
|
306
|
+
const stmts = (storage as any).stmts
|
|
307
|
+
const originalRun = stmts.deleteTombstonesBefore.run
|
|
308
|
+
stmts.deleteTombstonesBefore.run = () => {
|
|
309
|
+
throw new Error('SQLITE_BUSY: database is locked')
|
|
310
|
+
}
|
|
311
|
+
prune()
|
|
312
|
+
expect(() => prune.flush()).not.toThrow()
|
|
313
|
+
stmts.deleteTombstonesBefore.run = originalRun
|
|
314
|
+
|
|
315
|
+
// neither half of the prune applied: the history start was not advanced past
|
|
316
|
+
// tombstones that still exist
|
|
317
|
+
const snapshot = storage.getSnapshot()
|
|
318
|
+
expect(snapshot.tombstoneHistoryStartsAtClock).toBe(0)
|
|
319
|
+
expect(Object.keys(snapshot.tombstones!).length).toBe(MAX_TOMBSTONES + 500)
|
|
320
|
+
} finally {
|
|
321
|
+
consoleSpy.mockRestore()
|
|
322
|
+
}
|
|
323
|
+
})
|
|
324
|
+
})
|
|
266
325
|
})
|
|
@@ -394,7 +394,6 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
394
394
|
const documentClock = snapshot.documentClock ?? snapshot.clock ?? 0
|
|
395
395
|
const tombstoneHistoryStartsAtClock = snapshot.tombstoneHistoryStartsAtClock ?? documentClock
|
|
396
396
|
|
|
397
|
-
// Clear existing data
|
|
398
397
|
this.sql.exec(`
|
|
399
398
|
DELETE FROM ${documentsTable};
|
|
400
399
|
DELETE FROM ${objectsTable};
|
|
@@ -410,14 +409,12 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
410
409
|
table.insert.run(doc.state.id, encodeState(doc.state), doc.lastChangedClock)
|
|
411
410
|
}
|
|
412
411
|
|
|
413
|
-
// Insert tombstones
|
|
414
412
|
if (snapshot.tombstones) {
|
|
415
413
|
for (const [id, clock] of objectMapEntries(snapshot.tombstones)) {
|
|
416
414
|
this.stmts.insertTombstone.run(id, clock)
|
|
417
415
|
}
|
|
418
416
|
}
|
|
419
417
|
|
|
420
|
-
// Insert metadata row
|
|
421
418
|
this.stmts.updateMetadata.run(
|
|
422
419
|
documentClock,
|
|
423
420
|
tombstoneHistoryStartsAtClock,
|
|
@@ -513,18 +510,29 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
513
510
|
/** @internal */
|
|
514
511
|
pruneTombstones = throttle(
|
|
515
512
|
() => {
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
this.stmts.
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
513
|
+
// Runs from a timer, so a host that closed the database right after the last delete
|
|
514
|
+
// (e.g. `room.close(); db.close()` in onSessionRemoved) would otherwise get an uncaught
|
|
515
|
+
// throw from here; pruning is best-effort and simply runs again on the next delete.
|
|
516
|
+
try {
|
|
517
|
+
const tombstoneCount = this.stmts.countTombstones.all()[0].count as number
|
|
518
|
+
if (tombstoneCount > MAX_TOMBSTONES) {
|
|
519
|
+
// Get all tombstones sorted by clock ascending (oldest first)
|
|
520
|
+
const tombstones = this.stmts.iterateTombstones.all()
|
|
521
|
+
|
|
522
|
+
const result = computeTombstonePruning({ tombstones, documentClock: this.getClock() })
|
|
523
|
+
if (result) {
|
|
524
|
+
this.sql.transaction(() => {
|
|
525
|
+
this.stmts.setTombstoneHistoryStartsAtClock.run(
|
|
526
|
+
result.newTombstoneHistoryStartsAtClock
|
|
527
|
+
)
|
|
528
|
+
// Delete all tombstones with clock < newTombstoneHistoryStartsAtClock in one operation.
|
|
529
|
+
// This works because computeTombstonePruning ensures we never split a clock value.
|
|
530
|
+
this.stmts.deleteTombstonesBefore.run(result.newTombstoneHistoryStartsAtClock)
|
|
531
|
+
})
|
|
532
|
+
}
|
|
527
533
|
}
|
|
534
|
+
} catch (e) {
|
|
535
|
+
console.error('Failed to prune tombstones', e)
|
|
528
536
|
}
|
|
529
537
|
},
|
|
530
538
|
1000,
|
package/src/lib/TLSocketRoom.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { SerializedSchema, StoreSchema, UnknownRecord } from '@tldraw/store'
|
|
2
2
|
import { createTLSchema, TLInstancePresence, TLStoreSnapshot } from '@tldraw/tlschema'
|
|
3
3
|
import { getOwnProperty, hasOwnProperty, isEqual, structuredClone } from '@tldraw/utils'
|
|
4
|
-
import { JsonChunkAssembler } from './chunk'
|
|
4
|
+
import { JsonChunkAssembler, MessageTooLargeError } from './chunk'
|
|
5
5
|
import { DEFAULT_INITIAL_SNAPSHOT, InMemorySyncStorage } from './InMemorySyncStorage'
|
|
6
6
|
import { TLObjectStoreAccess, TLSocketServerSentEvent } from './protocol'
|
|
7
7
|
import { RoomSessionState } from './RoomSession'
|
|
@@ -273,13 +273,16 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
|
|
|
273
273
|
})
|
|
274
274
|
)
|
|
275
275
|
}
|
|
276
|
+
// The default logger belongs to both layers: TLSyncRoom is where authorizer throws are
|
|
277
|
+
// logged, and a host that passes no `log` would otherwise never see them.
|
|
278
|
+
this.log = 'log' in opts ? opts.log : { error: console.error }
|
|
276
279
|
this.room = new TLSyncRoom<R, SessionMeta>({
|
|
277
280
|
onPresenceChange: opts.onPresenceChange,
|
|
278
281
|
onCommittedChanges: opts.onCommittedChanges,
|
|
279
282
|
objectTypes: opts.objectTypes,
|
|
280
283
|
authorizeRecord: opts.authorizeRecord,
|
|
281
284
|
schema: opts.schema ?? (createTLSchema() as any),
|
|
282
|
-
log:
|
|
285
|
+
log: this.log,
|
|
283
286
|
storage,
|
|
284
287
|
clientTimeout: opts.clientTimeout,
|
|
285
288
|
})
|
|
@@ -295,7 +298,6 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
|
|
|
295
298
|
})
|
|
296
299
|
}
|
|
297
300
|
})
|
|
298
|
-
this.log = 'log' in opts ? opts.log : { error: console.error }
|
|
299
301
|
}
|
|
300
302
|
|
|
301
303
|
/**
|
|
@@ -475,6 +477,11 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
|
|
|
475
477
|
})
|
|
476
478
|
this.room.pruneSessions()
|
|
477
479
|
this.scheduleDebouncedSnapshot(sessionId)
|
|
480
|
+
} else if (res.error instanceof MessageTooLargeError) {
|
|
481
|
+
this.log?.error?.('Error assembling message', res.error)
|
|
482
|
+
// Resetting the connection would have the client reconnect and re-send the same
|
|
483
|
+
// oversized message, so reject the session instead and let the client surface it.
|
|
484
|
+
this.room.rejectSession(sessionId, TLSyncErrorCloseEventReason.MESSAGE_TOO_LARGE)
|
|
478
485
|
} else {
|
|
479
486
|
this.log?.error?.('Error assembling message', res.error)
|
|
480
487
|
// close the socket to reset the connection
|
|
@@ -854,12 +861,11 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
|
|
|
854
861
|
if (this.isClosed()) {
|
|
855
862
|
throw new Error('Cannot update store on a closed room')
|
|
856
863
|
}
|
|
864
|
+
// read through the transaction rather than getSnapshot(): the document snapshot excludes the
|
|
865
|
+
// object-store lane, which would make object-lane records invisible (and undeletable) here
|
|
866
|
+
const records = this.storage.transaction((txn) => Object.fromEntries(txn.entries())).result
|
|
857
867
|
// eslint-disable-next-line @typescript-eslint/no-deprecated
|
|
858
|
-
const ctx = new StoreUpdateContext<R>(
|
|
859
|
-
// eslint-disable-next-line @typescript-eslint/no-deprecated
|
|
860
|
-
Object.fromEntries(this.getCurrentSnapshot().documents.map((d) => [d.state.id, d.state])),
|
|
861
|
-
this.room.schema
|
|
862
|
-
)
|
|
868
|
+
const ctx = new StoreUpdateContext<R>(records, this.room.schema)
|
|
863
869
|
try {
|
|
864
870
|
await updater(ctx)
|
|
865
871
|
} finally {
|
|
@@ -908,7 +914,8 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
|
|
|
908
914
|
* The client will attempt to reconnect automatically unless a fatal reason is provided.
|
|
909
915
|
*
|
|
910
916
|
* @param sessionId - Session identifier to remove
|
|
911
|
-
* @param fatalReason - Optional fatal error reason that prevents reconnection
|
|
917
|
+
* @param fatalReason - Optional fatal error reason that prevents reconnection. WebSocket close
|
|
918
|
+
* reasons are capped at 123 UTF-8 bytes, so a longer reason is truncated and ends with `... (+N bytes)`.
|
|
912
919
|
*
|
|
913
920
|
* @example
|
|
914
921
|
* ```ts
|
package/src/lib/TLSyncClient.ts
CHANGED
|
@@ -122,6 +122,8 @@ export const TLSyncErrorCloseEventReason = {
|
|
|
122
122
|
RATE_LIMITED: 'RATE_LIMITED',
|
|
123
123
|
/** Room has reached maximum capacity */
|
|
124
124
|
ROOM_FULL: 'ROOM_FULL',
|
|
125
|
+
/** A single message exceeded the size the server will assemble */
|
|
126
|
+
MESSAGE_TOO_LARGE: 'MESSAGE_TOO_LARGE',
|
|
125
127
|
} as const
|
|
126
128
|
|
|
127
129
|
/**
|
package/src/lib/TLSyncRoom.ts
CHANGED
|
@@ -88,6 +88,29 @@ export const DATA_MESSAGE_DEBOUNCE_INTERVAL = 1000 / 60
|
|
|
88
88
|
|
|
89
89
|
const timeSince = (time: number) => Date.now() - time
|
|
90
90
|
|
|
91
|
+
// WebSocket close frames cap the reason at 123 UTF-8 bytes; a longer one makes `close()` throw
|
|
92
|
+
const MAX_CLOSE_REASON_BYTES = 123
|
|
93
|
+
const closeReasonEncoder = new TextEncoder()
|
|
94
|
+
|
|
95
|
+
const truncatedSuffix = (droppedBytes: number) => `... (+${droppedBytes} bytes)`
|
|
96
|
+
|
|
97
|
+
function truncateCloseReason(reason: string) {
|
|
98
|
+
const totalBytes = closeReasonEncoder.encode(reason).length
|
|
99
|
+
if (totalBytes <= MAX_CLOSE_REASON_BYTES) return reason
|
|
100
|
+
// the dropped count can't have more digits than the total, so reserving room for it always fits
|
|
101
|
+
const budget = MAX_CLOSE_REASON_BYTES - truncatedSuffix(totalBytes).length
|
|
102
|
+
let out = ''
|
|
103
|
+
let outBytes = 0
|
|
104
|
+
// step by code point so a multi-byte character is never cut in half
|
|
105
|
+
for (const char of reason) {
|
|
106
|
+
const charBytes = closeReasonEncoder.encode(char).length
|
|
107
|
+
if (outBytes + charBytes > budget) break
|
|
108
|
+
out += char
|
|
109
|
+
outBytes += charBytes
|
|
110
|
+
}
|
|
111
|
+
return out + truncatedSuffix(totalBytes - outBytes)
|
|
112
|
+
}
|
|
113
|
+
|
|
91
114
|
/**
|
|
92
115
|
* Snapshot of a room's complete state that can be persisted and restored.
|
|
93
116
|
* Contains all documents, tombstones, and metadata needed to reconstruct the room.
|
|
@@ -258,7 +281,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
258
281
|
}, 1000)
|
|
259
282
|
|
|
260
283
|
private scheduleFollowUpPrune() {
|
|
261
|
-
|
|
284
|
+
// don't leave a stray timer on a room the host has already torn down
|
|
285
|
+
if (this._isClosed || this.pruneTimer) return
|
|
262
286
|
this.pruneTimer = setTimeout(this.pruneSessions, SESSION_REMOVAL_WAIT_TIME + 100)
|
|
263
287
|
}
|
|
264
288
|
|
|
@@ -273,11 +297,27 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
273
297
|
* and stops background processes.
|
|
274
298
|
*/
|
|
275
299
|
close() {
|
|
300
|
+
this._isClosed = true
|
|
276
301
|
this.disposables.forEach((d) => d())
|
|
277
302
|
this.sessions.forEach((session) => {
|
|
278
|
-
|
|
303
|
+
this.clearDebounceTimer(session)
|
|
304
|
+
try {
|
|
305
|
+
session.socket.close()
|
|
306
|
+
} catch {
|
|
307
|
+
// noop, one bad socket must not leave the rest open
|
|
308
|
+
}
|
|
279
309
|
})
|
|
280
|
-
|
|
310
|
+
// forgetting the sessions is what makes late socket close/error events no-ops, so
|
|
311
|
+
// nothing can emit session_removed / room_became_empty on a closed room
|
|
312
|
+
this.sessions.clear()
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
private clearDebounceTimer(session: RoomSession<R, SessionMeta>) {
|
|
316
|
+
if (session.state === RoomSessionState.Connected && session.debounceTimer !== null) {
|
|
317
|
+
clearTimeout(session.debounceTimer)
|
|
318
|
+
session.debounceTimer = null
|
|
319
|
+
session.outstandingDataMessages = []
|
|
320
|
+
}
|
|
281
321
|
}
|
|
282
322
|
|
|
283
323
|
/**
|
|
@@ -519,6 +559,12 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
519
559
|
// place, so sockets that defer serialization don't see an emptied array
|
|
520
560
|
const data = session.outstandingDataMessages
|
|
521
561
|
session.outstandingDataMessages = []
|
|
562
|
+
if (!session.socket.isOpen) {
|
|
563
|
+
// same as the immediate-send path: send() into a closed socket throws on
|
|
564
|
+
// some runtimes (Cloudflare), and here that would be from inside a timer
|
|
565
|
+
this.cancelSession(sessionId)
|
|
566
|
+
return
|
|
567
|
+
}
|
|
522
568
|
session.socket.sendMessage({ type: 'data', data })
|
|
523
569
|
}
|
|
524
570
|
}
|
|
@@ -532,10 +578,12 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
532
578
|
}
|
|
533
579
|
|
|
534
580
|
this.sessions.delete(sessionId)
|
|
581
|
+
this.clearDebounceTimer(session)
|
|
535
582
|
|
|
536
583
|
try {
|
|
537
584
|
if (fatalReason) {
|
|
538
|
-
session
|
|
585
|
+
// the session is already gone, so a throwing close() would leave its socket open
|
|
586
|
+
session.socket.close(TLSyncErrorCloseEventCode, truncateCloseReason(fatalReason))
|
|
539
587
|
} else {
|
|
540
588
|
session.socket.close()
|
|
541
589
|
}
|
|
@@ -570,6 +618,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
570
618
|
return
|
|
571
619
|
}
|
|
572
620
|
|
|
621
|
+
this.clearDebounceTimer(session)
|
|
573
622
|
this.sessions.set(sessionId, {
|
|
574
623
|
state: RoomSessionState.AwaitingRemoval,
|
|
575
624
|
sessionId,
|
|
@@ -692,7 +741,13 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
692
741
|
objectAccess?: TLObjectStoreAccess
|
|
693
742
|
}) {
|
|
694
743
|
const { sessionId, socket, meta, isReadonly, objectAccess } = opts
|
|
744
|
+
// a connect racing close() would otherwise create a session nothing ever prunes
|
|
745
|
+
if (this._isClosed) {
|
|
746
|
+
socket.close()
|
|
747
|
+
return this
|
|
748
|
+
}
|
|
695
749
|
const existing = this.sessions.get(sessionId)
|
|
750
|
+
if (existing) this.clearDebounceTimer(existing)
|
|
696
751
|
this.sessions.set(sessionId, {
|
|
697
752
|
state: RoomSessionState.AwaitingConnectMessage,
|
|
698
753
|
sessionId,
|
|
@@ -741,6 +796,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
741
796
|
supportsStringAppend,
|
|
742
797
|
} = opts
|
|
743
798
|
|
|
799
|
+
if (this._isClosed) {
|
|
800
|
+
socket.close()
|
|
801
|
+
return
|
|
802
|
+
}
|
|
803
|
+
|
|
744
804
|
const migrations = this.schema.getMigrationsSince(serializedSchema)
|
|
745
805
|
const requiresDownMigrations = migrations.ok ? migrations.value.length > 0 : false
|
|
746
806
|
|
|
@@ -905,7 +965,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
905
965
|
* Sends appropriate error messages before closing the connection.
|
|
906
966
|
*
|
|
907
967
|
* @param sessionId - The session to reject
|
|
908
|
-
* @param fatalReason - The reason for rejection (optional)
|
|
968
|
+
* @param fatalReason - The reason for rejection (optional). WebSocket close reasons are capped
|
|
969
|
+
* at 123 UTF-8 bytes, so a longer reason is truncated and ends with `... (+N bytes)`.
|
|
909
970
|
* @example
|
|
910
971
|
* ```ts
|
|
911
972
|
* // Reject due to version mismatch
|
|
@@ -962,6 +1023,10 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
962
1023
|
|
|
963
1024
|
private forceAllReconnect() {
|
|
964
1025
|
for (const session of this.sessions.values()) {
|
|
1026
|
+
// Only connected clients hold state at a clock we can no longer diff from. A session
|
|
1027
|
+
// mid-handshake gets hydrated from its own lastServerClock in the same transaction, so
|
|
1028
|
+
// removing it would close its socket and then re-add it as Connected (resurrected).
|
|
1029
|
+
if (session.state !== RoomSessionState.Connected) continue
|
|
965
1030
|
this.removeSession(session.sessionId)
|
|
966
1031
|
}
|
|
967
1032
|
}
|
|
@@ -1055,7 +1120,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1055
1120
|
|
|
1056
1121
|
const requiresDownMigrations = migrations.value.length > 0
|
|
1057
1122
|
|
|
1058
|
-
const connect =
|
|
1123
|
+
const connect = (msg: Extract<TLSocketServerSentEvent<R>, { type: 'connect' }>) => {
|
|
1059
1124
|
this.sessions.set(session.sessionId, {
|
|
1060
1125
|
state: RoomSessionState.Connected,
|
|
1061
1126
|
sessionId: session.sessionId,
|
package/src/lib/chunk.test.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { assert } from 'tldraw'
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
JsonChunkAssembler,
|
|
4
|
+
MAX_ASSEMBLED_MESSAGE_CHARS,
|
|
5
|
+
MAX_CHUNK_COUNT,
|
|
6
|
+
MessageTooLargeError,
|
|
7
|
+
chunk,
|
|
8
|
+
} from './chunk'
|
|
3
9
|
|
|
4
10
|
describe('chunk (CH1–CH3)', () => {
|
|
5
11
|
describe('size boundary (CH1)', () => {
|
|
@@ -134,6 +140,40 @@ describe('chunk (CH1–CH3)', () => {
|
|
|
134
140
|
`)
|
|
135
141
|
})
|
|
136
142
|
})
|
|
143
|
+
|
|
144
|
+
describe('surrogate pairs (CH9)', () => {
|
|
145
|
+
it('[CH9] never splits a surrogate pair across two chunks', () => {
|
|
146
|
+
// WebSocket.send converts each chunk to a USVString, so a lone surrogate would arrive
|
|
147
|
+
// as U+FFFD and the reassembled message would carry corrupted text
|
|
148
|
+
const emoji = '\u{1F600}'
|
|
149
|
+
const isHigh = (code: number) => code >= 0xd800 && code <= 0xdbff
|
|
150
|
+
const isLow = (code: number) => code >= 0xdc00 && code <= 0xdfff
|
|
151
|
+
const msgs = [
|
|
152
|
+
...Array.from(
|
|
153
|
+
{ length: 8 },
|
|
154
|
+
(_, padding) => 'x'.repeat(padding) + emoji + 'y'.repeat(6) + emoji
|
|
155
|
+
),
|
|
156
|
+
// adjacent pairs: after a nudge the next boundary lands on another pair
|
|
157
|
+
'x' + emoji.repeat(3),
|
|
158
|
+
]
|
|
159
|
+
for (const msg of msgs) {
|
|
160
|
+
for (const maxSize of [3, 4, 5, 6, 7, 8]) {
|
|
161
|
+
const chunks = chunk(msg, maxSize)
|
|
162
|
+
const bodies = chunks.map((c) => c.slice(c.indexOf('_') + 1))
|
|
163
|
+
expect(bodies.join('')).toBe(msg)
|
|
164
|
+
for (const [i, body] of bodies.entries()) {
|
|
165
|
+
expect(body.length).toBeGreaterThan(0)
|
|
166
|
+
expect(isHigh(body.charCodeAt(body.length - 1))).toBe(false)
|
|
167
|
+
expect(isLow(body.charCodeAt(0))).toBe(false)
|
|
168
|
+
// keeping a pair whole may cost one character over the CH3 bound, never more
|
|
169
|
+
expect(chunks[i].length).toBeLessThanOrEqual(
|
|
170
|
+
Math.max(maxSize, chunks[i].indexOf('_') + 2) + 1
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
})
|
|
176
|
+
})
|
|
137
177
|
})
|
|
138
178
|
|
|
139
179
|
const testObject = {} as any
|
|
@@ -369,4 +409,51 @@ describe('JsonChunkAssembler (CH4–CH8)', () => {
|
|
|
369
409
|
})
|
|
370
410
|
})
|
|
371
411
|
})
|
|
412
|
+
|
|
413
|
+
describe('assembly bounds (CH10)', () => {
|
|
414
|
+
it('[CH10] rejects a first chunk declaring more than MAX_CHUNK_COUNT chunks', () => {
|
|
415
|
+
const unchunker = new JsonChunkAssembler()
|
|
416
|
+
|
|
417
|
+
const result = unchunker.handleMessage(`${MAX_CHUNK_COUNT}_{"a":`)
|
|
418
|
+
expect(result).toMatchObject({
|
|
419
|
+
error: expect.objectContaining({ message: `Too many chunks: ${MAX_CHUNK_COUNT + 1}` }),
|
|
420
|
+
})
|
|
421
|
+
expect(unchunker.state).toBe('idle')
|
|
422
|
+
})
|
|
423
|
+
|
|
424
|
+
it('[CH10] accepts a declared count at the limit', () => {
|
|
425
|
+
const unchunker = new JsonChunkAssembler()
|
|
426
|
+
|
|
427
|
+
expect(unchunker.handleMessage(`${MAX_CHUNK_COUNT - 1}_{"a":`)).toBeNull()
|
|
428
|
+
expect(unchunker.state).not.toBe('idle')
|
|
429
|
+
})
|
|
430
|
+
|
|
431
|
+
it('[CH10] rejects an assembly once its accumulated bodies pass MAX_ASSEMBLED_MESSAGE_CHARS', () => {
|
|
432
|
+
const unchunker = new JsonChunkAssembler()
|
|
433
|
+
// A chunk stream that never sends its final chunk. Before CH10 this accumulated
|
|
434
|
+
// without limit for as long as the sender kept sending.
|
|
435
|
+
const body = 'x'.repeat(1024 * 1024)
|
|
436
|
+
const chunksNeeded = Math.ceil(MAX_ASSEMBLED_MESSAGE_CHARS / body.length) + 1
|
|
437
|
+
|
|
438
|
+
let result = null
|
|
439
|
+
for (let i = 0; i < chunksNeeded; i++) {
|
|
440
|
+
result = unchunker.handleMessage(`${chunksNeeded - i}_${body}`)
|
|
441
|
+
if (result) break
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
expect(result).toMatchObject({ error: expect.any(MessageTooLargeError) })
|
|
445
|
+
expect(unchunker.state).toBe('idle')
|
|
446
|
+
})
|
|
447
|
+
|
|
448
|
+
it('[CH10] leaves the assembler usable after rejecting an oversized assembly', () => {
|
|
449
|
+
const unchunker = new JsonChunkAssembler()
|
|
450
|
+
const body = 'x'.repeat(1024 * 1024)
|
|
451
|
+
const chunksNeeded = Math.ceil(MAX_ASSEMBLED_MESSAGE_CHARS / body.length) + 1
|
|
452
|
+
for (let i = 0; i < chunksNeeded; i++) {
|
|
453
|
+
if (unchunker.handleMessage(`${chunksNeeded - i}_${body}`)) break
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
expect(unchunker.handleMessage('{"test": true}')).toMatchObject({ data: { test: true } })
|
|
457
|
+
})
|
|
458
|
+
})
|
|
372
459
|
})
|
package/src/lib/chunk.ts
CHANGED
|
@@ -8,6 +8,35 @@ const MAX_BYTES_PER_CHAR = 4
|
|
|
8
8
|
// in the (admittedly impossible) worst case, the max size is 1/4 of a megabyte
|
|
9
9
|
const MAX_SAFE_MESSAGE_SIZE = MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES / MAX_BYTES_PER_CHAR
|
|
10
10
|
|
|
11
|
+
// JsonChunkAssembler holds every chunk of a message until the last one arrives, and the sender
|
|
12
|
+
// decides how many that is — the count is just a prefix on the first chunk. Nothing in the
|
|
13
|
+
// protocol obliges a sender to ever send the last chunk, so without a ceiling a single connection
|
|
14
|
+
// can make the receiver hold an arbitrarily large buffer for as long as the socket stays open.
|
|
15
|
+
//
|
|
16
|
+
// The cap counts UTF-16 code units, which is what a JS string costs in memory: 16M of them is the
|
|
17
|
+
// 32MB of retained string data a room can afford to hold per session and stay inside a Durable
|
|
18
|
+
// Object's 128MB instance limit with room for the document itself. It is far above anything
|
|
19
|
+
// chunk() produces for a real document push. The character cap is the real bound; the chunk-count
|
|
20
|
+
// cap only rejects an absurd declared count on arrival instead of waiting for the characters to
|
|
21
|
+
// accumulate, and is loose enough not to constrain a sender that chunks more finely than the
|
|
22
|
+
// default.
|
|
23
|
+
export const MAX_ASSEMBLED_MESSAGE_CHARS = 16 * 1024 * 1024
|
|
24
|
+
export const MAX_CHUNK_COUNT = 100_000
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Thrown when a message would exceed MAX_ASSEMBLED_MESSAGE_CHARS, on either side of the wire.
|
|
28
|
+
* Distinct from the other assembly errors because it is fatal: retrying sends the same
|
|
29
|
+
* oversized message, so the session is rejected rather than reset.
|
|
30
|
+
*
|
|
31
|
+
* @internal
|
|
32
|
+
*/
|
|
33
|
+
export class MessageTooLargeError extends Error {
|
|
34
|
+
constructor(chars: number) {
|
|
35
|
+
super(`Message too large: ${chars} characters, max ${MAX_ASSEMBLED_MESSAGE_CHARS}`)
|
|
36
|
+
this.name = 'MessageTooLargeError'
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
11
40
|
/**
|
|
12
41
|
* Splits a string into smaller chunks suitable for transmission over WebSockets.
|
|
13
42
|
* This function ensures messages don't exceed size limits imposed by platforms like Cloudflare Workers (1MB max).
|
|
@@ -38,7 +67,18 @@ export function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
|
|
|
38
67
|
let offset = msg.length
|
|
39
68
|
while (offset > 0) {
|
|
40
69
|
const prefix = `${chunkNumber}_`
|
|
41
|
-
|
|
70
|
+
let chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1)
|
|
71
|
+
// Never split a surrogate pair across chunks: WebSocket.send converts each chunk to a
|
|
72
|
+
// USVString, turning a lone surrogate into U+FFFD, so the reassembled JSON silently
|
|
73
|
+
// carries corrupted text. Shrink by one so the pair moves whole into the preceding
|
|
74
|
+
// chunk; a single-character chunk grows by one instead (one over the CH3 bound, see CH9).
|
|
75
|
+
if (
|
|
76
|
+
chunkSize < offset &&
|
|
77
|
+
isLowSurrogate(msg.charCodeAt(offset - chunkSize)) &&
|
|
78
|
+
isHighSurrogate(msg.charCodeAt(offset - chunkSize - 1))
|
|
79
|
+
) {
|
|
80
|
+
chunkSize += chunkSize > 1 ? -1 : 1
|
|
81
|
+
}
|
|
42
82
|
chunks.unshift(prefix + msg.slice(offset - chunkSize, offset))
|
|
43
83
|
offset -= chunkSize
|
|
44
84
|
chunkNumber++
|
|
@@ -47,6 +87,14 @@ export function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
|
|
|
47
87
|
}
|
|
48
88
|
}
|
|
49
89
|
|
|
90
|
+
function isHighSurrogate(code: number) {
|
|
91
|
+
return code >= 0xd800 && code <= 0xdbff
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function isLowSurrogate(code: number) {
|
|
95
|
+
return code >= 0xdc00 && code <= 0xdfff
|
|
96
|
+
}
|
|
97
|
+
|
|
50
98
|
// The 's' flag (dotAll) makes '.' match any character including line terminators
|
|
51
99
|
// like U+2028 and U+2029, which are commonly introduced via copy/paste from Word
|
|
52
100
|
const chunkRe = /^(\d+)_(.*)$/s
|
|
@@ -81,6 +129,7 @@ export class JsonChunkAssembler {
|
|
|
81
129
|
| {
|
|
82
130
|
chunksReceived: string[]
|
|
83
131
|
totalChunks: number
|
|
132
|
+
charsReceived: number
|
|
84
133
|
} = 'idle'
|
|
85
134
|
|
|
86
135
|
/**
|
|
@@ -91,7 +140,8 @@ export class JsonChunkAssembler {
|
|
|
91
140
|
* @param msg - The message to process, either JSON or chunk format
|
|
92
141
|
* @returns Result object with data/stringified on success, error object on failure, or null for incomplete chunks
|
|
93
142
|
* - `\{ data: object, stringified: string \}` - Successfully parsed complete message
|
|
94
|
-
* - `\{ error: Error \}` - Parse error
|
|
143
|
+
* - `\{ error: Error \}` - Parse error, invalid chunk sequence, or a message exceeding
|
|
144
|
+
* MAX_ASSEMBLED_MESSAGE_CHARS / MAX_CHUNK_COUNT
|
|
95
145
|
* - `null` - Chunk received but more chunks expected
|
|
96
146
|
*
|
|
97
147
|
* @example
|
|
@@ -125,17 +175,30 @@ export class JsonChunkAssembler {
|
|
|
125
175
|
const data = match[2]
|
|
126
176
|
|
|
127
177
|
if (this.state === 'idle') {
|
|
178
|
+
const totalChunks = numChunksRemaining + 1
|
|
179
|
+
if (totalChunks > MAX_CHUNK_COUNT) {
|
|
180
|
+
// Nothing has been buffered yet, so the state is already idle.
|
|
181
|
+
return { error: new Error(`Too many chunks: ${totalChunks}`) }
|
|
182
|
+
}
|
|
128
183
|
this.state = {
|
|
129
184
|
chunksReceived: [data],
|
|
130
|
-
totalChunks
|
|
185
|
+
totalChunks,
|
|
186
|
+
charsReceived: data.length,
|
|
131
187
|
}
|
|
132
188
|
} else {
|
|
133
189
|
this.state.chunksReceived.push(data)
|
|
190
|
+
this.state.charsReceived += data.length
|
|
134
191
|
if (numChunksRemaining !== this.state.totalChunks - this.state.chunksReceived.length) {
|
|
135
192
|
this.state = 'idle'
|
|
136
193
|
return { error: new Error(`Chunks received in wrong order`) }
|
|
137
194
|
}
|
|
138
195
|
}
|
|
196
|
+
if (this.state.charsReceived > MAX_ASSEMBLED_MESSAGE_CHARS) {
|
|
197
|
+
// Drop what was buffered rather than keeping it until the socket closes.
|
|
198
|
+
const charsReceived = this.state.charsReceived
|
|
199
|
+
this.state = 'idle'
|
|
200
|
+
return { error: new MessageTooLargeError(charsReceived) }
|
|
201
|
+
}
|
|
139
202
|
if (this.state.chunksReceived.length === this.state.totalChunks) {
|
|
140
203
|
try {
|
|
141
204
|
const stringified = this.state.chunksReceived.join('')
|