@tldraw/sync-core 5.2.0-canary.fe03bcdddf34 → 5.2.0-canary.fff413eea248

package/src/lib/TLSyncClient.ts

@@ -12,7 +12,6 @@ import {
 	exhaustiveSwitchError,
 	isEqual,
 	objectMapEntries,
-	structuredClone,
 	uniqueId,
 } from '@tldraw/utils'
 import {
@@ -853,10 +852,11 @@ export class TLSyncClient<R extends UnknownRecord, S extends Store<R> = Store<R>
 		// in offline mode, we only accumulate in speculativeChanges
 		if (!this.isConnectedToRoom) return
 		if (!this.unsentChanges.nextDiff) {
-			this.unsentChanges.nextDiff = structuredClone(change)
-		} else {
-			squashRecordDiffsMutable(this.unsentChanges.nextDiff, [change])
+			this.unsentChanges.nextDiff = { added: {} as any, updated: {} as any, removed: {} as any }
 		}
+		// records are immutable, so sharing their references with `change` is fine — the
+		// squash gives nextDiff its own containers and tuples without deep-cloning records
+		squashRecordDiffsMutable(this.unsentChanges.nextDiff, [change])
 		this.sendUnsentChanges()
 	}
 

package/src/lib/TLSyncRoom.ts

@@ -379,8 +379,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
 		session.debounceTimer = null
 
 		if (session.outstandingDataMessages.length > 0) {
-			session.socket.sendMessage({ type: 'data', data: session.outstandingDataMessages })
-			session.outstandingDataMessages.length = 0
+			// hand the buffer over and start a fresh one, rather than truncating in
+			// place, so sockets that defer serialization don't see an emptied array
+			const data = session.outstandingDataMessages
+			session.outstandingDataMessages = []
+			session.socket.sendMessage({ type: 'data', data })
 		}
 	}
 

package/src/lib/TLSyncStorage.test.ts

@@ -0,0 +1,225 @@
+import {
+	BaseRecord,
+	createMigrationSequence,
+	createRecordType,
+	RecordId,
+	StoreSchema,
+} from '@tldraw/store'
+import { TLDOCUMENT_ID, TLRecord } from '@tldraw/tlschema'
+import { describe, expect, it, vi } from 'vitest'
+import {
+	contractRecords,
+	contractSchema,
+	makeContractSnapshot,
+	makePage,
+} from '../test/storageContractSuite'
+import { InMemorySyncStorage } from './InMemorySyncStorage'
+import {
+	convertStoreSnapshotToRoomSnapshot,
+	loadSnapshotIntoStorage,
+	toNetworkDiff,
+	TLSyncForwardDiff,
+} from './TLSyncStorage'
+
+describe('toNetworkDiff', () => {
+	const pageA = makePage('a', 'A')
+	const pageB = { ...makePage('a', 'A'), name: 'B' } as TLRecord
+
+	it('[ND2] maps plain puts, update tuples, and deletes to network ops', () => {
+		const diff: TLSyncForwardDiff<TLRecord> = {
+			puts: {
+				[contractRecords[0].id]: contractRecords[0],
+				[pageA.id]: [pageA, pageB],
+			},
+			deletes: ['shape:gone'],
+		}
+		expect(toNetworkDiff(diff)).toEqual({
+			[contractRecords[0].id]: ['put', contractRecords[0]],
+			[pageA.id]: ['patch', { name: ['put', 'B'] }],
+			'shape:gone': ['remove'],
+		})
+	})
+
+	it('[ND2] omits update tuples that compute to no diff', () => {
+		const diff: TLSyncForwardDiff<TLRecord> = {
+			puts: { [pageA.id]: [pageA, { ...pageA } as TLRecord] },
+			deletes: [],
+		}
+		expect(toNetworkDiff(diff)).toEqual({})
+	})
+
+	it('[ND2] returns an (possibly empty) object, never null', () => {
+		expect(toNetworkDiff({ puts: {}, deletes: [] })).toEqual({})
+	})
+})
+
+describe('convertStoreSnapshotToRoomSnapshot', () => {
+	it('[SL1] passes a RoomSnapshot through by reference', () => {
+		const roomSnapshot = makeContractSnapshot(contractRecords, { documentClock: 42 })
+		expect(convertStoreSnapshotToRoomSnapshot(roomSnapshot)).toBe(roomSnapshot)
+	})
+
+	it('[SL1] converts a StoreSnapshot to a clock-zero room snapshot', () => {
+		const storeSnapshot = {
+			store: {
+				[TLDOCUMENT_ID]: contractRecords[0],
+				[contractRecords[1].id]: contractRecords[1],
+			} as any,
+			schema: contractSchema.serialize(),
+		}
+		expect(convertStoreSnapshotToRoomSnapshot(storeSnapshot)).toEqual({
+			clock: 0,
+			documentClock: 0,
+			documents: [
+				{ state: contractRecords[0], lastChangedClock: 0 },
+				{ state: contractRecords[1], lastChangedClock: 0 },
+			],
+			schema: storeSnapshot.schema,
+			tombstones: {},
+		})
+	})
+})
+
+describe('loadSnapshotIntoStorage', () => {
+	it('[SL2] throws when the snapshot has no schema', () => {
+		const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+		const storage = new InMemorySyncStorage<TLRecord>({
+			snapshot: makeContractSnapshot(contractRecords),
+		})
+		expect(() =>
+			storage.transaction((txn) => {
+				loadSnapshotIntoStorage(txn, contractSchema, { documents: [], clock: 0 } as any)
+			})
+		).toThrow('Schema is required')
+		consoleSpy.mockRestore()
+	})
+
+	it('[SL3] writes new and changed documents and tombstones the rest', () => {
+		const extraPage = makePage('extra', 'Extra')
+		const storage = new InMemorySyncStorage<TLRecord>({
+			snapshot: makeContractSnapshot([...contractRecords, extraPage]),
+		})
+
+		const changedPage = { ...contractRecords[1], name: 'Renamed' } as TLRecord
+		const newSnapshot = makeContractSnapshot([contractRecords[0], changedPage])
+		storage.transaction((txn) => {
+			loadSnapshotIntoStorage(txn, contractSchema, newSnapshot)
+		})
+
+		expect(storage.documents.get(contractRecords[1].id)?.state).toEqual(changedPage)
+		expect(storage.documents.has(extraPage.id)).toBe(false)
+		expect(storage.tombstones.has(extraPage.id)).toBe(true)
+	})
+
+	it('[SL3] skips documents that are deep-equal to the stored state', () => {
+		const storage = new InMemorySyncStorage<TLRecord>({
+			snapshot: makeContractSnapshot(contractRecords, { documentClock: 5 }),
+		})
+		storage.transaction((txn) => {
+			loadSnapshotIntoStorage(txn, contractSchema, makeContractSnapshot(contractRecords))
+		})
+		// nothing changed, so the clock did not advance
+		expect(storage.getClock()).toBe(5)
+	})
+
+	it('[SL4] persists the snapshot schema and migrates loaded records to the current version', () => {
+		interface TestRecord extends BaseRecord<'test', RecordId<TestRecord>> {
+			value: number
+			migrated?: boolean
+		}
+		const TestRecordType = createRecordType<TestRecord>('test', {
+			validator: { validate: (r) => r as TestRecord },
+			scope: 'document',
+		})
+		const migrations = createMigrationSequence({
+			sequenceId: 'com.test.record',
+			retroactive: true,
+			sequence: [
+				{
+					id: 'com.test.record/1',
+					scope: 'record',
+					filter: (r: any) => r.typeName === 'test',
+					up: (record: any) => {
+						record.migrated = true
+					},
+				},
+			],
+		})
+		const oldSchema = StoreSchema.create({ test: TestRecordType })
+		const newSchema = StoreSchema.create({ test: TestRecordType }, { migrations: [migrations] })
+
+		const storage = new InMemorySyncStorage<TestRecord>({
+			snapshot: {
+				documents: [],
+				clock: 0,
+				documentClock: 0,
+				schema: newSchema.serialize(),
+			},
+		})
+
+		const record: TestRecord = { id: 'test:1' as RecordId<TestRecord>, typeName: 'test', value: 1 }
+		storage.transaction((txn) => {
+			loadSnapshotIntoStorage(txn, newSchema, {
+				documents: [{ state: record, lastChangedClock: 0 }],
+				clock: 0,
+				documentClock: 0,
+				schema: oldSchema.serialize(),
+			})
+		})
+
+		expect((storage.documents.get('test:1')!.state as TestRecord).migrated).toBe(true)
+		// the schema was migrated up to the current version too
+		expect(storage.schema.get()).toEqual(newSchema.serialize())
+	})
+
+	it('[SL4] applies record migrations exactly once per record', () => {
+		const migrationCounts = new Map<string, number>()
+
+		interface TestRecord extends BaseRecord<'test', RecordId<TestRecord>> {
+			value: number
+		}
+		const TestRecordType = createRecordType<TestRecord>('test', {
+			validator: { validate: (r) => r as TestRecord },
+			scope: 'document',
+		})
+		const migrations = createMigrationSequence({
+			sequenceId: 'com.test.record',
+			retroactive: true,
+			sequence: [
+				{
+					id: 'com.test.record/1',
+					scope: 'record',
+					filter: (r: any) => r.typeName === 'test',
+					up: (record: any) => {
+						migrationCounts.set(record.id, (migrationCounts.get(record.id) ?? 0) + 1)
+					},
+				},
+			],
+		})
+		const oldSchema = StoreSchema.create({ test: TestRecordType })
+		const newSchema = StoreSchema.create({ test: TestRecordType }, { migrations: [migrations] })
+
+		const numRecords = 100
+		const testRecords: TestRecord[] = Array.from({ length: numRecords }, (_, i) => ({
+			id: `test:${i}` as RecordId<TestRecord>,
+			typeName: 'test' as const,
+			value: i,
+		}))
+
+		const storage = new InMemorySyncStorage<TestRecord>({
+			snapshot: {
+				documents: testRecords.map((r) => ({ state: r, lastChangedClock: 0 })),
+				clock: 0,
+				documentClock: 0,
+				schema: oldSchema.serialize(),
+			},
+		})
+
+		storage.transaction((txn) => {
+			newSchema.migrateStorage(txn)
+		})
+
+		expect(migrationCounts.size).toBe(numRecords)
+		expect([...migrationCounts.values()].every((count) => count === 1)).toBe(true)
+	})
+})

package/src/lib/chunk.test.ts

@@ -0,0 +1,372 @@
+import { assert } from 'tldraw'
+import { JsonChunkAssembler, chunk } from './chunk'
+
+describe('chunk (CH1–CH3)', () => {
+	describe('size boundary (CH1)', () => {
+		it('[CH1] returns the message as a single chunk when its length is strictly less than maxSize', () => {
+			expect(chunk('hello', 100)).toEqual(['hello'])
+			expect(chunk('x'.repeat(9), 10)).toEqual(['x'.repeat(9)])
+		})
+
+		it('[CH1] chunks a message whose length equals maxSize', () => {
+			const msg = 'x'.repeat(10)
+			const result = chunk(msg, 10)
+			expect(result.length).toBeGreaterThan(1)
+			expect(result[0]).toMatch(/^\d+_x+$/)
+		})
+
+		it('[CH1] handles empty strings', () => {
+			expect(chunk('', 100)).toEqual([''])
+		})
+
+		it('[CH1] handles single character strings', () => {
+			expect(chunk('a', 100)).toEqual(['a'])
+		})
+
+		it('[CH1] uses the default max size when none is provided', () => {
+			const longString = 'x'.repeat(262145) // larger than the 262144 default
+			const result = chunk(longString)
+			expect(result.length).toBeGreaterThan(1)
+			expect(result[0]).toMatch(/^\d+_x+$/)
+		})
+	})
+
+	describe('chunk format and ordering (CH2)', () => {
+		it('[CH2] prefixes chunks with a countdown and puts the start of the message first', () => {
+			expect(chunk('hello there my good world', 5)).toMatchInlineSnapshot(`
+			[
+			  "8_h",
+			  "7_ell",
+			  "6_o t",
+			  "5_her",
+			  "4_e m",
+			  "3_y g",
+			  "2_ood",
+			  "1_ wo",
+			  "0_rld",
+			]
+		`)
+
+			expect(chunk('hello there my good world', 10)).toMatchInlineSnapshot(`
+			[
+			  "3_h",
+			  "2_ello the",
+			  "1_re my go",
+			  "0_od world",
+			]
+		`)
+		})
+
+		it('[CH2] concatenating the chunk bodies in order reconstructs the message', () => {
+			const original = 'hello world this is a long message'
+			const reconstructed = chunk(original, 8)
+				.map((c) => /^(\d+)_(.*)$/s.exec(c)![2])
+				.join('')
+			expect(reconstructed).toEqual(original)
+		})
+
+		it('[CH2] preserves unicode characters across chunk boundaries', () => {
+			const unicodeString = '🎨'.repeat(10) + '📐'.repeat(10) + '🗼️'.repeat(10)
+			const result = chunk(unicodeString, 20)
+			expect(result.length).toBeGreaterThan(1)
+
+			const reconstructed = result.map((c) => /^(\d+)_(.*)$/s.exec(c)![2]).join('')
+			expect(reconstructed).toEqual(unicodeString)
+		})
+
+		it('[CH2] reconstructs very large strings', () => {
+			const veryLargeString = 'a'.repeat(1000000) // 1MB string
+			const result = chunk(veryLargeString, 10000)
+			expect(result.length).toBeGreaterThan(1)
+
+			const reconstructed = result.map((c) => /^(\d+)_(.*)$/s.exec(c)![2]).join('')
+			expect(reconstructed).toEqual(veryLargeString)
+		})
+	})
+
+	describe('chunk size limits (CH3)', () => {
+		it('[CH3] keeps each chunk at or below maxSize', () => {
+			const maxSize = 15
+			for (const c of chunk('hello world this is a long message', maxSize)) {
+				expect(c.length).toBeLessThanOrEqual(maxSize)
+			}
+
+			expect(chunk('dark and stormy tonight', 4)).toMatchInlineSnapshot(`
+			[
+			  "12_d",
+			  "11_a",
+			  "10_r",
+			  "9_k ",
+			  "8_an",
+			  "7_d ",
+			  "6_st",
+			  "5_or",
+			  "4_my",
+			  "3_ t",
+			  "2_on",
+			  "1_ig",
+			  "0_ht",
+			]
+		`)
+		})
+
+		it('[CH3] carries at least one content character per chunk even when the prefix alone exceeds maxSize', () => {
+			const chunks = chunk('once upon a time', 1)
+			expect(chunks).toMatchInlineSnapshot(`
+			[
+			  "15_o",
+			  "14_n",
+			  "13_c",
+			  "12_e",
+			  "11_ ",
+			  "10_u",
+			  "9_p",
+			  "8_o",
+			  "7_n",
+			  "6_ ",
+			  "5_a",
+			  "4_ ",
+			  "3_t",
+			  "2_i",
+			  "1_m",
+			  "0_e",
+			]
+		`)
+		})
+	})
+})
+
+const testObject = {} as any
+for (let i = 0; i < 1000; i++) {
+	testObject['key_' + i] = 'value_' + i
+}
+
+describe('JsonChunkAssembler (CH4–CH8)', () => {
+	describe('whole JSON messages while idle (CH4)', () => {
+		it('[CH4] parses a message starting with { immediately and returns data and stringified', () => {
+			const unchunker = new JsonChunkAssembler()
+			expect(unchunker.handleMessage('{"key": "value"}')).toMatchObject({
+				data: { key: 'value' },
+				stringified: '{"key": "value"}',
+			})
+		})
+
+		it('[CH4] handles an empty JSON object', () => {
+			const unchunker = new JsonChunkAssembler()
+			expect(unchunker.handleMessage('{}')).toMatchObject({ data: {}, stringified: '{}' })
+		})
+
+		it('[CH4] throws synchronously on invalid JSON starting with {', () => {
+			const unchunker = new JsonChunkAssembler()
+			expect(() => unchunker.handleMessage('{not valid json')).toThrow()
+
+			// the assembler is still usable afterwards
+			expect(unchunker.handleMessage('{"ok": true}')).toMatchObject({ data: { ok: true } })
+		})
+	})
+
+	describe('JSON messages mid-sequence (CH5)', () => {
+		it('[CH5] returns an error when a { message interrupts a chunk sequence and resets to idle', () => {
+			const unchunker = new JsonChunkAssembler()
+
+			// start a chunk sequence
+			expect(unchunker.handleMessage('2_hello')).toBeNull()
+
+			// interrupt with a JSON message: both the partial sequence and this
+			// message are discarded
+			const result = unchunker.handleMessage('{"interrupt": true}')
+			assert(result && 'error' in result, 'expected error result')
+			expect(result.error.message).toBe('Unexpected non-chunk message')
+
+			// the assembler reset to idle, so the next message works
+			expect(unchunker.handleMessage('{"ok": true}')).toMatchObject({ data: { ok: true } })
+		})
+
+		it('[CH5] returns an error when the chunk stream ends abruptly', () => {
+			const chunks = chunk('{"hello": world"}', 10)
+
+			const unchunker = new JsonChunkAssembler()
+			expect(unchunker.handleMessage(chunks[0])).toBeNull()
+			expect(unchunker.handleMessage(chunks[1])).toBeNull()
+
+			const res = unchunker.handleMessage('{"hello": "world"}')
+			assert(res, 'expected a result')
+			assert('error' in res, 'expected an error')
+			expect(res.error.message).toBe('Unexpected non-chunk message')
+
+			// and the next one should be fine
+			expect(unchunker.handleMessage('{"ok": true}')).toMatchObject({ data: { ok: true } })
+		})
+	})
+
+	describe('chunk accumulation (CH6)', () => {
+		it.each([1, 5, 20, 200])(
+			'[CH6] returns null until the final chunk, then the parsed json (split at %s bytes)',
+			(size) => {
+				const chunks = chunk(JSON.stringify(testObject), size)
+
+				const unchunker = new JsonChunkAssembler()
+				for (const c of chunks.slice(0, -1)) {
+					expect(unchunker.handleMessage(c)).toBeNull()
+				}
+				expect(unchunker.handleMessage(chunks[chunks.length - 1])).toMatchObject({
+					data: testObject,
+				})
+
+				// resets to idle: the next one should be fine
+				expect(unchunker.handleMessage('{"ok": true}')).toMatchObject({ data: { ok: true } })
+			}
+		)
+
+		it('[CH6] handles complex nested JSON objects', () => {
+			const complexObject = {
+				array: [1, 2, { nested: true }],
+				null: null,
+				boolean: false,
+				number: 3.14,
+				string: 'hello world',
+				unicode: '🎨🗼️📐',
+				deep: {
+					nested: {
+						object: {
+							with: 'many levels',
+						},
+					},
+				},
+			}
+
+			const chunks = chunk(JSON.stringify(complexObject), 50)
+			const unchunker = new JsonChunkAssembler()
+
+			for (const c of chunks.slice(0, -1)) {
+				expect(unchunker.handleMessage(c)).toBeNull()
+			}
+
+			expect(unchunker.handleMessage(chunks[chunks.length - 1])).toMatchObject({
+				data: complexObject,
+			})
+		})
+
+		it('[CH6] handles a single chunk with a 0_ prefix', () => {
+			const unchunker = new JsonChunkAssembler()
+			expect(unchunker.handleMessage('0_{"single": "chunk"}')).toMatchObject({
+				data: { single: 'chunk' },
+				stringified: '{"single": "chunk"}',
+			})
+		})
+
+		it('[CH6] handles chunks with empty data parts', () => {
+			const unchunker = new JsonChunkAssembler()
+
+			expect(unchunker.handleMessage('1_')).toBeNull() // empty first chunk
+			expect(unchunker.handleMessage('0_{"test": true}')).toMatchObject({ data: { test: true } })
+		})
+
+		it('[CH6] returns an error when the assembled json is invalid, then resets to idle', () => {
+			const chunks = chunk('{"hello": world"}', 5)
+			const unchunker = new JsonChunkAssembler()
+			for (const c of chunks.slice(0, -1)) {
+				expect(unchunker.handleMessage(c)).toBeNull()
+			}
+
+			const node18Error = `Unexpected token w in JSON at position 10`
+			const node20Error = `Unexpected token 'w', "\\{"hello": world"}" is not valid JSON`
+			const res = unchunker.handleMessage(chunks[chunks.length - 1])
+			assert(res, 'expected a result')
+			assert('error' in res, 'expected an error')
+			expect(res.error.message).toMatch(new RegExp(`${node18Error}|${node20Error}`))
+
+			// and the next one should be fine
+			expect(unchunker.handleMessage('{"ok": true}')).toMatchObject({ data: { ok: true } })
+		})
+
+		it('[CH6] returns an error object when chunks form invalid json', () => {
+			const unchunker = new JsonChunkAssembler()
+
+			expect(unchunker.handleMessage('1_{"invalid":')).toBeNull()
+			const result = unchunker.handleMessage('0_ }')
+
+			assert(result && 'error' in result, 'expected error result')
+			expect(result.error).toBeInstanceOf(Error)
+		})
+	})
+
+	describe('malformed chunk sequences (CH7)', () => {
+		it('[CH7] returns an error when a chunk is missing from the sequence, then resets to idle', () => {
+			const chunks = chunk('{"hello": world"}', 10)
+
+			const unchunker = new JsonChunkAssembler()
+			expect(unchunker.handleMessage(chunks[0])).toBeNull()
+			const res = unchunker.handleMessage(chunks[2])
+			assert(res, 'expected a result')
+			assert('error' in res, 'expected an error')
+			expect(res.error.message).toBe('Chunks received in wrong order')
+
+			// and the next one should be fine
+			expect(unchunker.handleMessage('{"ok": true}')).toMatchObject({ data: { ok: true } })
+		})
+
+		it('[CH7] returns an error for duplicate or out-of-order chunk numbers', () => {
+			const unchunker = new JsonChunkAssembler()
+
+			expect(unchunker.handleMessage('2_part1')).toBeNull()
+
+			// the countdown should be 1 here, not 0
+			const result = unchunker.handleMessage('0_part3')
+			assert(result && 'error' in result, 'expected error result')
+			expect(result.error.message).toBe('Chunks received in wrong order')
+		})
+
+		it('[CH7] returns an invalid chunk error for non-json, non-chunk messages', () => {
+			const unchunker = new JsonChunkAssembler()
+			const res = unchunker.handleMessage('["yo"]')
+			assert(res, 'expected a result')
+			assert('error' in res, 'expected an error')
+			expect(res.error.message).toContain('Invalid chunk')
+
+			// and the next one should be fine
+			expect(unchunker.handleMessage('{"ok": true}')).toMatchObject({ data: { ok: true } })
+		})
+
+		it('[CH7] returns an invalid chunk error for a malformed chunk number', () => {
+			const unchunker = new JsonChunkAssembler()
+			const result = unchunker.handleMessage('abc_invalid_number')
+			assert(result && 'error' in result, 'expected error result')
+			expect(result.error.message).toContain('Invalid chunk')
+		})
+
+		it('[CH7] resets to idle after an invalid chunk mid-sequence', () => {
+			const unchunker = new JsonChunkAssembler()
+
+			// start a chunk sequence
+			expect(unchunker.handleMessage('1_hello')).toBeNull()
+
+			// send a malformed chunk to trigger the error
+			const result = unchunker.handleMessage('invalid_chunk_format')
+			assert(result && 'error' in result, 'expected error result')
+			expect(result.error.message).toContain('Invalid chunk')
+
+			// normal messages work again
+			expect(unchunker.handleMessage('{"test": true}')).toMatchObject({ data: { test: true } })
+		})
+	})
+
+	describe('chunk body contents (CH8)', () => {
+		it('[CH8] handles unicode line separators U+2028 and U+2029 in chunks', () => {
+			// These characters are common when copy/pasting from Microsoft Word
+			// and are valid JSON string content but act as line terminators in JS
+			const unchunker = new JsonChunkAssembler()
+
+			const jsonWithLineSeparators = '{"text": "hello\u2028world\u2029end"}'
+			const chunks = chunk(jsonWithLineSeparators, 10)
+
+			for (const c of chunks.slice(0, -1)) {
+				expect(unchunker.handleMessage(c)).toBeNull()
+			}
+
+			expect(unchunker.handleMessage(chunks[chunks.length - 1])).toMatchObject({
+				data: { text: 'hello\u2028world\u2029end' },
+			})
+		})
+	})
+})