@tldraw/sync-core 4.2.2 → 4.2.3

package/src/lib/TLSyncStorage.ts

@@ -1,216 +0,0 @@
-import { StoreSchema, SynchronousStorage, UnknownRecord } from '@tldraw/store'
-import { isEqual, objectMapEntriesIterable, objectMapValues } from '@tldraw/utils'
-import { TLStoreSnapshot } from 'tldraw'
-import { diffRecord, NetworkDiff, RecordOpType } from './diff'
-import { RoomSnapshot } from './TLSyncRoom'
-
-/**
- * Transaction interface for storage operations. Provides methods to read and modify
- * documents, tombstones, and metadata within a transaction.
- *
- * @public
- */
-export interface TLSyncStorageTransaction<R extends UnknownRecord> extends SynchronousStorage<R> {
-	/**
-	 * Get the current clock value.
-	 * If the clock has incremented during the transaction,
-	 * the incremented value will be returned.
-	 *
-	 * @returns The current clock value
-	 */
-	getClock(): number
-
-	/**
-	 * Get all changes (document updates and deletions) since a given clock time.
-	 * This is the main method for calculating diffs for client sync.
-	 *
-	 * @param sinceClock - The clock time to get changes since
-	 * @returns Changes since the specified clock time
-	 */
-	getChangesSince(sinceClock: number): TLSyncStorageGetChangesSinceResult<R> | undefined
-}
-
-/**
- * Options for a transaction.
- * @public
- */
-export interface TLSyncStorageTransactionOptions {
-	/**
-	 * Use this if you need to identify the transaction for logging or debugging purposes
-	 * or for ignoring certain changes in onChange callbacks
-	 */
-	id?: string
-	/**
-	 * Controls when the storage layer should emit the actual changes that occurred during the transaction.
-	 *
-	 * - `'always'` - Always emit the changes, regardless of whether they were applied verbatim
-	 * - `'when-different'` - Only emit changes if the storage layer modified/embellished the records
-	 *   (e.g., added server timestamps, normalized data, etc.)
-	 *
-	 * When changes are emitted, they will be available in the `changes` field of the transaction result.
-	 * This is useful when the storage layer may transform records and the caller needs to know
-	 * what actually changed rather than what was requested.
-	 */
-	emitChanges?: 'always' | 'when-different'
-}
-
-/**
- * Callback type for a transaction.
- * The conditional return type ensures that the callback is synchronous.
- * @public
- */
-export type TLSyncStorageTransactionCallback<R extends UnknownRecord, T> = (
-	txn: TLSyncStorageTransaction<R>
-) => T extends Promise<any>
-	? {
-			__error: 'Transaction callbacks cannot be async. Use synchronous operations only.'
-		}
-	: T
-
-/**
- * Pluggable synchronous transactional storage layer for TLSyncRoom.
- * Provides methods for managing documents, tombstones, and clocks within transactions.
- *
- * @public
- */
-export interface TLSyncStorage<R extends UnknownRecord> {
-	transaction<T>(
-		callback: TLSyncStorageTransactionCallback<R, T>,
-		opts?: TLSyncStorageTransactionOptions
-	): TLSyncStorageTransactionResult<T, R>
-
-	getClock(): number
-
-	onChange(callback: (arg: TLSyncStorageOnChangeCallbackProps) => unknown): () => void
-
-	getSnapshot?(): RoomSnapshot
-}
-
-/**
- * Properties passed to the onChange callback.
- * @public
- */
-export interface TLSyncStorageOnChangeCallbackProps {
-	/**
-	 * The ID of the transaction that caused the change.
-	 * This is useful for ignoring certain changes in onChange callbacks.
-	 */
-	id?: string
-	documentClock: number
-}
-
-/**
- * Result returned from a storage transaction.
- * @public
- */
-export interface TLSyncStorageTransactionResult<T, R extends UnknownRecord = UnknownRecord> {
-	documentClock: number
-	didChange: boolean
-	result: T
-	/**
-	 * The actual changes that occurred during the transaction, if requested via `emitChanges` option.
-	 * This is a RecordsDiff where:
-	 * - `added` contains records that were put (we don't have "from" state for emitted changes)
-	 * - `removed` contains records that were deleted (with placeholder values since we only have IDs)
-	 * - `updated` is empty (emitted changes don't track before/after pairs)
-	 *
-	 * Only populated when:
-	 * - `emitChanges: 'always'` was specified, or
-	 * - `emitChanges: 'when-different'` was specified and the storage layer modified records
-	 */
-	changes?: TLSyncForwardDiff<R>
-}
-
-/**
- * Respresents a diff of puts and deletes.
- * @public
- */
-export interface TLSyncForwardDiff<R extends UnknownRecord> {
-	puts: Record<string, R | [before: R, after: R]>
-	deletes: string[]
-}
-
-/**
- * @internal
- */
-export function toNetworkDiff<R extends UnknownRecord>(diff: TLSyncForwardDiff<R>): NetworkDiff<R> {
-	const networkDiff: NetworkDiff<R> = {}
-	for (const [id, put] of objectMapEntriesIterable(diff.puts)) {
-		if (Array.isArray(put)) {
-			const patch = diffRecord(put[0], put[1])
-			if (patch) {
-				networkDiff[id] = [RecordOpType.Patch, patch]
-			}
-		} else {
-			networkDiff[id] = [RecordOpType.Put, put]
-		}
-	}
-	for (const id of diff.deletes) {
-		networkDiff[id] = [RecordOpType.Remove]
-	}
-	return networkDiff
-}
-
-/**
- * Result returned from getChangesSince, containing all changes since a given clock time.
- * @public
- */
-export interface TLSyncStorageGetChangesSinceResult<R extends UnknownRecord> {
-	/**
-	 * The changes as a TLSyncForwardDiff.
-	 */
-	diff: TLSyncForwardDiff<R>
-	/**
-	 * If true, the client should wipe all local data and replace with the server's state.
-	 * This happens when the client's clock is too old and we've lost tombstone history.
-	 */
-	wipeAll: boolean
-}
-
-/**
- * Loads a snapshot into storage during a transaction.
- * Migrates the snapshot to the current schema and loads it into storage.
- *
- * @public
- * @param txn - The transaction to load the snapshot into
- * @param schema - The current schema
- * @param snapshot - The snapshot to load
- */
-export function loadSnapshotIntoStorage<R extends UnknownRecord>(
-	txn: TLSyncStorageTransaction<R>,
-	schema: StoreSchema<R, any>,
-	snapshot: RoomSnapshot | TLStoreSnapshot
-) {
-	snapshot = convertStoreSnapshotToRoomSnapshot(snapshot)
-	assert(snapshot.schema, 'Schema is required')
-	const docIds = new Set<string>()
-	for (const doc of snapshot.documents) {
-		docIds.add(doc.state.id)
-		const existing = txn.get(doc.state.id)
-		if (isEqual(existing, doc.state)) continue
-		txn.set(doc.state.id, doc.state as R)
-	}
-	for (const id of txn.keys()) {
-		if (!docIds.has(id)) {
-			txn.delete(id)
-		}
-	}
-	txn.setSchema(snapshot.schema)
-	schema.migrateStorage(txn)
-}
-
-export function convertStoreSnapshotToRoomSnapshot(
-	snapshot: RoomSnapshot | TLStoreSnapshot
-): RoomSnapshot {
-	if ('documents' in snapshot) return snapshot
-	return {
-		clock: 0,
-		documentClock: 0,
-		documents: objectMapValues(snapshot.store).map((state) => ({
-			state,
-			lastChangedClock: 0,
-		})),
-		schema: snapshot.schema,
-		tombstones: {},
-	}
-}

package/src/lib/computeTombstonePruning.test.ts

@@ -1,352 +0,0 @@
-import { describe, expect, it } from 'vitest'
-import {
-	computeTombstonePruning,
-	MAX_TOMBSTONES,
-	TOMBSTONE_PRUNE_BUFFER_SIZE,
-} from './InMemorySyncStorage'
-
-// Helper to create tombstone array
-function makeTombstones(
-	count: number,
-	clockFn: (i: number) => number = (i) => i + 1
-): Array<{ id: string; clock: number }> {
-	return Array.from({ length: count }, (_, i) => ({
-		id: `doc${i}`,
-		clock: clockFn(i),
-	}))
-}
-
-describe('computeTombstonePruning', () => {
-	describe('basic behavior', () => {
-		it('returns null when tombstone count is below threshold', () => {
-			const tombstones = makeTombstones(100)
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 1000,
-				maxTombstones: 500,
-			})
-			expect(result).toBeNull()
-		})
-
-		it('returns null when tombstone count equals threshold exactly', () => {
-			const tombstones = makeTombstones(500)
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 1000,
-				maxTombstones: 500,
-			})
-			expect(result).toBeNull()
-		})
-
-		it('returns null for empty tombstones array', () => {
-			const result = computeTombstonePruning({ tombstones: [], documentClock: 1000 })
-			expect(result).toBeNull()
-		})
-
-		it('returns pruning result when exceeding threshold', () => {
-			const tombstones = makeTombstones(600)
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 1000,
-				maxTombstones: 500,
-				pruneBufferSize: 50,
-			})
-			expect(result).not.toBeNull()
-			expect(result!.idsToDelete.length).toBeGreaterThan(0)
-		})
-	})
-
-	describe('pruning calculation', () => {
-		it('deletes oldest tombstones first', () => {
-			// 10 tombstones, max 5, buffer 2 => delete 10 - 5 + 2 = 7
-			const tombstones = makeTombstones(10)
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 100,
-				maxTombstones: 5,
-				pruneBufferSize: 2,
-			})
-
-			expect(result).not.toBeNull()
-			expect(result!.idsToDelete).toEqual(['doc0', 'doc1', 'doc2', 'doc3', 'doc4', 'doc5', 'doc6'])
-			expect(result!.newTombstoneHistoryStartsAtClock).toBe(8) // clock of doc7
-		})
-
-		it('keeps newest tombstones', () => {
-			const tombstones = makeTombstones(10)
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 100,
-				maxTombstones: 5,
-				pruneBufferSize: 2,
-			})
-
-			// Should keep doc7, doc8, doc9
-			const keptIds = tombstones.map((t) => t.id).filter((id) => !result!.idsToDelete.includes(id))
-			expect(keptIds).toEqual(['doc7', 'doc8', 'doc9'])
-		})
-
-		it('sets newTombstoneHistoryStartsAtClock to oldest remaining tombstone clock', () => {
-			// Tombstones with clocks 1-20
-			const tombstones = makeTombstones(20)
-			// max 10, buffer 5 => delete 20 - 10 + 5 = 15
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 1000,
-				maxTombstones: 10,
-				pruneBufferSize: 5,
-			})
-
-			expect(result).not.toBeNull()
-			// After deleting 15, oldest remaining is doc15 with clock 16
-			expect(result!.newTombstoneHistoryStartsAtClock).toBe(16)
-		})
-	})
-
-	describe('duplicate clock handling', () => {
-		it('avoids splitting tombstones with the same clock value', () => {
-			// Create tombstones where multiple have the same clock
-			// Clocks: [1, 1, 1, 2, 2, 2, 3, 3, 3, 4]
-			const tombstones = [
-				{ id: 'a1', clock: 1 },
-				{ id: 'a2', clock: 1 },
-				{ id: 'a3', clock: 1 },
-				{ id: 'b1', clock: 2 },
-				{ id: 'b2', clock: 2 },
-				{ id: 'b3', clock: 2 },
-				{ id: 'c1', clock: 3 },
-				{ id: 'c2', clock: 3 },
-				{ id: 'c3', clock: 3 },
-				{ id: 'd1', clock: 4 },
-			]
-
-			// max 5, buffer 1 => initial cutoff = 10 - 5 + 1 = 6
-			// cutoff 6 would split clock=2 (b3) from clock=3 (c1)
-			// But tombstones[5].clock (2) !== tombstones[6].clock (3), so no adjustment needed
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 100,
-				maxTombstones: 5,
-				pruneBufferSize: 1,
-			})
-
-			expect(result).not.toBeNull()
-			expect(result!.idsToDelete).toEqual(['a1', 'a2', 'a3', 'b1', 'b2', 'b3'])
-			expect(result!.newTombstoneHistoryStartsAtClock).toBe(3) // clock of c1
-		})
-
-		it('extends cutoff when it would split a clock value', () => {
-			// Clocks: [1, 2, 2, 2, 2, 3]
-			const tombstones = [
-				{ id: 'a', clock: 1 },
-				{ id: 'b1', clock: 2 },
-				{ id: 'b2', clock: 2 },
-				{ id: 'b3', clock: 2 },
-				{ id: 'b4', clock: 2 },
-				{ id: 'c', clock: 3 },
-			]
-
-			// max 4, buffer 1 => initial cutoff = 6 - 4 + 1 = 3
-			// cutoff 3 points to b3 (clock 2), cutoff-1 points to b2 (clock 2)
-			// Same clock, so extend cutoff until boundary
-			// cutoff 4: b4 (clock 2), cutoff-1: b3 (clock 2) - same, extend
-			// cutoff 5: c (clock 3), cutoff-1: b4 (clock 2) - different, stop
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 100,
-				maxTombstones: 4,
-				pruneBufferSize: 1,
-			})
-
-			expect(result).not.toBeNull()
-			// Must delete all clock=2 tombstones to avoid split
-			expect(result!.idsToDelete).toEqual(['a', 'b1', 'b2', 'b3', 'b4'])
-			expect(result!.newTombstoneHistoryStartsAtClock).toBe(3)
-		})
-
-		it('handles all tombstones with same clock value', () => {
-			// All tombstones have clock=5
-			const tombstones = makeTombstones(100, () => 5)
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 1000,
-				maxTombstones: 50,
-				pruneBufferSize: 10,
-			})
-
-			expect(result).not.toBeNull()
-			// Since all have same clock, cutoff extends to delete all
-			expect(result!.idsToDelete.length).toBe(100)
-			// Falls back to documentClock since no remaining tombstones
-			expect(result!.newTombstoneHistoryStartsAtClock).toBe(1000)
-		})
-	})
-
-	describe('edge cases', () => {
-		it('handles exactly one tombstone over threshold', () => {
-			const tombstones = makeTombstones(101)
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 1000,
-				maxTombstones: 100,
-				pruneBufferSize: 0,
-			})
-
-			expect(result).not.toBeNull()
-			expect(result!.idsToDelete).toEqual(['doc0'])
-			expect(result!.newTombstoneHistoryStartsAtClock).toBe(2)
-		})
-
-		it('handles buffer size larger than excess', () => {
-			// 105 tombstones, max 100, buffer 50
-			// cutoff = 50 + 105 - 100 = 55
-			const tombstones = makeTombstones(105)
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 1000,
-				maxTombstones: 100,
-				pruneBufferSize: 50,
-			})
-
-			expect(result).not.toBeNull()
-			expect(result!.idsToDelete.length).toBe(55)
-		})
-
-		it('uses documentClock when all tombstones are deleted', () => {
-			// Small array, aggressive pruning
-			const tombstones = makeTombstones(10)
-			// max 5, buffer 10 => cutoff = 10 + 10 - 5 = 15, but only 10 items
-			// So all get deleted
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 999,
-				maxTombstones: 5,
-				pruneBufferSize: 10,
-			})
-
-			expect(result).not.toBeNull()
-			expect(result!.idsToDelete.length).toBe(10)
-			expect(result!.newTombstoneHistoryStartsAtClock).toBe(999)
-		})
-
-		it('handles non-contiguous clock values', () => {
-			// Clocks with gaps: [1, 5, 10, 50, 100]
-			const tombstones = [
-				{ id: 'a', clock: 1 },
-				{ id: 'b', clock: 5 },
-				{ id: 'c', clock: 10 },
-				{ id: 'd', clock: 50 },
-				{ id: 'e', clock: 100 },
-			]
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 200,
-				maxTombstones: 3,
-				pruneBufferSize: 1,
-			})
-
-			expect(result).not.toBeNull()
-			// cutoff = 1 + 5 - 3 = 3
-			expect(result!.idsToDelete).toEqual(['a', 'b', 'c'])
-			expect(result!.newTombstoneHistoryStartsAtClock).toBe(50)
-		})
-
-		it('handles large clock values', () => {
-			const tombstones = [
-				{ id: 'a', clock: 1_000_000_000 },
-				{ id: 'b', clock: 1_000_000_001 },
-				{ id: 'c', clock: 1_000_000_002 },
-			]
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 2000000000,
-				maxTombstones: 2,
-				pruneBufferSize: 0,
-			})
-
-			expect(result).not.toBeNull()
-			expect(result!.idsToDelete).toEqual(['a'])
-			expect(result!.newTombstoneHistoryStartsAtClock).toBe(1_000_000_001)
-		})
-	})
-
-	describe('with default constants', () => {
-		it('does not prune at exactly MAX_TOMBSTONES', () => {
-			const tombstones = makeTombstones(MAX_TOMBSTONES)
-			const result = computeTombstonePruning({ tombstones, documentClock: 100000 })
-			expect(result).toBeNull()
-		})
-
-		it('prunes when exceeding MAX_TOMBSTONES by 1', () => {
-			const tombstones = makeTombstones(MAX_TOMBSTONES + 1)
-			const result = computeTombstonePruning({ tombstones, documentClock: 100000 })
-
-			expect(result).not.toBeNull()
-			// cutoff = BUFFER + (MAX+1) - MAX = BUFFER + 1
-			expect(result!.idsToDelete.length).toBe(TOMBSTONE_PRUNE_BUFFER_SIZE + 1)
-		})
-
-		it('prunes correctly with large excess', () => {
-			const totalTombstones = MAX_TOMBSTONES * 2
-			const tombstones = makeTombstones(totalTombstones)
-			const result = computeTombstonePruning({ tombstones, documentClock: 100000 })
-
-			expect(result).not.toBeNull()
-			// cutoff = BUFFER + 2*MAX - MAX = BUFFER + MAX
-			const expectedDeletes = TOMBSTONE_PRUNE_BUFFER_SIZE + MAX_TOMBSTONES
-			expect(result!.idsToDelete.length).toBe(expectedDeletes)
-
-			// Remaining should be MAX - BUFFER
-			const remaining = totalTombstones - result!.idsToDelete.length
-			expect(remaining).toBe(MAX_TOMBSTONES - TOMBSTONE_PRUNE_BUFFER_SIZE)
-		})
-	})
-
-	describe('input validation assumptions', () => {
-		it('assumes tombstones are sorted by clock ascending', () => {
-			// If not sorted, results are undefined - this documents the assumption
-			const sortedTombstones = [
-				{ id: 'a', clock: 1 },
-				{ id: 'b', clock: 2 },
-				{ id: 'c', clock: 3 },
-			]
-			const result = computeTombstonePruning({
-				tombstones: sortedTombstones,
-				documentClock: 100,
-				maxTombstones: 2,
-				pruneBufferSize: 0,
-			})
-			expect(result).not.toBeNull()
-			expect(result!.idsToDelete).toEqual(['a'])
-		})
-
-		it('works with zero buffer size', () => {
-			const tombstones = makeTombstones(10)
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 100,
-				maxTombstones: 5,
-				pruneBufferSize: 0,
-			})
-
-			expect(result).not.toBeNull()
-			// cutoff = 0 + 10 - 5 = 5
-			expect(result!.idsToDelete.length).toBe(5)
-		})
-
-		it('works with zero max tombstones (aggressive pruning)', () => {
-			const tombstones = makeTombstones(10)
-			const result = computeTombstonePruning({
-				tombstones,
-				documentClock: 100,
-				maxTombstones: 0,
-				pruneBufferSize: 0,
-			})
-
-			expect(result).not.toBeNull()
-			// cutoff = 0 + 10 - 0 = 10
-			expect(result!.idsToDelete.length).toBe(10)
-			expect(result!.newTombstoneHistoryStartsAtClock).toBe(100)
-		})
-	})
-})

package/src/lib/recordDiff.ts

@@ -1,73 +0,0 @@
-import { RecordType, UnknownRecord } from '@tldraw/store'
-import { ObjectDiff, applyObjectDiff, diffRecord } from './diff'
-import { TLSyncError, TLSyncErrorCloseEventReason } from './TLSyncClient'
-
-/**
- * Validate a record and compute the diff between two states.
- * Returns null if the states are identical.
- *
- * @param prevState - The previous record state
- * @param newState - The new record state
- * @param recordType - The record type definition for validation
- * @param legacyAppendMode - If true, string append operations will be converted to Put operations
- * @returns Result containing the diff and new state, or null if no changes, or validation error
- *
- * @internal
- */
-export function diffAndValidateRecord<R extends UnknownRecord>(
-	prevState: R,
-	newState: R,
-	recordType: RecordType<R, any>,
-	legacyAppendMode = false
-) {
-	const diff = diffRecord(prevState, newState, legacyAppendMode)
-	if (!diff) return
-	try {
-		recordType.validate(newState)
-	} catch (error: any) {
-		throw new TLSyncError(error.message, TLSyncErrorCloseEventReason.INVALID_RECORD)
-	}
-	return diff
-}
-
-/**
- * Apply a diff to a record state, validate the result, and compute the final diff.
- * Returns null if the diff produces no changes.
- *
- * @param prevState - The previous record state
- * @param diff - The object diff to apply
- * @param recordType - The record type definition for validation
- * @param legacyAppendMode - If true, string append operations will be converted to Put operations
- * @returns Result containing the final diff and new state, or null if no changes, or validation error
- *
- * @internal
- */
-export function applyAndDiffRecord<R extends UnknownRecord>(
-	prevState: R,
-	diff: ObjectDiff,
-	recordType: RecordType<R, any>,
-	legacyAppendMode = false
-): [ObjectDiff, R] | undefined {
-	const newState = applyObjectDiff(prevState, diff)
-	if (newState === prevState) return
-	const actualDiff = diffAndValidateRecord(prevState, newState, recordType, legacyAppendMode)
-	if (!actualDiff) return
-	return [actualDiff, newState]
-}
-
-/**
- * Validate a record without computing a diff. Used when creating new records.
- *
- * @param state - The record state to validate
- * @param recordType - The record type definition for validation
- * @returns Result indicating success or validation error
- *
- * @internal
- */
-export function validateRecord<R extends UnknownRecord>(state: R, recordType: RecordType<R, any>) {
-	try {
-		recordType.validate(state)
-	} catch (error: any) {
-		throw new TLSyncError(error.message, TLSyncErrorCloseEventReason.INVALID_RECORD)
-	}
-}