@tldraw/sync-core 4.3.0-next.2d181ae353a2 → 4.3.0-next.40e4536afc8e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/dist-cjs/index.d.ts +481 -56
  2. package/dist-cjs/index.js +13 -3
  3. package/dist-cjs/index.js.map +2 -2
  4. package/dist-cjs/lib/DurableObjectSqliteSyncWrapper.js +55 -0
  5. package/dist-cjs/lib/DurableObjectSqliteSyncWrapper.js.map +7 -0
  6. package/dist-cjs/lib/InMemorySyncStorage.js +287 -0
  7. package/dist-cjs/lib/InMemorySyncStorage.js.map +7 -0
  8. package/dist-cjs/lib/MicrotaskNotifier.js +50 -0
  9. package/dist-cjs/lib/MicrotaskNotifier.js.map +7 -0
  10. package/dist-cjs/lib/NodeSqliteWrapper.js +48 -0
  11. package/dist-cjs/lib/NodeSqliteWrapper.js.map +7 -0
  12. package/dist-cjs/lib/RoomSession.js.map +1 -1
  13. package/dist-cjs/lib/SQLiteSyncStorage.js +428 -0
  14. package/dist-cjs/lib/SQLiteSyncStorage.js.map +7 -0
  15. package/dist-cjs/lib/TLSocketRoom.js +117 -69
  16. package/dist-cjs/lib/TLSocketRoom.js.map +2 -2
  17. package/dist-cjs/lib/TLSyncClient.js +7 -0
  18. package/dist-cjs/lib/TLSyncClient.js.map +2 -2
  19. package/dist-cjs/lib/TLSyncRoom.js +357 -688
  20. package/dist-cjs/lib/TLSyncRoom.js.map +3 -3
  21. package/dist-cjs/lib/TLSyncStorage.js +76 -0
  22. package/dist-cjs/lib/TLSyncStorage.js.map +7 -0
  23. package/dist-cjs/lib/recordDiff.js +52 -0
  24. package/dist-cjs/lib/recordDiff.js.map +7 -0
  25. package/dist-esm/index.d.mts +481 -56
  26. package/dist-esm/index.mjs +20 -5
  27. package/dist-esm/index.mjs.map +2 -2
  28. package/dist-esm/lib/DurableObjectSqliteSyncWrapper.mjs +35 -0
  29. package/dist-esm/lib/DurableObjectSqliteSyncWrapper.mjs.map +7 -0
  30. package/dist-esm/lib/InMemorySyncStorage.mjs +272 -0
  31. package/dist-esm/lib/InMemorySyncStorage.mjs.map +7 -0
  32. package/dist-esm/lib/MicrotaskNotifier.mjs +30 -0
  33. package/dist-esm/lib/MicrotaskNotifier.mjs.map +7 -0
  34. package/dist-esm/lib/NodeSqliteWrapper.mjs +28 -0
  35. package/dist-esm/lib/NodeSqliteWrapper.mjs.map +7 -0
  36. package/dist-esm/lib/RoomSession.mjs.map +1 -1
  37. package/dist-esm/lib/SQLiteSyncStorage.mjs +414 -0
  38. package/dist-esm/lib/SQLiteSyncStorage.mjs.map +7 -0
  39. package/dist-esm/lib/TLSocketRoom.mjs +121 -70
  40. package/dist-esm/lib/TLSocketRoom.mjs.map +2 -2
  41. package/dist-esm/lib/TLSyncClient.mjs +7 -0
  42. package/dist-esm/lib/TLSyncClient.mjs.map +2 -2
  43. package/dist-esm/lib/TLSyncRoom.mjs +370 -702
  44. package/dist-esm/lib/TLSyncRoom.mjs.map +3 -3
  45. package/dist-esm/lib/TLSyncStorage.mjs +56 -0
  46. package/dist-esm/lib/TLSyncStorage.mjs.map +7 -0
  47. package/dist-esm/lib/recordDiff.mjs +32 -0
  48. package/dist-esm/lib/recordDiff.mjs.map +7 -0
  49. package/package.json +12 -11
  50. package/src/index.ts +32 -3
  51. package/src/lib/ClientWebSocketAdapter.test.ts +3 -0
  52. package/src/lib/DurableObjectSqliteSyncWrapper.ts +95 -0
  53. package/src/lib/InMemorySyncStorage.ts +387 -0
  54. package/src/lib/MicrotaskNotifier.test.ts +429 -0
  55. package/src/lib/MicrotaskNotifier.ts +38 -0
  56. package/src/lib/NodeSqliteSyncWrapper.integration.test.ts +270 -0
  57. package/src/lib/NodeSqliteSyncWrapper.test.ts +272 -0
  58. package/src/lib/NodeSqliteWrapper.ts +99 -0
  59. package/src/lib/RoomSession.test.ts +1 -0
  60. package/src/lib/RoomSession.ts +2 -0
  61. package/src/lib/SQLiteSyncStorage.ts +627 -0
  62. package/src/lib/TLSocketRoom.ts +228 -114
  63. package/src/lib/TLSyncClient.ts +12 -0
  64. package/src/lib/TLSyncRoom.ts +473 -913
  65. package/src/lib/TLSyncStorage.ts +216 -0
  66. package/src/lib/computeTombstonePruning.test.ts +352 -0
  67. package/src/lib/recordDiff.ts +73 -0
  68. package/src/test/InMemorySyncStorage.test.ts +1779 -0
  69. package/src/test/SQLiteSyncStorage.test.ts +1484 -0
  70. package/src/test/TLSocketRoom.test.ts +255 -49
  71. package/src/test/TLSyncRoom.test.ts +1023 -533
  72. package/src/test/TestServer.ts +12 -1
  73. package/src/test/customMessages.test.ts +1 -1
  74. package/src/test/presenceMode.test.ts +6 -6
  75. package/src/test/upgradeDowngrade.test.ts +290 -8
  76. package/src/test/validation.test.ts +15 -10
  77. package/src/test/pruneTombstones.test.ts +0 -178
@@ -0,0 +1,1484 @@
1
+ import {
2
+ BaseRecord,
3
+ createMigrationSequence,
4
+ createRecordType,
5
+ RecordId,
6
+ StoreSchema,
7
+ } from '@tldraw/store'
8
+ import {
9
+ createTLSchema,
10
+ DocumentRecordType,
11
+ PageRecordType,
12
+ TLDOCUMENT_ID,
13
+ TLRecord,
14
+ } from '@tldraw/tlschema'
15
+ import { IndexKey, ZERO_INDEX_KEY } from '@tldraw/utils'
16
+ import { DatabaseSync } from 'node:sqlite'
17
+ import { vi } from 'vitest'
18
+ import { MAX_TOMBSTONES, TOMBSTONE_PRUNE_BUFFER_SIZE } from '../lib/InMemorySyncStorage'
19
+ import { NodeSqliteWrapper } from '../lib/NodeSqliteWrapper'
20
+ import { SQLiteSyncStorage } from '../lib/SQLiteSyncStorage'
21
+ import { RoomSnapshot } from '../lib/TLSyncRoom'
22
+
23
+ const tlSchema = createTLSchema()
24
+
25
+ const makeSnapshot = (records: TLRecord[], others: Partial<RoomSnapshot> = {}): RoomSnapshot => ({
26
+ documents: records.map((r) => ({ state: r, lastChangedClock: 0 })),
27
+ clock: 0,
28
+ documentClock: 0,
29
+ schema: tlSchema.serialize(),
30
+ ...others,
31
+ })
32
+
33
+ // Helper to create legacy snapshots without documentClock field
34
+ const makeLegacySnapshot = (
35
+ records: TLRecord[],
36
+ others: Partial<Omit<RoomSnapshot, 'documentClock'>> = {}
37
+ ): Omit<RoomSnapshot, 'documentClock'> & { schema: RoomSnapshot['schema'] } => ({
38
+ documents: records.map((r) => ({ state: r, lastChangedClock: 0 })),
39
+ clock: 0,
40
+ schema: tlSchema.serialize(),
41
+ ...others,
42
+ })
43
+
44
+ const defaultRecords = [
45
+ DocumentRecordType.create({ id: TLDOCUMENT_ID }),
46
+ PageRecordType.create({
47
+ index: ZERO_INDEX_KEY,
48
+ name: 'Page 1',
49
+ id: PageRecordType.createId('page_1'),
50
+ }),
51
+ ]
52
+
53
+ function createWrapper(config?: { tablePrefix?: string }) {
54
+ const db = new DatabaseSync(':memory:')
55
+ return new NodeSqliteWrapper(db, config)
56
+ }
57
+
58
+ function getStorage(snapshot: RoomSnapshot, wrapperConfig?: { tablePrefix?: string }) {
59
+ const sql = createWrapper(wrapperConfig)
60
+ return new SQLiteSyncStorage<TLRecord>({ sql, snapshot })
61
+ }
62
+
63
+ describe('SQLiteSyncStorage', () => {
64
+ describe('Static methods', () => {
65
+ describe('hasBeenInitialized', () => {
66
+ it('returns false for empty database', () => {
67
+ const sql = createWrapper()
68
+ expect(SQLiteSyncStorage.hasBeenInitialized(sql)).toBe(false)
69
+ })
70
+
71
+ it('returns true after storage is initialized', () => {
72
+ const sql = createWrapper()
73
+ new SQLiteSyncStorage<TLRecord>({ sql, snapshot: makeSnapshot(defaultRecords) })
74
+ expect(SQLiteSyncStorage.hasBeenInitialized(sql)).toBe(true)
75
+ })
76
+
77
+ it('respects table prefix', () => {
78
+ const sql = createWrapper({ tablePrefix: 'test_' })
79
+ expect(SQLiteSyncStorage.hasBeenInitialized(sql)).toBe(false)
80
+ new SQLiteSyncStorage<TLRecord>({ sql, snapshot: makeSnapshot(defaultRecords) })
81
+ expect(SQLiteSyncStorage.hasBeenInitialized(sql)).toBe(true)
82
+ })
83
+ })
84
+
85
+ describe('getDocumentClock', () => {
86
+ it('returns null for empty database', () => {
87
+ const sql = createWrapper()
88
+ expect(SQLiteSyncStorage.getDocumentClock(sql)).toBe(null)
89
+ })
90
+
91
+ it('returns 0 for newly initialized storage with default snapshot', () => {
92
+ const sql = createWrapper()
93
+ new SQLiteSyncStorage<TLRecord>({ sql, snapshot: makeSnapshot(defaultRecords) })
94
+ expect(SQLiteSyncStorage.getDocumentClock(sql)).toBe(0)
95
+ })
96
+
97
+ it('returns the documentClock value from snapshot', () => {
98
+ const sql = createWrapper()
99
+ const snapshot = makeSnapshot(defaultRecords)
100
+ snapshot.documentClock = 42
101
+ new SQLiteSyncStorage<TLRecord>({ sql, snapshot })
102
+ expect(SQLiteSyncStorage.getDocumentClock(sql)).toBe(42)
103
+ })
104
+
105
+ it('returns updated clock after transactions', () => {
106
+ const sql = createWrapper()
107
+ const storage = new SQLiteSyncStorage<TLRecord>({
108
+ sql,
109
+ snapshot: makeSnapshot(defaultRecords),
110
+ })
111
+ expect(SQLiteSyncStorage.getDocumentClock(sql)).toBe(0)
112
+
113
+ const newPage = PageRecordType.create({
114
+ id: PageRecordType.createId('test_page'),
115
+ name: 'Test Page',
116
+ index: 'a1' as IndexKey,
117
+ })
118
+ storage.transaction((txn) => {
119
+ txn.set(newPage.id, newPage)
120
+ })
121
+ expect(SQLiteSyncStorage.getDocumentClock(sql)).toBe(1)
122
+ })
123
+
124
+ it('respects table prefix', () => {
125
+ const sql = createWrapper({ tablePrefix: 'test_' })
126
+ expect(SQLiteSyncStorage.getDocumentClock(sql)).toBe(null)
127
+ new SQLiteSyncStorage<TLRecord>({ sql, snapshot: makeSnapshot(defaultRecords) })
128
+ expect(SQLiteSyncStorage.getDocumentClock(sql)).toBe(0)
129
+ })
130
+ })
131
+ })
132
+
133
+ describe('Constructor', () => {
134
+ it('initializes documents from snapshot', () => {
135
+ const storage = getStorage(makeSnapshot(defaultRecords))
136
+
137
+ const snapshot = storage.getSnapshot()
138
+ expect(snapshot.documents.length).toBe(2)
139
+ expect(snapshot.documents.find((d) => d.state.id === TLDOCUMENT_ID)).toBeDefined()
140
+ })
141
+
142
+ it('initializes schema from snapshot', () => {
143
+ const snapshotIn = makeSnapshot(defaultRecords)
144
+ const storage = getStorage(snapshotIn)
145
+
146
+ const snapshot = storage.getSnapshot()
147
+ expect(snapshot.schema).toEqual(snapshotIn.schema)
148
+ })
149
+
150
+ it('initializes documentClock from snapshot', () => {
151
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 42 }))
152
+
153
+ expect(storage.getClock()).toBe(42)
154
+ })
155
+
156
+ it('falls back to clock when documentClock is not present (legacy snapshot)', () => {
157
+ const storage = getStorage(makeLegacySnapshot(defaultRecords, { clock: 15 }) as RoomSnapshot)
158
+
159
+ expect(storage.getClock()).toBe(15)
160
+ })
161
+
162
+ it('falls back to 0 when neither documentClock nor clock is present', () => {
163
+ const snapshot = {
164
+ documents: defaultRecords.map((r) => ({ state: r, lastChangedClock: 0 })),
165
+ schema: tlSchema.serialize(),
166
+ } as RoomSnapshot
167
+
168
+ const storage = getStorage(snapshot)
169
+
170
+ expect(storage.getClock()).toBe(0)
171
+ })
172
+
173
+ it('initializes tombstones from snapshot', () => {
174
+ const storage = getStorage(
175
+ makeSnapshot(defaultRecords, {
176
+ tombstones: { 'shape:deleted1': 5, 'shape:deleted2': 10 },
177
+ tombstoneHistoryStartsAtClock: 0,
178
+ documentClock: 15,
179
+ })
180
+ )
181
+
182
+ const snapshot = storage.getSnapshot()
183
+ expect(Object.keys(snapshot.tombstones!).length).toBe(2)
184
+ expect(snapshot.tombstones?.['shape:deleted1']).toBe(5)
185
+ expect(snapshot.tombstones?.['shape:deleted2']).toBe(10)
186
+ })
187
+
188
+ it('sets tombstoneHistoryStartsAtClock from snapshot', () => {
189
+ const storage = getStorage(
190
+ makeSnapshot(defaultRecords, {
191
+ tombstoneHistoryStartsAtClock: 5,
192
+ documentClock: 10,
193
+ })
194
+ )
195
+
196
+ const snapshot = storage.getSnapshot()
197
+ expect(snapshot.tombstoneHistoryStartsAtClock).toBe(5)
198
+ })
199
+
200
+ it('defaults tombstoneHistoryStartsAtClock to documentClock when not provided', () => {
201
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 20 }))
202
+
203
+ const snapshot = storage.getSnapshot()
204
+ expect(snapshot.tombstoneHistoryStartsAtClock).toBe(20)
205
+ })
206
+
207
+ it('handles empty documents array', () => {
208
+ const storage = getStorage(makeSnapshot([]))
209
+
210
+ const snapshot = storage.getSnapshot()
211
+ expect(snapshot.documents.length).toBe(0)
212
+ })
213
+
214
+ it('works with table prefix', () => {
215
+ const storage = getStorage(makeSnapshot(defaultRecords), { tablePrefix: 'myapp_' })
216
+
217
+ const snapshot = storage.getSnapshot()
218
+ expect(snapshot.documents.length).toBe(2)
219
+ })
220
+
221
+ it('reinitializes storage when snapshot provided to existing tables', () => {
222
+ const sql = createWrapper()
223
+
224
+ // First initialization
225
+ new SQLiteSyncStorage<TLRecord>({
226
+ sql,
227
+ snapshot: makeSnapshot(defaultRecords, { documentClock: 10 }),
228
+ })
229
+
230
+ // Second initialization with different data
231
+ const newRecords = [DocumentRecordType.create({ id: TLDOCUMENT_ID })]
232
+ const storage2 = new SQLiteSyncStorage<TLRecord>({
233
+ sql,
234
+ snapshot: makeSnapshot(newRecords, { documentClock: 20 }),
235
+ })
236
+
237
+ const snapshot = storage2.getSnapshot()
238
+ expect(snapshot.documents.length).toBe(1)
239
+ expect(snapshot.documentClock).toBe(20)
240
+ })
241
+ })
242
+
243
+ describe('Transaction', () => {
244
+ describe('get()', () => {
245
+ it('returns record by id', () => {
246
+ const storage = getStorage(makeSnapshot(defaultRecords))
247
+
248
+ storage.transaction((txn) => {
249
+ const doc = txn.get(TLDOCUMENT_ID)
250
+ expect(doc).toBeDefined()
251
+ expect(doc?.id).toBe(TLDOCUMENT_ID)
252
+ })
253
+ })
254
+
255
+ it('returns undefined for non-existent record', () => {
256
+ const storage = getStorage(makeSnapshot(defaultRecords))
257
+
258
+ storage.transaction((txn) => {
259
+ expect(txn.get('nonexistent')).toBeUndefined()
260
+ })
261
+ })
262
+ })
263
+
264
+ describe('set()', () => {
265
+ it('creates new records', () => {
266
+ const storage = getStorage(makeSnapshot(defaultRecords))
267
+
268
+ const newPage = PageRecordType.create({
269
+ id: PageRecordType.createId('new_page'),
270
+ name: 'New Page',
271
+ index: 'a2' as IndexKey,
272
+ })
273
+
274
+ storage.transaction((txn) => {
275
+ txn.set(newPage.id, newPage)
276
+ })
277
+
278
+ const snapshot = storage.getSnapshot()
279
+ expect(snapshot.documents.length).toBe(3)
280
+ expect(snapshot.documents.find((d) => d.state.id === newPage.id)?.state).toEqual(newPage)
281
+ })
282
+
283
+ it('updates existing records', () => {
284
+ const storage = getStorage(makeSnapshot(defaultRecords))
285
+
286
+ const pageId = PageRecordType.createId('page_1')
287
+ const updatedPage = PageRecordType.create({
288
+ id: pageId,
289
+ name: 'Updated Page',
290
+ index: ZERO_INDEX_KEY,
291
+ })
292
+
293
+ storage.transaction((txn) => {
294
+ txn.set(pageId, updatedPage)
295
+ })
296
+
297
+ const snapshot = storage.getSnapshot()
298
+ expect(snapshot.documents.find((d) => d.state.id === pageId)?.state).toEqual(updatedPage)
299
+ })
300
+
301
+ it('clears tombstone when re-creating a deleted record', () => {
302
+ const pageId = PageRecordType.createId('page_to_delete')
303
+ const page = PageRecordType.create({
304
+ id: pageId,
305
+ name: 'Page',
306
+ index: 'a2' as IndexKey,
307
+ })
308
+
309
+ const storage = getStorage(makeSnapshot([...defaultRecords, page]))
310
+
311
+ // Delete the page
312
+ storage.transaction((txn) => {
313
+ txn.delete(pageId)
314
+ })
315
+
316
+ let snapshot = storage.getSnapshot()
317
+ expect(snapshot.tombstones?.[pageId]).toBeDefined()
318
+ expect(snapshot.documents.find((d) => d.state.id === pageId)).toBeUndefined()
319
+
320
+ // Re-create the page
321
+ storage.transaction((txn) => {
322
+ txn.set(pageId, page)
323
+ })
324
+
325
+ snapshot = storage.getSnapshot()
326
+ expect(snapshot.tombstones?.[pageId]).toBeUndefined()
327
+ expect(snapshot.documents.find((d) => d.state.id === pageId)).toBeDefined()
328
+ })
329
+
330
+ it('sets lastChangedClock to the incremented clock', () => {
331
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 5 }))
332
+
333
+ const newPage = PageRecordType.create({
334
+ id: PageRecordType.createId('new_page'),
335
+ name: 'New Page',
336
+ index: 'a2' as IndexKey,
337
+ })
338
+
339
+ storage.transaction((txn) => {
340
+ txn.set(newPage.id, newPage)
341
+ })
342
+
343
+ const snapshot = storage.getSnapshot()
344
+ expect(snapshot.documents.find((d) => d.state.id === newPage.id)?.lastChangedClock).toBe(6)
345
+ })
346
+ })
347
+
348
+ describe('delete()', () => {
349
+ it('removes records', () => {
350
+ const storage = getStorage(makeSnapshot(defaultRecords))
351
+
352
+ const pageId = PageRecordType.createId('page_1')
353
+
354
+ storage.transaction((txn) => {
355
+ txn.delete(pageId)
356
+ })
357
+
358
+ const snapshot = storage.getSnapshot()
359
+ expect(snapshot.documents.find((d) => d.state.id === pageId)).toBeUndefined()
360
+ })
361
+
362
+ it('creates tombstones', () => {
363
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 10 }))
364
+
365
+ const pageId = PageRecordType.createId('page_1')
366
+
367
+ storage.transaction((txn) => {
368
+ txn.delete(pageId)
369
+ })
370
+
371
+ const snapshot = storage.getSnapshot()
372
+ expect(snapshot.tombstones?.[pageId]).toBe(11) // clock incremented to 11
373
+ })
374
+ })
375
+
376
+ describe('getClock()', () => {
377
+ it('returns current clock at start of transaction', () => {
378
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 42 }))
379
+
380
+ storage.transaction((txn) => {
381
+ expect(txn.getClock()).toBe(42)
382
+ })
383
+ })
384
+
385
+ it('returns incremented clock after a write', () => {
386
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 42 }))
387
+
388
+ storage.transaction((txn) => {
389
+ expect(txn.getClock()).toBe(42)
390
+ const newPage = PageRecordType.create({
391
+ id: PageRecordType.createId('new'),
392
+ name: 'New',
393
+ index: 'a2' as IndexKey,
394
+ })
395
+ txn.set(newPage.id, newPage)
396
+ expect(txn.getClock()).toBe(43)
397
+ })
398
+ })
399
+
400
+ it('increments clock only once per transaction', () => {
401
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 10 }))
402
+
403
+ storage.transaction((txn) => {
404
+ const page1 = PageRecordType.create({
405
+ id: PageRecordType.createId('p1'),
406
+ name: 'P1',
407
+ index: 'a2' as IndexKey,
408
+ })
409
+ const page2 = PageRecordType.create({
410
+ id: PageRecordType.createId('p2'),
411
+ name: 'P2',
412
+ index: 'a3' as IndexKey,
413
+ })
414
+
415
+ txn.set(page1.id, page1)
416
+ expect(txn.getClock()).toBe(11)
417
+
418
+ txn.set(page2.id, page2)
419
+ expect(txn.getClock()).toBe(11) // Still 11, not 12
420
+
421
+ txn.delete(PageRecordType.createId('page_1'))
422
+ expect(txn.getClock()).toBe(11) // Still 11
423
+ })
424
+
425
+ expect(storage.getClock()).toBe(11)
426
+ })
427
+ })
428
+
429
+ describe('entries()', () => {
430
+ it('iterates over all documents', () => {
431
+ const storage = getStorage(makeSnapshot(defaultRecords))
432
+
433
+ storage.transaction((txn) => {
434
+ const entries = Array.from(txn.entries())
435
+ expect(entries.length).toBe(2)
436
+ expect(entries.map(([id]) => id).sort()).toEqual(defaultRecords.map((r) => r.id).sort())
437
+ })
438
+ })
439
+ })
440
+
441
+ describe('keys()', () => {
442
+ it('iterates over all document ids', () => {
443
+ const storage = getStorage(makeSnapshot(defaultRecords))
444
+
445
+ storage.transaction((txn) => {
446
+ const keys = Array.from(txn.keys())
447
+ expect(keys.sort()).toEqual(defaultRecords.map((r) => r.id).sort())
448
+ })
449
+ })
450
+ })
451
+
452
+ describe('values()', () => {
453
+ it('iterates over all document states', () => {
454
+ const storage = getStorage(makeSnapshot(defaultRecords))
455
+
456
+ storage.transaction((txn) => {
457
+ const values = Array.from(txn.values())
458
+ expect(values.length).toBe(2)
459
+ expect(values.map((v) => v.id).sort()).toEqual(defaultRecords.map((r) => r.id).sort())
460
+ })
461
+ })
462
+ })
463
+
464
+ describe('iterator consumption after transaction ends', () => {
465
+ it('throws when entries() iterator is consumed after transaction ends', () => {
466
+ const storage = getStorage(makeSnapshot(defaultRecords))
467
+
468
+ let iterator: Iterator<[string, TLRecord]>
469
+
470
+ storage.transaction((txn) => {
471
+ iterator = txn.entries()[Symbol.iterator]()
472
+ // Consume one item inside the transaction - should work
473
+ iterator.next()
474
+ })
475
+
476
+ // Trying to consume more after transaction ends should throw
477
+ expect(() => iterator.next()).toThrow('Transaction has ended')
478
+ })
479
+
480
+ it('throws when keys() iterator is consumed after transaction ends', () => {
481
+ const storage = getStorage(makeSnapshot(defaultRecords))
482
+
483
+ let iterator: Iterator<string>
484
+
485
+ storage.transaction((txn) => {
486
+ iterator = txn.keys()[Symbol.iterator]()
487
+ iterator.next()
488
+ })
489
+
490
+ expect(() => iterator.next()).toThrow('Transaction has ended')
491
+ })
492
+
493
+ it('throws when values() iterator is consumed after transaction ends', () => {
494
+ const storage = getStorage(makeSnapshot(defaultRecords))
495
+
496
+ let iterator: Iterator<TLRecord>
497
+
498
+ storage.transaction((txn) => {
499
+ iterator = txn.values()[Symbol.iterator]()
500
+ iterator.next()
501
+ })
502
+
503
+ expect(() => iterator.next()).toThrow('Transaction has ended')
504
+ })
505
+
506
+ it('allows full consumption of iterator within transaction', () => {
507
+ const storage = getStorage(makeSnapshot(defaultRecords))
508
+
509
+ storage.transaction((txn) => {
510
+ // Should be able to fully consume all iterators
511
+ const entries = Array.from(txn.entries())
512
+ const keys = Array.from(txn.keys())
513
+ const values = Array.from(txn.values())
514
+
515
+ expect(entries.length).toBe(2)
516
+ expect(keys.length).toBe(2)
517
+ expect(values.length).toBe(2)
518
+ })
519
+ })
520
+ })
521
+
522
+ describe('getSchema() / setSchema()', () => {
523
+ it('gets the current schema', () => {
524
+ const snapshotIn = makeSnapshot(defaultRecords)
525
+ const storage = getStorage(snapshotIn)
526
+
527
+ storage.transaction((txn) => {
528
+ expect(txn.getSchema()).toEqual(snapshotIn.schema)
529
+ })
530
+ })
531
+
532
+ it('sets the schema', () => {
533
+ const storage = getStorage(makeSnapshot(defaultRecords))
534
+
535
+ const newSchema = { ...tlSchema.serialize(), schemaVersion: 99 as any }
536
+
537
+ storage.transaction((txn) => {
538
+ txn.setSchema(newSchema)
539
+ })
540
+
541
+ const snapshot = storage.getSnapshot()
542
+ expect(snapshot.schema?.schemaVersion).toBe(99)
543
+ })
544
+ })
545
+
546
+ describe('transaction result', () => {
547
+ it('returns result from callback', () => {
548
+ const storage = getStorage(makeSnapshot(defaultRecords))
549
+
550
+ const { result } = storage.transaction((txn) => {
551
+ return txn.get(TLDOCUMENT_ID)
552
+ })
553
+
554
+ expect(result?.id).toBe(TLDOCUMENT_ID)
555
+ })
556
+
557
+ it('returns didChange: false when no writes occur', () => {
558
+ const storage = getStorage(makeSnapshot(defaultRecords))
559
+
560
+ const { didChange, documentClock } = storage.transaction((txn) => {
561
+ txn.get(TLDOCUMENT_ID)
562
+ })
563
+
564
+ expect(didChange).toBe(false)
565
+ expect(documentClock).toBe(0)
566
+ })
567
+
568
+ it('returns didChange: true when writes occur', () => {
569
+ const storage = getStorage(makeSnapshot(defaultRecords))
570
+
571
+ const newPage = PageRecordType.create({
572
+ id: PageRecordType.createId('new'),
573
+ name: 'New',
574
+ index: 'a2' as IndexKey,
575
+ })
576
+
577
+ const { didChange, documentClock } = storage.transaction((txn) => {
578
+ txn.set(newPage.id, newPage)
579
+ })
580
+
581
+ expect(didChange).toBe(true)
582
+ expect(documentClock).toBe(1)
583
+ })
584
+
585
+ it('throws when callback returns a promise', () => {
586
+ const storage = getStorage(makeSnapshot(defaultRecords))
587
+
588
+ expect(() => {
589
+ storage.transaction(() => Promise.resolve() as any)
590
+ }).toThrow('Transaction must return a value, not a promise')
591
+ })
592
+ })
593
+ })
594
+
595
+ describe('getChangesSince', () => {
596
+ it('returns puts for records changed after sinceClock', () => {
597
+ const storage = getStorage(
598
+ makeSnapshot(defaultRecords, {
599
+ documents: [
600
+ { state: defaultRecords[0], lastChangedClock: 5 },
601
+ { state: defaultRecords[1], lastChangedClock: 10 },
602
+ ],
603
+ documentClock: 15,
604
+ tombstoneHistoryStartsAtClock: 0,
605
+ })
606
+ )
607
+
608
+ storage.transaction((txn) => {
609
+ const changes = txn.getChangesSince(7)!
610
+ const puts = Object.values(changes.diff.puts)
611
+
612
+ expect(puts.length).toBe(1)
613
+ expect((puts[0] as TLRecord).id).toBe(defaultRecords[1].id) // only record with clock 10 > 7
614
+ expect(changes.wipeAll).toBe(false)
615
+ })
616
+ })
617
+
618
+ it('returns all records when sinceClock is before all changes', () => {
619
+ const storage = getStorage(
620
+ makeSnapshot(defaultRecords, {
621
+ documents: [
622
+ { state: defaultRecords[0], lastChangedClock: 5 },
623
+ { state: defaultRecords[1], lastChangedClock: 10 },
624
+ ],
625
+ documentClock: 15,
626
+ tombstoneHistoryStartsAtClock: 0,
627
+ })
628
+ )
629
+
630
+ storage.transaction((txn) => {
631
+ const changes = txn.getChangesSince(0)!
632
+ const puts = Object.values(changes.diff.puts)
633
+
634
+ expect(puts.length).toBe(2)
635
+ })
636
+ })
637
+
638
+ it('returns deletes for tombstones after sinceClock', () => {
639
+ const storage = getStorage(
640
+ makeSnapshot(defaultRecords, {
641
+ tombstones: {
642
+ 'shape:deleted1': 5,
643
+ 'shape:deleted2': 12,
644
+ },
645
+ documentClock: 15,
646
+ tombstoneHistoryStartsAtClock: 0,
647
+ })
648
+ )
649
+
650
+ storage.transaction((txn) => {
651
+ const changes = txn.getChangesSince(7)!
652
+
653
+ expect(changes.diff.deletes).toEqual(['shape:deleted2']) // only tombstone with clock 12 > 7
654
+ })
655
+ })
656
+
657
+ it('returns wipeAll: true when sinceClock < tombstoneHistoryStartsAtClock', () => {
658
+ const storage = getStorage(
659
+ makeSnapshot(defaultRecords, {
660
+ documentClock: 20,
661
+ tombstoneHistoryStartsAtClock: 10,
662
+ })
663
+ )
664
+
665
+ storage.transaction((txn) => {
666
+ const changes = txn.getChangesSince(5)! // 5 < 10
667
+
668
+ expect(changes.wipeAll).toBe(true)
669
+ // When wipeAll is true, all documents are returned
670
+ const puts = Object.values(changes.diff.puts)
671
+ expect(puts.length).toBe(2)
672
+ })
673
+ })
674
+
675
+ it('returns wipeAll: false when sinceClock >= tombstoneHistoryStartsAtClock', () => {
676
+ const storage = getStorage(
677
+ makeSnapshot(defaultRecords, {
678
+ documentClock: 20,
679
+ tombstoneHistoryStartsAtClock: 10,
680
+ })
681
+ )
682
+
683
+ storage.transaction((txn) => {
684
+ const changes = txn.getChangesSince(15)!
685
+
686
+ expect(changes.wipeAll).toBe(false)
687
+ })
688
+ })
689
+
690
+ it('returns undefined when no changes since clock', () => {
691
+ const storage = getStorage(
692
+ makeSnapshot(defaultRecords, {
693
+ documents: [
694
+ { state: defaultRecords[0], lastChangedClock: 5 },
695
+ { state: defaultRecords[1], lastChangedClock: 10 },
696
+ ],
697
+ documentClock: 15,
698
+ tombstoneHistoryStartsAtClock: 0,
699
+ })
700
+ )
701
+
702
+ storage.transaction((txn) => {
703
+ const changes = txn.getChangesSince(15)
704
+ expect(changes).toBeUndefined()
705
+ })
706
+ })
707
+ })
708
+
709
+ describe('onChange', () => {
710
+ it('accepts onChange callback in constructor', async () => {
711
+ const listener = vi.fn()
712
+ const sql = createWrapper()
713
+ const storage = new SQLiteSyncStorage<TLRecord>({
714
+ sql,
715
+ snapshot: makeSnapshot(defaultRecords),
716
+ onChange: listener,
717
+ })
718
+
719
+ await Promise.resolve()
720
+
721
+ const newPage = PageRecordType.create({
722
+ id: PageRecordType.createId('new'),
723
+ name: 'New',
724
+ index: 'a2' as IndexKey,
725
+ })
726
+
727
+ storage.transaction((txn) => {
728
+ txn.set(newPage.id, newPage)
729
+ })
730
+
731
+ await Promise.resolve()
732
+
733
+ expect(listener).toHaveBeenCalledTimes(1)
734
+ expect(listener).toHaveBeenCalledWith(
735
+ expect.objectContaining({
736
+ documentClock: 1,
737
+ })
738
+ )
739
+ })
740
+
741
+ it('notifies listeners after changes', async () => {
742
+ const storage = getStorage(makeSnapshot(defaultRecords))
743
+
744
+ const listener = vi.fn()
745
+ storage.onChange(listener)
746
+
747
+ // Wait for listener registration (microtask)
748
+ await Promise.resolve()
749
+
750
+ const newPage = PageRecordType.create({
751
+ id: PageRecordType.createId('new'),
752
+ name: 'New',
753
+ index: 'a2' as IndexKey,
754
+ })
755
+
756
+ storage.transaction((txn) => {
757
+ txn.set(newPage.id, newPage)
758
+ })
759
+
760
+ // Wait for notification (microtask)
761
+ await Promise.resolve()
762
+
763
+ expect(listener).toHaveBeenCalledTimes(1)
764
+ })
765
+
766
+ it('receives correct documentClock', async () => {
767
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 10 }))
768
+
769
+ const listener = vi.fn()
770
+ storage.onChange(listener)
771
+
772
+ await Promise.resolve()
773
+
774
+ const newPage = PageRecordType.create({
775
+ id: PageRecordType.createId('new'),
776
+ name: 'New',
777
+ index: 'a2' as IndexKey,
778
+ })
779
+
780
+ storage.transaction((txn) => {
781
+ txn.set(newPage.id, newPage)
782
+ })
783
+
784
+ await Promise.resolve()
785
+
786
+ expect(listener).toHaveBeenCalledWith(
787
+ expect.objectContaining({
788
+ documentClock: 11,
789
+ })
790
+ )
791
+ })
792
+
793
+ it('receives transaction id when provided', async () => {
794
+ const storage = getStorage(makeSnapshot(defaultRecords))
795
+
796
+ const listener = vi.fn()
797
+ storage.onChange(listener)
798
+
799
+ await Promise.resolve()
800
+
801
+ const newPage = PageRecordType.create({
802
+ id: PageRecordType.createId('new'),
803
+ name: 'New',
804
+ index: 'a2' as IndexKey,
805
+ })
806
+
807
+ storage.transaction(
808
+ (txn) => {
809
+ txn.set(newPage.id, newPage)
810
+ },
811
+ { id: 'my-transaction-id' }
812
+ )
813
+
814
+ await Promise.resolve()
815
+
816
+ expect(listener).toHaveBeenCalledWith(
817
+ expect.objectContaining({
818
+ id: 'my-transaction-id',
819
+ })
820
+ )
821
+ })
822
+
823
+ it('unsubscribe prevents future notifications', async () => {
824
+ const storage = getStorage(makeSnapshot(defaultRecords))
825
+
826
+ const listener = vi.fn()
827
+ const unsubscribe = storage.onChange(listener)
828
+
829
+ await Promise.resolve()
830
+
831
+ // Unsubscribe immediately
832
+ unsubscribe()
833
+
834
+ const newPage = PageRecordType.create({
835
+ id: PageRecordType.createId('new'),
836
+ name: 'New',
837
+ index: 'a2' as IndexKey,
838
+ })
839
+
840
+ storage.transaction((txn) => {
841
+ txn.set(newPage.id, newPage)
842
+ })
843
+
844
+ await Promise.resolve()
845
+
846
+ expect(listener).not.toHaveBeenCalled()
847
+ })
848
+
849
+ it('does not notify for read-only transactions', async () => {
850
+ const storage = getStorage(makeSnapshot(defaultRecords))
851
+
852
+ const listener = vi.fn()
853
+ storage.onChange(listener)
854
+
855
+ await Promise.resolve()
856
+
857
+ storage.transaction((txn) => {
858
+ txn.get(TLDOCUMENT_ID) // read only
859
+ })
860
+
861
+ await Promise.resolve()
862
+
863
+ expect(listener).not.toHaveBeenCalled()
864
+ })
865
+ })
866
+
867
+ describe('getSnapshot', () => {
868
+ it('returns correct snapshot structure', () => {
869
+ const storage = getStorage(
870
+ makeSnapshot(defaultRecords, {
871
+ documentClock: 15,
872
+ tombstoneHistoryStartsAtClock: 5,
873
+ tombstones: { 'shape:deleted': 10 },
874
+ })
875
+ )
876
+
877
+ const snapshot = storage.getSnapshot()
878
+
879
+ expect(snapshot.documentClock).toBe(15)
880
+ expect(snapshot.tombstoneHistoryStartsAtClock).toBe(5)
881
+ expect(snapshot.documents.length).toBe(2)
882
+ expect(snapshot.tombstones).toEqual({ 'shape:deleted': 10 })
883
+ expect(snapshot.schema).toEqual(tlSchema.serialize())
884
+ })
885
+
886
+ it('reflects changes from transactions', () => {
887
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 0 }))
888
+
889
+ const newPage = PageRecordType.create({
890
+ id: PageRecordType.createId('new'),
891
+ name: 'New',
892
+ index: 'a2' as IndexKey,
893
+ })
894
+
895
+ storage.transaction((txn) => {
896
+ txn.set(newPage.id, newPage)
897
+ })
898
+
899
+ const snapshot = storage.getSnapshot()
900
+
901
+ expect(snapshot.documentClock).toBe(1)
902
+ expect(snapshot.documents.length).toBe(3)
903
+ expect(snapshot.documents.find((d) => d.state.id === newPage.id)).toBeDefined()
904
+ })
905
+ })
906
+
907
+ describe('Edge cases', () => {
908
+ describe('Transaction error handling', () => {
909
+ it('does not increment clock if transaction throws', () => {
910
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 10 }))
911
+
912
+ expect(() => {
913
+ storage.transaction(() => {
914
+ throw new Error('Oops!')
915
+ })
916
+ }).toThrow('Oops!')
917
+
918
+ // Clock should not have changed
919
+ expect(storage.getClock()).toBe(10)
920
+ })
921
+
922
+ it('rolls back changes if transaction throws after a write', () => {
923
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 10 }))
924
+
925
+ const newPage = PageRecordType.create({
926
+ id: PageRecordType.createId('new'),
927
+ name: 'New',
928
+ index: 'a2' as IndexKey,
929
+ })
930
+
931
+ expect(() => {
932
+ storage.transaction((txn) => {
933
+ txn.set(newPage.id, newPage)
934
+ throw new Error('Oops after write!')
935
+ })
936
+ }).toThrow('Oops after write!')
937
+
938
+ // Document should not have been added (rolled back)
939
+ const snapshot = storage.getSnapshot()
940
+ expect(snapshot.documents.find((d) => d.state.id === newPage.id)).toBeUndefined()
941
+ // Clock should not have changed
942
+ expect(storage.getClock()).toBe(10)
943
+ })
944
+ })
945
+
946
+ describe('Deleting non-existent records', () => {
947
+ it('does not create a tombstone for records that never existed', () => {
948
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 5 }))
949
+
950
+ storage.transaction((txn) => {
951
+ txn.delete('nonexistent:record')
952
+ })
953
+
954
+ // No tombstone should be created for a record that never existed
955
+ const snapshot = storage.getSnapshot()
956
+ expect(snapshot.tombstones?.['nonexistent:record']).toBeUndefined()
957
+ // Clock should not be incremented since nothing changed
958
+ expect(storage.getClock()).toBe(5)
959
+ })
960
+
961
+ it('does not increment clock when deleting non-existent record', () => {
962
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 10 }))
963
+
964
+ const { didChange, documentClock } = storage.transaction((txn) => {
965
+ txn.delete('nonexistent:record')
966
+ })
967
+
968
+ expect(didChange).toBe(false)
969
+ expect(documentClock).toBe(10)
970
+ })
971
+ })
972
+
973
+ describe('Set with mismatched ID', () => {
974
+ it('throws when key does not match record.id', () => {
975
+ const storage = getStorage(makeSnapshot(defaultRecords))
976
+
977
+ const page = PageRecordType.create({
978
+ id: PageRecordType.createId('actual_id'),
979
+ name: 'Test',
980
+ index: 'a2' as IndexKey,
981
+ })
982
+
983
+ // Attempting to store with a different key than the record's id should throw
984
+ expect(() => {
985
+ storage.transaction((txn) => {
986
+ txn.set('different:key', page)
987
+ })
988
+ }).toThrow('Record id mismatch: key does not match record.id')
989
+ })
990
+
991
+ it('succeeds when key matches record.id', () => {
992
+ const storage = getStorage(makeSnapshot(defaultRecords))
993
+
994
+ const page = PageRecordType.create({
995
+ id: PageRecordType.createId('my_page'),
996
+ name: 'Test',
997
+ index: 'a2' as IndexKey,
998
+ })
999
+
1000
+ // Store with matching key
1001
+ storage.transaction((txn) => {
1002
+ txn.set(page.id, page)
1003
+ })
1004
+
1005
+ const snapshot = storage.getSnapshot()
1006
+ expect(snapshot.documents.find((d) => d.state.id === page.id)).toBeDefined()
1007
+ })
1008
+ })
1009
+
1010
+ describe('getChangesSince boundary conditions', () => {
1011
+ it('sinceClock exactly equal to tombstoneHistoryStartsAtClock is NOT wipeAll', () => {
1012
+ const storage = getStorage(
1013
+ makeSnapshot(defaultRecords, {
1014
+ documentClock: 20,
1015
+ tombstoneHistoryStartsAtClock: 10,
1016
+ })
1017
+ )
1018
+
1019
+ storage.transaction((txn) => {
1020
+ // sinceClock === tombstoneHistoryStartsAtClock
1021
+ const changes = txn.getChangesSince(10)!
1022
+ expect(changes.wipeAll).toBe(false)
1023
+ })
1024
+ })
1025
+
1026
+ it('sinceClock one less than tombstoneHistoryStartsAtClock IS wipeAll', () => {
1027
+ const storage = getStorage(
1028
+ makeSnapshot(defaultRecords, {
1029
+ documentClock: 20,
1030
+ tombstoneHistoryStartsAtClock: 10,
1031
+ })
1032
+ )
1033
+
1034
+ storage.transaction((txn) => {
1035
+ const changes = txn.getChangesSince(9)!
1036
+ expect(changes.wipeAll).toBe(true)
1037
+ })
1038
+ })
1039
+
1040
+ it('handles negative sinceClock', () => {
1041
+ const storage = getStorage(
1042
+ makeSnapshot(defaultRecords, {
1043
+ documentClock: 10,
1044
+ tombstoneHistoryStartsAtClock: 0,
1045
+ })
1046
+ )
1047
+
1048
+ storage.transaction((txn) => {
1049
+ const changes = txn.getChangesSince(-1)!
1050
+ // -1 < 0, so wipeAll should be true
1051
+ expect(changes.wipeAll).toBe(true)
1052
+ // All documents should be returned
1053
+ expect(Object.values(changes.diff.puts).length).toBe(2)
1054
+ })
1055
+ })
1056
+
1057
+ it('returns undefined when sinceClock equals current documentClock', () => {
1058
+ const storage = getStorage(
1059
+ makeSnapshot(defaultRecords, {
1060
+ documents: [
1061
+ { state: defaultRecords[0], lastChangedClock: 5 },
1062
+ { state: defaultRecords[1], lastChangedClock: 10 },
1063
+ ],
1064
+ documentClock: 10,
1065
+ tombstoneHistoryStartsAtClock: 0,
1066
+ })
1067
+ )
1068
+
1069
+ storage.transaction((txn) => {
1070
+ const changes = txn.getChangesSince(10)
1071
+ expect(changes).toBeUndefined()
1072
+ })
1073
+ })
1074
+
1075
+ it('returns all changes when sinceClock is greater than documentClock', () => {
1076
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 10 }))
1077
+
1078
+ storage.transaction((txn) => {
1079
+ const changes = txn.getChangesSince(100)!
1080
+ expect(Object.values(changes.diff.puts).length).toBe(2)
1081
+ expect(changes.wipeAll).toBe(true)
1082
+ })
1083
+ })
1084
+ })
1085
+
1086
+ describe('Transaction result consistency', () => {
1087
+ it('didChange reflects whether clock was incremented', () => {
1088
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 10 }))
1089
+
1090
+ // Read-only transaction
1091
+ const readResult = storage.transaction((txn) => {
1092
+ txn.get(TLDOCUMENT_ID)
1093
+ return 'read'
1094
+ })
1095
+
1096
+ expect(readResult.didChange).toBe(false)
1097
+ expect(readResult.documentClock).toBe(10)
1098
+
1099
+ // Write transaction
1100
+ const writeResult = storage.transaction((txn) => {
1101
+ txn.set(TLDOCUMENT_ID, defaultRecords[0])
1102
+ return 'write'
1103
+ })
1104
+
1105
+ expect(writeResult.didChange).toBe(true)
1106
+ expect(writeResult.documentClock).toBe(11)
1107
+ })
1108
+
1109
+ it('documentClock in result matches storage.getClock()', () => {
1110
+ const storage = getStorage(makeSnapshot(defaultRecords, { documentClock: 5 }))
1111
+
1112
+ const result = storage.transaction((txn) => {
1113
+ const page = PageRecordType.create({
1114
+ id: PageRecordType.createId('new'),
1115
+ name: 'New',
1116
+ index: 'a2' as IndexKey,
1117
+ })
1118
+ txn.set(page.id, page)
1119
+ })
1120
+
1121
+ expect(result.documentClock).toBe(storage.getClock())
1122
+ expect(result.documentClock).toBe(6)
1123
+ })
1124
+ })
1125
+ })
1126
+
1127
+ describe('pruneTombstones', () => {
1128
+ // Helper to create a snapshot with many tombstones
1129
+ function makeTombstoneMap(count: number, clockFn: (i: number) => number = (i) => i + 1) {
1130
+ const tombstones: Record<string, number> = {}
1131
+ for (let i = 0; i < count; i++) {
1132
+ tombstones[`doc${i}`] = clockFn(i)
1133
+ }
1134
+ return tombstones
1135
+ }
1136
+
1137
+ it('does not prune when below MAX_TOMBSTONES', () => {
1138
+ const tombstoneCount = Math.floor(MAX_TOMBSTONES / 2)
1139
+ const storage = getStorage(
1140
+ makeSnapshot(defaultRecords, {
1141
+ documentClock: tombstoneCount + 1,
1142
+ tombstoneHistoryStartsAtClock: 0,
1143
+ tombstones: makeTombstoneMap(tombstoneCount),
1144
+ })
1145
+ )
1146
+
1147
+ const snapshotBefore = storage.getSnapshot()
1148
+ const initialCount = Object.keys(snapshotBefore.tombstones!).length
1149
+ const initialHistoryClock = snapshotBefore.tombstoneHistoryStartsAtClock
1150
+
1151
+ // Schedule the throttled function then force it to run
1152
+ storage.pruneTombstones()
1153
+ storage.pruneTombstones.flush()
1154
+
1155
+ const snapshotAfter = storage.getSnapshot()
1156
+ expect(Object.keys(snapshotAfter.tombstones!).length).toBe(initialCount)
1157
+ expect(snapshotAfter.tombstoneHistoryStartsAtClock).toBe(initialHistoryClock)
1158
+ })
1159
+
1160
+ it('prunes when exceeding MAX_TOMBSTONES', () => {
1161
+ const totalTombstones = MAX_TOMBSTONES + 500
1162
+ const storage = getStorage(
1163
+ makeSnapshot(defaultRecords, {
1164
+ documentClock: totalTombstones + 1,
1165
+ tombstoneHistoryStartsAtClock: 0,
1166
+ tombstones: makeTombstoneMap(totalTombstones),
1167
+ })
1168
+ )
1169
+
1170
+ const snapshotBefore = storage.getSnapshot()
1171
+ expect(Object.keys(snapshotBefore.tombstones!).length).toBe(totalTombstones)
1172
+
1173
+ // Schedule the throttled function then force it to run
1174
+ storage.pruneTombstones()
1175
+ storage.pruneTombstones.flush()
1176
+
1177
+ const snapshotAfter = storage.getSnapshot()
1178
+ expect(Object.keys(snapshotAfter.tombstones!).length).toBeLessThan(totalTombstones)
1179
+ expect(Object.keys(snapshotAfter.tombstones!).length).toBeLessThanOrEqual(
1180
+ MAX_TOMBSTONES - TOMBSTONE_PRUNE_BUFFER_SIZE
1181
+ )
1182
+ expect(snapshotAfter.tombstoneHistoryStartsAtClock).toBeGreaterThan(0)
1183
+ })
1184
+
1185
+ it('updates tombstoneHistoryStartsAtClock correctly', () => {
1186
+ const totalTombstones = MAX_TOMBSTONES * 2
1187
+ const storage = getStorage(
1188
+ makeSnapshot(defaultRecords, {
1189
+ documentClock: totalTombstones + 1000,
1190
+ tombstoneHistoryStartsAtClock: 0,
1191
+ tombstones: makeTombstoneMap(totalTombstones),
1192
+ })
1193
+ )
1194
+
1195
+ const snapshotBefore = storage.getSnapshot()
1196
+ const initialHistoryClock = snapshotBefore.tombstoneHistoryStartsAtClock
1197
+
1198
+ // Schedule the throttled function then force it to run
1199
+ storage.pruneTombstones()
1200
+ storage.pruneTombstones.flush()
1201
+
1202
+ const snapshotAfter = storage.getSnapshot()
1203
+
1204
+ // History clock should have advanced
1205
+ expect(snapshotAfter.tombstoneHistoryStartsAtClock).toBeGreaterThan(initialHistoryClock!)
1206
+
1207
+ // The algorithm deletes the OLDEST tombstones and keeps the NEWEST ones.
1208
+ // tombstoneHistoryStartsAtClock is set to the oldest REMAINING clock.
1209
+ // Remaining tombstones have clocks >= tombstoneHistoryStartsAtClock.
1210
+ const historyClock = snapshotAfter.tombstoneHistoryStartsAtClock
1211
+ for (const clock of Object.values(snapshotAfter.tombstones!)) {
1212
+ expect(clock).toBeGreaterThanOrEqual(historyClock!)
1213
+ }
1214
+ })
1215
+
1216
+ it('handles duplicate clock values across tombstones', () => {
1217
+ const totalTombstones = MAX_TOMBSTONES + 1
1218
+ const expectedCutoff = TOMBSTONE_PRUNE_BUFFER_SIZE + 1
1219
+ const overflow = 10
1220
+ const boundary = expectedCutoff + overflow
1221
+ const lowerClockVal = 1
1222
+ const upperClockVal = 2
1223
+
1224
+ const tombstones: Record<string, number> = {}
1225
+ for (let i = 0; i < totalTombstones; i++) {
1226
+ tombstones[`doc${i}`] = i < boundary ? lowerClockVal : upperClockVal
1227
+ }
1228
+
1229
+ const storage = getStorage(
1230
+ makeSnapshot(defaultRecords, {
1231
+ documentClock: 3,
1232
+ tombstoneHistoryStartsAtClock: 0,
1233
+ tombstones,
1234
+ })
1235
+ )
1236
+
1237
+ // Schedule the throttled function then force it to run
1238
+ storage.pruneTombstones()
1239
+ storage.pruneTombstones.flush()
1240
+
1241
+ const snapshot = storage.getSnapshot()
1242
+ expect(Object.keys(snapshot.tombstones!).length).toEqual(
1243
+ MAX_TOMBSTONES - TOMBSTONE_PRUNE_BUFFER_SIZE - overflow
1244
+ )
1245
+ expect(Object.values(snapshot.tombstones!).every((clock) => clock === upperClockVal)).toBe(
1246
+ true
1247
+ )
1248
+ })
1249
+
1250
+ it('handles all tombstones with same clock value', () => {
1251
+ const totalTombstones = MAX_TOMBSTONES * 2
1252
+ const sameClock = 100
1253
+
1254
+ const storage = getStorage(
1255
+ makeSnapshot(defaultRecords, {
1256
+ documentClock: 1000,
1257
+ tombstoneHistoryStartsAtClock: 0,
1258
+ tombstones: makeTombstoneMap(totalTombstones, () => sameClock),
1259
+ })
1260
+ )
1261
+
1262
+ // Schedule the throttled function then force it to run
1263
+ storage.pruneTombstones()
1264
+ storage.pruneTombstones.flush()
1265
+
1266
+ const snapshot = storage.getSnapshot()
1267
+ // When all have same clock, the cutoff extends to include all of them,
1268
+ // so all are pruned and history starts at documentClock
1269
+ expect(Object.keys(snapshot.tombstones!).length).toBe(0)
1270
+ expect(snapshot.tombstoneHistoryStartsAtClock).toBe(1000) // documentClock
1271
+ })
1272
+
1273
+ it('does not prune at exactly MAX_TOMBSTONES', () => {
1274
+ const storage = getStorage(
1275
+ makeSnapshot(defaultRecords, {
1276
+ documentClock: MAX_TOMBSTONES + 1,
1277
+ tombstoneHistoryStartsAtClock: 0,
1278
+ tombstones: makeTombstoneMap(MAX_TOMBSTONES),
1279
+ })
1280
+ )
1281
+
1282
+ // Schedule the throttled function then force it to run
1283
+ storage.pruneTombstones()
1284
+ storage.pruneTombstones.flush()
1285
+
1286
+ const snapshot = storage.getSnapshot()
1287
+ // Should not prune at exactly the threshold
1288
+ expect(Object.keys(snapshot.tombstones!).length).toBe(MAX_TOMBSTONES)
1289
+ })
1290
+ })
1291
+
1292
+ describe('Migration from TEXT to BLOB', () => {
1293
+ it('migrates existing TEXT data to BLOB format', () => {
1294
+ // Simulate a database created with the old schema (migration version 1, TEXT column)
1295
+ const db = new DatabaseSync(':memory:')
1296
+
1297
+ // Create old schema with TEXT column (migration version 1)
1298
+ db.exec(`
1299
+ CREATE TABLE documents (
1300
+ id TEXT PRIMARY KEY,
1301
+ state TEXT NOT NULL,
1302
+ lastChangedClock INTEGER NOT NULL
1303
+ );
1304
+ CREATE INDEX idx_documents_lastChangedClock ON documents(lastChangedClock);
1305
+ CREATE TABLE tombstones (
1306
+ id TEXT PRIMARY KEY,
1307
+ clock INTEGER NOT NULL
1308
+ );
1309
+ CREATE INDEX idx_tombstones_clock ON tombstones(clock);
1310
+ CREATE TABLE metadata (
1311
+ migrationVersion INTEGER NOT NULL,
1312
+ documentClock INTEGER NOT NULL,
1313
+ tombstoneHistoryStartsAtClock INTEGER NOT NULL,
1314
+ schema TEXT NOT NULL
1315
+ );
1316
+ INSERT INTO metadata (migrationVersion, documentClock, tombstoneHistoryStartsAtClock, schema)
1317
+ VALUES (1, 5, 0, '${JSON.stringify(tlSchema.serialize()).replace(/'/g, "''")}');
1318
+ `)
1319
+
1320
+ // Insert documents using the old TEXT format
1321
+ const doc1 = DocumentRecordType.create({ id: TLDOCUMENT_ID })
1322
+ const page1 = PageRecordType.create({
1323
+ id: PageRecordType.createId('migrated_page'),
1324
+ name: 'Migrated Page',
1325
+ index: ZERO_INDEX_KEY,
1326
+ })
1327
+
1328
+ const insertStmt = db.prepare(
1329
+ 'INSERT INTO documents (id, state, lastChangedClock) VALUES (?, ?, ?)'
1330
+ )
1331
+ insertStmt.run(doc1.id, JSON.stringify(doc1), 1)
1332
+ insertStmt.run(page1.id, JSON.stringify(page1), 2)
1333
+
1334
+ // Add a tombstone
1335
+ db.exec("INSERT INTO tombstones (id, clock) VALUES ('shape:deleted', 3)")
1336
+
1337
+ // Now create SQLiteSyncStorage which should trigger the migration
1338
+ const sql = new NodeSqliteWrapper(db)
1339
+ const storage = new SQLiteSyncStorage<TLRecord>({ sql })
1340
+
1341
+ // Verify data is accessible after migration
1342
+ const snapshot = storage.getSnapshot()
1343
+ expect(snapshot.documents.length).toBe(2)
1344
+ expect(snapshot.documentClock).toBe(5)
1345
+ expect(snapshot.tombstones?.['shape:deleted']).toBe(3)
1346
+
1347
+ // Verify we can read specific records
1348
+ storage.transaction((txn) => {
1349
+ const doc = txn.get(TLDOCUMENT_ID)
1350
+ expect(doc).toBeDefined()
1351
+ expect(doc?.id).toBe(TLDOCUMENT_ID)
1352
+
1353
+ const page = txn.get(page1.id)
1354
+ expect(page).toBeDefined()
1355
+ expect((page as any)?.name).toBe('Migrated Page')
1356
+ })
1357
+
1358
+ // Verify we can still write new records
1359
+ const newPage = PageRecordType.create({
1360
+ id: PageRecordType.createId('new_after_migration'),
1361
+ name: 'New After Migration',
1362
+ index: 'a2' as IndexKey,
1363
+ })
1364
+
1365
+ storage.transaction((txn) => {
1366
+ txn.set(newPage.id, newPage)
1367
+ })
1368
+
1369
+ const snapshotAfter = storage.getSnapshot()
1370
+ expect(snapshotAfter.documents.length).toBe(3)
1371
+ expect(snapshotAfter.documents.find((d) => d.state.id === newPage.id)).toBeDefined()
1372
+ })
1373
+
1374
+ it('preserves migration version 2 for fresh databases', () => {
1375
+ const sql = createWrapper()
1376
+ new SQLiteSyncStorage<TLRecord>({ sql, snapshot: makeSnapshot(defaultRecords) })
1377
+
1378
+ // Check the migration version is 2
1379
+ const row = sql
1380
+ .prepare<{ migrationVersion: number }>('SELECT migrationVersion FROM metadata')
1381
+ .all()[0]
1382
+ expect(row?.migrationVersion).toBe(2)
1383
+ })
1384
+ })
1385
+
1386
+ describe('Schema migrations via migrateStorage', () => {
1387
+ /**
1388
+ * Regression test for a bug where record-level migrations were applied multiple
1389
+ * times to the same record when using SQLite storage.
1390
+ *
1391
+ * Root cause: SQLite iterators are live cursors. When a record is updated via
1392
+ * storage.set(), the lastChangedClock column changes, which can reposition the
1393
+ * row in the B-tree index and cause it to be visited again by the iterator.
1394
+ *
1395
+ * Fix: migrateStorage now collects updates during iteration and applies them
1396
+ * after the iteration completes.
1397
+ */
1398
+ it('should apply record migrations exactly once per record', () => {
1399
+ // Track how many times each record is migrated
1400
+ const migrationCounts = new Map<string, number>()
1401
+
1402
+ interface TestRecord extends BaseRecord<'test', RecordId<TestRecord>> {
1403
+ value: number
1404
+ migrated?: boolean
1405
+ }
1406
+
1407
+ const TestRecordType = createRecordType<TestRecord>('test', {
1408
+ validator: { validate: (r) => r as TestRecord },
1409
+ scope: 'document',
1410
+ })
1411
+
1412
+ const testMigrations = createMigrationSequence({
1413
+ sequenceId: 'com.test.record',
1414
+ retroactive: true,
1415
+ sequence: [
1416
+ {
1417
+ id: 'com.test.record/1',
1418
+ scope: 'record',
1419
+ filter: (r: any) => r.typeName === 'test',
1420
+ up: (record: any) => {
1421
+ // Track how many times this record is migrated
1422
+ const count = migrationCounts.get(record.id) ?? 0
1423
+ migrationCounts.set(record.id, count + 1)
1424
+ record.migrated = true
1425
+ },
1426
+ },
1427
+ ],
1428
+ })
1429
+
1430
+ const oldSchema = StoreSchema.create({ test: TestRecordType })
1431
+ const newSchema = StoreSchema.create(
1432
+ { test: TestRecordType },
1433
+ { migrations: [testMigrations] }
1434
+ )
1435
+
1436
+ // Create storage with the old schema (no migrations applied)
1437
+ const sql = createWrapper()
1438
+ const numRecords = 100
1439
+
1440
+ // Build initial snapshot with many test records
1441
+ const testRecords: TestRecord[] = []
1442
+ for (let i = 0; i < numRecords; i++) {
1443
+ testRecords.push({
1444
+ id: `test:${i}` as RecordId<TestRecord>,
1445
+ typeName: 'test',
1446
+ value: i,
1447
+ })
1448
+ }
1449
+
1450
+ const snapshot = {
1451
+ documents: testRecords.map((r) => ({ state: r, lastChangedClock: 0 })),
1452
+ clock: 0,
1453
+ documentClock: 0,
1454
+ schema: oldSchema.serialize(),
1455
+ }
1456
+
1457
+ const storage = new SQLiteSyncStorage<TestRecord>({ sql, snapshot })
1458
+
1459
+ // Run the migration within a transaction
1460
+ storage.transaction((txn) => {
1461
+ newSchema.migrateStorage(txn)
1462
+ })
1463
+
1464
+ // Verify each record was migrated exactly once
1465
+ const migrationCountValues = Array.from(migrationCounts.values())
1466
+ const recordsMigratedMoreThanOnce = migrationCountValues.filter((count) => count > 1)
1467
+
1468
+ // This assertion ensures migrations are applied exactly once per record
1469
+ expect(recordsMigratedMoreThanOnce).toEqual([])
1470
+
1471
+ // Additional check: verify the total number of migrations equals number of records
1472
+ expect(migrationCounts.size).toBe(numRecords)
1473
+ for (const [_id, count] of migrationCounts) {
1474
+ expect(count).toBe(1)
1475
+ }
1476
+
1477
+ // Verify all records now have the migrated flag
1478
+ const finalSnapshot = storage.getSnapshot()
1479
+ for (const doc of finalSnapshot.documents) {
1480
+ expect((doc.state as TestRecord).migrated).toBe(true)
1481
+ }
1482
+ })
1483
+ })
1484
+ })