@tldraw/sync-core 5.3.0-canary.fceaae5e9feb → 5.3.0-internal.fba91ed55f6c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-cjs/index.d.ts +154 -3
- package/dist-cjs/index.js +1 -1
- package/dist-cjs/index.js.map +2 -2
- package/dist-cjs/lib/InMemorySyncStorage.js +55 -20
- package/dist-cjs/lib/InMemorySyncStorage.js.map +2 -2
- package/dist-cjs/lib/RoomSession.js.map +1 -1
- package/dist-cjs/lib/SQLiteSyncStorage.js +115 -52
- package/dist-cjs/lib/SQLiteSyncStorage.js.map +2 -2
- package/dist-cjs/lib/TLSocketRoom.js +27 -2
- package/dist-cjs/lib/TLSocketRoom.js.map +2 -2
- package/dist-cjs/lib/TLSyncClient.js +6 -1
- package/dist-cjs/lib/TLSyncClient.js.map +2 -2
- package/dist-cjs/lib/TLSyncRoom.js +160 -10
- package/dist-cjs/lib/TLSyncRoom.js.map +3 -3
- package/dist-cjs/lib/TLSyncStorage.js.map +2 -2
- package/dist-cjs/lib/protocol.js.map +2 -2
- package/dist-esm/index.d.mts +154 -3
- package/dist-esm/index.mjs +1 -1
- package/dist-esm/index.mjs.map +2 -2
- package/dist-esm/lib/InMemorySyncStorage.mjs +55 -20
- package/dist-esm/lib/InMemorySyncStorage.mjs.map +2 -2
- package/dist-esm/lib/RoomSession.mjs.map +1 -1
- package/dist-esm/lib/SQLiteSyncStorage.mjs +115 -52
- package/dist-esm/lib/SQLiteSyncStorage.mjs.map +2 -2
- package/dist-esm/lib/TLSocketRoom.mjs +27 -2
- package/dist-esm/lib/TLSocketRoom.mjs.map +2 -2
- package/dist-esm/lib/TLSyncClient.mjs +6 -1
- package/dist-esm/lib/TLSyncClient.mjs.map +2 -2
- package/dist-esm/lib/TLSyncRoom.mjs +160 -10
- package/dist-esm/lib/TLSyncRoom.mjs.map +3 -3
- package/dist-esm/lib/TLSyncStorage.mjs.map +2 -2
- package/dist-esm/lib/protocol.mjs.map +2 -2
- package/package.json +6 -6
- package/src/index.ts +3 -0
- package/src/lib/InMemorySyncStorage.ts +74 -21
- package/src/lib/RoomSession.ts +7 -2
- package/src/lib/SQLiteSyncStorage.test.ts +29 -2
- package/src/lib/SQLiteSyncStorage.ts +158 -59
- package/src/lib/TLSocketRoom.ts +61 -3
- package/src/lib/TLSyncClient.test.ts +9 -2
- package/src/lib/TLSyncClient.ts +15 -3
- package/src/lib/TLSyncRoom.ts +294 -10
- package/src/lib/TLSyncStorage.ts +8 -0
- package/src/lib/protocol.ts +17 -0
- package/src/test/TLSocketRoom.test.ts +37 -2
- package/src/test/TLSyncClientRebase.test.ts +285 -0
- package/src/test/TLSyncRoom.test.ts +25 -0
- package/src/test/TestServer.ts +14 -4
- package/src/test/objectStore.test.ts +630 -0
- package/src/test/storageContractSuite.ts +79 -0
- package/src/test/upgradeDowngrade.test.ts +144 -2
|
@@ -117,6 +117,14 @@ export const DEFAULT_INITIAL_SNAPSHOT = {
|
|
|
117
117
|
export class InMemorySyncStorage<R extends UnknownRecord> implements TLSyncStorage<R> {
|
|
118
118
|
/** @internal */
|
|
119
119
|
documents: AtomMap<string, { state: R; lastChangedClock: number }>
|
|
120
|
+
/**
|
|
121
|
+
* Object-store lane records, partitioned out of `documents` so they never appear in the
|
|
122
|
+
* document snapshot. They share the same clock, tombstones, and transactions as documents.
|
|
123
|
+
* @internal
|
|
124
|
+
*/
|
|
125
|
+
objects: AtomMap<string, { state: R; lastChangedClock: number }>
|
|
126
|
+
/** @internal */
|
|
127
|
+
readonly objectTypes: ReadonlySet<string>
|
|
120
128
|
/** @internal */
|
|
121
129
|
tombstones: AtomMap<string, number>
|
|
122
130
|
/** @internal */
|
|
@@ -133,22 +141,38 @@ export class InMemorySyncStorage<R extends UnknownRecord> implements TLSyncStora
|
|
|
133
141
|
|
|
134
142
|
constructor({
|
|
135
143
|
snapshot = DEFAULT_INITIAL_SNAPSHOT,
|
|
144
|
+
objectTypes,
|
|
136
145
|
onChange,
|
|
137
146
|
}: {
|
|
138
147
|
snapshot?: RoomSnapshot
|
|
148
|
+
/**
|
|
149
|
+
* Record type names stored in the object-store lane. Records of these types are routed to
|
|
150
|
+
* a separate partition: excluded from `getSnapshot()`, returned by `getObjectsSnapshot()`.
|
|
151
|
+
*/
|
|
152
|
+
objectTypes?: readonly string[]
|
|
139
153
|
onChange?(arg: TLSyncStorageOnChangeCallbackProps): unknown
|
|
140
154
|
} = {}) {
|
|
155
|
+
this.objectTypes = new Set(objectTypes ?? [])
|
|
141
156
|
const maxClockValue = Math.max(
|
|
142
157
|
0,
|
|
143
158
|
...Object.values(snapshot.tombstones ?? {}),
|
|
144
159
|
...Object.values(snapshot.documents.map((d) => d.lastChangedClock))
|
|
145
160
|
)
|
|
161
|
+
// route snapshot entries into their partitions (a seed snapshot may carry object-lane
|
|
162
|
+
// records merged in, e.g. loaded from a separate persistence lane)
|
|
163
|
+
const toEntry = (
|
|
164
|
+
d: RoomSnapshot['documents'][number]
|
|
165
|
+
): [string, { state: R; lastChangedClock: number }] => [
|
|
166
|
+
d.state.id,
|
|
167
|
+
{ state: devFreeze(d.state) as R, lastChangedClock: d.lastChangedClock },
|
|
168
|
+
]
|
|
146
169
|
this.documents = new AtomMap(
|
|
147
170
|
'room documents',
|
|
148
|
-
snapshot.documents.
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
171
|
+
snapshot.documents.filter((d) => !this.objectTypes.has(d.state.typeName)).map(toEntry)
|
|
172
|
+
)
|
|
173
|
+
this.objects = new AtomMap(
|
|
174
|
+
'room objects',
|
|
175
|
+
snapshot.documents.filter((d) => this.objectTypes.has(d.state.typeName)).map(toEntry)
|
|
152
176
|
)
|
|
153
177
|
const documentClock = Math.max(maxClockValue, snapshot.documentClock ?? snapshot.clock ?? 0)
|
|
154
178
|
|
|
@@ -249,6 +273,8 @@ export class InMemorySyncStorage<R extends UnknownRecord> implements TLSyncStora
|
|
|
249
273
|
)
|
|
250
274
|
|
|
251
275
|
getSnapshot(): RoomSnapshot {
|
|
276
|
+
// object-lane records live in their own partition, so the document snapshot is
|
|
277
|
+
// pure-document by construction
|
|
252
278
|
return {
|
|
253
279
|
tombstoneHistoryStartsAtClock: this.tombstoneHistoryStartsAtClock.get(),
|
|
254
280
|
documentClock: this.documentClock.get(),
|
|
@@ -257,6 +283,10 @@ export class InMemorySyncStorage<R extends UnknownRecord> implements TLSyncStora
|
|
|
257
283
|
schema: this.schema.get(),
|
|
258
284
|
}
|
|
259
285
|
}
|
|
286
|
+
|
|
287
|
+
getObjectsSnapshot(): RoomSnapshot['documents'] {
|
|
288
|
+
return Array.from(this.objects.values())
|
|
289
|
+
}
|
|
260
290
|
}
|
|
261
291
|
|
|
262
292
|
/**
|
|
@@ -297,9 +327,16 @@ class InMemorySyncStorageTransaction<
|
|
|
297
327
|
return this._clock
|
|
298
328
|
}
|
|
299
329
|
|
|
330
|
+
/** The partition a record with this id currently lives in, if any. */
|
|
331
|
+
private mapContaining(id: string) {
|
|
332
|
+
if (this.storage.documents.has(id)) return this.storage.documents
|
|
333
|
+
if (this.storage.objects.has(id)) return this.storage.objects
|
|
334
|
+
return undefined
|
|
335
|
+
}
|
|
336
|
+
|
|
300
337
|
get(id: string): R | undefined {
|
|
301
338
|
this.assertNotClosed()
|
|
302
|
-
return this.storage.documents.get(id)?.state
|
|
339
|
+
return (this.storage.documents.get(id) ?? this.storage.objects.get(id))?.state
|
|
303
340
|
}
|
|
304
341
|
|
|
305
342
|
set(id: string, record: R): void {
|
|
@@ -310,7 +347,11 @@ class InMemorySyncStorageTransaction<
|
|
|
310
347
|
if (this.storage.tombstones.has(id)) {
|
|
311
348
|
this.storage.tombstones.delete(id)
|
|
312
349
|
}
|
|
313
|
-
|
|
350
|
+
// route by type: object-lane records live in their own partition
|
|
351
|
+
const map = this.storage.objectTypes.has(record.typeName)
|
|
352
|
+
? this.storage.objects
|
|
353
|
+
: this.storage.documents
|
|
354
|
+
map.set(id, {
|
|
314
355
|
state: devFreeze(record) as R,
|
|
315
356
|
lastChangedClock: clock,
|
|
316
357
|
})
|
|
@@ -319,34 +360,43 @@ class InMemorySyncStorageTransaction<
|
|
|
319
360
|
delete(id: string): void {
|
|
320
361
|
this.assertNotClosed()
|
|
321
362
|
// Only create a tombstone if the record actually exists
|
|
322
|
-
|
|
363
|
+
const map = this.mapContaining(id)
|
|
364
|
+
if (!map) return
|
|
323
365
|
const clock = this.getNextClock()
|
|
324
|
-
|
|
366
|
+
map.delete(id)
|
|
325
367
|
this.storage.tombstones.set(id, clock)
|
|
326
368
|
this.storage.pruneTombstones()
|
|
327
369
|
}
|
|
328
370
|
|
|
371
|
+
// iteration spans both partitions so schema migrations cover object-lane records too
|
|
372
|
+
|
|
329
373
|
*entries(): IterableIterator<[string, R]> {
|
|
330
374
|
this.assertNotClosed()
|
|
331
|
-
for (const
|
|
332
|
-
|
|
333
|
-
|
|
375
|
+
for (const map of [this.storage.documents, this.storage.objects]) {
|
|
376
|
+
for (const [id, record] of map.entries()) {
|
|
377
|
+
this.assertNotClosed()
|
|
378
|
+
yield [id, record.state]
|
|
379
|
+
}
|
|
334
380
|
}
|
|
335
381
|
}
|
|
336
382
|
|
|
337
383
|
*keys(): IterableIterator<string> {
|
|
338
384
|
this.assertNotClosed()
|
|
339
|
-
for (const
|
|
340
|
-
|
|
341
|
-
|
|
385
|
+
for (const map of [this.storage.documents, this.storage.objects]) {
|
|
386
|
+
for (const key of map.keys()) {
|
|
387
|
+
this.assertNotClosed()
|
|
388
|
+
yield key
|
|
389
|
+
}
|
|
342
390
|
}
|
|
343
391
|
}
|
|
344
392
|
|
|
345
393
|
*values(): IterableIterator<R> {
|
|
346
394
|
this.assertNotClosed()
|
|
347
|
-
for (const
|
|
348
|
-
|
|
349
|
-
|
|
395
|
+
for (const map of [this.storage.documents, this.storage.objects]) {
|
|
396
|
+
for (const record of map.values()) {
|
|
397
|
+
this.assertNotClosed()
|
|
398
|
+
yield record.state
|
|
399
|
+
}
|
|
350
400
|
}
|
|
351
401
|
}
|
|
352
402
|
|
|
@@ -370,10 +420,13 @@ class InMemorySyncStorageTransaction<
|
|
|
370
420
|
}
|
|
371
421
|
const diff: TLSyncForwardDiff<R> = { puts: {}, deletes: [] }
|
|
372
422
|
const wipeAll = sinceClock < this.storage.tombstoneHistoryStartsAtClock.get()
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
423
|
+
// both partitions share the clock and tombstones, so the change feed spans both
|
|
424
|
+
for (const map of [this.storage.documents, this.storage.objects]) {
|
|
425
|
+
for (const doc of map.values()) {
|
|
426
|
+
if (wipeAll || doc.lastChangedClock > sinceClock) {
|
|
427
|
+
// For historical changes, we don't have "from" state, so use added
|
|
428
|
+
diff.puts[doc.state.id] = doc.state as R
|
|
429
|
+
}
|
|
377
430
|
}
|
|
378
431
|
}
|
|
379
432
|
if (!wipeAll) {
|
package/src/lib/RoomSession.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { SerializedSchema, UnknownRecord } from '@tldraw/store'
|
|
2
|
-
import { TLSocketServerSentDataEvent } from './protocol'
|
|
2
|
+
import { TLObjectStoreAccess, TLSocketServerSentDataEvent } from './protocol'
|
|
3
3
|
import { TLRoomSocket } from './TLSyncRoom'
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -78,8 +78,13 @@ export interface RoomSessionBase<R extends UnknownRecord, Meta> {
|
|
|
78
78
|
socket: TLRoomSocket<R>
|
|
79
79
|
/** Custom metadata associated with this session */
|
|
80
80
|
meta: Meta
|
|
81
|
-
/** Whether this session has read-only permissions */
|
|
81
|
+
/** Whether this session has read-only permissions for document-lane records */
|
|
82
82
|
isReadonly: boolean
|
|
83
|
+
/**
|
|
84
|
+
* Write access for object-store lane record types (see `objectTypes` on the room).
|
|
85
|
+
* Deliberately independent of `isReadonly` so "can comment but not edit" is expressible.
|
|
86
|
+
*/
|
|
87
|
+
objectAccess: TLObjectStoreAccess
|
|
83
88
|
/** Whether this session requires legacy protocol rejection handling */
|
|
84
89
|
requiresLegacyRejection: boolean
|
|
85
90
|
/** Whether this session supports string append operations */
|
|
@@ -38,6 +38,33 @@ describe('SQLiteSyncStorage', () => {
|
|
|
38
38
|
).toEqual(newPage)
|
|
39
39
|
})
|
|
40
40
|
|
|
41
|
+
it('[SQ1] sweeps pre-partition object-lane rows out of the documents table on reopen', () => {
|
|
42
|
+
// a database written before the objects table existed: object-lane records (here
|
|
43
|
+
// 'page' stands in) live in the documents table
|
|
44
|
+
const sql = createWrapper()
|
|
45
|
+
const storage = new SQLiteSyncStorage<TLRecord>({
|
|
46
|
+
sql,
|
|
47
|
+
snapshot: makeContractSnapshot(contractRecords),
|
|
48
|
+
})
|
|
49
|
+
const extraPage = makePage('legacy')
|
|
50
|
+
storage.transaction((txn) => txn.set(extraPage.id, extraPage))
|
|
51
|
+
expect(storage.getSnapshot().documents).toHaveLength(3)
|
|
52
|
+
|
|
53
|
+
// reopening with objectTypes moves the rows into the objects partition
|
|
54
|
+
const reopened = new SQLiteSyncStorage<TLRecord>({ sql, objectTypes: ['page'] })
|
|
55
|
+
expect(reopened.getSnapshot().documents.map((d) => d.state.typeName)).toEqual(['document'])
|
|
56
|
+
expect(
|
|
57
|
+
reopened
|
|
58
|
+
.getObjectsSnapshot()
|
|
59
|
+
.map((d) => d.state.id)
|
|
60
|
+
.sort()
|
|
61
|
+
).toEqual([contractRecords[1].id, extraPage.id].sort())
|
|
62
|
+
// lastChangedClock survives the move
|
|
63
|
+
expect(
|
|
64
|
+
reopened.getObjectsSnapshot().find((d) => d.state.id === extraPage.id)?.lastChangedClock
|
|
65
|
+
).toBe(1)
|
|
66
|
+
})
|
|
67
|
+
|
|
41
68
|
it('[SQ1] honors the table prefix', () => {
|
|
42
69
|
const sql = createWrapper({ tablePrefix: 'tldraw_' })
|
|
43
70
|
const storage = new SQLiteSyncStorage<TLRecord>({
|
|
@@ -227,13 +254,13 @@ describe('SQLiteSyncStorage', () => {
|
|
|
227
254
|
expect(storage.getSnapshot().documents.length).toBe(3)
|
|
228
255
|
})
|
|
229
256
|
|
|
230
|
-
it('[SQ6] fresh databases start at migration version
|
|
257
|
+
it('[SQ6] fresh databases start at migration version 3', () => {
|
|
231
258
|
const sql = createWrapper()
|
|
232
259
|
new SQLiteSyncStorage<TLRecord>({ sql, snapshot: makeContractSnapshot(contractRecords) })
|
|
233
260
|
const row = sql
|
|
234
261
|
.prepare<{ migrationVersion: number }>('SELECT migrationVersion FROM metadata')
|
|
235
262
|
.all()[0]
|
|
236
|
-
expect(row?.migrationVersion).toBe(
|
|
263
|
+
expect(row?.migrationVersion).toBe(3)
|
|
237
264
|
})
|
|
238
265
|
})
|
|
239
266
|
})
|
|
@@ -86,9 +86,15 @@ export function migrateSqliteSyncStorage(
|
|
|
86
86
|
storage: TLSyncSqliteWrapper,
|
|
87
87
|
{
|
|
88
88
|
documentsTable = 'documents',
|
|
89
|
+
objectsTable = 'objects',
|
|
89
90
|
tombstonesTable = 'tombstones',
|
|
90
91
|
metadataTable = 'metadata',
|
|
91
|
-
}: {
|
|
92
|
+
}: {
|
|
93
|
+
documentsTable?: string
|
|
94
|
+
objectsTable?: string
|
|
95
|
+
tombstonesTable?: string
|
|
96
|
+
metadataTable?: string
|
|
97
|
+
} = {}
|
|
92
98
|
): void {
|
|
93
99
|
let migrationVersion = 0
|
|
94
100
|
try {
|
|
@@ -144,18 +150,34 @@ export function migrateSqliteSyncStorage(
|
|
|
144
150
|
state BLOB NOT NULL,
|
|
145
151
|
lastChangedClock INTEGER NOT NULL
|
|
146
152
|
);
|
|
147
|
-
|
|
153
|
+
|
|
148
154
|
INSERT INTO ${documentsTable}_new (id, state, lastChangedClock)
|
|
149
155
|
SELECT id, CAST(state AS BLOB), lastChangedClock FROM ${documentsTable};
|
|
150
|
-
|
|
156
|
+
|
|
151
157
|
DROP TABLE ${documentsTable};
|
|
152
|
-
|
|
158
|
+
|
|
153
159
|
ALTER TABLE ${documentsTable}_new RENAME TO ${documentsTable};
|
|
154
|
-
|
|
160
|
+
|
|
155
161
|
CREATE INDEX idx_${documentsTable}_lastChangedClock ON ${documentsTable}(lastChangedClock);
|
|
156
162
|
`)
|
|
157
163
|
}
|
|
158
164
|
|
|
165
|
+
if (migrationVersion === 2) {
|
|
166
|
+
// Migration 3: Add the object-store lane table. Object-lane records (e.g. comments) are
|
|
167
|
+
// partitioned out of the documents table so the document snapshot never contains them.
|
|
168
|
+
// They share the documents' clock and tombstones.
|
|
169
|
+
migrationVersion++
|
|
170
|
+
storage.exec(`
|
|
171
|
+
CREATE TABLE ${objectsTable} (
|
|
172
|
+
id TEXT PRIMARY KEY,
|
|
173
|
+
state BLOB NOT NULL,
|
|
174
|
+
lastChangedClock INTEGER NOT NULL
|
|
175
|
+
);
|
|
176
|
+
|
|
177
|
+
CREATE INDEX idx_${objectsTable}_lastChangedClock ON ${objectsTable}(lastChangedClock);
|
|
178
|
+
`)
|
|
179
|
+
}
|
|
180
|
+
|
|
159
181
|
// add more migrations here if and when needed
|
|
160
182
|
|
|
161
183
|
storage.exec(`UPDATE ${metadataTable} SET migrationVersion = ${migrationVersion}`)
|
|
@@ -250,22 +272,64 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
250
272
|
|
|
251
273
|
private readonly sql: TLSyncSqliteWrapper
|
|
252
274
|
|
|
275
|
+
/** @internal */
|
|
276
|
+
readonly objectTypes: ReadonlySet<string>
|
|
277
|
+
|
|
253
278
|
constructor({
|
|
254
279
|
sql,
|
|
255
280
|
snapshot,
|
|
281
|
+
objectTypes,
|
|
256
282
|
onChange,
|
|
257
283
|
}: {
|
|
258
284
|
sql: TLSyncSqliteWrapper
|
|
259
285
|
snapshot?: RoomSnapshot | StoreSnapshot<R>
|
|
286
|
+
/**
|
|
287
|
+
* Record type names stored in the object-store lane. Records of these types are routed to
|
|
288
|
+
* a separate table: excluded from `getSnapshot()`, returned by `getObjectsSnapshot()`.
|
|
289
|
+
* They share the documents' clock, tombstones, and transactions.
|
|
290
|
+
*/
|
|
291
|
+
objectTypes?: readonly string[]
|
|
260
292
|
onChange?(arg: TLSyncStorageOnChangeCallbackProps): unknown
|
|
261
293
|
}) {
|
|
262
294
|
this.sql = sql
|
|
295
|
+
this.objectTypes = new Set(objectTypes ?? [])
|
|
263
296
|
const prefix = sql.config?.tablePrefix ?? ''
|
|
264
297
|
const documentsTable = `${prefix}documents`
|
|
298
|
+
const objectsTable = `${prefix}objects`
|
|
265
299
|
const tombstonesTable = `${prefix}tombstones`
|
|
266
300
|
const metadataTable = `${prefix}metadata`
|
|
267
301
|
|
|
268
|
-
migrateSqliteSyncStorage(this.sql, {
|
|
302
|
+
migrateSqliteSyncStorage(this.sql, {
|
|
303
|
+
documentsTable,
|
|
304
|
+
objectsTable,
|
|
305
|
+
tombstonesTable,
|
|
306
|
+
metadataTable,
|
|
307
|
+
})
|
|
308
|
+
|
|
309
|
+
// Both record tables have identical shape; prepare the same statements for each.
|
|
310
|
+
const makeRecordTableStmts = (table: string) => ({
|
|
311
|
+
get: this.sql.prepare<{ state: Uint8Array }, [id: string]>(
|
|
312
|
+
`SELECT state FROM ${table} WHERE id = ?`
|
|
313
|
+
),
|
|
314
|
+
insert: this.sql.prepare<void, [id: string, state: Uint8Array, lastChangedClock: number]>(
|
|
315
|
+
`INSERT OR REPLACE INTO ${table} (id, state, lastChangedClock) VALUES (?, ?, ?)`
|
|
316
|
+
),
|
|
317
|
+
delete: this.sql.prepare<void, [id: string]>(`DELETE FROM ${table} WHERE id = ?`),
|
|
318
|
+
exists: this.sql.prepare<{ id: string }, [id: string]>(
|
|
319
|
+
`SELECT id FROM ${table} WHERE id = ?`
|
|
320
|
+
),
|
|
321
|
+
iterate: this.sql.prepare<{ state: Uint8Array; lastChangedClock: number }>(
|
|
322
|
+
`SELECT state, lastChangedClock FROM ${table}`
|
|
323
|
+
),
|
|
324
|
+
iterateEntries: this.sql.prepare<{ id: string; state: Uint8Array }>(
|
|
325
|
+
`SELECT id, state FROM ${table}`
|
|
326
|
+
),
|
|
327
|
+
iterateKeys: this.sql.prepare<{ id: string }>(`SELECT id FROM ${table}`),
|
|
328
|
+
iterateValues: this.sql.prepare<{ state: Uint8Array }>(`SELECT state FROM ${table}`),
|
|
329
|
+
changedSince: this.sql.prepare<{ state: Uint8Array }, [sinceClock: number]>(
|
|
330
|
+
`SELECT state FROM ${table} WHERE lastChangedClock > ?`
|
|
331
|
+
),
|
|
332
|
+
})
|
|
269
333
|
|
|
270
334
|
// Prepare all statements once
|
|
271
335
|
this.stmts = {
|
|
@@ -285,33 +349,9 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
285
349
|
`UPDATE ${metadataTable} SET documentClock = documentClock + 1`
|
|
286
350
|
),
|
|
287
351
|
|
|
288
|
-
//
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
),
|
|
292
|
-
insertDocument: this.sql.prepare<
|
|
293
|
-
void,
|
|
294
|
-
[id: string, state: Uint8Array, lastChangedClock: number]
|
|
295
|
-
>(`INSERT OR REPLACE INTO ${documentsTable} (id, state, lastChangedClock) VALUES (?, ?, ?)`),
|
|
296
|
-
deleteDocument: this.sql.prepare<void, [id: string]>(
|
|
297
|
-
`DELETE FROM ${documentsTable} WHERE id = ?`
|
|
298
|
-
),
|
|
299
|
-
documentExists: this.sql.prepare<{ id: string }, [id: string]>(
|
|
300
|
-
`SELECT id FROM ${documentsTable} WHERE id = ?`
|
|
301
|
-
),
|
|
302
|
-
iterateDocuments: this.sql.prepare<{ state: Uint8Array; lastChangedClock: number }>(
|
|
303
|
-
`SELECT state, lastChangedClock FROM ${documentsTable}`
|
|
304
|
-
),
|
|
305
|
-
iterateDocumentEntries: this.sql.prepare<{ id: string; state: Uint8Array }>(
|
|
306
|
-
`SELECT id, state FROM ${documentsTable}`
|
|
307
|
-
),
|
|
308
|
-
iterateDocumentKeys: this.sql.prepare<{ id: string }>(`SELECT id FROM ${documentsTable}`),
|
|
309
|
-
iterateDocumentValues: this.sql.prepare<{ state: Uint8Array }>(
|
|
310
|
-
`SELECT state FROM ${documentsTable}`
|
|
311
|
-
),
|
|
312
|
-
getDocumentsChangedSince: this.sql.prepare<{ state: Uint8Array }, [sinceClock: number]>(
|
|
313
|
-
`SELECT state FROM ${documentsTable} WHERE lastChangedClock > ?`
|
|
314
|
-
),
|
|
352
|
+
// Record tables (document lane + object-store lane)
|
|
353
|
+
documents: makeRecordTableStmts(documentsTable),
|
|
354
|
+
objects: makeRecordTableStmts(objectsTable),
|
|
315
355
|
|
|
316
356
|
// Tombstones
|
|
317
357
|
insertTombstone: this.sql.prepare<void, [id: string, clock: number]>(
|
|
@@ -354,12 +394,17 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
354
394
|
// Clear existing data
|
|
355
395
|
this.sql.exec(`
|
|
356
396
|
DELETE FROM ${documentsTable};
|
|
397
|
+
DELETE FROM ${objectsTable};
|
|
357
398
|
DELETE FROM ${tombstonesTable};
|
|
358
399
|
`)
|
|
359
400
|
|
|
360
|
-
// Insert
|
|
401
|
+
// Insert records, routing object-lane types into their partition (a seed snapshot may
|
|
402
|
+
// carry object-lane records merged in, e.g. loaded from a separate persistence lane)
|
|
361
403
|
for (const doc of snapshot.documents) {
|
|
362
|
-
this.
|
|
404
|
+
const table = this.objectTypes.has(doc.state.typeName)
|
|
405
|
+
? this.stmts.objects
|
|
406
|
+
: this.stmts.documents
|
|
407
|
+
table.insert.run(doc.state.id, encodeState(doc.state), doc.lastChangedClock)
|
|
363
408
|
}
|
|
364
409
|
|
|
365
410
|
// Insert tombstones
|
|
@@ -375,6 +420,21 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
375
420
|
tombstoneHistoryStartsAtClock,
|
|
376
421
|
JSON.stringify(snapshot.schema)
|
|
377
422
|
)
|
|
423
|
+
} else {
|
|
424
|
+
// One-time sweep: rooms written before the objects table existed may have object-lane
|
|
425
|
+
// records sitting in the documents table. Record ids are typeName-prefixed, so a PK
|
|
426
|
+
// range scan per object type moves them cheaply (no-op once clean).
|
|
427
|
+
for (const typeName of this.objectTypes) {
|
|
428
|
+
const lo = `${typeName}:`
|
|
429
|
+
// ';' is the next code point after ':' — [lo, hi) covers exactly the `type:` prefix
|
|
430
|
+
const hi = `${typeName};`
|
|
431
|
+
this.sql.exec(`
|
|
432
|
+
INSERT OR REPLACE INTO ${objectsTable} (id, state, lastChangedClock)
|
|
433
|
+
SELECT id, state, lastChangedClock FROM ${documentsTable}
|
|
434
|
+
WHERE id >= '${lo}' AND id < '${hi}';
|
|
435
|
+
DELETE FROM ${documentsTable} WHERE id >= '${lo}' AND id < '${hi}';
|
|
436
|
+
`)
|
|
437
|
+
}
|
|
378
438
|
}
|
|
379
439
|
if (onChange) {
|
|
380
440
|
this.onChange(onChange)
|
|
@@ -470,16 +530,25 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
470
530
|
)
|
|
471
531
|
|
|
472
532
|
getSnapshot(): RoomSnapshot {
|
|
533
|
+
// object-lane records live in their own table, so the document snapshot is
|
|
534
|
+
// pure-document by construction
|
|
473
535
|
return {
|
|
474
536
|
tombstoneHistoryStartsAtClock: this._getTombstoneHistoryStartsAtClock(),
|
|
475
537
|
documentClock: this.getClock(),
|
|
476
|
-
documents: Array.from(this.
|
|
538
|
+
documents: Array.from(this._iterateRecords(this.stmts.documents)),
|
|
477
539
|
tombstones: Object.fromEntries(this._iterateTombstones()),
|
|
478
540
|
schema: this._getSchema(),
|
|
479
541
|
}
|
|
480
542
|
}
|
|
481
|
-
|
|
482
|
-
|
|
543
|
+
|
|
544
|
+
getObjectsSnapshot(): RoomSnapshot['documents'] {
|
|
545
|
+
return Array.from(this._iterateRecords(this.stmts.objects))
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
private *_iterateRecords(
|
|
549
|
+
table: SQLiteSyncStorage<R>['stmts']['documents']
|
|
550
|
+
): IterableIterator<{ state: R; lastChangedClock: number }> {
|
|
551
|
+
for (const row of table.iterate.iterate()) {
|
|
483
552
|
yield { state: decodeState<R>(row.state), lastChangedClock: row.lastChangedClock }
|
|
484
553
|
}
|
|
485
554
|
}
|
|
@@ -531,9 +600,21 @@ class SQLiteSyncStorageTransaction<R extends UnknownRecord> implements TLSyncSto
|
|
|
531
600
|
return this._clock
|
|
532
601
|
}
|
|
533
602
|
|
|
603
|
+
/**
|
|
604
|
+
* Which record table an id belongs to. Record ids are typeName-prefixed (`comment:abc`),
|
|
605
|
+
* so the partition is derivable from the id alone.
|
|
606
|
+
*/
|
|
607
|
+
private tableFor(id: string) {
|
|
608
|
+
const sep = id.indexOf(':')
|
|
609
|
+
if (sep === -1) return this.stmts.documents
|
|
610
|
+
return this.storage.objectTypes.has(id.slice(0, sep))
|
|
611
|
+
? this.stmts.objects
|
|
612
|
+
: this.stmts.documents
|
|
613
|
+
}
|
|
614
|
+
|
|
534
615
|
get(id: string): R | undefined {
|
|
535
616
|
this.assertNotClosed()
|
|
536
|
-
const row = this.
|
|
617
|
+
const row = this.tableFor(id).get.all(id)[0]
|
|
537
618
|
if (!row) return undefined
|
|
538
619
|
return decodeState<R>(row.state)
|
|
539
620
|
}
|
|
@@ -544,41 +625,54 @@ class SQLiteSyncStorageTransaction<R extends UnknownRecord> implements TLSyncSto
|
|
|
544
625
|
const clock = this.getNextClock()
|
|
545
626
|
// Automatically clear tombstone if it exists
|
|
546
627
|
this.stmts.deleteTombstone.run(id)
|
|
547
|
-
|
|
628
|
+
// route by type: object-lane records live in their own table
|
|
629
|
+
const table = this.storage.objectTypes.has(record.typeName)
|
|
630
|
+
? this.stmts.objects
|
|
631
|
+
: this.stmts.documents
|
|
632
|
+
table.insert.run(id, encodeState(record), clock)
|
|
548
633
|
}
|
|
549
634
|
|
|
550
635
|
delete(id: string): void {
|
|
551
636
|
this.assertNotClosed()
|
|
637
|
+
const table = this.tableFor(id)
|
|
552
638
|
// Only create a tombstone if the record actually exists
|
|
553
|
-
const exists =
|
|
639
|
+
const exists = table.exists.all(id)[0]
|
|
554
640
|
if (!exists) return
|
|
555
641
|
const clock = this.getNextClock()
|
|
556
|
-
|
|
642
|
+
table.delete.run(id)
|
|
557
643
|
this.stmts.insertTombstone.run(id, clock)
|
|
558
644
|
this.storage.pruneTombstones()
|
|
559
645
|
}
|
|
560
646
|
|
|
647
|
+
// iteration spans both record tables so schema migrations cover object-lane records too
|
|
648
|
+
|
|
561
649
|
*entries(): IterableIterator<[string, R]> {
|
|
562
650
|
this.assertNotClosed()
|
|
563
|
-
for (const
|
|
564
|
-
|
|
565
|
-
|
|
651
|
+
for (const table of [this.stmts.documents, this.stmts.objects]) {
|
|
652
|
+
for (const row of table.iterateEntries.iterate()) {
|
|
653
|
+
this.assertNotClosed()
|
|
654
|
+
yield [row.id, decodeState<R>(row.state)]
|
|
655
|
+
}
|
|
566
656
|
}
|
|
567
657
|
}
|
|
568
658
|
|
|
569
659
|
*keys(): IterableIterator<string> {
|
|
570
660
|
this.assertNotClosed()
|
|
571
|
-
for (const
|
|
572
|
-
|
|
573
|
-
|
|
661
|
+
for (const table of [this.stmts.documents, this.stmts.objects]) {
|
|
662
|
+
for (const row of table.iterateKeys.iterate()) {
|
|
663
|
+
this.assertNotClosed()
|
|
664
|
+
yield row.id
|
|
665
|
+
}
|
|
574
666
|
}
|
|
575
667
|
}
|
|
576
668
|
|
|
577
669
|
*values(): IterableIterator<R> {
|
|
578
670
|
this.assertNotClosed()
|
|
579
|
-
for (const
|
|
580
|
-
|
|
581
|
-
|
|
671
|
+
for (const table of [this.stmts.documents, this.stmts.objects]) {
|
|
672
|
+
for (const row of table.iterateValues.iterate()) {
|
|
673
|
+
this.assertNotClosed()
|
|
674
|
+
yield decodeState<R>(row.state)
|
|
675
|
+
}
|
|
582
676
|
}
|
|
583
677
|
}
|
|
584
678
|
|
|
@@ -603,17 +697,22 @@ class SQLiteSyncStorageTransaction<R extends UnknownRecord> implements TLSyncSto
|
|
|
603
697
|
const diff: TLSyncForwardDiff<R> = { puts: {}, deletes: [] }
|
|
604
698
|
const wipeAll = sinceClock < this.storage._getTombstoneHistoryStartsAtClock()
|
|
605
699
|
|
|
700
|
+
// both record tables share the clock and tombstones, so the change feed spans both
|
|
606
701
|
if (wipeAll) {
|
|
607
|
-
// If wipeAll, include all
|
|
608
|
-
for (const
|
|
609
|
-
const
|
|
610
|
-
|
|
702
|
+
// If wipeAll, include all records
|
|
703
|
+
for (const table of [this.stmts.documents, this.stmts.objects]) {
|
|
704
|
+
for (const row of table.iterateValues.iterate()) {
|
|
705
|
+
const state = decodeState<R>(row.state)
|
|
706
|
+
diff.puts[state.id] = state
|
|
707
|
+
}
|
|
611
708
|
}
|
|
612
709
|
} else {
|
|
613
|
-
// Get
|
|
614
|
-
for (const
|
|
615
|
-
const
|
|
616
|
-
|
|
710
|
+
// Get records changed since clock
|
|
711
|
+
for (const table of [this.stmts.documents, this.stmts.objects]) {
|
|
712
|
+
for (const row of table.changedSince.iterate(sinceClock)) {
|
|
713
|
+
const state = decodeState<R>(row.state)
|
|
714
|
+
diff.puts[state.id] = state
|
|
715
|
+
}
|
|
617
716
|
}
|
|
618
717
|
// When wipeAll, deletes are redundant (full state is in puts). Only include tombstones otherwise.
|
|
619
718
|
for (const row of this.stmts.getTombstonesChangedSince.iterate(sinceClock)) {
|