@tldraw/sync-core 5.5.0-next.f9308d324e1c → 5.6.0-canary.5bfc37740468
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-cjs/index.d.ts +8 -4
- package/dist-cjs/index.js +1 -1
- package/dist-cjs/lib/ClientWebSocketAdapter.js +46 -18
- package/dist-cjs/lib/ClientWebSocketAdapter.js.map +2 -2
- package/dist-cjs/lib/InMemorySyncStorage.js +18 -18
- package/dist-cjs/lib/InMemorySyncStorage.js.map +2 -2
- package/dist-cjs/lib/SQLiteSyncStorage.js +15 -7
- package/dist-cjs/lib/SQLiteSyncStorage.js.map +2 -2
- package/dist-cjs/lib/TLSocketRoom.js +9 -8
- package/dist-cjs/lib/TLSocketRoom.js.map +2 -2
- package/dist-cjs/lib/TLSyncClient.js +8 -1
- package/dist-cjs/lib/TLSyncClient.js.map +2 -2
- package/dist-cjs/lib/TLSyncRoom.js +82 -50
- package/dist-cjs/lib/TLSyncRoom.js.map +2 -2
- package/dist-cjs/lib/TLSyncStorage.js +2 -2
- package/dist-cjs/lib/TLSyncStorage.js.map +2 -2
- package/dist-cjs/lib/chunk.js +35 -3
- package/dist-cjs/lib/chunk.js.map +2 -2
- package/dist-cjs/lib/diff.js +15 -7
- package/dist-cjs/lib/diff.js.map +2 -2
- package/dist-esm/index.d.mts +8 -4
- package/dist-esm/index.mjs +1 -1
- package/dist-esm/lib/ClientWebSocketAdapter.mjs +47 -19
- package/dist-esm/lib/ClientWebSocketAdapter.mjs.map +2 -2
- package/dist-esm/lib/InMemorySyncStorage.mjs +18 -18
- package/dist-esm/lib/InMemorySyncStorage.mjs.map +2 -2
- package/dist-esm/lib/SQLiteSyncStorage.mjs +15 -7
- package/dist-esm/lib/SQLiteSyncStorage.mjs.map +2 -2
- package/dist-esm/lib/TLSocketRoom.mjs +10 -9
- package/dist-esm/lib/TLSocketRoom.mjs.map +2 -2
- package/dist-esm/lib/TLSyncClient.mjs +8 -1
- package/dist-esm/lib/TLSyncClient.mjs.map +2 -2
- package/dist-esm/lib/TLSyncRoom.mjs +82 -50
- package/dist-esm/lib/TLSyncRoom.mjs.map +2 -2
- package/dist-esm/lib/TLSyncStorage.mjs +2 -2
- package/dist-esm/lib/TLSyncStorage.mjs.map +2 -2
- package/dist-esm/lib/chunk.mjs +35 -3
- package/dist-esm/lib/chunk.mjs.map +2 -2
- package/dist-esm/lib/diff.mjs +16 -8
- package/dist-esm/lib/diff.mjs.map +2 -2
- package/package.json +22 -18
- package/src/lib/ClientWebSocketAdapter.test.ts +221 -16
- package/src/lib/ClientWebSocketAdapter.ts +62 -23
- package/src/lib/InMemorySyncStorage.test.ts +21 -0
- package/src/lib/InMemorySyncStorage.ts +22 -24
- package/src/lib/SQLiteSyncStorage.test.ts +60 -1
- package/src/lib/SQLiteSyncStorage.ts +24 -16
- package/src/lib/TLSocketRoom.ts +16 -9
- package/src/lib/TLSyncClient.test.ts +61 -0
- package/src/lib/TLSyncClient.ts +11 -0
- package/src/lib/TLSyncRoom.ts +122 -54
- package/src/lib/TLSyncStorage.ts +7 -2
- package/src/lib/chunk.test.ts +88 -1
- package/src/lib/chunk.ts +66 -3
- package/src/lib/diff.test.ts +91 -1
- package/src/lib/diff.ts +30 -9
- package/src/test/TLSocketRoom.test.ts +63 -0
- package/src/test/TLSyncRoom.test.ts +300 -0
- package/src/test/objectStore.test.ts +35 -0
- package/src/test/upgradeDowngrade.test.ts +21 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { atom, Atom } from '@tldraw/state'
|
|
2
2
|
import { TLRecord } from '@tldraw/tlschema'
|
|
3
3
|
import { assert, warnOnce } from '@tldraw/utils'
|
|
4
|
-
import { chunk } from './chunk'
|
|
4
|
+
import { chunk, MAX_ASSEMBLED_MESSAGE_CHARS } from './chunk'
|
|
5
5
|
import { TLSocketClientSentEvent, TLSocketServerSentEvent } from './protocol'
|
|
6
6
|
import {
|
|
7
7
|
TLPersistentClientSocket,
|
|
@@ -11,6 +11,16 @@ import {
|
|
|
11
11
|
TLSyncErrorCloseEventReason,
|
|
12
12
|
} from './TLSyncClient'
|
|
13
13
|
|
|
14
|
+
// The ready states from the WebSocket spec, kept local instead of read off the global
|
|
15
|
+
// constructor: an extension or page teardown can leave `WebSocket` undefined by the time a
|
|
16
|
+
// window `online` or close handler fires, and `WebSocket.OPEN` then throws (#10106).
|
|
17
|
+
const ReadyState = {
|
|
18
|
+
CONNECTING: 0,
|
|
19
|
+
OPEN: 1,
|
|
20
|
+
CLOSING: 2,
|
|
21
|
+
CLOSED: 3,
|
|
22
|
+
} as const
|
|
23
|
+
|
|
14
24
|
function listenTo<T extends EventTarget>(target: T, event: string, handler: () => void) {
|
|
15
25
|
target.addEventListener(event, handler)
|
|
16
26
|
return () => {
|
|
@@ -93,8 +103,13 @@ export class ClientWebSocketAdapter implements TLPersistentClientSocket<
|
|
|
93
103
|
close() {
|
|
94
104
|
this.isDisposed = true
|
|
95
105
|
this._reconnectManager.close()
|
|
106
|
+
// orphan the socket before closing it (as _closeSocket does) so its onclose can't run
|
|
107
|
+
// _handleDisconnect after disposal — that would notify listeners and schedule a reconnect
|
|
108
|
+
const ws = this._ws
|
|
109
|
+
this._ws = null
|
|
96
110
|
// WebSocket.close() is idempotent
|
|
97
|
-
|
|
111
|
+
ws?.close()
|
|
112
|
+
this._connectionStatus.set('offline')
|
|
98
113
|
}
|
|
99
114
|
|
|
100
115
|
/**
|
|
@@ -171,8 +186,8 @@ export class ClientWebSocketAdapter implements TLPersistentClientSocket<
|
|
|
171
186
|
assert(!this.isDisposed, 'Tried to set a new websocket on a disposed socket')
|
|
172
187
|
assert(
|
|
173
188
|
this._ws === null ||
|
|
174
|
-
this._ws.readyState ===
|
|
175
|
-
this._ws.readyState ===
|
|
189
|
+
this._ws.readyState === ReadyState.CLOSED ||
|
|
190
|
+
this._ws.readyState === ReadyState.CLOSING,
|
|
176
191
|
`Tried to set a new websocket in when the existing one was ${this._ws?.readyState}`
|
|
177
192
|
)
|
|
178
193
|
|
|
@@ -240,8 +255,6 @@ export class ClientWebSocketAdapter implements TLPersistentClientSocket<
|
|
|
240
255
|
this._handleDisconnect('manual')
|
|
241
256
|
}
|
|
242
257
|
|
|
243
|
-
// TLPersistentClientSocket stuff
|
|
244
|
-
|
|
245
258
|
_connectionStatus: Atom<TLPersistentClientSocketStatus | 'initial'> = atom(
|
|
246
259
|
'websocket connection status',
|
|
247
260
|
'initial'
|
|
@@ -277,7 +290,20 @@ export class ClientWebSocketAdapter implements TLPersistentClientSocket<
|
|
|
277
290
|
|
|
278
291
|
if (!this._ws) return
|
|
279
292
|
if (this.connectionStatus === 'online') {
|
|
280
|
-
const
|
|
293
|
+
const stringified = JSON.stringify(msg)
|
|
294
|
+
// The server rejects a session whose assembled message exceeds this, so sending would
|
|
295
|
+
// only earn a close and a reconnect that re-sends the same message. Fail here the way
|
|
296
|
+
// the server would, so onSyncError runs once instead of the client looping forever.
|
|
297
|
+
if (stringified.length > MAX_ASSEMBLED_MESSAGE_CHARS) {
|
|
298
|
+
this._handleDisconnect(
|
|
299
|
+
'closed',
|
|
300
|
+
TLSyncErrorCloseEventCode,
|
|
301
|
+
true,
|
|
302
|
+
TLSyncErrorCloseEventReason.MESSAGE_TOO_LARGE
|
|
303
|
+
)
|
|
304
|
+
return
|
|
305
|
+
}
|
|
306
|
+
const chunks = chunk(stringified)
|
|
281
307
|
for (const part of chunks) {
|
|
282
308
|
this._ws.send(part)
|
|
283
309
|
}
|
|
@@ -513,21 +539,33 @@ export class ReconnectManager {
|
|
|
513
539
|
}
|
|
514
540
|
}
|
|
515
541
|
|
|
516
|
-
private scheduleAttempt() {
|
|
542
|
+
private async scheduleAttempt() {
|
|
517
543
|
assert(this.state === 'pendingAttempt')
|
|
544
|
+
if (this.isDisposed) return
|
|
518
545
|
debug('scheduling a connection attempt')
|
|
519
|
-
|
|
520
|
-
|
|
546
|
+
|
|
547
|
+
let uri: string
|
|
548
|
+
try {
|
|
549
|
+
uri = await this.getUri()
|
|
550
|
+
} catch (error) {
|
|
551
|
+
// A failed auth-token fetch must retry instead of leaving us with no socket or timer.
|
|
521
552
|
if (this.state !== 'pendingAttempt' || this.isDisposed) return
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
553
|
+
console.error('Failed to get the websocket URI, retrying', error)
|
|
554
|
+
this.state = 'delay'
|
|
555
|
+
this.reconnectTimeout = setTimeout(() => this.disconnected(), this.intendedDelay)
|
|
556
|
+
return
|
|
557
|
+
}
|
|
526
558
|
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
559
|
+
// this can happen if the promise gets resolved too late
|
|
560
|
+
if (this.state !== 'pendingAttempt' || this.isDisposed) return
|
|
561
|
+
assert(
|
|
562
|
+
this.socketAdapter._ws?.readyState !== ReadyState.OPEN,
|
|
563
|
+
'There should be no connection attempts while already connected'
|
|
564
|
+
)
|
|
565
|
+
|
|
566
|
+
this.lastAttemptStart = Date.now()
|
|
567
|
+
this.socketAdapter._setNewSocket(new WebSocket(httpToWs(uri)))
|
|
568
|
+
this.state = 'pendingAttemptResult'
|
|
531
569
|
}
|
|
532
570
|
|
|
533
571
|
private getMaxDelay() {
|
|
@@ -575,13 +613,13 @@ export class ReconnectManager {
|
|
|
575
613
|
this.clearRecheckConnectingTimeout()
|
|
576
614
|
|
|
577
615
|
// readyState can be CONNECTING, OPEN, CLOSING, CLOSED, or null (if getUri() is still pending)
|
|
578
|
-
if (this.socketAdapter._ws?.readyState ===
|
|
616
|
+
if (this.socketAdapter._ws?.readyState === ReadyState.OPEN) {
|
|
579
617
|
debug('ReconnectManager.maybeReconnected: already connected')
|
|
580
618
|
// nothing to do, we're already OK
|
|
581
619
|
return
|
|
582
620
|
}
|
|
583
621
|
|
|
584
|
-
if (this.socketAdapter._ws?.readyState ===
|
|
622
|
+
if (this.socketAdapter._ws?.readyState === ReadyState.CONNECTING) {
|
|
585
623
|
debug('ReconnectManager.maybeReconnected: connecting')
|
|
586
624
|
// We might be waiting for a TCP connection that sent SYN out and will never get it back,
|
|
587
625
|
// while a new connection appeared. On the other hand, we might have just started connecting
|
|
@@ -641,11 +679,12 @@ export class ReconnectManager {
|
|
|
641
679
|
debug('ReconnectManager.disconnected')
|
|
642
680
|
// This either means we're freshly disconnected, or the last connection attempt failed;
|
|
643
681
|
// either way, time to try again.
|
|
682
|
+
if (this.isDisposed) return
|
|
644
683
|
|
|
645
684
|
// Guard against delayed notifications and recheck synchronously
|
|
646
685
|
if (
|
|
647
|
-
this.socketAdapter._ws?.readyState !==
|
|
648
|
-
this.socketAdapter._ws?.readyState !==
|
|
686
|
+
this.socketAdapter._ws?.readyState !== ReadyState.OPEN &&
|
|
687
|
+
this.socketAdapter._ws?.readyState !== ReadyState.CONNECTING
|
|
649
688
|
) {
|
|
650
689
|
debug('ReconnectManager.disconnected: websocket is not OPEN or CONNECTING')
|
|
651
690
|
this.clearReconnectTimeout()
|
|
@@ -702,7 +741,7 @@ export class ReconnectManager {
|
|
|
702
741
|
connected() {
|
|
703
742
|
debug('ReconnectManager.connected')
|
|
704
743
|
// this notification could've been delayed, recheck synchronously
|
|
705
|
-
if (this.socketAdapter._ws?.readyState ===
|
|
744
|
+
if (this.socketAdapter._ws?.readyState === ReadyState.OPEN) {
|
|
706
745
|
debug('ReconnectManager.connected: websocket is OPEN')
|
|
707
746
|
this.state = 'connected'
|
|
708
747
|
this.clearReconnectTimeout()
|
|
@@ -51,6 +51,27 @@ describe('InMemorySyncStorage', () => {
|
|
|
51
51
|
expect(storage.getClock()).toBe(25)
|
|
52
52
|
})
|
|
53
53
|
|
|
54
|
+
it('[IM1] handles snapshots with far more records and tombstones than fit in a spread call', () => {
|
|
55
|
+
const documents = []
|
|
56
|
+
for (let i = 0; i < 150_000; i++) {
|
|
57
|
+
documents.push({
|
|
58
|
+
state: { ...contractRecords[0], id: `shape:${i}` } as TLRecord,
|
|
59
|
+
lastChangedClock: i,
|
|
60
|
+
})
|
|
61
|
+
}
|
|
62
|
+
const tombstones: Record<string, number> = {}
|
|
63
|
+
for (let i = 0; i < 150_000; i++) tombstones[`shape:gone-${i}`] = 150_000 + i
|
|
64
|
+
const storage = new InMemorySyncStorage<TLRecord>({
|
|
65
|
+
snapshot: makeContractSnapshot(contractRecords, {
|
|
66
|
+
documents,
|
|
67
|
+
tombstones,
|
|
68
|
+
documentClock: 0,
|
|
69
|
+
tombstoneHistoryStartsAtClock: 0,
|
|
70
|
+
}),
|
|
71
|
+
})
|
|
72
|
+
expect(storage.getClock()).toBe(299_999)
|
|
73
|
+
})
|
|
74
|
+
|
|
54
75
|
it('[IM2] clamps tombstoneHistoryStartsAtClock down to the document clock', () => {
|
|
55
76
|
const storage = new InMemorySyncStorage<TLRecord>({
|
|
56
77
|
snapshot: makeContractSnapshot(contractRecords, {
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
TLDOCUMENT_ID,
|
|
8
8
|
TLPageId,
|
|
9
9
|
} from '@tldraw/tlschema'
|
|
10
|
-
import { assert, IndexKey, objectMapEntries, throttle } from '@tldraw/utils'
|
|
10
|
+
import { assert, IndexKey, objectMapEntries, throttle, type ThrottledFunction } from '@tldraw/utils'
|
|
11
11
|
import { MicrotaskNotifier } from './MicrotaskNotifier'
|
|
12
12
|
import { RoomSnapshot } from './TLSyncRoom'
|
|
13
13
|
import {
|
|
@@ -153,27 +153,27 @@ export class InMemorySyncStorage<R extends UnknownRecord> implements TLSyncStora
|
|
|
153
153
|
onChange?(arg: TLSyncStorageOnChangeCallbackProps): unknown
|
|
154
154
|
} = {}) {
|
|
155
155
|
this.objectTypes = new Set(objectTypes ?? [])
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
...Object.values(snapshot.documents.map((d) => d.lastChangedClock))
|
|
160
|
-
)
|
|
156
|
+
// a loop rather than `Math.max(0, ...clocks)`: spreading a large room's clocks as call
|
|
157
|
+
// arguments overflows the stack at roughly 100k+ records/tombstones
|
|
158
|
+
let maxClockValue = 0
|
|
161
159
|
// route snapshot entries into their partitions (a seed snapshot may carry object-lane
|
|
162
160
|
// records merged in, e.g. loaded from a separate persistence lane)
|
|
163
|
-
const
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
d.
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
161
|
+
const documentEntries: [string, { state: R; lastChangedClock: number }][] = []
|
|
162
|
+
const objectEntries: [string, { state: R; lastChangedClock: number }][] = []
|
|
163
|
+
for (const d of snapshot.documents) {
|
|
164
|
+
if (d.lastChangedClock > maxClockValue) maxClockValue = d.lastChangedClock
|
|
165
|
+
const entries = this.objectTypes.has(d.state.typeName) ? objectEntries : documentEntries
|
|
166
|
+
entries.push([
|
|
167
|
+
d.state.id,
|
|
168
|
+
{ state: devFreeze(d.state) as R, lastChangedClock: d.lastChangedClock },
|
|
169
|
+
])
|
|
170
|
+
}
|
|
171
|
+
const tombstoneEntries = objectMapEntries(snapshot.tombstones ?? {})
|
|
172
|
+
for (const [, clock] of tombstoneEntries) {
|
|
173
|
+
if (clock > maxClockValue) maxClockValue = clock
|
|
174
|
+
}
|
|
175
|
+
this.documents = new AtomMap('room documents', documentEntries)
|
|
176
|
+
this.objects = new AtomMap('room objects', objectEntries)
|
|
177
177
|
const documentClock = Math.max(maxClockValue, snapshot.documentClock ?? snapshot.clock ?? 0)
|
|
178
178
|
|
|
179
179
|
this.documentClock = atom('document clock', documentClock)
|
|
@@ -192,9 +192,7 @@ export class InMemorySyncStorage<R extends UnknownRecord> implements TLSyncStora
|
|
|
192
192
|
'room tombstones',
|
|
193
193
|
// If the tombstone history starts now (or we didn't have the
|
|
194
194
|
// tombstoneHistoryStartsAtClock) then there are no tombstones
|
|
195
|
-
tombstoneHistoryStartsAtClock === documentClock
|
|
196
|
-
? []
|
|
197
|
-
: objectMapEntries(snapshot.tombstones ?? {})
|
|
195
|
+
tombstoneHistoryStartsAtClock === documentClock ? [] : tombstoneEntries
|
|
198
196
|
)
|
|
199
197
|
if (onChange) {
|
|
200
198
|
this.onChange(onChange)
|
|
@@ -249,7 +247,7 @@ export class InMemorySyncStorage<R extends UnknownRecord> implements TLSyncStora
|
|
|
249
247
|
}
|
|
250
248
|
|
|
251
249
|
/** @internal */
|
|
252
|
-
pruneTombstones = throttle(
|
|
250
|
+
pruneTombstones: ThrottledFunction<() => void> = throttle(
|
|
253
251
|
() => {
|
|
254
252
|
if (this.tombstones.size > MAX_TOMBSTONES) {
|
|
255
253
|
// Convert to array and sort by clock ascending (oldest first)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite'
|
|
2
2
|
import { DocumentRecordType, PageRecordType, TLDOCUMENT_ID, TLRecord } from '@tldraw/tlschema'
|
|
3
3
|
import { ZERO_INDEX_KEY } from '@tldraw/utils'
|
|
4
|
-
import { describe, expect, it } from 'vitest'
|
|
4
|
+
import { describe, expect, it, vi } from 'vitest'
|
|
5
5
|
import {
|
|
6
6
|
contractRecords,
|
|
7
7
|
contractSchema,
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
makePage,
|
|
10
10
|
registerStorageContractTests,
|
|
11
11
|
} from '../test/storageContractSuite'
|
|
12
|
+
import { MAX_TOMBSTONES } from './InMemorySyncStorage'
|
|
12
13
|
import { NodeSqliteWrapper } from './NodeSqliteWrapper'
|
|
13
14
|
import { SQLiteSyncStorage } from './SQLiteSyncStorage'
|
|
14
15
|
|
|
@@ -263,4 +264,62 @@ describe('SQLiteSyncStorage', () => {
|
|
|
263
264
|
expect(row?.migrationVersion).toBe(3)
|
|
264
265
|
})
|
|
265
266
|
})
|
|
267
|
+
|
|
268
|
+
describe('tombstone pruning robustness', () => {
|
|
269
|
+
const makeTombstones = (count: number) => {
|
|
270
|
+
const tombstones: Record<string, number> = {}
|
|
271
|
+
for (let i = 0; i < count; i++) tombstones[`shape:doc${i}`] = i + 1
|
|
272
|
+
return tombstones
|
|
273
|
+
}
|
|
274
|
+
function makeOverfullStorage() {
|
|
275
|
+
const db = new DatabaseSync(':memory:')
|
|
276
|
+
const storage = new SQLiteSyncStorage<TLRecord>({
|
|
277
|
+
sql: new NodeSqliteWrapper(db),
|
|
278
|
+
snapshot: makeContractSnapshot(contractRecords, {
|
|
279
|
+
tombstones: makeTombstones(MAX_TOMBSTONES + 500),
|
|
280
|
+
documentClock: MAX_TOMBSTONES + 501,
|
|
281
|
+
tombstoneHistoryStartsAtClock: 0,
|
|
282
|
+
}),
|
|
283
|
+
})
|
|
284
|
+
return { db, storage, prune: (storage as any).pruneTombstones as { (): void; flush(): void } }
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
it('[SS18] a prune that fires after the database was closed does not throw', () => {
|
|
288
|
+
const { db, storage, prune } = makeOverfullStorage()
|
|
289
|
+
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
|
290
|
+
try {
|
|
291
|
+
// a delete schedules the throttled prune; the host then closes everything
|
|
292
|
+
storage.transaction((txn) => txn.delete(contractRecords[1].id))
|
|
293
|
+
db.close()
|
|
294
|
+
// the throttle timer fires later, from outside any caller that could catch it
|
|
295
|
+
expect(() => prune.flush()).not.toThrow()
|
|
296
|
+
expect(consoleSpy).toHaveBeenCalledWith('Failed to prune tombstones', expect.anything())
|
|
297
|
+
} finally {
|
|
298
|
+
consoleSpy.mockRestore()
|
|
299
|
+
}
|
|
300
|
+
})
|
|
301
|
+
|
|
302
|
+
it('[SS18] a prune whose delete fails leaves the history clock where it was', () => {
|
|
303
|
+
const { storage, prune } = makeOverfullStorage()
|
|
304
|
+
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
|
305
|
+
try {
|
|
306
|
+
const stmts = (storage as any).stmts
|
|
307
|
+
const originalRun = stmts.deleteTombstonesBefore.run
|
|
308
|
+
stmts.deleteTombstonesBefore.run = () => {
|
|
309
|
+
throw new Error('SQLITE_BUSY: database is locked')
|
|
310
|
+
}
|
|
311
|
+
prune()
|
|
312
|
+
expect(() => prune.flush()).not.toThrow()
|
|
313
|
+
stmts.deleteTombstonesBefore.run = originalRun
|
|
314
|
+
|
|
315
|
+
// neither half of the prune applied: the history start was not advanced past
|
|
316
|
+
// tombstones that still exist
|
|
317
|
+
const snapshot = storage.getSnapshot()
|
|
318
|
+
expect(snapshot.tombstoneHistoryStartsAtClock).toBe(0)
|
|
319
|
+
expect(Object.keys(snapshot.tombstones!).length).toBe(MAX_TOMBSTONES + 500)
|
|
320
|
+
} finally {
|
|
321
|
+
consoleSpy.mockRestore()
|
|
322
|
+
}
|
|
323
|
+
})
|
|
324
|
+
})
|
|
266
325
|
})
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { transaction } from '@tldraw/state'
|
|
2
2
|
import { SerializedSchema, StoreSnapshot, UnknownRecord } from '@tldraw/store'
|
|
3
|
-
import { assert, objectMapEntries, throttle } from '@tldraw/utils'
|
|
3
|
+
import { assert, objectMapEntries, throttle, type ThrottledFunction } from '@tldraw/utils'
|
|
4
4
|
import {
|
|
5
5
|
computeTombstonePruning,
|
|
6
6
|
DEFAULT_INITIAL_SNAPSHOT,
|
|
@@ -394,7 +394,6 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
394
394
|
const documentClock = snapshot.documentClock ?? snapshot.clock ?? 0
|
|
395
395
|
const tombstoneHistoryStartsAtClock = snapshot.tombstoneHistoryStartsAtClock ?? documentClock
|
|
396
396
|
|
|
397
|
-
// Clear existing data
|
|
398
397
|
this.sql.exec(`
|
|
399
398
|
DELETE FROM ${documentsTable};
|
|
400
399
|
DELETE FROM ${objectsTable};
|
|
@@ -410,14 +409,12 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
410
409
|
table.insert.run(doc.state.id, encodeState(doc.state), doc.lastChangedClock)
|
|
411
410
|
}
|
|
412
411
|
|
|
413
|
-
// Insert tombstones
|
|
414
412
|
if (snapshot.tombstones) {
|
|
415
413
|
for (const [id, clock] of objectMapEntries(snapshot.tombstones)) {
|
|
416
414
|
this.stmts.insertTombstone.run(id, clock)
|
|
417
415
|
}
|
|
418
416
|
}
|
|
419
417
|
|
|
420
|
-
// Insert metadata row
|
|
421
418
|
this.stmts.updateMetadata.run(
|
|
422
419
|
documentClock,
|
|
423
420
|
tombstoneHistoryStartsAtClock,
|
|
@@ -511,20 +508,31 @@ export class SQLiteSyncStorage<R extends UnknownRecord> implements TLSyncStorage
|
|
|
511
508
|
}
|
|
512
509
|
|
|
513
510
|
/** @internal */
|
|
514
|
-
pruneTombstones = throttle(
|
|
511
|
+
pruneTombstones: ThrottledFunction<() => void> = throttle(
|
|
515
512
|
() => {
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
this.stmts.
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
513
|
+
// Runs from a timer, so a host that closed the database right after the last delete
|
|
514
|
+
// (e.g. `room.close(); db.close()` in onSessionRemoved) would otherwise get an uncaught
|
|
515
|
+
// throw from here; pruning is best-effort and simply runs again on the next delete.
|
|
516
|
+
try {
|
|
517
|
+
const tombstoneCount = this.stmts.countTombstones.all()[0].count as number
|
|
518
|
+
if (tombstoneCount > MAX_TOMBSTONES) {
|
|
519
|
+
// Get all tombstones sorted by clock ascending (oldest first)
|
|
520
|
+
const tombstones = this.stmts.iterateTombstones.all()
|
|
521
|
+
|
|
522
|
+
const result = computeTombstonePruning({ tombstones, documentClock: this.getClock() })
|
|
523
|
+
if (result) {
|
|
524
|
+
this.sql.transaction(() => {
|
|
525
|
+
this.stmts.setTombstoneHistoryStartsAtClock.run(
|
|
526
|
+
result.newTombstoneHistoryStartsAtClock
|
|
527
|
+
)
|
|
528
|
+
// Delete all tombstones with clock < newTombstoneHistoryStartsAtClock in one operation.
|
|
529
|
+
// This works because computeTombstonePruning ensures we never split a clock value.
|
|
530
|
+
this.stmts.deleteTombstonesBefore.run(result.newTombstoneHistoryStartsAtClock)
|
|
531
|
+
})
|
|
532
|
+
}
|
|
527
533
|
}
|
|
534
|
+
} catch (e) {
|
|
535
|
+
console.error('Failed to prune tombstones', e)
|
|
528
536
|
}
|
|
529
537
|
},
|
|
530
538
|
1000,
|
package/src/lib/TLSocketRoom.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { SerializedSchema, StoreSchema, UnknownRecord } from '@tldraw/store'
|
|
2
2
|
import { createTLSchema, TLInstancePresence, TLStoreSnapshot } from '@tldraw/tlschema'
|
|
3
3
|
import { getOwnProperty, hasOwnProperty, isEqual, structuredClone } from '@tldraw/utils'
|
|
4
|
-
import { JsonChunkAssembler } from './chunk'
|
|
4
|
+
import { JsonChunkAssembler, MessageTooLargeError } from './chunk'
|
|
5
5
|
import { DEFAULT_INITIAL_SNAPSHOT, InMemorySyncStorage } from './InMemorySyncStorage'
|
|
6
6
|
import { TLObjectStoreAccess, TLSocketServerSentEvent } from './protocol'
|
|
7
7
|
import { RoomSessionState } from './RoomSession'
|
|
@@ -273,13 +273,16 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
|
|
|
273
273
|
})
|
|
274
274
|
)
|
|
275
275
|
}
|
|
276
|
+
// The default logger belongs to both layers: TLSyncRoom is where authorizer throws are
|
|
277
|
+
// logged, and a host that passes no `log` would otherwise never see them.
|
|
278
|
+
this.log = 'log' in opts ? opts.log : { error: console.error }
|
|
276
279
|
this.room = new TLSyncRoom<R, SessionMeta>({
|
|
277
280
|
onPresenceChange: opts.onPresenceChange,
|
|
278
281
|
onCommittedChanges: opts.onCommittedChanges,
|
|
279
282
|
objectTypes: opts.objectTypes,
|
|
280
283
|
authorizeRecord: opts.authorizeRecord,
|
|
281
284
|
schema: opts.schema ?? (createTLSchema() as any),
|
|
282
|
-
log:
|
|
285
|
+
log: this.log,
|
|
283
286
|
storage,
|
|
284
287
|
clientTimeout: opts.clientTimeout,
|
|
285
288
|
})
|
|
@@ -295,7 +298,6 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
|
|
|
295
298
|
})
|
|
296
299
|
}
|
|
297
300
|
})
|
|
298
|
-
this.log = 'log' in opts ? opts.log : { error: console.error }
|
|
299
301
|
}
|
|
300
302
|
|
|
301
303
|
/**
|
|
@@ -475,6 +477,11 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
|
|
|
475
477
|
})
|
|
476
478
|
this.room.pruneSessions()
|
|
477
479
|
this.scheduleDebouncedSnapshot(sessionId)
|
|
480
|
+
} else if (res.error instanceof MessageTooLargeError) {
|
|
481
|
+
this.log?.error?.('Error assembling message', res.error)
|
|
482
|
+
// Resetting the connection would have the client reconnect and re-send the same
|
|
483
|
+
// oversized message, so reject the session instead and let the client surface it.
|
|
484
|
+
this.room.rejectSession(sessionId, TLSyncErrorCloseEventReason.MESSAGE_TOO_LARGE)
|
|
478
485
|
} else {
|
|
479
486
|
this.log?.error?.('Error assembling message', res.error)
|
|
480
487
|
// close the socket to reset the connection
|
|
@@ -854,12 +861,11 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
|
|
|
854
861
|
if (this.isClosed()) {
|
|
855
862
|
throw new Error('Cannot update store on a closed room')
|
|
856
863
|
}
|
|
864
|
+
// read through the transaction rather than getSnapshot(): the document snapshot excludes the
|
|
865
|
+
// object-store lane, which would make object-lane records invisible (and undeletable) here
|
|
866
|
+
const records = this.storage.transaction((txn) => Object.fromEntries(txn.entries())).result
|
|
857
867
|
// eslint-disable-next-line @typescript-eslint/no-deprecated
|
|
858
|
-
const ctx = new StoreUpdateContext<R>(
|
|
859
|
-
// eslint-disable-next-line @typescript-eslint/no-deprecated
|
|
860
|
-
Object.fromEntries(this.getCurrentSnapshot().documents.map((d) => [d.state.id, d.state])),
|
|
861
|
-
this.room.schema
|
|
862
|
-
)
|
|
868
|
+
const ctx = new StoreUpdateContext<R>(records, this.room.schema)
|
|
863
869
|
try {
|
|
864
870
|
await updater(ctx)
|
|
865
871
|
} finally {
|
|
@@ -908,7 +914,8 @@ export class TLSocketRoom<R extends UnknownRecord = UnknownRecord, SessionMeta =
|
|
|
908
914
|
* The client will attempt to reconnect automatically unless a fatal reason is provided.
|
|
909
915
|
*
|
|
910
916
|
* @param sessionId - Session identifier to remove
|
|
911
|
-
* @param fatalReason - Optional fatal error reason that prevents reconnection
|
|
917
|
+
* @param fatalReason - Optional fatal error reason that prevents reconnection. WebSocket close
|
|
918
|
+
* reasons are capped at 123 UTF-8 bytes, so a longer reason is truncated and ends with `... (+N bytes)`.
|
|
912
919
|
*
|
|
913
920
|
* @example
|
|
914
921
|
* ```ts
|
|
@@ -375,6 +375,67 @@ describe('TLSyncClient', () => {
|
|
|
375
375
|
expect(pushWithPage!.diff![localPage.id][0]).toBe(RecordOpType.Put)
|
|
376
376
|
})
|
|
377
377
|
|
|
378
|
+
it('[CL5] a reconnect flushes pending store history first, so a change queued behind the frame throttle is neither lost nor resurrected', () => {
|
|
379
|
+
connectClient()
|
|
380
|
+
|
|
381
|
+
socket.mockConnectionStatus('offline')
|
|
382
|
+
const localPage = makePage('Offline Page')
|
|
383
|
+
store.put([localPage]) // reaches speculativeChanges
|
|
384
|
+
|
|
385
|
+
// In the browser the store delivers history on the next animation frame, so the client's
|
|
386
|
+
// view of speculative changes can lag the store. Hold the next flush back the same way.
|
|
387
|
+
const flushHistory = store._flushHistory
|
|
388
|
+
store._flushHistory = () => {}
|
|
389
|
+
store.remove([localPage.id]) // ...so this delete is still queued when we reconnect
|
|
390
|
+
store._flushHistory = flushHistory
|
|
391
|
+
|
|
392
|
+
socket.mockConnectionStatus('online')
|
|
393
|
+
socket.mockServerMessage(
|
|
394
|
+
createConnectMessage({ hydrationType: 'wipe_presence', serverClock: 2, diff: {} })
|
|
395
|
+
)
|
|
396
|
+
vi.advanceTimersByTime(100)
|
|
397
|
+
|
|
398
|
+
// the user deleted the page: it must not come back, and must not be pushed as a put
|
|
399
|
+
expect(store.get(localPage.id)).toBeUndefined()
|
|
400
|
+
expect(
|
|
401
|
+
getSentPushes().some((msg) => msg.diff?.[localPage.id]?.[0] === RecordOpType.Put)
|
|
402
|
+
).toBe(false)
|
|
403
|
+
})
|
|
404
|
+
|
|
405
|
+
it('[CL5] a store listener that throws during the reconnect flush does not stall the connect', () => {
|
|
406
|
+
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
|
|
407
|
+
try {
|
|
408
|
+
client = createClient()
|
|
409
|
+
let listenerThrown = false
|
|
410
|
+
store.listen(() => {
|
|
411
|
+
if (listenerThrown) return
|
|
412
|
+
listenerThrown = true
|
|
413
|
+
throw new Error('listener error')
|
|
414
|
+
})
|
|
415
|
+
|
|
416
|
+
// queue a local change behind the frame throttle so the connect flush delivers it
|
|
417
|
+
const flushHistory = store._flushHistory
|
|
418
|
+
store._flushHistory = () => {}
|
|
419
|
+
const localPage = makePage('Local Page', 'a5')
|
|
420
|
+
store.put([localPage])
|
|
421
|
+
store._flushHistory = flushHistory
|
|
422
|
+
|
|
423
|
+
const serverDiff = documentScopeDiff()
|
|
424
|
+
delete serverDiff[localPage.id] // the server never saw it
|
|
425
|
+
socket.mockServerMessage(createConnectMessage({ diff: serverDiff }))
|
|
426
|
+
vi.advanceTimersByTime(100)
|
|
427
|
+
|
|
428
|
+
expect(client.isConnectedToRoom).toBe(true)
|
|
429
|
+
expect(onLoad).toHaveBeenCalled()
|
|
430
|
+
expect(store.get(localPage.id)).toEqual(localPage)
|
|
431
|
+
expect(
|
|
432
|
+
getSentPushes().some((msg) => msg.diff?.[localPage.id]?.[0] === RecordOpType.Put)
|
|
433
|
+
).toBe(true)
|
|
434
|
+
} finally {
|
|
435
|
+
consoleSpy.mockRestore()
|
|
436
|
+
}
|
|
437
|
+
})
|
|
438
|
+
|
|
378
439
|
it('[CL6] wipe_all reconnect wipes document records before applying the server diff', () => {
|
|
379
440
|
client = createClient()
|
|
380
441
|
|
package/src/lib/TLSyncClient.ts
CHANGED
|
@@ -122,6 +122,8 @@ export const TLSyncErrorCloseEventReason = {
|
|
|
122
122
|
RATE_LIMITED: 'RATE_LIMITED',
|
|
123
123
|
/** Room has reached maximum capacity */
|
|
124
124
|
ROOM_FULL: 'ROOM_FULL',
|
|
125
|
+
/** A single message exceeded the size the server will assemble */
|
|
126
|
+
MESSAGE_TOO_LARGE: 'MESSAGE_TOO_LARGE',
|
|
125
127
|
} as const
|
|
126
128
|
|
|
127
129
|
/**
|
|
@@ -752,6 +754,15 @@ export class TLSyncClient<R extends UnknownRecord, S extends Store<R> = Store<R>
|
|
|
752
754
|
this.resetConnection(true)
|
|
753
755
|
return
|
|
754
756
|
}
|
|
757
|
+
// Local changes reach `push` (and so `speculativeChanges`) on the store's frame-throttled
|
|
758
|
+
// history flush. Anything still queued there would be neither reverted below nor
|
|
759
|
+
// re-applied on top of the server state, so bring speculativeChanges up to date first.
|
|
760
|
+
try {
|
|
761
|
+
this.store._flushHistory()
|
|
762
|
+
} catch (e) {
|
|
763
|
+
console.error(e)
|
|
764
|
+
}
|
|
765
|
+
|
|
755
766
|
// at the end of this process we want to have at most one pending push request
|
|
756
767
|
// based on anything inside this.speculativeChanges
|
|
757
768
|
transact(() => {
|