@tldraw/sync-core 5.5.0-next.f9308d324e1c → 5.6.0-canary.5bfc37740468
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-cjs/index.d.ts +8 -4
- package/dist-cjs/index.js +1 -1
- package/dist-cjs/lib/ClientWebSocketAdapter.js +46 -18
- package/dist-cjs/lib/ClientWebSocketAdapter.js.map +2 -2
- package/dist-cjs/lib/InMemorySyncStorage.js +18 -18
- package/dist-cjs/lib/InMemorySyncStorage.js.map +2 -2
- package/dist-cjs/lib/SQLiteSyncStorage.js +15 -7
- package/dist-cjs/lib/SQLiteSyncStorage.js.map +2 -2
- package/dist-cjs/lib/TLSocketRoom.js +9 -8
- package/dist-cjs/lib/TLSocketRoom.js.map +2 -2
- package/dist-cjs/lib/TLSyncClient.js +8 -1
- package/dist-cjs/lib/TLSyncClient.js.map +2 -2
- package/dist-cjs/lib/TLSyncRoom.js +82 -50
- package/dist-cjs/lib/TLSyncRoom.js.map +2 -2
- package/dist-cjs/lib/TLSyncStorage.js +2 -2
- package/dist-cjs/lib/TLSyncStorage.js.map +2 -2
- package/dist-cjs/lib/chunk.js +35 -3
- package/dist-cjs/lib/chunk.js.map +2 -2
- package/dist-cjs/lib/diff.js +15 -7
- package/dist-cjs/lib/diff.js.map +2 -2
- package/dist-esm/index.d.mts +8 -4
- package/dist-esm/index.mjs +1 -1
- package/dist-esm/lib/ClientWebSocketAdapter.mjs +47 -19
- package/dist-esm/lib/ClientWebSocketAdapter.mjs.map +2 -2
- package/dist-esm/lib/InMemorySyncStorage.mjs +18 -18
- package/dist-esm/lib/InMemorySyncStorage.mjs.map +2 -2
- package/dist-esm/lib/SQLiteSyncStorage.mjs +15 -7
- package/dist-esm/lib/SQLiteSyncStorage.mjs.map +2 -2
- package/dist-esm/lib/TLSocketRoom.mjs +10 -9
- package/dist-esm/lib/TLSocketRoom.mjs.map +2 -2
- package/dist-esm/lib/TLSyncClient.mjs +8 -1
- package/dist-esm/lib/TLSyncClient.mjs.map +2 -2
- package/dist-esm/lib/TLSyncRoom.mjs +82 -50
- package/dist-esm/lib/TLSyncRoom.mjs.map +2 -2
- package/dist-esm/lib/TLSyncStorage.mjs +2 -2
- package/dist-esm/lib/TLSyncStorage.mjs.map +2 -2
- package/dist-esm/lib/chunk.mjs +35 -3
- package/dist-esm/lib/chunk.mjs.map +2 -2
- package/dist-esm/lib/diff.mjs +16 -8
- package/dist-esm/lib/diff.mjs.map +2 -2
- package/package.json +22 -18
- package/src/lib/ClientWebSocketAdapter.test.ts +221 -16
- package/src/lib/ClientWebSocketAdapter.ts +62 -23
- package/src/lib/InMemorySyncStorage.test.ts +21 -0
- package/src/lib/InMemorySyncStorage.ts +22 -24
- package/src/lib/SQLiteSyncStorage.test.ts +60 -1
- package/src/lib/SQLiteSyncStorage.ts +24 -16
- package/src/lib/TLSocketRoom.ts +16 -9
- package/src/lib/TLSyncClient.test.ts +61 -0
- package/src/lib/TLSyncClient.ts +11 -0
- package/src/lib/TLSyncRoom.ts +122 -54
- package/src/lib/TLSyncStorage.ts +7 -2
- package/src/lib/chunk.test.ts +88 -1
- package/src/lib/chunk.ts +66 -3
- package/src/lib/diff.test.ts +91 -1
- package/src/lib/diff.ts +30 -9
- package/src/test/TLSocketRoom.test.ts +63 -0
- package/src/test/TLSyncRoom.test.ts +300 -0
- package/src/test/objectStore.test.ts +35 -0
- package/src/test/upgradeDowngrade.test.ts +21 -0
package/src/lib/TLSyncRoom.ts
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
objectMapEntriesIterable,
|
|
17
17
|
Result,
|
|
18
18
|
throttle,
|
|
19
|
+
type ThrottledFunction,
|
|
19
20
|
} from '@tldraw/utils'
|
|
20
21
|
import { createNanoEvents } from 'nanoevents'
|
|
21
22
|
import {
|
|
@@ -88,6 +89,29 @@ export const DATA_MESSAGE_DEBOUNCE_INTERVAL = 1000 / 60
|
|
|
88
89
|
|
|
89
90
|
const timeSince = (time: number) => Date.now() - time
|
|
90
91
|
|
|
92
|
+
// WebSocket close frames cap the reason at 123 UTF-8 bytes; a longer one makes `close()` throw
|
|
93
|
+
const MAX_CLOSE_REASON_BYTES = 123
|
|
94
|
+
const closeReasonEncoder = new TextEncoder()
|
|
95
|
+
|
|
96
|
+
const truncatedSuffix = (droppedBytes: number) => `... (+${droppedBytes} bytes)`
|
|
97
|
+
|
|
98
|
+
function truncateCloseReason(reason: string) {
|
|
99
|
+
const totalBytes = closeReasonEncoder.encode(reason).length
|
|
100
|
+
if (totalBytes <= MAX_CLOSE_REASON_BYTES) return reason
|
|
101
|
+
// the dropped count can't have more digits than the total, so reserving room for it always fits
|
|
102
|
+
const budget = MAX_CLOSE_REASON_BYTES - truncatedSuffix(totalBytes).length
|
|
103
|
+
let out = ''
|
|
104
|
+
let outBytes = 0
|
|
105
|
+
// step by code point so a multi-byte character is never cut in half
|
|
106
|
+
for (const char of reason) {
|
|
107
|
+
const charBytes = closeReasonEncoder.encode(char).length
|
|
108
|
+
if (outBytes + charBytes > budget) break
|
|
109
|
+
out += char
|
|
110
|
+
outBytes += charBytes
|
|
111
|
+
}
|
|
112
|
+
return out + truncatedSuffix(totalBytes - outBytes)
|
|
113
|
+
}
|
|
114
|
+
|
|
91
115
|
/**
|
|
92
116
|
* Snapshot of a room's complete state that can be persisted and restored.
|
|
93
117
|
* Contains all documents, tombstones, and metadata needed to reconstruct the room.
|
|
@@ -217,7 +241,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
217
241
|
|
|
218
242
|
private pruneTimer: ReturnType<typeof setTimeout> | null = null
|
|
219
243
|
|
|
220
|
-
pruneSessions = throttle(() => {
|
|
244
|
+
pruneSessions: ThrottledFunction<() => void> = throttle(() => {
|
|
221
245
|
if (this.pruneTimer) {
|
|
222
246
|
clearTimeout(this.pruneTimer)
|
|
223
247
|
this.pruneTimer = null
|
|
@@ -258,7 +282,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
258
282
|
}, 1000)
|
|
259
283
|
|
|
260
284
|
private scheduleFollowUpPrune() {
|
|
261
|
-
|
|
285
|
+
// don't leave a stray timer on a room the host has already torn down
|
|
286
|
+
if (this._isClosed || this.pruneTimer) return
|
|
262
287
|
this.pruneTimer = setTimeout(this.pruneSessions, SESSION_REMOVAL_WAIT_TIME + 100)
|
|
263
288
|
}
|
|
264
289
|
|
|
@@ -273,11 +298,27 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
273
298
|
* and stops background processes.
|
|
274
299
|
*/
|
|
275
300
|
close() {
|
|
301
|
+
this._isClosed = true
|
|
276
302
|
this.disposables.forEach((d) => d())
|
|
277
303
|
this.sessions.forEach((session) => {
|
|
278
|
-
|
|
304
|
+
this.clearDebounceTimer(session)
|
|
305
|
+
try {
|
|
306
|
+
session.socket.close()
|
|
307
|
+
} catch {
|
|
308
|
+
// noop, one bad socket must not leave the rest open
|
|
309
|
+
}
|
|
279
310
|
})
|
|
280
|
-
|
|
311
|
+
// forgetting the sessions is what makes late socket close/error events no-ops, so
|
|
312
|
+
// nothing can emit session_removed / room_became_empty on a closed room
|
|
313
|
+
this.sessions.clear()
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
private clearDebounceTimer(session: RoomSession<R, SessionMeta>) {
|
|
317
|
+
if (session.state === RoomSessionState.Connected && session.debounceTimer !== null) {
|
|
318
|
+
clearTimeout(session.debounceTimer)
|
|
319
|
+
session.debounceTimer = null
|
|
320
|
+
session.outstandingDataMessages = []
|
|
321
|
+
}
|
|
281
322
|
}
|
|
282
323
|
|
|
283
324
|
/**
|
|
@@ -519,6 +560,12 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
519
560
|
// place, so sockets that defer serialization don't see an emptied array
|
|
520
561
|
const data = session.outstandingDataMessages
|
|
521
562
|
session.outstandingDataMessages = []
|
|
563
|
+
if (!session.socket.isOpen) {
|
|
564
|
+
// same as the immediate-send path: send() into a closed socket throws on
|
|
565
|
+
// some runtimes (Cloudflare), and here that would be from inside a timer
|
|
566
|
+
this.cancelSession(sessionId)
|
|
567
|
+
return
|
|
568
|
+
}
|
|
522
569
|
session.socket.sendMessage({ type: 'data', data })
|
|
523
570
|
}
|
|
524
571
|
}
|
|
@@ -532,10 +579,12 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
532
579
|
}
|
|
533
580
|
|
|
534
581
|
this.sessions.delete(sessionId)
|
|
582
|
+
this.clearDebounceTimer(session)
|
|
535
583
|
|
|
536
584
|
try {
|
|
537
585
|
if (fatalReason) {
|
|
538
|
-
session
|
|
586
|
+
// the session is already gone, so a throwing close() would leave its socket open
|
|
587
|
+
session.socket.close(TLSyncErrorCloseEventCode, truncateCloseReason(fatalReason))
|
|
539
588
|
} else {
|
|
540
589
|
session.socket.close()
|
|
541
590
|
}
|
|
@@ -570,6 +619,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
570
619
|
return
|
|
571
620
|
}
|
|
572
621
|
|
|
622
|
+
this.clearDebounceTimer(session)
|
|
573
623
|
this.sessions.set(sessionId, {
|
|
574
624
|
state: RoomSessionState.AwaitingRemoval,
|
|
575
625
|
sessionId,
|
|
@@ -607,9 +657,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
607
657
|
networkDiff?: NetworkDiff<R> | null,
|
|
608
658
|
sourceSessionId?: string
|
|
609
659
|
) {
|
|
610
|
-
//
|
|
611
|
-
|
|
612
|
-
|
|
660
|
+
// Computed once and shared by every session that needs no down-migration; the push path
|
|
661
|
+
// hands us the diff it already computed (in legacy append mode when needed), and re-deriving
|
|
662
|
+
// it per session would both repeat the diffing work and lose that legacy handling.
|
|
663
|
+
const legacyAppendMode = !this.getCanEmitStringAppend()
|
|
664
|
+
const unmigrated = networkDiff ?? toNetworkDiff(diff, legacyAppendMode)
|
|
613
665
|
|
|
614
666
|
this.sessions.forEach((session) => {
|
|
615
667
|
if (session.state !== RoomSessionState.Connected) return
|
|
@@ -623,7 +675,9 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
623
675
|
session.sessionId,
|
|
624
676
|
session.serializedSchema,
|
|
625
677
|
session.requiresDownMigrations,
|
|
626
|
-
diff
|
|
678
|
+
diff,
|
|
679
|
+
unmigrated,
|
|
680
|
+
legacyAppendMode
|
|
627
681
|
)
|
|
628
682
|
if (!diffResult.ok) return
|
|
629
683
|
|
|
@@ -692,7 +746,13 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
692
746
|
objectAccess?: TLObjectStoreAccess
|
|
693
747
|
}) {
|
|
694
748
|
const { sessionId, socket, meta, isReadonly, objectAccess } = opts
|
|
749
|
+
// a connect racing close() would otherwise create a session nothing ever prunes
|
|
750
|
+
if (this._isClosed) {
|
|
751
|
+
socket.close()
|
|
752
|
+
return this
|
|
753
|
+
}
|
|
695
754
|
const existing = this.sessions.get(sessionId)
|
|
755
|
+
if (existing) this.clearDebounceTimer(existing)
|
|
696
756
|
this.sessions.set(sessionId, {
|
|
697
757
|
state: RoomSessionState.AwaitingConnectMessage,
|
|
698
758
|
sessionId,
|
|
@@ -741,6 +801,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
741
801
|
supportsStringAppend,
|
|
742
802
|
} = opts
|
|
743
803
|
|
|
804
|
+
if (this._isClosed) {
|
|
805
|
+
socket.close()
|
|
806
|
+
return
|
|
807
|
+
}
|
|
808
|
+
|
|
744
809
|
const migrations = this.schema.getMigrationsSince(serializedSchema)
|
|
745
810
|
const requiresDownMigrations = migrations.ok ? migrations.value.length > 0 : false
|
|
746
811
|
|
|
@@ -761,9 +826,22 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
761
826
|
supportsStringAppend,
|
|
762
827
|
})
|
|
763
828
|
|
|
829
|
+
// Restore presence before the schema checks: other clients still hold it from before the
|
|
830
|
+
// socket slept, and a rejection only broadcasts its removal if it's in the store
|
|
764
831
|
if (presenceRecord && presenceId) {
|
|
765
832
|
this.presenceStore.set(presenceId, presenceRecord as R)
|
|
766
833
|
}
|
|
834
|
+
|
|
835
|
+
// The server may have changed builds while the socket slept, so re-run the handshake's
|
|
836
|
+
// schema checks (HS3): a schema we can no longer reconcile must not be served raw diffs.
|
|
837
|
+
if (!migrations.ok) {
|
|
838
|
+
this.rejectSession(sessionId, this.getVersionMismatchReason(serializedSchema))
|
|
839
|
+
return
|
|
840
|
+
}
|
|
841
|
+
if (migrations.value.some((m) => m.scope !== 'record' || !m.down)) {
|
|
842
|
+
this.rejectSession(sessionId, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD)
|
|
843
|
+
return
|
|
844
|
+
}
|
|
767
845
|
}
|
|
768
846
|
|
|
769
847
|
/**
|
|
@@ -798,6 +876,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
798
876
|
* @param requiresDownMigrations - Whether the client needs down migrations
|
|
799
877
|
* @param diff - The TLSyncForwardDiff containing full records to migrate
|
|
800
878
|
* @param unmigrated - Optional pre-computed NetworkDiff for when no migration is needed
|
|
879
|
+
* @param legacyAppendMode - Emit string appends as puts (SES5); defaults to the room-wide state
|
|
801
880
|
* @returns A NetworkDiff with migrated records, or a migration failure
|
|
802
881
|
*/
|
|
803
882
|
private migrateDiffOrRejectSession(
|
|
@@ -805,10 +884,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
805
884
|
serializedSchema: SerializedSchema,
|
|
806
885
|
requiresDownMigrations: boolean,
|
|
807
886
|
diff: TLSyncForwardDiff<R>,
|
|
808
|
-
unmigrated?: NetworkDiff<R
|
|
887
|
+
unmigrated?: NetworkDiff<R>,
|
|
888
|
+
legacyAppendMode = !this.getCanEmitStringAppend()
|
|
809
889
|
): Result<NetworkDiff<R>, MigrationFailureReason> {
|
|
810
890
|
if (!requiresDownMigrations) {
|
|
811
|
-
return Result.ok(unmigrated ?? toNetworkDiff(diff
|
|
891
|
+
return Result.ok(unmigrated ?? toNetworkDiff(diff, legacyAppendMode))
|
|
812
892
|
}
|
|
813
893
|
|
|
814
894
|
const result: NetworkDiff<R> = {}
|
|
@@ -828,7 +908,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
828
908
|
this.rejectSession(sessionId, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD)
|
|
829
909
|
return Result.err(toResult.reason)
|
|
830
910
|
}
|
|
831
|
-
const patch = diffRecord(fromResult.value, toResult.value)
|
|
911
|
+
const patch = diffRecord(fromResult.value, toResult.value, legacyAppendMode)
|
|
832
912
|
if (patch) {
|
|
833
913
|
result[id] = [RecordOpType.Patch, patch]
|
|
834
914
|
}
|
|
@@ -905,7 +985,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
905
985
|
* Sends appropriate error messages before closing the connection.
|
|
906
986
|
*
|
|
907
987
|
* @param sessionId - The session to reject
|
|
908
|
-
* @param fatalReason - The reason for rejection (optional)
|
|
988
|
+
* @param fatalReason - The reason for rejection (optional). WebSocket close reasons are capped
|
|
989
|
+
* at 123 UTF-8 bytes, so a longer reason is truncated and ends with `... (+N bytes)`.
|
|
909
990
|
* @example
|
|
910
991
|
* ```ts
|
|
911
992
|
* // Reject due to version mismatch
|
|
@@ -962,6 +1043,10 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
962
1043
|
|
|
963
1044
|
private forceAllReconnect() {
|
|
964
1045
|
for (const session of this.sessions.values()) {
|
|
1046
|
+
// Only connected clients hold state at a clock we can no longer diff from. A session
|
|
1047
|
+
// mid-handshake gets hydrated from its own lastServerClock in the same transaction, so
|
|
1048
|
+
// removing it would close its socket and then re-add it as Connected (resurrected).
|
|
1049
|
+
if (session.state !== RoomSessionState.Connected) continue
|
|
965
1050
|
this.removeSession(session.sessionId)
|
|
966
1051
|
}
|
|
967
1052
|
}
|
|
@@ -991,7 +1076,13 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
991
1076
|
return TLSyncErrorCloseEventReason.SERVER_TOO_OLD
|
|
992
1077
|
}
|
|
993
1078
|
|
|
994
|
-
if (
|
|
1079
|
+
if (
|
|
1080
|
+
theirSchema.schemaVersion === 2 &&
|
|
1081
|
+
ourSchema.schemaVersion === 2 &&
|
|
1082
|
+
// a malformed client schema must reject the session, not throw
|
|
1083
|
+
typeof theirSchema.sequences === 'object' &&
|
|
1084
|
+
theirSchema.sequences !== null
|
|
1085
|
+
) {
|
|
995
1086
|
for (const [sequenceId, theirVersion] of Object.entries(theirSchema.sequences)) {
|
|
996
1087
|
const ourVersion = ourSchema.sequences[sequenceId]
|
|
997
1088
|
if (ourVersion === undefined || theirVersion > ourVersion) {
|
|
@@ -1055,7 +1146,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1055
1146
|
|
|
1056
1147
|
const requiresDownMigrations = migrations.value.length > 0
|
|
1057
1148
|
|
|
1058
|
-
const connect =
|
|
1149
|
+
const connect = (msg: Extract<TLSocketServerSentEvent<R>, { type: 'connect' }>) => {
|
|
1059
1150
|
this.sessions.set(session.sessionId, {
|
|
1060
1151
|
state: RoomSessionState.Connected,
|
|
1061
1152
|
sessionId: session.sessionId,
|
|
@@ -1196,7 +1287,6 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1196
1287
|
}
|
|
1197
1288
|
let { value: state } = res
|
|
1198
1289
|
|
|
1199
|
-
// Get the existing document, if any
|
|
1200
1290
|
const doc =
|
|
1201
1291
|
prevDoc !== undefined ? (prevDoc ?? undefined) : (storage.get(id) as R | undefined)
|
|
1202
1292
|
|
|
@@ -1212,7 +1302,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1212
1302
|
// If there's an existing document, replace it with the new state
|
|
1213
1303
|
// but propagate a diff rather than the entire value
|
|
1214
1304
|
const recordType = assertExists(getOwnProperty(this.schema.types, doc.typeName))
|
|
1215
|
-
const diff = diffAndValidateRecord(doc, state, recordType)
|
|
1305
|
+
const diff = diffAndValidateRecord(doc, state, recordType, legacyAppendMode)
|
|
1216
1306
|
if (diff) {
|
|
1217
1307
|
storage.set(id, state)
|
|
1218
1308
|
propagateOp(changes, id, [RecordOpType.Patch, diff], doc, state)
|
|
@@ -1330,6 +1420,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1330
1420
|
!session ||
|
|
1331
1421
|
(this.objectTypes.has(typeName) ? session.objectAccess !== 'read' : !session.isReadonly)
|
|
1332
1422
|
|
|
1423
|
+
// what authorizers see of the pushing session, shared by every authorized op in this push
|
|
1424
|
+
const authSession = session
|
|
1425
|
+
? { sessionId: session.sessionId, isReadonly: session.isReadonly, meta: session.meta }
|
|
1426
|
+
: null
|
|
1427
|
+
|
|
1333
1428
|
if (message.diff) {
|
|
1334
1429
|
// The push request was for the document scope.
|
|
1335
1430
|
for (const [id, op] of objectMapEntriesIterable(message.diff!)) {
|
|
@@ -1380,26 +1475,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1380
1475
|
authorize = (prevRec, next) => {
|
|
1381
1476
|
const result = authorizePut(
|
|
1382
1477
|
prevRec
|
|
1383
|
-
? {
|
|
1384
|
-
|
|
1385
|
-
sessionId: session.sessionId,
|
|
1386
|
-
isReadonly: session.isReadonly,
|
|
1387
|
-
meta: session.meta,
|
|
1388
|
-
},
|
|
1389
|
-
type: 'update',
|
|
1390
|
-
prev: prevRec,
|
|
1391
|
-
next,
|
|
1392
|
-
}
|
|
1393
|
-
: {
|
|
1394
|
-
session: {
|
|
1395
|
-
sessionId: session.sessionId,
|
|
1396
|
-
isReadonly: session.isReadonly,
|
|
1397
|
-
meta: session.meta,
|
|
1398
|
-
},
|
|
1399
|
-
type: 'create',
|
|
1400
|
-
prev: null,
|
|
1401
|
-
next,
|
|
1402
|
-
}
|
|
1478
|
+
? { session: authSession!, type: 'update', prev: prevRec, next }
|
|
1479
|
+
: { session: authSession!, type: 'create', prev: null, next }
|
|
1403
1480
|
)
|
|
1404
1481
|
if (!result) {
|
|
1405
1482
|
this.log?.warn?.(
|
|
@@ -1431,11 +1508,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1431
1508
|
const authorize = authorizePatch
|
|
1432
1509
|
? (prev: R, next: R) => {
|
|
1433
1510
|
const result = authorizePatch({
|
|
1434
|
-
session:
|
|
1435
|
-
sessionId: session.sessionId,
|
|
1436
|
-
isReadonly: session.isReadonly,
|
|
1437
|
-
meta: session.meta,
|
|
1438
|
-
},
|
|
1511
|
+
session: authSession!,
|
|
1439
1512
|
type: 'update',
|
|
1440
1513
|
prev,
|
|
1441
1514
|
next,
|
|
@@ -1469,16 +1542,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1469
1542
|
const authorizeRemove = session && this.authorizerFor(doc.typeName)
|
|
1470
1543
|
if (
|
|
1471
1544
|
authorizeRemove &&
|
|
1472
|
-
!authorizeRemove({
|
|
1473
|
-
session: {
|
|
1474
|
-
sessionId: session.sessionId,
|
|
1475
|
-
isReadonly: session.isReadonly,
|
|
1476
|
-
meta: session.meta,
|
|
1477
|
-
},
|
|
1478
|
-
type: 'delete',
|
|
1479
|
-
prev: doc,
|
|
1480
|
-
next: null,
|
|
1481
|
-
})
|
|
1545
|
+
!authorizeRemove({ session: authSession!, type: 'delete', prev: doc, next: null })
|
|
1482
1546
|
) {
|
|
1483
1547
|
this.log?.warn?.(
|
|
1484
1548
|
'authorizer vetoed delete',
|
|
@@ -1510,7 +1574,10 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1510
1574
|
let pushResult: TLSocketServerSentEvent<R> | undefined
|
|
1511
1575
|
if (changes && session) {
|
|
1512
1576
|
// txn did not apply verbatim so we should broadcast the actual changes
|
|
1513
|
-
result.docChanges.diffs = {
|
|
1577
|
+
result.docChanges.diffs = {
|
|
1578
|
+
networkDiff: toNetworkDiff(changes, legacyAppendMode),
|
|
1579
|
+
diff: changes,
|
|
1580
|
+
}
|
|
1514
1581
|
}
|
|
1515
1582
|
|
|
1516
1583
|
if (isEqual(result.docChanges.diffs?.networkDiff, message.diff)) {
|
|
@@ -1535,7 +1602,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1535
1602
|
session.serializedSchema,
|
|
1536
1603
|
session.requiresDownMigrations,
|
|
1537
1604
|
result.docChanges.diffs.diff,
|
|
1538
|
-
result.docChanges.diffs.networkDiff
|
|
1605
|
+
result.docChanges.diffs.networkDiff,
|
|
1606
|
+
legacyAppendMode
|
|
1539
1607
|
)
|
|
1540
1608
|
if (diff.ok) {
|
|
1541
1609
|
pushResult = {
|
package/src/lib/TLSyncStorage.ts
CHANGED
|
@@ -146,13 +146,18 @@ export interface TLSyncForwardDiff<R extends UnknownRecord> {
|
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
/**
|
|
149
|
+
* @param legacyAppendMode - When true, string appends are emitted as puts (see `diffRecord`); the
|
|
150
|
+
* room sets this whenever a connected client predates string-append support.
|
|
149
151
|
* @internal
|
|
150
152
|
*/
|
|
151
|
-
export function toNetworkDiff<R extends UnknownRecord>(
|
|
153
|
+
export function toNetworkDiff<R extends UnknownRecord>(
|
|
154
|
+
diff: TLSyncForwardDiff<R>,
|
|
155
|
+
legacyAppendMode = false
|
|
156
|
+
): NetworkDiff<R> {
|
|
152
157
|
const networkDiff: NetworkDiff<R> = {}
|
|
153
158
|
for (const [id, put] of objectMapEntriesIterable(diff.puts)) {
|
|
154
159
|
if (Array.isArray(put)) {
|
|
155
|
-
const patch = diffRecord(put[0], put[1])
|
|
160
|
+
const patch = diffRecord(put[0], put[1], legacyAppendMode)
|
|
156
161
|
if (patch) {
|
|
157
162
|
networkDiff[id] = [RecordOpType.Patch, patch]
|
|
158
163
|
}
|
package/src/lib/chunk.test.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { assert } from 'tldraw'
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
JsonChunkAssembler,
|
|
4
|
+
MAX_ASSEMBLED_MESSAGE_CHARS,
|
|
5
|
+
MAX_CHUNK_COUNT,
|
|
6
|
+
MessageTooLargeError,
|
|
7
|
+
chunk,
|
|
8
|
+
} from './chunk'
|
|
3
9
|
|
|
4
10
|
describe('chunk (CH1–CH3)', () => {
|
|
5
11
|
describe('size boundary (CH1)', () => {
|
|
@@ -134,6 +140,40 @@ describe('chunk (CH1–CH3)', () => {
|
|
|
134
140
|
`)
|
|
135
141
|
})
|
|
136
142
|
})
|
|
143
|
+
|
|
144
|
+
describe('surrogate pairs (CH9)', () => {
|
|
145
|
+
it('[CH9] never splits a surrogate pair across two chunks', () => {
|
|
146
|
+
// WebSocket.send converts each chunk to a USVString, so a lone surrogate would arrive
|
|
147
|
+
// as U+FFFD and the reassembled message would carry corrupted text
|
|
148
|
+
const emoji = '\u{1F600}'
|
|
149
|
+
const isHigh = (code: number) => code >= 0xd800 && code <= 0xdbff
|
|
150
|
+
const isLow = (code: number) => code >= 0xdc00 && code <= 0xdfff
|
|
151
|
+
const msgs = [
|
|
152
|
+
...Array.from(
|
|
153
|
+
{ length: 8 },
|
|
154
|
+
(_, padding) => 'x'.repeat(padding) + emoji + 'y'.repeat(6) + emoji
|
|
155
|
+
),
|
|
156
|
+
// adjacent pairs: after a nudge the next boundary lands on another pair
|
|
157
|
+
'x' + emoji.repeat(3),
|
|
158
|
+
]
|
|
159
|
+
for (const msg of msgs) {
|
|
160
|
+
for (const maxSize of [3, 4, 5, 6, 7, 8]) {
|
|
161
|
+
const chunks = chunk(msg, maxSize)
|
|
162
|
+
const bodies = chunks.map((c) => c.slice(c.indexOf('_') + 1))
|
|
163
|
+
expect(bodies.join('')).toBe(msg)
|
|
164
|
+
for (const [i, body] of bodies.entries()) {
|
|
165
|
+
expect(body.length).toBeGreaterThan(0)
|
|
166
|
+
expect(isHigh(body.charCodeAt(body.length - 1))).toBe(false)
|
|
167
|
+
expect(isLow(body.charCodeAt(0))).toBe(false)
|
|
168
|
+
// keeping a pair whole may cost one character over the CH3 bound, never more
|
|
169
|
+
expect(chunks[i].length).toBeLessThanOrEqual(
|
|
170
|
+
Math.max(maxSize, chunks[i].indexOf('_') + 2) + 1
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
})
|
|
176
|
+
})
|
|
137
177
|
})
|
|
138
178
|
|
|
139
179
|
const testObject = {} as any
|
|
@@ -369,4 +409,51 @@ describe('JsonChunkAssembler (CH4–CH8)', () => {
|
|
|
369
409
|
})
|
|
370
410
|
})
|
|
371
411
|
})
|
|
412
|
+
|
|
413
|
+
describe('assembly bounds (CH10)', () => {
|
|
414
|
+
it('[CH10] rejects a first chunk declaring more than MAX_CHUNK_COUNT chunks', () => {
|
|
415
|
+
const unchunker = new JsonChunkAssembler()
|
|
416
|
+
|
|
417
|
+
const result = unchunker.handleMessage(`${MAX_CHUNK_COUNT}_{"a":`)
|
|
418
|
+
expect(result).toMatchObject({
|
|
419
|
+
error: expect.objectContaining({ message: `Too many chunks: ${MAX_CHUNK_COUNT + 1}` }),
|
|
420
|
+
})
|
|
421
|
+
expect(unchunker.state).toBe('idle')
|
|
422
|
+
})
|
|
423
|
+
|
|
424
|
+
it('[CH10] accepts a declared count at the limit', () => {
|
|
425
|
+
const unchunker = new JsonChunkAssembler()
|
|
426
|
+
|
|
427
|
+
expect(unchunker.handleMessage(`${MAX_CHUNK_COUNT - 1}_{"a":`)).toBeNull()
|
|
428
|
+
expect(unchunker.state).not.toBe('idle')
|
|
429
|
+
})
|
|
430
|
+
|
|
431
|
+
it('[CH10] rejects an assembly once its accumulated bodies pass MAX_ASSEMBLED_MESSAGE_CHARS', () => {
|
|
432
|
+
const unchunker = new JsonChunkAssembler()
|
|
433
|
+
// A chunk stream that never sends its final chunk. Before CH10 this accumulated
|
|
434
|
+
// without limit for as long as the sender kept sending.
|
|
435
|
+
const body = 'x'.repeat(1024 * 1024)
|
|
436
|
+
const chunksNeeded = Math.ceil(MAX_ASSEMBLED_MESSAGE_CHARS / body.length) + 1
|
|
437
|
+
|
|
438
|
+
let result = null
|
|
439
|
+
for (let i = 0; i < chunksNeeded; i++) {
|
|
440
|
+
result = unchunker.handleMessage(`${chunksNeeded - i}_${body}`)
|
|
441
|
+
if (result) break
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
expect(result).toMatchObject({ error: expect.any(MessageTooLargeError) })
|
|
445
|
+
expect(unchunker.state).toBe('idle')
|
|
446
|
+
})
|
|
447
|
+
|
|
448
|
+
it('[CH10] leaves the assembler usable after rejecting an oversized assembly', () => {
|
|
449
|
+
const unchunker = new JsonChunkAssembler()
|
|
450
|
+
const body = 'x'.repeat(1024 * 1024)
|
|
451
|
+
const chunksNeeded = Math.ceil(MAX_ASSEMBLED_MESSAGE_CHARS / body.length) + 1
|
|
452
|
+
for (let i = 0; i < chunksNeeded; i++) {
|
|
453
|
+
if (unchunker.handleMessage(`${chunksNeeded - i}_${body}`)) break
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
expect(unchunker.handleMessage('{"test": true}')).toMatchObject({ data: { test: true } })
|
|
457
|
+
})
|
|
458
|
+
})
|
|
372
459
|
})
|
package/src/lib/chunk.ts
CHANGED
|
@@ -8,6 +8,35 @@ const MAX_BYTES_PER_CHAR = 4
|
|
|
8
8
|
// in the (admittedly impossible) worst case, the max size is 1/4 of a megabyte
|
|
9
9
|
const MAX_SAFE_MESSAGE_SIZE = MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES / MAX_BYTES_PER_CHAR
|
|
10
10
|
|
|
11
|
+
// JsonChunkAssembler holds every chunk of a message until the last one arrives, and the sender
|
|
12
|
+
// decides how many that is — the count is just a prefix on the first chunk. Nothing in the
|
|
13
|
+
// protocol obliges a sender to ever send the last chunk, so without a ceiling a single connection
|
|
14
|
+
// can make the receiver hold an arbitrarily large buffer for as long as the socket stays open.
|
|
15
|
+
//
|
|
16
|
+
// The cap counts UTF-16 code units, which is what a JS string costs in memory: 16M of them is the
|
|
17
|
+
// 32MB of retained string data a room can afford to hold per session and stay inside a Durable
|
|
18
|
+
// Object's 128MB instance limit with room for the document itself. It is far above anything
|
|
19
|
+
// chunk() produces for a real document push. The character cap is the real bound; the chunk-count
|
|
20
|
+
// cap only rejects an absurd declared count on arrival instead of waiting for the characters to
|
|
21
|
+
// accumulate, and is loose enough not to constrain a sender that chunks more finely than the
|
|
22
|
+
// default.
|
|
23
|
+
export const MAX_ASSEMBLED_MESSAGE_CHARS = 16 * 1024 * 1024
|
|
24
|
+
export const MAX_CHUNK_COUNT = 100_000
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Thrown when a message would exceed MAX_ASSEMBLED_MESSAGE_CHARS, on either side of the wire.
|
|
28
|
+
* Distinct from the other assembly errors because it is fatal: retrying sends the same
|
|
29
|
+
* oversized message, so the session is rejected rather than reset.
|
|
30
|
+
*
|
|
31
|
+
* @internal
|
|
32
|
+
*/
|
|
33
|
+
export class MessageTooLargeError extends Error {
|
|
34
|
+
constructor(chars: number) {
|
|
35
|
+
super(`Message too large: ${chars} characters, max ${MAX_ASSEMBLED_MESSAGE_CHARS}`)
|
|
36
|
+
this.name = 'MessageTooLargeError'
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
11
40
|
/**
|
|
12
41
|
* Splits a string into smaller chunks suitable for transmission over WebSockets.
|
|
13
42
|
* This function ensures messages don't exceed size limits imposed by platforms like Cloudflare Workers (1MB max).
|
|
@@ -38,7 +67,18 @@ export function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
|
|
|
38
67
|
let offset = msg.length
|
|
39
68
|
while (offset > 0) {
|
|
40
69
|
const prefix = `${chunkNumber}_`
|
|
41
|
-
|
|
70
|
+
let chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1)
|
|
71
|
+
// Never split a surrogate pair across chunks: WebSocket.send converts each chunk to a
|
|
72
|
+
// USVString, turning a lone surrogate into U+FFFD, so the reassembled JSON silently
|
|
73
|
+
// carries corrupted text. Shrink by one so the pair moves whole into the preceding
|
|
74
|
+
// chunk; a single-character chunk grows by one instead (one over the CH3 bound, see CH9).
|
|
75
|
+
if (
|
|
76
|
+
chunkSize < offset &&
|
|
77
|
+
isLowSurrogate(msg.charCodeAt(offset - chunkSize)) &&
|
|
78
|
+
isHighSurrogate(msg.charCodeAt(offset - chunkSize - 1))
|
|
79
|
+
) {
|
|
80
|
+
chunkSize += chunkSize > 1 ? -1 : 1
|
|
81
|
+
}
|
|
42
82
|
chunks.unshift(prefix + msg.slice(offset - chunkSize, offset))
|
|
43
83
|
offset -= chunkSize
|
|
44
84
|
chunkNumber++
|
|
@@ -47,6 +87,14 @@ export function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
|
|
|
47
87
|
}
|
|
48
88
|
}
|
|
49
89
|
|
|
90
|
+
function isHighSurrogate(code: number) {
|
|
91
|
+
return code >= 0xd800 && code <= 0xdbff
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function isLowSurrogate(code: number) {
|
|
95
|
+
return code >= 0xdc00 && code <= 0xdfff
|
|
96
|
+
}
|
|
97
|
+
|
|
50
98
|
// The 's' flag (dotAll) makes '.' match any character including line terminators
|
|
51
99
|
// like U+2028 and U+2029, which are commonly introduced via copy/paste from Word
|
|
52
100
|
const chunkRe = /^(\d+)_(.*)$/s
|
|
@@ -81,6 +129,7 @@ export class JsonChunkAssembler {
|
|
|
81
129
|
| {
|
|
82
130
|
chunksReceived: string[]
|
|
83
131
|
totalChunks: number
|
|
132
|
+
charsReceived: number
|
|
84
133
|
} = 'idle'
|
|
85
134
|
|
|
86
135
|
/**
|
|
@@ -91,7 +140,8 @@ export class JsonChunkAssembler {
|
|
|
91
140
|
* @param msg - The message to process, either JSON or chunk format
|
|
92
141
|
* @returns Result object with data/stringified on success, error object on failure, or null for incomplete chunks
|
|
93
142
|
* - `\{ data: object, stringified: string \}` - Successfully parsed complete message
|
|
94
|
-
* - `\{ error: Error \}` - Parse error
|
|
143
|
+
* - `\{ error: Error \}` - Parse error, invalid chunk sequence, or a message exceeding
|
|
144
|
+
* MAX_ASSEMBLED_MESSAGE_CHARS / MAX_CHUNK_COUNT
|
|
95
145
|
* - `null` - Chunk received but more chunks expected
|
|
96
146
|
*
|
|
97
147
|
* @example
|
|
@@ -125,17 +175,30 @@ export class JsonChunkAssembler {
|
|
|
125
175
|
const data = match[2]
|
|
126
176
|
|
|
127
177
|
if (this.state === 'idle') {
|
|
178
|
+
const totalChunks = numChunksRemaining + 1
|
|
179
|
+
if (totalChunks > MAX_CHUNK_COUNT) {
|
|
180
|
+
// Nothing has been buffered yet, so the state is already idle.
|
|
181
|
+
return { error: new Error(`Too many chunks: ${totalChunks}`) }
|
|
182
|
+
}
|
|
128
183
|
this.state = {
|
|
129
184
|
chunksReceived: [data],
|
|
130
|
-
totalChunks
|
|
185
|
+
totalChunks,
|
|
186
|
+
charsReceived: data.length,
|
|
131
187
|
}
|
|
132
188
|
} else {
|
|
133
189
|
this.state.chunksReceived.push(data)
|
|
190
|
+
this.state.charsReceived += data.length
|
|
134
191
|
if (numChunksRemaining !== this.state.totalChunks - this.state.chunksReceived.length) {
|
|
135
192
|
this.state = 'idle'
|
|
136
193
|
return { error: new Error(`Chunks received in wrong order`) }
|
|
137
194
|
}
|
|
138
195
|
}
|
|
196
|
+
if (this.state.charsReceived > MAX_ASSEMBLED_MESSAGE_CHARS) {
|
|
197
|
+
// Drop what was buffered rather than keeping it until the socket closes.
|
|
198
|
+
const charsReceived = this.state.charsReceived
|
|
199
|
+
this.state = 'idle'
|
|
200
|
+
return { error: new MessageTooLargeError(charsReceived) }
|
|
201
|
+
}
|
|
139
202
|
if (this.state.chunksReceived.length === this.state.totalChunks) {
|
|
140
203
|
try {
|
|
141
204
|
const stringified = this.state.chunksReceived.join('')
|