@tldraw/sync-core 5.5.0-canary.fdb2d37fcb2a → 5.5.0-canary.fdd9b4dee0d9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-cjs/index.d.ts +8 -4
- package/dist-cjs/index.js +1 -1
- package/dist-cjs/lib/ClientWebSocketAdapter.js +23 -7
- package/dist-cjs/lib/ClientWebSocketAdapter.js.map +2 -2
- package/dist-cjs/lib/InMemorySyncStorage.js +18 -18
- package/dist-cjs/lib/InMemorySyncStorage.js.map +2 -2
- package/dist-cjs/lib/SQLiteSyncStorage.js.map +2 -2
- package/dist-cjs/lib/TLSocketRoom.js +5 -1
- package/dist-cjs/lib/TLSocketRoom.js.map +2 -2
- package/dist-cjs/lib/TLSyncClient.js +3 -1
- package/dist-cjs/lib/TLSyncClient.js.map +2 -2
- package/dist-cjs/lib/TLSyncRoom.js +52 -47
- package/dist-cjs/lib/TLSyncRoom.js.map +2 -2
- package/dist-cjs/lib/TLSyncStorage.js +2 -2
- package/dist-cjs/lib/TLSyncStorage.js.map +2 -2
- package/dist-cjs/lib/chunk.js +25 -2
- package/dist-cjs/lib/chunk.js.map +2 -2
- package/dist-cjs/lib/diff.js +15 -7
- package/dist-cjs/lib/diff.js.map +2 -2
- package/dist-esm/index.d.mts +8 -4
- package/dist-esm/index.mjs +1 -1
- package/dist-esm/lib/ClientWebSocketAdapter.mjs +24 -8
- package/dist-esm/lib/ClientWebSocketAdapter.mjs.map +2 -2
- package/dist-esm/lib/InMemorySyncStorage.mjs +18 -18
- package/dist-esm/lib/InMemorySyncStorage.mjs.map +2 -2
- package/dist-esm/lib/SQLiteSyncStorage.mjs.map +2 -2
- package/dist-esm/lib/TLSocketRoom.mjs +6 -2
- package/dist-esm/lib/TLSocketRoom.mjs.map +2 -2
- package/dist-esm/lib/TLSyncClient.mjs +3 -1
- package/dist-esm/lib/TLSyncClient.mjs.map +2 -2
- package/dist-esm/lib/TLSyncRoom.mjs +52 -47
- package/dist-esm/lib/TLSyncRoom.mjs.map +2 -2
- package/dist-esm/lib/TLSyncStorage.mjs +2 -2
- package/dist-esm/lib/TLSyncStorage.mjs.map +2 -2
- package/dist-esm/lib/chunk.mjs +25 -2
- package/dist-esm/lib/chunk.mjs.map +2 -2
- package/dist-esm/lib/diff.mjs +16 -8
- package/dist-esm/lib/diff.mjs.map +2 -2
- package/package.json +20 -17
- package/src/lib/ClientWebSocketAdapter.test.ts +58 -0
- package/src/lib/ClientWebSocketAdapter.ts +33 -10
- package/src/lib/InMemorySyncStorage.test.ts +21 -0
- package/src/lib/InMemorySyncStorage.ts +22 -24
- package/src/lib/SQLiteSyncStorage.ts +2 -2
- package/src/lib/TLSocketRoom.ts +8 -2
- package/src/lib/TLSyncClient.ts +2 -0
- package/src/lib/TLSyncRoom.ts +83 -51
- package/src/lib/TLSyncStorage.ts +7 -2
- package/src/lib/chunk.test.ts +54 -1
- package/src/lib/chunk.ts +46 -2
- package/src/lib/diff.test.ts +91 -1
- package/src/lib/diff.ts +30 -9
- package/src/test/TLSocketRoom.test.ts +24 -0
- package/src/test/TLSyncRoom.test.ts +195 -0
- package/src/test/upgradeDowngrade.test.ts +21 -0
package/src/lib/TLSyncRoom.ts
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
objectMapEntriesIterable,
|
|
17
17
|
Result,
|
|
18
18
|
throttle,
|
|
19
|
+
type ThrottledFunction,
|
|
19
20
|
} from '@tldraw/utils'
|
|
20
21
|
import { createNanoEvents } from 'nanoevents'
|
|
21
22
|
import {
|
|
@@ -88,6 +89,29 @@ export const DATA_MESSAGE_DEBOUNCE_INTERVAL = 1000 / 60
|
|
|
88
89
|
|
|
89
90
|
const timeSince = (time: number) => Date.now() - time
|
|
90
91
|
|
|
92
|
+
// WebSocket close frames cap the reason at 123 UTF-8 bytes; a longer one makes `close()` throw
|
|
93
|
+
const MAX_CLOSE_REASON_BYTES = 123
|
|
94
|
+
const closeReasonEncoder = new TextEncoder()
|
|
95
|
+
|
|
96
|
+
const truncatedSuffix = (droppedBytes: number) => `... (+${droppedBytes} bytes)`
|
|
97
|
+
|
|
98
|
+
function truncateCloseReason(reason: string) {
|
|
99
|
+
const totalBytes = closeReasonEncoder.encode(reason).length
|
|
100
|
+
if (totalBytes <= MAX_CLOSE_REASON_BYTES) return reason
|
|
101
|
+
// the dropped count can't have more digits than the total, so reserving room for it always fits
|
|
102
|
+
const budget = MAX_CLOSE_REASON_BYTES - truncatedSuffix(totalBytes).length
|
|
103
|
+
let out = ''
|
|
104
|
+
let outBytes = 0
|
|
105
|
+
// step by code point so a multi-byte character is never cut in half
|
|
106
|
+
for (const char of reason) {
|
|
107
|
+
const charBytes = closeReasonEncoder.encode(char).length
|
|
108
|
+
if (outBytes + charBytes > budget) break
|
|
109
|
+
out += char
|
|
110
|
+
outBytes += charBytes
|
|
111
|
+
}
|
|
112
|
+
return out + truncatedSuffix(totalBytes - outBytes)
|
|
113
|
+
}
|
|
114
|
+
|
|
91
115
|
/**
|
|
92
116
|
* Snapshot of a room's complete state that can be persisted and restored.
|
|
93
117
|
* Contains all documents, tombstones, and metadata needed to reconstruct the room.
|
|
@@ -217,7 +241,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
217
241
|
|
|
218
242
|
private pruneTimer: ReturnType<typeof setTimeout> | null = null
|
|
219
243
|
|
|
220
|
-
pruneSessions = throttle(() => {
|
|
244
|
+
pruneSessions: ThrottledFunction<() => void> = throttle(() => {
|
|
221
245
|
if (this.pruneTimer) {
|
|
222
246
|
clearTimeout(this.pruneTimer)
|
|
223
247
|
this.pruneTimer = null
|
|
@@ -559,7 +583,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
559
583
|
|
|
560
584
|
try {
|
|
561
585
|
if (fatalReason) {
|
|
562
|
-
session
|
|
586
|
+
// the session is already gone, so a throwing close() would leave its socket open
|
|
587
|
+
session.socket.close(TLSyncErrorCloseEventCode, truncateCloseReason(fatalReason))
|
|
563
588
|
} else {
|
|
564
589
|
session.socket.close()
|
|
565
590
|
}
|
|
@@ -632,9 +657,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
632
657
|
networkDiff?: NetworkDiff<R> | null,
|
|
633
658
|
sourceSessionId?: string
|
|
634
659
|
) {
|
|
635
|
-
//
|
|
636
|
-
|
|
637
|
-
|
|
660
|
+
// Computed once and shared by every session that needs no down-migration; the push path
|
|
661
|
+
// hands us the diff it already computed (in legacy append mode when needed), and re-deriving
|
|
662
|
+
// it per session would both repeat the diffing work and lose that legacy handling.
|
|
663
|
+
const legacyAppendMode = !this.getCanEmitStringAppend()
|
|
664
|
+
const unmigrated = networkDiff ?? toNetworkDiff(diff, legacyAppendMode)
|
|
638
665
|
|
|
639
666
|
this.sessions.forEach((session) => {
|
|
640
667
|
if (session.state !== RoomSessionState.Connected) return
|
|
@@ -648,7 +675,9 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
648
675
|
session.sessionId,
|
|
649
676
|
session.serializedSchema,
|
|
650
677
|
session.requiresDownMigrations,
|
|
651
|
-
diff
|
|
678
|
+
diff,
|
|
679
|
+
unmigrated,
|
|
680
|
+
legacyAppendMode
|
|
652
681
|
)
|
|
653
682
|
if (!diffResult.ok) return
|
|
654
683
|
|
|
@@ -797,9 +826,22 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
797
826
|
supportsStringAppend,
|
|
798
827
|
})
|
|
799
828
|
|
|
829
|
+
// Restore presence before the schema checks: other clients still hold it from before the
|
|
830
|
+
// socket slept, and a rejection only broadcasts its removal if it's in the store
|
|
800
831
|
if (presenceRecord && presenceId) {
|
|
801
832
|
this.presenceStore.set(presenceId, presenceRecord as R)
|
|
802
833
|
}
|
|
834
|
+
|
|
835
|
+
// The server may have changed builds while the socket slept, so re-run the handshake's
|
|
836
|
+
// schema checks (HS3): a schema we can no longer reconcile must not be served raw diffs.
|
|
837
|
+
if (!migrations.ok) {
|
|
838
|
+
this.rejectSession(sessionId, this.getVersionMismatchReason(serializedSchema))
|
|
839
|
+
return
|
|
840
|
+
}
|
|
841
|
+
if (migrations.value.some((m) => m.scope !== 'record' || !m.down)) {
|
|
842
|
+
this.rejectSession(sessionId, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD)
|
|
843
|
+
return
|
|
844
|
+
}
|
|
803
845
|
}
|
|
804
846
|
|
|
805
847
|
/**
|
|
@@ -834,6 +876,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
834
876
|
* @param requiresDownMigrations - Whether the client needs down migrations
|
|
835
877
|
* @param diff - The TLSyncForwardDiff containing full records to migrate
|
|
836
878
|
* @param unmigrated - Optional pre-computed NetworkDiff for when no migration is needed
|
|
879
|
+
* @param legacyAppendMode - Emit string appends as puts (SES5); defaults to the room-wide state
|
|
837
880
|
* @returns A NetworkDiff with migrated records, or a migration failure
|
|
838
881
|
*/
|
|
839
882
|
private migrateDiffOrRejectSession(
|
|
@@ -841,10 +884,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
841
884
|
serializedSchema: SerializedSchema,
|
|
842
885
|
requiresDownMigrations: boolean,
|
|
843
886
|
diff: TLSyncForwardDiff<R>,
|
|
844
|
-
unmigrated?: NetworkDiff<R
|
|
887
|
+
unmigrated?: NetworkDiff<R>,
|
|
888
|
+
legacyAppendMode = !this.getCanEmitStringAppend()
|
|
845
889
|
): Result<NetworkDiff<R>, MigrationFailureReason> {
|
|
846
890
|
if (!requiresDownMigrations) {
|
|
847
|
-
return Result.ok(unmigrated ?? toNetworkDiff(diff
|
|
891
|
+
return Result.ok(unmigrated ?? toNetworkDiff(diff, legacyAppendMode))
|
|
848
892
|
}
|
|
849
893
|
|
|
850
894
|
const result: NetworkDiff<R> = {}
|
|
@@ -864,7 +908,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
864
908
|
this.rejectSession(sessionId, TLSyncErrorCloseEventReason.CLIENT_TOO_OLD)
|
|
865
909
|
return Result.err(toResult.reason)
|
|
866
910
|
}
|
|
867
|
-
const patch = diffRecord(fromResult.value, toResult.value)
|
|
911
|
+
const patch = diffRecord(fromResult.value, toResult.value, legacyAppendMode)
|
|
868
912
|
if (patch) {
|
|
869
913
|
result[id] = [RecordOpType.Patch, patch]
|
|
870
914
|
}
|
|
@@ -941,7 +985,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
941
985
|
* Sends appropriate error messages before closing the connection.
|
|
942
986
|
*
|
|
943
987
|
* @param sessionId - The session to reject
|
|
944
|
-
* @param fatalReason - The reason for rejection (optional)
|
|
988
|
+
* @param fatalReason - The reason for rejection (optional). WebSocket close reasons are capped
|
|
989
|
+
* at 123 UTF-8 bytes, so a longer reason is truncated and ends with `... (+N bytes)`.
|
|
945
990
|
* @example
|
|
946
991
|
* ```ts
|
|
947
992
|
* // Reject due to version mismatch
|
|
@@ -998,6 +1043,10 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
998
1043
|
|
|
999
1044
|
private forceAllReconnect() {
|
|
1000
1045
|
for (const session of this.sessions.values()) {
|
|
1046
|
+
// Only connected clients hold state at a clock we can no longer diff from. A session
|
|
1047
|
+
// mid-handshake gets hydrated from its own lastServerClock in the same transaction, so
|
|
1048
|
+
// removing it would close its socket and then re-add it as Connected (resurrected).
|
|
1049
|
+
if (session.state !== RoomSessionState.Connected) continue
|
|
1001
1050
|
this.removeSession(session.sessionId)
|
|
1002
1051
|
}
|
|
1003
1052
|
}
|
|
@@ -1027,7 +1076,13 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1027
1076
|
return TLSyncErrorCloseEventReason.SERVER_TOO_OLD
|
|
1028
1077
|
}
|
|
1029
1078
|
|
|
1030
|
-
if (
|
|
1079
|
+
if (
|
|
1080
|
+
theirSchema.schemaVersion === 2 &&
|
|
1081
|
+
ourSchema.schemaVersion === 2 &&
|
|
1082
|
+
// a malformed client schema must reject the session, not throw
|
|
1083
|
+
typeof theirSchema.sequences === 'object' &&
|
|
1084
|
+
theirSchema.sequences !== null
|
|
1085
|
+
) {
|
|
1031
1086
|
for (const [sequenceId, theirVersion] of Object.entries(theirSchema.sequences)) {
|
|
1032
1087
|
const ourVersion = ourSchema.sequences[sequenceId]
|
|
1033
1088
|
if (ourVersion === undefined || theirVersion > ourVersion) {
|
|
@@ -1091,7 +1146,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1091
1146
|
|
|
1092
1147
|
const requiresDownMigrations = migrations.value.length > 0
|
|
1093
1148
|
|
|
1094
|
-
const connect =
|
|
1149
|
+
const connect = (msg: Extract<TLSocketServerSentEvent<R>, { type: 'connect' }>) => {
|
|
1095
1150
|
this.sessions.set(session.sessionId, {
|
|
1096
1151
|
state: RoomSessionState.Connected,
|
|
1097
1152
|
sessionId: session.sessionId,
|
|
@@ -1232,7 +1287,6 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1232
1287
|
}
|
|
1233
1288
|
let { value: state } = res
|
|
1234
1289
|
|
|
1235
|
-
// Get the existing document, if any
|
|
1236
1290
|
const doc =
|
|
1237
1291
|
prevDoc !== undefined ? (prevDoc ?? undefined) : (storage.get(id) as R | undefined)
|
|
1238
1292
|
|
|
@@ -1248,7 +1302,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1248
1302
|
// If there's an existing document, replace it with the new state
|
|
1249
1303
|
// but propagate a diff rather than the entire value
|
|
1250
1304
|
const recordType = assertExists(getOwnProperty(this.schema.types, doc.typeName))
|
|
1251
|
-
const diff = diffAndValidateRecord(doc, state, recordType)
|
|
1305
|
+
const diff = diffAndValidateRecord(doc, state, recordType, legacyAppendMode)
|
|
1252
1306
|
if (diff) {
|
|
1253
1307
|
storage.set(id, state)
|
|
1254
1308
|
propagateOp(changes, id, [RecordOpType.Patch, diff], doc, state)
|
|
@@ -1366,6 +1420,11 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1366
1420
|
!session ||
|
|
1367
1421
|
(this.objectTypes.has(typeName) ? session.objectAccess !== 'read' : !session.isReadonly)
|
|
1368
1422
|
|
|
1423
|
+
// what authorizers see of the pushing session, shared by every authorized op in this push
|
|
1424
|
+
const authSession = session
|
|
1425
|
+
? { sessionId: session.sessionId, isReadonly: session.isReadonly, meta: session.meta }
|
|
1426
|
+
: null
|
|
1427
|
+
|
|
1369
1428
|
if (message.diff) {
|
|
1370
1429
|
// The push request was for the document scope.
|
|
1371
1430
|
for (const [id, op] of objectMapEntriesIterable(message.diff!)) {
|
|
@@ -1416,26 +1475,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1416
1475
|
authorize = (prevRec, next) => {
|
|
1417
1476
|
const result = authorizePut(
|
|
1418
1477
|
prevRec
|
|
1419
|
-
? {
|
|
1420
|
-
|
|
1421
|
-
sessionId: session.sessionId,
|
|
1422
|
-
isReadonly: session.isReadonly,
|
|
1423
|
-
meta: session.meta,
|
|
1424
|
-
},
|
|
1425
|
-
type: 'update',
|
|
1426
|
-
prev: prevRec,
|
|
1427
|
-
next,
|
|
1428
|
-
}
|
|
1429
|
-
: {
|
|
1430
|
-
session: {
|
|
1431
|
-
sessionId: session.sessionId,
|
|
1432
|
-
isReadonly: session.isReadonly,
|
|
1433
|
-
meta: session.meta,
|
|
1434
|
-
},
|
|
1435
|
-
type: 'create',
|
|
1436
|
-
prev: null,
|
|
1437
|
-
next,
|
|
1438
|
-
}
|
|
1478
|
+
? { session: authSession!, type: 'update', prev: prevRec, next }
|
|
1479
|
+
: { session: authSession!, type: 'create', prev: null, next }
|
|
1439
1480
|
)
|
|
1440
1481
|
if (!result) {
|
|
1441
1482
|
this.log?.warn?.(
|
|
@@ -1467,11 +1508,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1467
1508
|
const authorize = authorizePatch
|
|
1468
1509
|
? (prev: R, next: R) => {
|
|
1469
1510
|
const result = authorizePatch({
|
|
1470
|
-
session:
|
|
1471
|
-
sessionId: session.sessionId,
|
|
1472
|
-
isReadonly: session.isReadonly,
|
|
1473
|
-
meta: session.meta,
|
|
1474
|
-
},
|
|
1511
|
+
session: authSession!,
|
|
1475
1512
|
type: 'update',
|
|
1476
1513
|
prev,
|
|
1477
1514
|
next,
|
|
@@ -1505,16 +1542,7 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1505
1542
|
const authorizeRemove = session && this.authorizerFor(doc.typeName)
|
|
1506
1543
|
if (
|
|
1507
1544
|
authorizeRemove &&
|
|
1508
|
-
!authorizeRemove({
|
|
1509
|
-
session: {
|
|
1510
|
-
sessionId: session.sessionId,
|
|
1511
|
-
isReadonly: session.isReadonly,
|
|
1512
|
-
meta: session.meta,
|
|
1513
|
-
},
|
|
1514
|
-
type: 'delete',
|
|
1515
|
-
prev: doc,
|
|
1516
|
-
next: null,
|
|
1517
|
-
})
|
|
1545
|
+
!authorizeRemove({ session: authSession!, type: 'delete', prev: doc, next: null })
|
|
1518
1546
|
) {
|
|
1519
1547
|
this.log?.warn?.(
|
|
1520
1548
|
'authorizer vetoed delete',
|
|
@@ -1546,7 +1574,10 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1546
1574
|
let pushResult: TLSocketServerSentEvent<R> | undefined
|
|
1547
1575
|
if (changes && session) {
|
|
1548
1576
|
// txn did not apply verbatim so we should broadcast the actual changes
|
|
1549
|
-
result.docChanges.diffs = {
|
|
1577
|
+
result.docChanges.diffs = {
|
|
1578
|
+
networkDiff: toNetworkDiff(changes, legacyAppendMode),
|
|
1579
|
+
diff: changes,
|
|
1580
|
+
}
|
|
1550
1581
|
}
|
|
1551
1582
|
|
|
1552
1583
|
if (isEqual(result.docChanges.diffs?.networkDiff, message.diff)) {
|
|
@@ -1571,7 +1602,8 @@ export class TLSyncRoom<R extends UnknownRecord, SessionMeta> {
|
|
|
1571
1602
|
session.serializedSchema,
|
|
1572
1603
|
session.requiresDownMigrations,
|
|
1573
1604
|
result.docChanges.diffs.diff,
|
|
1574
|
-
result.docChanges.diffs.networkDiff
|
|
1605
|
+
result.docChanges.diffs.networkDiff,
|
|
1606
|
+
legacyAppendMode
|
|
1575
1607
|
)
|
|
1576
1608
|
if (diff.ok) {
|
|
1577
1609
|
pushResult = {
|
package/src/lib/TLSyncStorage.ts
CHANGED
|
@@ -146,13 +146,18 @@ export interface TLSyncForwardDiff<R extends UnknownRecord> {
|
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
/**
|
|
149
|
+
* @param legacyAppendMode - When true, string appends are emitted as puts (see `diffRecord`); the
|
|
150
|
+
* room sets this whenever a connected client predates string-append support.
|
|
149
151
|
* @internal
|
|
150
152
|
*/
|
|
151
|
-
export function toNetworkDiff<R extends UnknownRecord>(
|
|
153
|
+
export function toNetworkDiff<R extends UnknownRecord>(
|
|
154
|
+
diff: TLSyncForwardDiff<R>,
|
|
155
|
+
legacyAppendMode = false
|
|
156
|
+
): NetworkDiff<R> {
|
|
152
157
|
const networkDiff: NetworkDiff<R> = {}
|
|
153
158
|
for (const [id, put] of objectMapEntriesIterable(diff.puts)) {
|
|
154
159
|
if (Array.isArray(put)) {
|
|
155
|
-
const patch = diffRecord(put[0], put[1])
|
|
160
|
+
const patch = diffRecord(put[0], put[1], legacyAppendMode)
|
|
156
161
|
if (patch) {
|
|
157
162
|
networkDiff[id] = [RecordOpType.Patch, patch]
|
|
158
163
|
}
|
package/src/lib/chunk.test.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { assert } from 'tldraw'
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
JsonChunkAssembler,
|
|
4
|
+
MAX_ASSEMBLED_MESSAGE_CHARS,
|
|
5
|
+
MAX_CHUNK_COUNT,
|
|
6
|
+
MessageTooLargeError,
|
|
7
|
+
chunk,
|
|
8
|
+
} from './chunk'
|
|
3
9
|
|
|
4
10
|
describe('chunk (CH1–CH3)', () => {
|
|
5
11
|
describe('size boundary (CH1)', () => {
|
|
@@ -403,4 +409,51 @@ describe('JsonChunkAssembler (CH4–CH8)', () => {
|
|
|
403
409
|
})
|
|
404
410
|
})
|
|
405
411
|
})
|
|
412
|
+
|
|
413
|
+
describe('assembly bounds (CH10)', () => {
|
|
414
|
+
it('[CH10] rejects a first chunk declaring more than MAX_CHUNK_COUNT chunks', () => {
|
|
415
|
+
const unchunker = new JsonChunkAssembler()
|
|
416
|
+
|
|
417
|
+
const result = unchunker.handleMessage(`${MAX_CHUNK_COUNT}_{"a":`)
|
|
418
|
+
expect(result).toMatchObject({
|
|
419
|
+
error: expect.objectContaining({ message: `Too many chunks: ${MAX_CHUNK_COUNT + 1}` }),
|
|
420
|
+
})
|
|
421
|
+
expect(unchunker.state).toBe('idle')
|
|
422
|
+
})
|
|
423
|
+
|
|
424
|
+
it('[CH10] accepts a declared count at the limit', () => {
|
|
425
|
+
const unchunker = new JsonChunkAssembler()
|
|
426
|
+
|
|
427
|
+
expect(unchunker.handleMessage(`${MAX_CHUNK_COUNT - 1}_{"a":`)).toBeNull()
|
|
428
|
+
expect(unchunker.state).not.toBe('idle')
|
|
429
|
+
})
|
|
430
|
+
|
|
431
|
+
it('[CH10] rejects an assembly once its accumulated bodies pass MAX_ASSEMBLED_MESSAGE_CHARS', () => {
|
|
432
|
+
const unchunker = new JsonChunkAssembler()
|
|
433
|
+
// A chunk stream that never sends its final chunk. Before CH10 this accumulated
|
|
434
|
+
// without limit for as long as the sender kept sending.
|
|
435
|
+
const body = 'x'.repeat(1024 * 1024)
|
|
436
|
+
const chunksNeeded = Math.ceil(MAX_ASSEMBLED_MESSAGE_CHARS / body.length) + 1
|
|
437
|
+
|
|
438
|
+
let result = null
|
|
439
|
+
for (let i = 0; i < chunksNeeded; i++) {
|
|
440
|
+
result = unchunker.handleMessage(`${chunksNeeded - i}_${body}`)
|
|
441
|
+
if (result) break
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
expect(result).toMatchObject({ error: expect.any(MessageTooLargeError) })
|
|
445
|
+
expect(unchunker.state).toBe('idle')
|
|
446
|
+
})
|
|
447
|
+
|
|
448
|
+
it('[CH10] leaves the assembler usable after rejecting an oversized assembly', () => {
|
|
449
|
+
const unchunker = new JsonChunkAssembler()
|
|
450
|
+
const body = 'x'.repeat(1024 * 1024)
|
|
451
|
+
const chunksNeeded = Math.ceil(MAX_ASSEMBLED_MESSAGE_CHARS / body.length) + 1
|
|
452
|
+
for (let i = 0; i < chunksNeeded; i++) {
|
|
453
|
+
if (unchunker.handleMessage(`${chunksNeeded - i}_${body}`)) break
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
expect(unchunker.handleMessage('{"test": true}')).toMatchObject({ data: { test: true } })
|
|
457
|
+
})
|
|
458
|
+
})
|
|
406
459
|
})
|
package/src/lib/chunk.ts
CHANGED
|
@@ -8,6 +8,35 @@ const MAX_BYTES_PER_CHAR = 4
|
|
|
8
8
|
// in the (admittedly impossible) worst case, the max size is 1/4 of a megabyte
|
|
9
9
|
const MAX_SAFE_MESSAGE_SIZE = MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES / MAX_BYTES_PER_CHAR
|
|
10
10
|
|
|
11
|
+
// JsonChunkAssembler holds every chunk of a message until the last one arrives, and the sender
|
|
12
|
+
// decides how many that is — the count is just a prefix on the first chunk. Nothing in the
|
|
13
|
+
// protocol obliges a sender to ever send the last chunk, so without a ceiling a single connection
|
|
14
|
+
// can make the receiver hold an arbitrarily large buffer for as long as the socket stays open.
|
|
15
|
+
//
|
|
16
|
+
// The cap counts UTF-16 code units, which is what a JS string costs in memory: 16M of them is the
|
|
17
|
+
// 32MB of retained string data a room can afford to hold per session and stay inside a Durable
|
|
18
|
+
// Object's 128MB instance limit with room for the document itself. It is far above anything
|
|
19
|
+
// chunk() produces for a real document push. The character cap is the real bound; the chunk-count
|
|
20
|
+
// cap only rejects an absurd declared count on arrival instead of waiting for the characters to
|
|
21
|
+
// accumulate, and is loose enough not to constrain a sender that chunks more finely than the
|
|
22
|
+
// default.
|
|
23
|
+
export const MAX_ASSEMBLED_MESSAGE_CHARS = 16 * 1024 * 1024
|
|
24
|
+
export const MAX_CHUNK_COUNT = 100_000
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Thrown when a message would exceed MAX_ASSEMBLED_MESSAGE_CHARS, on either side of the wire.
|
|
28
|
+
* Distinct from the other assembly errors because it is fatal: retrying sends the same
|
|
29
|
+
* oversized message, so the session is rejected rather than reset.
|
|
30
|
+
*
|
|
31
|
+
* @internal
|
|
32
|
+
*/
|
|
33
|
+
export class MessageTooLargeError extends Error {
|
|
34
|
+
constructor(chars: number) {
|
|
35
|
+
super(`Message too large: ${chars} characters, max ${MAX_ASSEMBLED_MESSAGE_CHARS}`)
|
|
36
|
+
this.name = 'MessageTooLargeError'
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
11
40
|
/**
|
|
12
41
|
* Splits a string into smaller chunks suitable for transmission over WebSockets.
|
|
13
42
|
* This function ensures messages don't exceed size limits imposed by platforms like Cloudflare Workers (1MB max).
|
|
@@ -100,6 +129,7 @@ export class JsonChunkAssembler {
|
|
|
100
129
|
| {
|
|
101
130
|
chunksReceived: string[]
|
|
102
131
|
totalChunks: number
|
|
132
|
+
charsReceived: number
|
|
103
133
|
} = 'idle'
|
|
104
134
|
|
|
105
135
|
/**
|
|
@@ -110,7 +140,8 @@ export class JsonChunkAssembler {
|
|
|
110
140
|
* @param msg - The message to process, either JSON or chunk format
|
|
111
141
|
* @returns Result object with data/stringified on success, error object on failure, or null for incomplete chunks
|
|
112
142
|
* - `\{ data: object, stringified: string \}` - Successfully parsed complete message
|
|
113
|
-
* - `\{ error: Error \}` - Parse error
|
|
143
|
+
* - `\{ error: Error \}` - Parse error, invalid chunk sequence, or a message exceeding
|
|
144
|
+
* MAX_ASSEMBLED_MESSAGE_CHARS / MAX_CHUNK_COUNT
|
|
114
145
|
* - `null` - Chunk received but more chunks expected
|
|
115
146
|
*
|
|
116
147
|
* @example
|
|
@@ -144,17 +175,30 @@ export class JsonChunkAssembler {
|
|
|
144
175
|
const data = match[2]
|
|
145
176
|
|
|
146
177
|
if (this.state === 'idle') {
|
|
178
|
+
const totalChunks = numChunksRemaining + 1
|
|
179
|
+
if (totalChunks > MAX_CHUNK_COUNT) {
|
|
180
|
+
// Nothing has been buffered yet, so the state is already idle.
|
|
181
|
+
return { error: new Error(`Too many chunks: ${totalChunks}`) }
|
|
182
|
+
}
|
|
147
183
|
this.state = {
|
|
148
184
|
chunksReceived: [data],
|
|
149
|
-
totalChunks
|
|
185
|
+
totalChunks,
|
|
186
|
+
charsReceived: data.length,
|
|
150
187
|
}
|
|
151
188
|
} else {
|
|
152
189
|
this.state.chunksReceived.push(data)
|
|
190
|
+
this.state.charsReceived += data.length
|
|
153
191
|
if (numChunksRemaining !== this.state.totalChunks - this.state.chunksReceived.length) {
|
|
154
192
|
this.state = 'idle'
|
|
155
193
|
return { error: new Error(`Chunks received in wrong order`) }
|
|
156
194
|
}
|
|
157
195
|
}
|
|
196
|
+
if (this.state.charsReceived > MAX_ASSEMBLED_MESSAGE_CHARS) {
|
|
197
|
+
// Drop what was buffered rather than keeping it until the socket closes.
|
|
198
|
+
const charsReceived = this.state.charsReceived
|
|
199
|
+
this.state = 'idle'
|
|
200
|
+
return { error: new MessageTooLargeError(charsReceived) }
|
|
201
|
+
}
|
|
158
202
|
if (this.state.chunksReceived.length === this.state.totalChunks) {
|
|
159
203
|
try {
|
|
160
204
|
const stringified = this.state.chunksReceived.join('')
|
package/src/lib/diff.test.ts
CHANGED
|
@@ -39,6 +39,52 @@ describe('computing diffs (D)', () => {
|
|
|
39
39
|
expect(patch).toEqual({ c: [ValueOpType.Put, 3] })
|
|
40
40
|
expect(applyObjectDiff(a, patch!)).toEqual(b)
|
|
41
41
|
})
|
|
42
|
+
|
|
43
|
+
it('[D2] treats a key whose value is undefined as absent', () => {
|
|
44
|
+
// undefined does not survive JSON, so `['put', undefined]` would arrive as
|
|
45
|
+
// `['put', null]` and fail validators that only accept the key being missing
|
|
46
|
+
const present = { a: 1, props: { foo: 'x' } }
|
|
47
|
+
const explicitlyUndefined = { a: 1, props: { foo: undefined } }
|
|
48
|
+
const absent = { a: 1, props: {} }
|
|
49
|
+
|
|
50
|
+
expect(diffRecord(present, explicitlyUndefined)).toEqual({
|
|
51
|
+
props: [ValueOpType.Patch, { foo: [ValueOpType.Delete] }],
|
|
52
|
+
})
|
|
53
|
+
expect(diffRecord(explicitlyUndefined, present)).toEqual({
|
|
54
|
+
props: [ValueOpType.Patch, { foo: [ValueOpType.Put, 'x'] }],
|
|
55
|
+
})
|
|
56
|
+
expect(diffRecord(absent, explicitlyUndefined)).toBeNull()
|
|
57
|
+
expect(diffRecord(explicitlyUndefined, absent)).toBeNull()
|
|
58
|
+
expect(diffRecord({ a: undefined }, { a: undefined })).toBeNull()
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
it('[D2] only considers own keys, not Object.prototype members', () => {
|
|
62
|
+
const withCtor = { meta: { constructor: 1, toString: 2 } }
|
|
63
|
+
const without = { meta: {} }
|
|
64
|
+
|
|
65
|
+
expect(diffRecord(withCtor, without)).toEqual({
|
|
66
|
+
meta: [
|
|
67
|
+
ValueOpType.Patch,
|
|
68
|
+
{ constructor: [ValueOpType.Delete], toString: [ValueOpType.Delete] },
|
|
69
|
+
],
|
|
70
|
+
})
|
|
71
|
+
expect(diffRecord(without, withCtor)).toEqual({
|
|
72
|
+
meta: [
|
|
73
|
+
ValueOpType.Patch,
|
|
74
|
+
{ constructor: [ValueOpType.Put, 1], toString: [ValueOpType.Put, 2] },
|
|
75
|
+
],
|
|
76
|
+
})
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
it('[D2] ignores __proto__ keys, which applyObjectDiff would drop (AD8)', () => {
|
|
80
|
+
// JSON.parse creates '__proto__' as an own key; diffing it would store it on the server
|
|
81
|
+
// while peers receive an op they drop
|
|
82
|
+
const polluted = JSON.parse('{"__proto__":{"evil":1},"meta":{"__proto__":{"evil":1}}}')
|
|
83
|
+
const clean = { meta: {} }
|
|
84
|
+
|
|
85
|
+
expect(diffRecord(clean, polluted)).toBeNull()
|
|
86
|
+
expect(diffRecord(polluted, clean)).toBeNull()
|
|
87
|
+
})
|
|
42
88
|
})
|
|
43
89
|
|
|
44
90
|
describe('top-level keys vs props/meta (D3)', () => {
|
|
@@ -102,7 +148,8 @@ describe('computing diffs (D)', () => {
|
|
|
102
148
|
|
|
103
149
|
const diff = diffRecord(prev, next)
|
|
104
150
|
expect(diff).toBeTruthy()
|
|
105
|
-
|
|
151
|
+
// undefined is not representable on the wire, so it reads as the key being absent (D2)
|
|
152
|
+
expect(diff!.optional).toEqual([ValueOpType.Delete])
|
|
106
153
|
expect(diff!.nullable).toEqual([ValueOpType.Put, 'value'])
|
|
107
154
|
})
|
|
108
155
|
})
|
|
@@ -378,6 +425,25 @@ describe('computing diffs (D)', () => {
|
|
|
378
425
|
arr: [ValueOpType.Patch, { '0': [ValueOpType.Put, null] }],
|
|
379
426
|
})
|
|
380
427
|
})
|
|
428
|
+
|
|
429
|
+
it('[D6] puts a changed index when an item switches between array and object', () => {
|
|
430
|
+
// index deletes and named puts don't apply cleanly across shapes, so a patch would
|
|
431
|
+
// leave the receiver with a sparse array / an object where an array is expected
|
|
432
|
+
const arrayItem = { meta: { items: [[1, 2]] } }
|
|
433
|
+
const objectItem = { meta: { items: [{ x: 1 }] } }
|
|
434
|
+
|
|
435
|
+
const toObject = diffRecord(arrayItem, objectItem)!
|
|
436
|
+
expect(toObject).toEqual({
|
|
437
|
+
meta: [
|
|
438
|
+
ValueOpType.Patch,
|
|
439
|
+
{ items: [ValueOpType.Patch, { '0': [ValueOpType.Put, { x: 1 }] }] },
|
|
440
|
+
],
|
|
441
|
+
})
|
|
442
|
+
expect(applyObjectDiff(arrayItem, toObject)).toEqual(objectItem)
|
|
443
|
+
|
|
444
|
+
const toArray = diffRecord(objectItem, arrayItem)!
|
|
445
|
+
expect(applyObjectDiff(objectItem, toArray)).toEqual(arrayItem)
|
|
446
|
+
})
|
|
381
447
|
})
|
|
382
448
|
|
|
383
449
|
describe('different-length arrays (D7)', () => {
|
|
@@ -747,6 +813,30 @@ describe('applying diffs (AD)', () => {
|
|
|
747
813
|
|
|
748
814
|
expect(applyObjectDiff(obj, diff)).toBe(obj)
|
|
749
815
|
})
|
|
816
|
+
|
|
817
|
+
it('[AD5] deleting an inherited key has no effect', () => {
|
|
818
|
+
const obj = { a: 1 }
|
|
819
|
+
// `toString` collides with Object.prototype's own member in the literal's type, hence the cast
|
|
820
|
+
const diff = { toString: [ValueOpType.Delete] } as unknown as ObjectDiff
|
|
821
|
+
|
|
822
|
+
expect(applyObjectDiff(obj, diff)).toBe(obj)
|
|
823
|
+
})
|
|
824
|
+
})
|
|
825
|
+
|
|
826
|
+
describe('untrusted keys (AD8)', () => {
|
|
827
|
+
it('[AD8] ignores __proto__ keys instead of changing the prototype', () => {
|
|
828
|
+
// JSON.parse produces '__proto__' as an own key, so a peer can put it in a diff
|
|
829
|
+
const obj = { a: 1, props: { b: 2 } }
|
|
830
|
+
const diff: ObjectDiff = JSON.parse(
|
|
831
|
+
'{"__proto__":["put",{"isAdmin":true}],"props":["patch",{"__proto__":["put",{"polluted":1}]}]}'
|
|
832
|
+
)
|
|
833
|
+
|
|
834
|
+
const result: any = applyObjectDiff(obj, diff)
|
|
835
|
+
expect(result).toBe(obj)
|
|
836
|
+
expect(Object.getPrototypeOf(result)).toBe(Object.prototype)
|
|
837
|
+
expect(result.isAdmin).toBeUndefined()
|
|
838
|
+
expect(result.props.polluted).toBeUndefined()
|
|
839
|
+
})
|
|
750
840
|
})
|
|
751
841
|
|
|
752
842
|
describe('patching non-objects (AD6)', () => {
|
package/src/lib/diff.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { RecordsDiff, UnknownRecord } from '@tldraw/store'
|
|
2
|
-
import { isEqual, objectMapEntries, objectMapValues } from '@tldraw/utils'
|
|
2
|
+
import { hasOwnProperty, isEqual, objectMapEntries, objectMapValues } from '@tldraw/utils'
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Constants representing the types of operations that can be applied to records in network diffs.
|
|
@@ -200,15 +200,21 @@ function diffObject(
|
|
|
200
200
|
return null
|
|
201
201
|
}
|
|
202
202
|
let result: ObjectDiff | null = null
|
|
203
|
+
// Own-property checks throughout: `in` would see Object.prototype members, so a user key named
|
|
204
|
+
// e.g. 'constructor' would never diff as an add or delete. And a key whose value is `undefined`
|
|
205
|
+
// is treated as absent: `undefined` doesn't survive JSON, so `['put', undefined]` would arrive
|
|
206
|
+
// as `['put', null]` and fail validators that accept only the missing key.
|
|
203
207
|
for (const key of Object.keys(prev)) {
|
|
204
|
-
//
|
|
205
|
-
if (
|
|
208
|
+
// `result['__proto__'] = op` would set the diff's prototype, and applyObjectDiff drops the key (AD8)
|
|
209
|
+
if (key === '__proto__') continue
|
|
210
|
+
const prevValue = (prev as any)[key]
|
|
211
|
+
if (prevValue === undefined) continue
|
|
212
|
+
const nextValue = hasOwnProperty(next, key) ? (next as any)[key] : undefined
|
|
213
|
+
if (nextValue === undefined) {
|
|
206
214
|
if (!result) result = {}
|
|
207
215
|
result[key] = [ValueOpType.Delete]
|
|
208
216
|
continue
|
|
209
217
|
}
|
|
210
|
-
const prevValue = (prev as any)[key]
|
|
211
|
-
const nextValue = (next as any)[key]
|
|
212
218
|
if (
|
|
213
219
|
nestedKeys?.has(key) ||
|
|
214
220
|
(Array.isArray(prevValue) && Array.isArray(nextValue)) ||
|
|
@@ -226,10 +232,13 @@ function diffObject(
|
|
|
226
232
|
}
|
|
227
233
|
}
|
|
228
234
|
for (const key of Object.keys(next)) {
|
|
235
|
+
if (key === '__proto__') continue
|
|
236
|
+
const nextValue = (next as any)[key]
|
|
237
|
+
if (nextValue === undefined) continue
|
|
229
238
|
// if key is in next but not in prev then it was added
|
|
230
|
-
if (!(key
|
|
239
|
+
if (!hasOwnProperty(prev, key) || (prev as any)[key] === undefined) {
|
|
231
240
|
if (!result) result = {}
|
|
232
|
-
result[key] = [ValueOpType.Put,
|
|
241
|
+
result[key] = [ValueOpType.Put, nextValue]
|
|
233
242
|
}
|
|
234
243
|
}
|
|
235
244
|
return result
|
|
@@ -245,7 +254,15 @@ function diffValue(valueA: unknown, valueB: unknown, legacyAppendMode: boolean):
|
|
|
245
254
|
return [ValueOpType.Append, appendedText, valueA.length]
|
|
246
255
|
}
|
|
247
256
|
return [ValueOpType.Put, valueB]
|
|
248
|
-
} else if (
|
|
257
|
+
} else if (
|
|
258
|
+
!valueA ||
|
|
259
|
+
!valueB ||
|
|
260
|
+
typeof valueA !== 'object' ||
|
|
261
|
+
typeof valueB !== 'object' ||
|
|
262
|
+
// an array on one side and a plain object on the other can't be expressed as a patch:
|
|
263
|
+
// index deletes and named puts don't apply cleanly to either shape
|
|
264
|
+
Array.isArray(valueA) !== Array.isArray(valueB)
|
|
265
|
+
) {
|
|
249
266
|
return isEqual(valueA, valueB) ? null : [ValueOpType.Put, valueB]
|
|
250
267
|
} else {
|
|
251
268
|
const diff = diffObject(valueA, valueB, undefined, legacyAppendMode)
|
|
@@ -348,6 +365,10 @@ export function applyObjectDiff<T extends object>(object: T, objectDiff: ObjectD
|
|
|
348
365
|
}
|
|
349
366
|
}
|
|
350
367
|
for (const [key, op] of Object.entries(objectDiff)) {
|
|
368
|
+
// Diffs come off the wire from untrusted peers. `newObject['__proto__'] = v` would set the
|
|
369
|
+
// record's prototype (JSON.parse creates '__proto__' as an own key), so a client could give
|
|
370
|
+
// server-held records attacker-controlled inherited fields; skip such keys outright.
|
|
371
|
+
if (key === '__proto__') continue
|
|
351
372
|
switch (op[0]) {
|
|
352
373
|
case ValueOpType.Put: {
|
|
353
374
|
const value = op[1]
|
|
@@ -383,7 +404,7 @@ export function applyObjectDiff<T extends object>(object: T, objectDiff: ObjectD
|
|
|
383
404
|
break
|
|
384
405
|
}
|
|
385
406
|
case ValueOpType.Delete: {
|
|
386
|
-
if (key
|
|
407
|
+
if (hasOwnProperty(object, key)) {
|
|
387
408
|
if (!newObject) {
|
|
388
409
|
if (isArray) {
|
|
389
410
|
console.error("Can't delete array item yet (this should never happen)")
|