@tldraw/sync-core 5.5.0-canary.d06d59e62d6c → 5.5.0-canary.d64d005dcf06
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-cjs/index.js +1 -1
- package/dist-cjs/lib/chunk.js +10 -1
- package/dist-cjs/lib/chunk.js.map +2 -2
- package/dist-esm/index.mjs +1 -1
- package/dist-esm/lib/chunk.mjs +10 -1
- package/dist-esm/lib/chunk.mjs.map +2 -2
- package/package.json +6 -6
- package/src/lib/chunk.test.ts +34 -0
- package/src/lib/chunk.ts +20 -1
package/dist-cjs/index.js
CHANGED
|
@@ -61,7 +61,7 @@ var import_TLSyncRoom = require("./lib/TLSyncRoom");
|
|
|
61
61
|
var import_TLSyncStorage = require("./lib/TLSyncStorage");
|
|
62
62
|
(0, import_utils.registerTldrawLibraryVersion)(
|
|
63
63
|
"@tldraw/sync-core",
|
|
64
|
-
"5.5.0-canary.
|
|
64
|
+
"5.5.0-canary.d64d005dcf06",
|
|
65
65
|
"cjs"
|
|
66
66
|
);
|
|
67
67
|
//# sourceMappingURL=index.js.map
|
package/dist-cjs/lib/chunk.js
CHANGED
|
@@ -34,7 +34,10 @@ function chunk(msg, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
|
|
|
34
34
|
let offset = msg.length;
|
|
35
35
|
while (offset > 0) {
|
|
36
36
|
const prefix = `${chunkNumber}_`;
|
|
37
|
-
|
|
37
|
+
let chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1);
|
|
38
|
+
if (chunkSize < offset && isLowSurrogate(msg.charCodeAt(offset - chunkSize)) && isHighSurrogate(msg.charCodeAt(offset - chunkSize - 1))) {
|
|
39
|
+
chunkSize += chunkSize > 1 ? -1 : 1;
|
|
40
|
+
}
|
|
38
41
|
chunks.unshift(prefix + msg.slice(offset - chunkSize, offset));
|
|
39
42
|
offset -= chunkSize;
|
|
40
43
|
chunkNumber++;
|
|
@@ -42,6 +45,12 @@ function chunk(msg, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
|
|
|
42
45
|
return chunks;
|
|
43
46
|
}
|
|
44
47
|
}
|
|
48
|
+
function isHighSurrogate(code) {
|
|
49
|
+
return code >= 55296 && code <= 56319;
|
|
50
|
+
}
|
|
51
|
+
function isLowSurrogate(code) {
|
|
52
|
+
return code >= 56320 && code <= 57343;
|
|
53
|
+
}
|
|
45
54
|
const chunkRe = /^(\d+)_(.*)$/s;
|
|
46
55
|
class JsonChunkAssembler {
|
|
47
56
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/lib/chunk.ts"],
|
|
4
|
-
"sourcesContent": ["// quarter of a megabyte, max possible utf-8 string size\n\n// cloudflare workers only accept messages of max 1mb\nconst MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES = 1024 * 1024\n// utf-8 is max 4 bytes per char\nconst MAX_BYTES_PER_CHAR = 4\n\n// in the (admittedly impossible) worst case, the max size is 1/4 of a megabyte\nconst MAX_SAFE_MESSAGE_SIZE = MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES / MAX_BYTES_PER_CHAR\n\n/**\n * Splits a string into smaller chunks suitable for transmission over WebSockets.\n * This function ensures messages don't exceed size limits imposed by platforms like Cloudflare Workers (1MB max).\n * Each chunk is prefixed with a number indicating how many more chunks follow.\n *\n * @param msg - The string to split into chunks\n * @param maxSafeMessageSize - Maximum safe size for each chunk in characters. Defaults to quarter megabyte to account for UTF-8 encoding\n * @returns Array of chunked strings, each prefixed with \"\\{number\\}_\" where number indicates remaining chunks\n *\n * @example\n * ```ts\n * // Small message - returns as single chunk\n * chunk('hello world') // ['hello world']\n *\n * // Large message - splits into multiple chunks\n * chunk('very long message...', 10)\n * // ['2_very long', '1_ message', '0_...']\n * ```\n *\n * @internal\n */\nexport function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {\n\tif (msg.length < maxSafeMessageSize) {\n\t\treturn [msg]\n\t} else {\n\t\tconst chunks = []\n\t\tlet chunkNumber = 0\n\t\tlet offset = msg.length\n\t\twhile (offset > 0) {\n\t\t\tconst prefix = `${chunkNumber}_`\n\t\t\
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,MAAM,qCAAqC,OAAO;AAElD,MAAM,qBAAqB;AAG3B,MAAM,wBAAwB,qCAAqC;AAuB5D,SAAS,MAAM,KAAa,qBAAqB,uBAAuB;AAC9E,MAAI,IAAI,SAAS,oBAAoB;AACpC,WAAO,CAAC,GAAG;AAAA,EACZ,OAAO;AACN,UAAM,SAAS,CAAC;AAChB,QAAI,cAAc;AAClB,QAAI,SAAS,IAAI;AACjB,WAAO,SAAS,GAAG;AAClB,YAAM,SAAS,GAAG,WAAW;AAC7B,
|
|
4
|
+
"sourcesContent": ["// quarter of a megabyte, max possible utf-8 string size\n\n// cloudflare workers only accept messages of max 1mb\nconst MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES = 1024 * 1024\n// utf-8 is max 4 bytes per char\nconst MAX_BYTES_PER_CHAR = 4\n\n// in the (admittedly impossible) worst case, the max size is 1/4 of a megabyte\nconst MAX_SAFE_MESSAGE_SIZE = MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES / MAX_BYTES_PER_CHAR\n\n/**\n * Splits a string into smaller chunks suitable for transmission over WebSockets.\n * This function ensures messages don't exceed size limits imposed by platforms like Cloudflare Workers (1MB max).\n * Each chunk is prefixed with a number indicating how many more chunks follow.\n *\n * @param msg - The string to split into chunks\n * @param maxSafeMessageSize - Maximum safe size for each chunk in characters. Defaults to quarter megabyte to account for UTF-8 encoding\n * @returns Array of chunked strings, each prefixed with \"\\{number\\}_\" where number indicates remaining chunks\n *\n * @example\n * ```ts\n * // Small message - returns as single chunk\n * chunk('hello world') // ['hello world']\n *\n * // Large message - splits into multiple chunks\n * chunk('very long message...', 10)\n * // ['2_very long', '1_ message', '0_...']\n * ```\n *\n * @internal\n */\nexport function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {\n\tif (msg.length < maxSafeMessageSize) {\n\t\treturn [msg]\n\t} else {\n\t\tconst chunks = []\n\t\tlet chunkNumber = 0\n\t\tlet offset = msg.length\n\t\twhile (offset > 0) {\n\t\t\tconst prefix = `${chunkNumber}_`\n\t\t\tlet chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1)\n\t\t\t// Never split a surrogate pair across chunks: WebSocket.send converts each chunk to a\n\t\t\t// USVString, turning a lone surrogate into U+FFFD, so the reassembled JSON silently\n\t\t\t// carries corrupted text. Shrink by one so the pair moves whole into the preceding\n\t\t\t// chunk; a single-character chunk grows by one instead (one over the CH3 bound, see CH9).\n\t\t\tif (\n\t\t\t\tchunkSize < offset &&\n\t\t\t\tisLowSurrogate(msg.charCodeAt(offset - chunkSize)) &&\n\t\t\t\tisHighSurrogate(msg.charCodeAt(offset - chunkSize - 1))\n\t\t\t) {\n\t\t\t\tchunkSize += chunkSize > 1 ? -1 : 1\n\t\t\t}\n\t\t\tchunks.unshift(prefix + msg.slice(offset - chunkSize, offset))\n\t\t\toffset -= chunkSize\n\t\t\tchunkNumber++\n\t\t}\n\t\treturn chunks\n\t}\n}\n\nfunction isHighSurrogate(code: number) {\n\treturn code >= 0xd800 && code <= 0xdbff\n}\n\nfunction isLowSurrogate(code: number) {\n\treturn code >= 0xdc00 && code <= 0xdfff\n}\n\n// The 's' flag (dotAll) makes '.' match any character including line terminators\n// like U+2028 and U+2029, which are commonly introduced via copy/paste from Word\nconst chunkRe = /^(\\d+)_(.*)$/s\n\n/**\n * Assembles chunked JSON messages back into complete objects.\n * Handles both regular JSON messages and chunked messages created by the chunk() function.\n * Maintains internal state to track partially received chunked messages.\n *\n * @example\n * ```ts\n * const assembler = new JsonChunkAssembler()\n *\n * // Handle regular JSON message\n * const result1 = assembler.handleMessage('{\"hello\": \"world\"}')\n * // Returns: { data: { hello: \"world\" }, stringified: '{\"hello\": \"world\"}' }\n *\n * // Handle chunked message\n * assembler.handleMessage('1_hello') // Returns: null (partial)\n * const result2 = assembler.handleMessage('0_ world')\n * // Returns: { data: \"hello world\", stringified: \"hello world\" }\n * ```\n *\n * @public\n */\nexport class JsonChunkAssembler {\n\t/**\n\t * Current assembly state - either 'idle' or tracking chunks being received\n\t */\n\tstate:\n\t\t| 'idle'\n\t\t| {\n\t\t\t\tchunksReceived: string[]\n\t\t\t\ttotalChunks: number\n\t\t } = 'idle'\n\n\t/**\n\t * Processes a single message, which can be either a complete JSON object or a chunk.\n\t * For complete JSON objects (starting with '\\{'), parses immediately.\n\t * For chunks (prefixed with \"\\{number\\}_\"), accumulates until all chunks received.\n\t *\n\t * @param msg - The message to process, either JSON or chunk format\n\t * @returns Result object with data/stringified on success, error object on failure, or null for incomplete chunks\n\t * \t- `\\{ data: object, stringified: string \\}` - Successfully parsed complete message\n\t * \t- `\\{ error: Error \\}` - Parse error or invalid chunk sequence\n\t * \t- `null` - Chunk received but more chunks expected\n\t *\n\t * @example\n\t * ```ts\n\t * const assembler = new JsonChunkAssembler()\n\t *\n\t * // Complete JSON message\n\t * const result = assembler.handleMessage('{\"key\": \"value\"}')\n\t * if (result && 'data' in result) {\n\t * console.log(result.data) // { key: \"value\" }\n\t * }\n\t *\n\t * // Chunked message sequence\n\t * assembler.handleMessage('2_hel') // null - more chunks expected\n\t * assembler.handleMessage('1_lo ') // null - more chunks expected\n\t * assembler.handleMessage('0_wor') // { data: \"hello wor\", stringified: \"hello wor\" }\n\t * ```\n\t */\n\thandleMessage(msg: string): { error: Error } | { stringified: string; data: object } | null {\n\t\tif (msg.startsWith('{')) {\n\t\t\tconst error = this.state === 'idle' ? undefined : new Error('Unexpected non-chunk message')\n\t\t\tthis.state = 'idle'\n\t\t\treturn error ? { error } : { data: JSON.parse(msg), stringified: msg }\n\t\t} else {\n\t\t\tconst match = chunkRe.exec(msg)!\n\t\t\tif (!match) {\n\t\t\t\tthis.state = 'idle'\n\t\t\t\treturn { error: new Error('Invalid chunk: ' + JSON.stringify(msg.slice(0, 20) + '...')) }\n\t\t\t}\n\t\t\tconst numChunksRemaining = Number(match[1])\n\t\t\tconst data = match[2]\n\n\t\t\tif (this.state === 'idle') {\n\t\t\t\tthis.state = {\n\t\t\t\t\tchunksReceived: [data],\n\t\t\t\t\ttotalChunks: numChunksRemaining + 1,\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tthis.state.chunksReceived.push(data)\n\t\t\t\tif (numChunksRemaining !== this.state.totalChunks - this.state.chunksReceived.length) {\n\t\t\t\t\tthis.state = 'idle'\n\t\t\t\t\treturn { error: new Error(`Chunks received in wrong order`) }\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (this.state.chunksReceived.length === this.state.totalChunks) {\n\t\t\t\ttry {\n\t\t\t\t\tconst stringified = this.state.chunksReceived.join('')\n\t\t\t\t\tconst data = JSON.parse(stringified)\n\t\t\t\t\treturn { data, stringified }\n\t\t\t\t} catch (e) {\n\t\t\t\t\treturn { error: e as Error }\n\t\t\t\t} finally {\n\t\t\t\t\tthis.state = 'idle'\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn null\n\t\t}\n\t}\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,MAAM,qCAAqC,OAAO;AAElD,MAAM,qBAAqB;AAG3B,MAAM,wBAAwB,qCAAqC;AAuB5D,SAAS,MAAM,KAAa,qBAAqB,uBAAuB;AAC9E,MAAI,IAAI,SAAS,oBAAoB;AACpC,WAAO,CAAC,GAAG;AAAA,EACZ,OAAO;AACN,UAAM,SAAS,CAAC;AAChB,QAAI,cAAc;AAClB,QAAI,SAAS,IAAI;AACjB,WAAO,SAAS,GAAG;AAClB,YAAM,SAAS,GAAG,WAAW;AAC7B,UAAI,YAAY,KAAK,IAAI,KAAK,IAAI,qBAAqB,OAAO,QAAQ,MAAM,GAAG,CAAC;AAKhF,UACC,YAAY,UACZ,eAAe,IAAI,WAAW,SAAS,SAAS,CAAC,KACjD,gBAAgB,IAAI,WAAW,SAAS,YAAY,CAAC,CAAC,GACrD;AACD,qBAAa,YAAY,IAAI,KAAK;AAAA,MACnC;AACA,aAAO,QAAQ,SAAS,IAAI,MAAM,SAAS,WAAW,MAAM,CAAC;AAC7D,gBAAU;AACV;AAAA,IACD;AACA,WAAO;AAAA,EACR;AACD;AAEA,SAAS,gBAAgB,MAAc;AACtC,SAAO,QAAQ,SAAU,QAAQ;AAClC;AAEA,SAAS,eAAe,MAAc;AACrC,SAAO,QAAQ,SAAU,QAAQ;AAClC;AAIA,MAAM,UAAU;AAuBT,MAAM,mBAAmB;AAAA;AAAA;AAAA;AAAA,EAI/B,QAKO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA6BP,cAAc,KAA8E;AAC3F,QAAI,IAAI,WAAW,GAAG,GAAG;AACxB,YAAM,QAAQ,KAAK,UAAU,SAAS,SAAY,IAAI,MAAM,8BAA8B;AAC1F,WAAK,QAAQ;AACb,aAAO,QAAQ,EAAE,MAAM,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,GAAG,aAAa,IAAI;AAAA,IACtE,OAAO;AACN,YAAM,QAAQ,QAAQ,KAAK,GAAG;AAC9B,UAAI,CAAC,OAAO;AACX,aAAK,QAAQ;AACb,eAAO,EAAE,OAAO,IAAI,MAAM,oBAAoB,KAAK,UAAU,IAAI,MAAM,GAAG,EAAE,IAAI,KAAK,CAAC,EAAE;AAAA,MACzF;AACA,YAAM,qBAAqB,OAAO,MAAM,CAAC,CAAC;AAC1C,YAAM,OAAO,MAAM,CAAC;AAEpB,UAAI,KAAK,UAAU,QAAQ;AAC1B,aAAK,QAAQ;AAAA,UACZ,gBAAgB,CAAC,IAAI;AAAA,UACrB,aAAa,qBAAqB;AAAA,QACnC;AAAA,MACD,OAAO;AACN,aAAK,MAAM,eAAe,KAAK,IAAI;AACnC,YAAI,uBAAuB,KAAK,MAAM,cAAc,KAAK,MAAM,eAAe,QAAQ;AACrF,eAAK,QAAQ;AACb,iBAAO,EAAE,OAAO,IAAI,MAAM,gCAAgC,EAAE;AAAA,QAC7D;AAAA,MACD;AACA,UAAI,KAAK,MAAM,eAAe,WAAW,KAAK,MAAM,aAAa;AAChE,YAAI;AACH,gBAAM,cAAc,KAAK,MAAM,eAAe,KAAK,EAAE;AACrD,gBAAMA,QAAO,KAAK,MAAM,WAAW;AACnC,iBAAO,EAAE,MAAAA,OAAM,YAAY;AAAA,QAC5B,SAAS,GAAG;AACX,iBAAO,EAAE,OAAO,EAAW;AAAA,QAC5B,UAAE;AACD,eAAK,QAAQ;AAAA,QACd;AAAA,MACD;AACA,aAAO;AAAA,IACR;AAAA,EACD;AACD;",
|
|
6
6
|
"names": ["data"]
|
|
7
7
|
}
|
package/dist-esm/index.mjs
CHANGED
package/dist-esm/lib/chunk.mjs
CHANGED
|
@@ -10,7 +10,10 @@ function chunk(msg, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
|
|
|
10
10
|
let offset = msg.length;
|
|
11
11
|
while (offset > 0) {
|
|
12
12
|
const prefix = `${chunkNumber}_`;
|
|
13
|
-
|
|
13
|
+
let chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1);
|
|
14
|
+
if (chunkSize < offset && isLowSurrogate(msg.charCodeAt(offset - chunkSize)) && isHighSurrogate(msg.charCodeAt(offset - chunkSize - 1))) {
|
|
15
|
+
chunkSize += chunkSize > 1 ? -1 : 1;
|
|
16
|
+
}
|
|
14
17
|
chunks.unshift(prefix + msg.slice(offset - chunkSize, offset));
|
|
15
18
|
offset -= chunkSize;
|
|
16
19
|
chunkNumber++;
|
|
@@ -18,6 +21,12 @@ function chunk(msg, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
|
|
|
18
21
|
return chunks;
|
|
19
22
|
}
|
|
20
23
|
}
|
|
24
|
+
function isHighSurrogate(code) {
|
|
25
|
+
return code >= 55296 && code <= 56319;
|
|
26
|
+
}
|
|
27
|
+
function isLowSurrogate(code) {
|
|
28
|
+
return code >= 56320 && code <= 57343;
|
|
29
|
+
}
|
|
21
30
|
const chunkRe = /^(\d+)_(.*)$/s;
|
|
22
31
|
class JsonChunkAssembler {
|
|
23
32
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/lib/chunk.ts"],
|
|
4
|
-
"sourcesContent": ["// quarter of a megabyte, max possible utf-8 string size\n\n// cloudflare workers only accept messages of max 1mb\nconst MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES = 1024 * 1024\n// utf-8 is max 4 bytes per char\nconst MAX_BYTES_PER_CHAR = 4\n\n// in the (admittedly impossible) worst case, the max size is 1/4 of a megabyte\nconst MAX_SAFE_MESSAGE_SIZE = MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES / MAX_BYTES_PER_CHAR\n\n/**\n * Splits a string into smaller chunks suitable for transmission over WebSockets.\n * This function ensures messages don't exceed size limits imposed by platforms like Cloudflare Workers (1MB max).\n * Each chunk is prefixed with a number indicating how many more chunks follow.\n *\n * @param msg - The string to split into chunks\n * @param maxSafeMessageSize - Maximum safe size for each chunk in characters. Defaults to quarter megabyte to account for UTF-8 encoding\n * @returns Array of chunked strings, each prefixed with \"\\{number\\}_\" where number indicates remaining chunks\n *\n * @example\n * ```ts\n * // Small message - returns as single chunk\n * chunk('hello world') // ['hello world']\n *\n * // Large message - splits into multiple chunks\n * chunk('very long message...', 10)\n * // ['2_very long', '1_ message', '0_...']\n * ```\n *\n * @internal\n */\nexport function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {\n\tif (msg.length < maxSafeMessageSize) {\n\t\treturn [msg]\n\t} else {\n\t\tconst chunks = []\n\t\tlet chunkNumber = 0\n\t\tlet offset = msg.length\n\t\twhile (offset > 0) {\n\t\t\tconst prefix = `${chunkNumber}_`\n\t\t\
|
|
5
|
-
"mappings": "AAGA,MAAM,qCAAqC,OAAO;AAElD,MAAM,qBAAqB;AAG3B,MAAM,wBAAwB,qCAAqC;AAuB5D,SAAS,MAAM,KAAa,qBAAqB,uBAAuB;AAC9E,MAAI,IAAI,SAAS,oBAAoB;AACpC,WAAO,CAAC,GAAG;AAAA,EACZ,OAAO;AACN,UAAM,SAAS,CAAC;AAChB,QAAI,cAAc;AAClB,QAAI,SAAS,IAAI;AACjB,WAAO,SAAS,GAAG;AAClB,YAAM,SAAS,GAAG,WAAW;AAC7B,
|
|
4
|
+
"sourcesContent": ["// quarter of a megabyte, max possible utf-8 string size\n\n// cloudflare workers only accept messages of max 1mb\nconst MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES = 1024 * 1024\n// utf-8 is max 4 bytes per char\nconst MAX_BYTES_PER_CHAR = 4\n\n// in the (admittedly impossible) worst case, the max size is 1/4 of a megabyte\nconst MAX_SAFE_MESSAGE_SIZE = MAX_CLIENT_SENT_MESSAGE_SIZE_BYTES / MAX_BYTES_PER_CHAR\n\n/**\n * Splits a string into smaller chunks suitable for transmission over WebSockets.\n * This function ensures messages don't exceed size limits imposed by platforms like Cloudflare Workers (1MB max).\n * Each chunk is prefixed with a number indicating how many more chunks follow.\n *\n * @param msg - The string to split into chunks\n * @param maxSafeMessageSize - Maximum safe size for each chunk in characters. Defaults to quarter megabyte to account for UTF-8 encoding\n * @returns Array of chunked strings, each prefixed with \"\\{number\\}_\" where number indicates remaining chunks\n *\n * @example\n * ```ts\n * // Small message - returns as single chunk\n * chunk('hello world') // ['hello world']\n *\n * // Large message - splits into multiple chunks\n * chunk('very long message...', 10)\n * // ['2_very long', '1_ message', '0_...']\n * ```\n *\n * @internal\n */\nexport function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {\n\tif (msg.length < maxSafeMessageSize) {\n\t\treturn [msg]\n\t} else {\n\t\tconst chunks = []\n\t\tlet chunkNumber = 0\n\t\tlet offset = msg.length\n\t\twhile (offset > 0) {\n\t\t\tconst prefix = `${chunkNumber}_`\n\t\t\tlet chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1)\n\t\t\t// Never split a surrogate pair across chunks: WebSocket.send converts each chunk to a\n\t\t\t// USVString, turning a lone surrogate into U+FFFD, so the reassembled JSON silently\n\t\t\t// carries corrupted text. Shrink by one so the pair moves whole into the preceding\n\t\t\t// chunk; a single-character chunk grows by one instead (one over the CH3 bound, see CH9).\n\t\t\tif (\n\t\t\t\tchunkSize < offset &&\n\t\t\t\tisLowSurrogate(msg.charCodeAt(offset - chunkSize)) &&\n\t\t\t\tisHighSurrogate(msg.charCodeAt(offset - chunkSize - 1))\n\t\t\t) {\n\t\t\t\tchunkSize += chunkSize > 1 ? -1 : 1\n\t\t\t}\n\t\t\tchunks.unshift(prefix + msg.slice(offset - chunkSize, offset))\n\t\t\toffset -= chunkSize\n\t\t\tchunkNumber++\n\t\t}\n\t\treturn chunks\n\t}\n}\n\nfunction isHighSurrogate(code: number) {\n\treturn code >= 0xd800 && code <= 0xdbff\n}\n\nfunction isLowSurrogate(code: number) {\n\treturn code >= 0xdc00 && code <= 0xdfff\n}\n\n// The 's' flag (dotAll) makes '.' match any character including line terminators\n// like U+2028 and U+2029, which are commonly introduced via copy/paste from Word\nconst chunkRe = /^(\\d+)_(.*)$/s\n\n/**\n * Assembles chunked JSON messages back into complete objects.\n * Handles both regular JSON messages and chunked messages created by the chunk() function.\n * Maintains internal state to track partially received chunked messages.\n *\n * @example\n * ```ts\n * const assembler = new JsonChunkAssembler()\n *\n * // Handle regular JSON message\n * const result1 = assembler.handleMessage('{\"hello\": \"world\"}')\n * // Returns: { data: { hello: \"world\" }, stringified: '{\"hello\": \"world\"}' }\n *\n * // Handle chunked message\n * assembler.handleMessage('1_hello') // Returns: null (partial)\n * const result2 = assembler.handleMessage('0_ world')\n * // Returns: { data: \"hello world\", stringified: \"hello world\" }\n * ```\n *\n * @public\n */\nexport class JsonChunkAssembler {\n\t/**\n\t * Current assembly state - either 'idle' or tracking chunks being received\n\t */\n\tstate:\n\t\t| 'idle'\n\t\t| {\n\t\t\t\tchunksReceived: string[]\n\t\t\t\ttotalChunks: number\n\t\t } = 'idle'\n\n\t/**\n\t * Processes a single message, which can be either a complete JSON object or a chunk.\n\t * For complete JSON objects (starting with '\\{'), parses immediately.\n\t * For chunks (prefixed with \"\\{number\\}_\"), accumulates until all chunks received.\n\t *\n\t * @param msg - The message to process, either JSON or chunk format\n\t * @returns Result object with data/stringified on success, error object on failure, or null for incomplete chunks\n\t * \t- `\\{ data: object, stringified: string \\}` - Successfully parsed complete message\n\t * \t- `\\{ error: Error \\}` - Parse error or invalid chunk sequence\n\t * \t- `null` - Chunk received but more chunks expected\n\t *\n\t * @example\n\t * ```ts\n\t * const assembler = new JsonChunkAssembler()\n\t *\n\t * // Complete JSON message\n\t * const result = assembler.handleMessage('{\"key\": \"value\"}')\n\t * if (result && 'data' in result) {\n\t * console.log(result.data) // { key: \"value\" }\n\t * }\n\t *\n\t * // Chunked message sequence\n\t * assembler.handleMessage('2_hel') // null - more chunks expected\n\t * assembler.handleMessage('1_lo ') // null - more chunks expected\n\t * assembler.handleMessage('0_wor') // { data: \"hello wor\", stringified: \"hello wor\" }\n\t * ```\n\t */\n\thandleMessage(msg: string): { error: Error } | { stringified: string; data: object } | null {\n\t\tif (msg.startsWith('{')) {\n\t\t\tconst error = this.state === 'idle' ? undefined : new Error('Unexpected non-chunk message')\n\t\t\tthis.state = 'idle'\n\t\t\treturn error ? { error } : { data: JSON.parse(msg), stringified: msg }\n\t\t} else {\n\t\t\tconst match = chunkRe.exec(msg)!\n\t\t\tif (!match) {\n\t\t\t\tthis.state = 'idle'\n\t\t\t\treturn { error: new Error('Invalid chunk: ' + JSON.stringify(msg.slice(0, 20) + '...')) }\n\t\t\t}\n\t\t\tconst numChunksRemaining = Number(match[1])\n\t\t\tconst data = match[2]\n\n\t\t\tif (this.state === 'idle') {\n\t\t\t\tthis.state = {\n\t\t\t\t\tchunksReceived: [data],\n\t\t\t\t\ttotalChunks: numChunksRemaining + 1,\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tthis.state.chunksReceived.push(data)\n\t\t\t\tif (numChunksRemaining !== this.state.totalChunks - this.state.chunksReceived.length) {\n\t\t\t\t\tthis.state = 'idle'\n\t\t\t\t\treturn { error: new Error(`Chunks received in wrong order`) }\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (this.state.chunksReceived.length === this.state.totalChunks) {\n\t\t\t\ttry {\n\t\t\t\t\tconst stringified = this.state.chunksReceived.join('')\n\t\t\t\t\tconst data = JSON.parse(stringified)\n\t\t\t\t\treturn { data, stringified }\n\t\t\t\t} catch (e) {\n\t\t\t\t\treturn { error: e as Error }\n\t\t\t\t} finally {\n\t\t\t\t\tthis.state = 'idle'\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn null\n\t\t}\n\t}\n}\n"],
|
|
5
|
+
"mappings": "AAGA,MAAM,qCAAqC,OAAO;AAElD,MAAM,qBAAqB;AAG3B,MAAM,wBAAwB,qCAAqC;AAuB5D,SAAS,MAAM,KAAa,qBAAqB,uBAAuB;AAC9E,MAAI,IAAI,SAAS,oBAAoB;AACpC,WAAO,CAAC,GAAG;AAAA,EACZ,OAAO;AACN,UAAM,SAAS,CAAC;AAChB,QAAI,cAAc;AAClB,QAAI,SAAS,IAAI;AACjB,WAAO,SAAS,GAAG;AAClB,YAAM,SAAS,GAAG,WAAW;AAC7B,UAAI,YAAY,KAAK,IAAI,KAAK,IAAI,qBAAqB,OAAO,QAAQ,MAAM,GAAG,CAAC;AAKhF,UACC,YAAY,UACZ,eAAe,IAAI,WAAW,SAAS,SAAS,CAAC,KACjD,gBAAgB,IAAI,WAAW,SAAS,YAAY,CAAC,CAAC,GACrD;AACD,qBAAa,YAAY,IAAI,KAAK;AAAA,MACnC;AACA,aAAO,QAAQ,SAAS,IAAI,MAAM,SAAS,WAAW,MAAM,CAAC;AAC7D,gBAAU;AACV;AAAA,IACD;AACA,WAAO;AAAA,EACR;AACD;AAEA,SAAS,gBAAgB,MAAc;AACtC,SAAO,QAAQ,SAAU,QAAQ;AAClC;AAEA,SAAS,eAAe,MAAc;AACrC,SAAO,QAAQ,SAAU,QAAQ;AAClC;AAIA,MAAM,UAAU;AAuBT,MAAM,mBAAmB;AAAA;AAAA;AAAA;AAAA,EAI/B,QAKO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA6BP,cAAc,KAA8E;AAC3F,QAAI,IAAI,WAAW,GAAG,GAAG;AACxB,YAAM,QAAQ,KAAK,UAAU,SAAS,SAAY,IAAI,MAAM,8BAA8B;AAC1F,WAAK,QAAQ;AACb,aAAO,QAAQ,EAAE,MAAM,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,GAAG,aAAa,IAAI;AAAA,IACtE,OAAO;AACN,YAAM,QAAQ,QAAQ,KAAK,GAAG;AAC9B,UAAI,CAAC,OAAO;AACX,aAAK,QAAQ;AACb,eAAO,EAAE,OAAO,IAAI,MAAM,oBAAoB,KAAK,UAAU,IAAI,MAAM,GAAG,EAAE,IAAI,KAAK,CAAC,EAAE;AAAA,MACzF;AACA,YAAM,qBAAqB,OAAO,MAAM,CAAC,CAAC;AAC1C,YAAM,OAAO,MAAM,CAAC;AAEpB,UAAI,KAAK,UAAU,QAAQ;AAC1B,aAAK,QAAQ;AAAA,UACZ,gBAAgB,CAAC,IAAI;AAAA,UACrB,aAAa,qBAAqB;AAAA,QACnC;AAAA,MACD,OAAO;AACN,aAAK,MAAM,eAAe,KAAK,IAAI;AACnC,YAAI,uBAAuB,KAAK,MAAM,cAAc,KAAK,MAAM,eAAe,QAAQ;AACrF,eAAK,QAAQ;AACb,iBAAO,EAAE,OAAO,IAAI,MAAM,gCAAgC,EAAE;AAAA,QAC7D;AAAA,MACD;AACA,UAAI,KAAK,MAAM,eAAe,WAAW,KAAK,MAAM,aAAa;AAChE,YAAI;AACH,gBAAM,cAAc,KAAK,MAAM,eAAe,KAAK,EAAE;AACrD,gBAAMA,QAAO,KAAK,MAAM,WAAW;AACnC,iBAAO,EAAE,MAAAA,OAAM,YAAY;AAAA,QAC5B,SAAS,GAAG;AACX,iBAAO,EAAE,OAAO,EAAW;AAAA,QAC5B,UAAE;AACD,eAAK,QAAQ;AAAA,QACd;AAAA,MACD;AACA,aAAO;AAAA,IACR;AAAA,EACD;AACD;",
|
|
6
6
|
"names": ["data"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tldraw/sync-core",
|
|
3
3
|
"description": "tldraw infinite canvas SDK (multiplayer sync).",
|
|
4
|
-
"version": "5.5.0-canary.
|
|
4
|
+
"version": "5.5.0-canary.d64d005dcf06",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "tldraw GB Ltd.",
|
|
7
7
|
"email": "hello@tldraw.com"
|
|
@@ -56,17 +56,17 @@
|
|
|
56
56
|
"@types/uuid-readable": "^0.0.3",
|
|
57
57
|
"react": "^19.2.1",
|
|
58
58
|
"react-dom": "^19.2.1",
|
|
59
|
-
"tldraw": "5.5.0-canary.
|
|
59
|
+
"tldraw": "5.5.0-canary.d64d005dcf06",
|
|
60
60
|
"typescript": "^7.0.2",
|
|
61
61
|
"uuid-by-string": "^4.0.0",
|
|
62
62
|
"uuid-readable": "^0.0.2",
|
|
63
63
|
"vitest": "^4.1.7"
|
|
64
64
|
},
|
|
65
65
|
"dependencies": {
|
|
66
|
-
"@tldraw/state": "5.5.0-canary.
|
|
67
|
-
"@tldraw/store": "5.5.0-canary.
|
|
68
|
-
"@tldraw/tlschema": "5.5.0-canary.
|
|
69
|
-
"@tldraw/utils": "5.5.0-canary.
|
|
66
|
+
"@tldraw/state": "5.5.0-canary.d64d005dcf06",
|
|
67
|
+
"@tldraw/store": "5.5.0-canary.d64d005dcf06",
|
|
68
|
+
"@tldraw/tlschema": "5.5.0-canary.d64d005dcf06",
|
|
69
|
+
"@tldraw/utils": "5.5.0-canary.d64d005dcf06",
|
|
70
70
|
"nanoevents": "^7.0.1",
|
|
71
71
|
"ws": "^8.18.0"
|
|
72
72
|
},
|
package/src/lib/chunk.test.ts
CHANGED
|
@@ -134,6 +134,40 @@ describe('chunk (CH1–CH3)', () => {
|
|
|
134
134
|
`)
|
|
135
135
|
})
|
|
136
136
|
})
|
|
137
|
+
|
|
138
|
+
describe('surrogate pairs (CH9)', () => {
|
|
139
|
+
it('[CH9] never splits a surrogate pair across two chunks', () => {
|
|
140
|
+
// WebSocket.send converts each chunk to a USVString, so a lone surrogate would arrive
|
|
141
|
+
// as U+FFFD and the reassembled message would carry corrupted text
|
|
142
|
+
const emoji = '\u{1F600}'
|
|
143
|
+
const isHigh = (code: number) => code >= 0xd800 && code <= 0xdbff
|
|
144
|
+
const isLow = (code: number) => code >= 0xdc00 && code <= 0xdfff
|
|
145
|
+
const msgs = [
|
|
146
|
+
...Array.from(
|
|
147
|
+
{ length: 8 },
|
|
148
|
+
(_, padding) => 'x'.repeat(padding) + emoji + 'y'.repeat(6) + emoji
|
|
149
|
+
),
|
|
150
|
+
// adjacent pairs: after a nudge the next boundary lands on another pair
|
|
151
|
+
'x' + emoji.repeat(3),
|
|
152
|
+
]
|
|
153
|
+
for (const msg of msgs) {
|
|
154
|
+
for (const maxSize of [3, 4, 5, 6, 7, 8]) {
|
|
155
|
+
const chunks = chunk(msg, maxSize)
|
|
156
|
+
const bodies = chunks.map((c) => c.slice(c.indexOf('_') + 1))
|
|
157
|
+
expect(bodies.join('')).toBe(msg)
|
|
158
|
+
for (const [i, body] of bodies.entries()) {
|
|
159
|
+
expect(body.length).toBeGreaterThan(0)
|
|
160
|
+
expect(isHigh(body.charCodeAt(body.length - 1))).toBe(false)
|
|
161
|
+
expect(isLow(body.charCodeAt(0))).toBe(false)
|
|
162
|
+
// keeping a pair whole may cost one character over the CH3 bound, never more
|
|
163
|
+
expect(chunks[i].length).toBeLessThanOrEqual(
|
|
164
|
+
Math.max(maxSize, chunks[i].indexOf('_') + 2) + 1
|
|
165
|
+
)
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
})
|
|
170
|
+
})
|
|
137
171
|
})
|
|
138
172
|
|
|
139
173
|
const testObject = {} as any
|
package/src/lib/chunk.ts
CHANGED
|
@@ -38,7 +38,18 @@ export function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
|
|
|
38
38
|
let offset = msg.length
|
|
39
39
|
while (offset > 0) {
|
|
40
40
|
const prefix = `${chunkNumber}_`
|
|
41
|
-
|
|
41
|
+
let chunkSize = Math.max(Math.min(maxSafeMessageSize - prefix.length, offset), 1)
|
|
42
|
+
// Never split a surrogate pair across chunks: WebSocket.send converts each chunk to a
|
|
43
|
+
// USVString, turning a lone surrogate into U+FFFD, so the reassembled JSON silently
|
|
44
|
+
// carries corrupted text. Shrink by one so the pair moves whole into the preceding
|
|
45
|
+
// chunk; a single-character chunk grows by one instead (one over the CH3 bound, see CH9).
|
|
46
|
+
if (
|
|
47
|
+
chunkSize < offset &&
|
|
48
|
+
isLowSurrogate(msg.charCodeAt(offset - chunkSize)) &&
|
|
49
|
+
isHighSurrogate(msg.charCodeAt(offset - chunkSize - 1))
|
|
50
|
+
) {
|
|
51
|
+
chunkSize += chunkSize > 1 ? -1 : 1
|
|
52
|
+
}
|
|
42
53
|
chunks.unshift(prefix + msg.slice(offset - chunkSize, offset))
|
|
43
54
|
offset -= chunkSize
|
|
44
55
|
chunkNumber++
|
|
@@ -47,6 +58,14 @@ export function chunk(msg: string, maxSafeMessageSize = MAX_SAFE_MESSAGE_SIZE) {
|
|
|
47
58
|
}
|
|
48
59
|
}
|
|
49
60
|
|
|
61
|
+
function isHighSurrogate(code: number) {
|
|
62
|
+
return code >= 0xd800 && code <= 0xdbff
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function isLowSurrogate(code: number) {
|
|
66
|
+
return code >= 0xdc00 && code <= 0xdfff
|
|
67
|
+
}
|
|
68
|
+
|
|
50
69
|
// The 's' flag (dotAll) makes '.' match any character including line terminators
|
|
51
70
|
// like U+2028 and U+2029, which are commonly introduced via copy/paste from Word
|
|
52
71
|
const chunkRe = /^(\d+)_(.*)$/s
|