@livekit/rtc-node 0.13.4 → 0.13.5

package/dist/utils.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/** convert bigints to numbers preserving undefined values */\nexport function bigIntToNumber<T extends bigint | undefined>(\n  value: T,\n): T extends bigint ? number : undefined {\n  return (value !== undefined ? Number(value) : undefined) as T extends bigint ? number : undefined;\n}\n\n/** convert numbers to bigints preserving undefined values */\nexport function numberToBigInt<T extends number | undefined>(\n  value: T,\n): T extends number ? bigint : undefined {\n  return (value !== undefined ? BigInt(value) : undefined) as T extends number ? bigint : undefined;\n}\n\nexport function splitUtf8(s: string, n: number): string[] {\n  // adapted from https://stackoverflow.com/a/6043797\n  const result: string[] = [];\n  while (s.length > n) {\n    let k = n;\n    // Move back to find the start of a UTF-8 character\n    while ((s.charCodeAt(k) & 0xc0) === 0x80) {\n      k--;\n    }\n    result.push(s.slice(0, k));\n    s = s.slice(k);\n  }\n  result.push(s);\n  return result;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKO,SAAS,eACd,OACuC;AACvC,SAAQ,UAAU,SAAY,OAAO,KAAK,IAAI;AAChD;AAGO,SAAS,eACd,OACuC;AACvC,SAAQ,UAAU,SAAY,OAAO,KAAK,IAAI;AAChD;AAEO,SAAS,UAAU,GAAW,GAAqB;AAExD,QAAM,SAAmB,CAAC;AAC1B,SAAO,EAAE,SAAS,GAAG;AACnB,QAAI,IAAI;AAER,YAAQ,EAAE,WAAW,CAAC,IAAI,SAAU,KAAM;AACxC;AAAA,IACF;AACA,WAAO,KAAK,EAAE,MAAM,GAAG,CAAC,CAAC;AACzB,QAAI,EAAE,MAAM,CAAC;AAAA,EACf;AACA,SAAO,KAAK,CAAC;AACb,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../src/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/** convert bigints to numbers preserving undefined values */\nexport function bigIntToNumber<T extends bigint | undefined>(\n  value: T,\n): T extends bigint ? number : undefined {\n  return (value !== undefined ? Number(value) : undefined) as T extends bigint ? number : undefined;\n}\n\n/** convert numbers to bigints preserving undefined values */\nexport function numberToBigInt<T extends number | undefined>(\n  value: T,\n): T extends number ? bigint : undefined {\n  return (value !== undefined ? BigInt(value) : undefined) as T extends number ? bigint : undefined;\n}\n\nexport function splitUtf8(s: string, n: number): Uint8Array[] {\n  if (n < 4) {\n    throw new Error('n must be at least 4 due to utf8 encoding rules');\n  }\n  // adapted from https://stackoverflow.com/a/6043797\n  const result: Uint8Array[] = [];\n  let encoded = new TextEncoder().encode(s);\n  while (encoded.length > n) {\n    let k = n;\n    while (k > 0) {\n      const byte = encoded[k];\n      if (byte !== undefined && (byte & 0xc0) !== 0x80) {\n        break;\n      }\n      k--;\n    }\n    result.push(encoded.slice(0, k));\n    encoded = encoded.slice(k);\n  }\n  if (encoded.length > 0) {\n    result.push(encoded);\n  }\n  return result;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKO,SAAS,eACd,OACuC;AACvC,SAAQ,UAAU,SAAY,OAAO,KAAK,IAAI;AAChD;AAGO,SAAS,eACd,OACuC;AACvC,SAAQ,UAAU,SAAY,OAAO,KAAK,IAAI;AAChD;AAEO,SAAS,UAAU,GAAW,GAAyB;AAC5D,MAAI,IAAI,GAAG;AACT,UAAM,IAAI,MAAM,iDAAiD;AAAA,EACnE;AAEA,QAAM,SAAuB,CAAC;AAC9B,MAAI,UAAU,IAAI,YAAY,EAAE,OAAO,CAAC;AACxC,SAAO,QAAQ,SAAS,GAAG;AACzB,QAAI,IAAI;AACR,WAAO,IAAI,GAAG;AACZ,YAAM,OAAO,QAAQ,CAAC;AACtB,UAAI,SAAS,WAAc,OAAO,SAAU,KAAM;AAChD;AAAA,MACF;AACA;AAAA,IACF;AACA,WAAO,KAAK,QAAQ,MAAM,GAAG,CAAC,CAAC;AAC/B,cAAU,QAAQ,MAAM,CAAC;AAAA,EAC3B;AACA,MAAI,QAAQ,SAAS,GAAG;AACtB,WAAO,KAAK,OAAO;AAAA,EACrB;AACA,SAAO;AACT;","names":[]}

package/dist/utils.d.cts

@@ -2,6 +2,6 @@
 declare function bigIntToNumber<T extends bigint | undefined>(value: T): T extends bigint ? number : undefined;
 /** convert numbers to bigints preserving undefined values */
 declare function numberToBigInt<T extends number | undefined>(value: T): T extends number ? bigint : undefined;
-declare function splitUtf8(s: string, n: number): string[];
+declare function splitUtf8(s: string, n: number): Uint8Array[];
 
 export { bigIntToNumber, numberToBigInt, splitUtf8 };

package/dist/utils.d.ts

@@ -2,6 +2,6 @@
 declare function bigIntToNumber<T extends bigint | undefined>(value: T): T extends bigint ? number : undefined;
 /** convert numbers to bigints preserving undefined values */
 declare function numberToBigInt<T extends number | undefined>(value: T): T extends number ? bigint : undefined;
-declare function splitUtf8(s: string, n: number): string[];
+declare function splitUtf8(s: string, n: number): Uint8Array[];
 
 export { bigIntToNumber, numberToBigInt, splitUtf8 };

package/dist/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAIA,6DAA6D;AAC7D,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,GAAG,SAAS,EACzD,KAAK,EAAE,CAAC,GACP,CAAC,SAAS,MAAM,GAAG,MAAM,GAAG,SAAS,CAEvC;AAED,6DAA6D;AAC7D,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,GAAG,SAAS,EACzD,KAAK,EAAE,CAAC,GACP,CAAC,SAAS,MAAM,GAAG,MAAM,GAAG,SAAS,CAEvC;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAcxD"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAIA,6DAA6D;AAC7D,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,GAAG,SAAS,EACzD,KAAK,EAAE,CAAC,GACP,CAAC,SAAS,MAAM,GAAG,MAAM,GAAG,SAAS,CAEvC;AAED,6DAA6D;AAC7D,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,GAAG,SAAS,EACzD,KAAK,EAAE,CAAC,GACP,CAAC,SAAS,MAAM,GAAG,MAAM,GAAG,SAAS,CAEvC;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,UAAU,EAAE,CAuB5D"}

package/dist/utils.js

@@ -5,16 +5,26 @@ function numberToBigInt(value) {
   return value !== void 0 ? BigInt(value) : void 0;
 }
 function splitUtf8(s, n) {
+  if (n < 4) {
+    throw new Error("n must be at least 4 due to utf8 encoding rules");
+  }
   const result = [];
-  while (s.length > n) {
+  let encoded = new TextEncoder().encode(s);
+  while (encoded.length > n) {
     let k = n;
-    while ((s.charCodeAt(k) & 192) === 128) {
+    while (k > 0) {
+      const byte = encoded[k];
+      if (byte !== void 0 && (byte & 192) !== 128) {
+        break;
+      }
       k--;
     }
-    result.push(s.slice(0, k));
-    s = s.slice(k);
+    result.push(encoded.slice(0, k));
+    encoded = encoded.slice(k);
+  }
+  if (encoded.length > 0) {
+    result.push(encoded);
   }
-  result.push(s);
   return result;
 }
 export {

package/dist/utils.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/** convert bigints to numbers preserving undefined values */\nexport function bigIntToNumber<T extends bigint | undefined>(\n  value: T,\n): T extends bigint ? number : undefined {\n  return (value !== undefined ? Number(value) : undefined) as T extends bigint ? number : undefined;\n}\n\n/** convert numbers to bigints preserving undefined values */\nexport function numberToBigInt<T extends number | undefined>(\n  value: T,\n): T extends number ? bigint : undefined {\n  return (value !== undefined ? BigInt(value) : undefined) as T extends number ? bigint : undefined;\n}\n\nexport function splitUtf8(s: string, n: number): string[] {\n  // adapted from https://stackoverflow.com/a/6043797\n  const result: string[] = [];\n  while (s.length > n) {\n    let k = n;\n    // Move back to find the start of a UTF-8 character\n    while ((s.charCodeAt(k) & 0xc0) === 0x80) {\n      k--;\n    }\n    result.push(s.slice(0, k));\n    s = s.slice(k);\n  }\n  result.push(s);\n  return result;\n}\n"],"mappings":"AAKO,SAAS,eACd,OACuC;AACvC,SAAQ,UAAU,SAAY,OAAO,KAAK,IAAI;AAChD;AAGO,SAAS,eACd,OACuC;AACvC,SAAQ,UAAU,SAAY,OAAO,KAAK,IAAI;AAChD;AAEO,SAAS,UAAU,GAAW,GAAqB;AAExD,QAAM,SAAmB,CAAC;AAC1B,SAAO,EAAE,SAAS,GAAG;AACnB,QAAI,IAAI;AAER,YAAQ,EAAE,WAAW,CAAC,IAAI,SAAU,KAAM;AACxC;AAAA,IACF;AACA,WAAO,KAAK,EAAE,MAAM,GAAG,CAAC,CAAC;AACzB,QAAI,EAAE,MAAM,CAAC;AAAA,EACf;AACA,SAAO,KAAK,CAAC;AACb,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../src/utils.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/** convert bigints to numbers preserving undefined values */\nexport function bigIntToNumber<T extends bigint | undefined>(\n  value: T,\n): T extends bigint ? number : undefined {\n  return (value !== undefined ? Number(value) : undefined) as T extends bigint ? number : undefined;\n}\n\n/** convert numbers to bigints preserving undefined values */\nexport function numberToBigInt<T extends number | undefined>(\n  value: T,\n): T extends number ? bigint : undefined {\n  return (value !== undefined ? BigInt(value) : undefined) as T extends number ? bigint : undefined;\n}\n\nexport function splitUtf8(s: string, n: number): Uint8Array[] {\n  if (n < 4) {\n    throw new Error('n must be at least 4 due to utf8 encoding rules');\n  }\n  // adapted from https://stackoverflow.com/a/6043797\n  const result: Uint8Array[] = [];\n  let encoded = new TextEncoder().encode(s);\n  while (encoded.length > n) {\n    let k = n;\n    while (k > 0) {\n      const byte = encoded[k];\n      if (byte !== undefined && (byte & 0xc0) !== 0x80) {\n        break;\n      }\n      k--;\n    }\n    result.push(encoded.slice(0, k));\n    encoded = encoded.slice(k);\n  }\n  if (encoded.length > 0) {\n    result.push(encoded);\n  }\n  return result;\n}\n"],"mappings":"AAKO,SAAS,eACd,OACuC;AACvC,SAAQ,UAAU,SAAY,OAAO,KAAK,IAAI;AAChD;AAGO,SAAS,eACd,OACuC;AACvC,SAAQ,UAAU,SAAY,OAAO,KAAK,IAAI;AAChD;AAEO,SAAS,UAAU,GAAW,GAAyB;AAC5D,MAAI,IAAI,GAAG;AACT,UAAM,IAAI,MAAM,iDAAiD;AAAA,EACnE;AAEA,QAAM,SAAuB,CAAC;AAC9B,MAAI,UAAU,IAAI,YAAY,EAAE,OAAO,CAAC;AACxC,SAAO,QAAQ,SAAS,GAAG;AACzB,QAAI,IAAI;AACR,WAAO,IAAI,GAAG;AACZ,YAAM,OAAO,QAAQ,CAAC;AACtB,UAAI,SAAS,WAAc,OAAO,SAAU,KAAM;AAChD;AAAA,MACF;AACA;AAAA,IACF;AACA,WAAO,KAAK,QAAQ,MAAM,GAAG,CAAC,CAAC;AAC/B,cAAU,QAAQ,MAAM,CAAC;AAAA,EAC3B;AACA,MAAI,QAAQ,SAAS,GAAG;AACtB,WAAO,KAAK,OAAO;AAAA,EACrB;AACA,SAAO;AACT;","names":[]}

package/dist/version.cjs

@@ -21,7 +21,7 @@ __export(version_exports, {
   SDK_VERSION: () => SDK_VERSION
 });
 module.exports = __toCommonJS(version_exports);
-const SDK_VERSION = "0.13.4";
+const SDK_VERSION = "0.13.5";
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   SDK_VERSION

package/dist/version.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const SDK_VERSION = \"0.13.4\";\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAO,MAAM,cAAc;","names":[]}
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const SDK_VERSION = \"0.13.5\";\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAO,MAAM,cAAc;","names":[]}

package/dist/version.d.cts

@@ -1,3 +1,3 @@
-declare const SDK_VERSION = "0.13.4";
+declare const SDK_VERSION = "0.13.5";
 
 export { SDK_VERSION };

package/dist/version.d.ts

@@ -1,3 +1,3 @@
-declare const SDK_VERSION = "0.13.4";
+declare const SDK_VERSION = "0.13.5";
 
 export { SDK_VERSION };

package/dist/version.js

@@ -1,4 +1,4 @@
-const SDK_VERSION = "0.13.4";
+const SDK_VERSION = "0.13.5";
 export {
   SDK_VERSION
 };

package/dist/version.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const SDK_VERSION = \"0.13.4\";\n"],"mappings":"AAAO,MAAM,cAAc;","names":[]}
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const SDK_VERSION = \"0.13.5\";\n"],"mappings":"AAAO,MAAM,cAAc;","names":[]}

package/package.json

@@ -3,7 +3,7 @@
   "description": "LiveKit RTC Node",
   "license": "Apache-2.0",
   "author": "LiveKit",
-  "version": "0.13.4",
+  "version": "0.13.5",
   "main": "dist/index.js",
   "require": "dist/index.cjs",
   "types": "dist/index.d.ts",
@@ -61,11 +61,11 @@
     "@bufbuild/protoc-gen-es": "^1.10.0"
   },
   "optionalDependencies": {
-    "@livekit/rtc-node-darwin-arm64": "0.13.4",
-    "@livekit/rtc-node-darwin-x64": "0.13.4",
-    "@livekit/rtc-node-linux-arm64-gnu": "0.13.4",
-    "@livekit/rtc-node-linux-x64-gnu": "0.13.4",
-    "@livekit/rtc-node-win32-x64-msvc": "0.13.4"
+    "@livekit/rtc-node-darwin-arm64": "0.13.5",
+    "@livekit/rtc-node-darwin-x64": "0.13.5",
+    "@livekit/rtc-node-linux-arm64-gnu": "0.13.5",
+    "@livekit/rtc-node-linux-x64-gnu": "0.13.5",
+    "@livekit/rtc-node-win32-x64-msvc": "0.13.5"
   },
   "engines": {
     "node": ">= 18"

package/src/data_streams/stream_reader.ts

@@ -5,7 +5,7 @@ import type { ReadableStream } from 'node:stream/web';
 import { log } from '../log.js';
 import type { DataStream_Chunk } from '../proto/room_pb.js';
 import { bigIntToNumber } from '../utils.js';
-import type { BaseStreamInfo, ByteStreamInfo, TextStreamChunk, TextStreamInfo } from './types.js';
+import type { BaseStreamInfo, ByteStreamInfo, TextStreamInfo } from './types.js';
 
 abstract class BaseStreamReader<T extends BaseStreamInfo> {
   protected reader: ReadableStream<DataStream_Chunk>;
@@ -124,7 +124,7 @@ export class TextStreamReader extends BaseStreamReader<TextStreamInfo> {
     const decoder = new TextDecoder();
 
     return {
-      next: async (): Promise<IteratorResult<TextStreamChunk>> => {
+      next: async (): Promise<IteratorResult<string>> => {
         try {
           const { done, value } = await reader.read();
           if (done) {
@@ -133,14 +133,7 @@ export class TextStreamReader extends BaseStreamReader<TextStreamInfo> {
             this.handleChunkReceived(value);
             return {
               done: false,
-              value: {
-                index: bigIntToNumber(value.chunkIndex)!,
-                current: decoder.decode(value.content!),
-                collected: Array.from(this.receivedChunks.values())
-                  .sort((a, b) => bigIntToNumber(a.chunkIndex!) - bigIntToNumber(b.chunkIndex!))
-                  .map((chunk) => decoder.decode(chunk.content!))
-                  .join(''),
-              },
+              value: decoder.decode(value.content!),
             };
           }
         } catch (error) {
@@ -149,7 +142,7 @@ export class TextStreamReader extends BaseStreamReader<TextStreamInfo> {
         }
       },
 
-      return(): IteratorResult<TextStreamChunk> {
+      return(): IteratorResult<string> {
         reader.releaseLock();
         return { done: true, value: undefined };
       },
@@ -157,10 +150,10 @@ export class TextStreamReader extends BaseStreamReader<TextStreamInfo> {
   }
 
   async readAll(): Promise<string> {
-    let latestString: string = '';
-    for await (const { collected } of this) {
-      latestString = collected;
+    let finalString: string = '';
+    for await (const chunk of this) {
+      finalString += chunk;
     }
-    return latestString;
+    return finalString;
   }
 }

package/src/data_streams/stream_writer.ts

@@ -5,23 +5,23 @@ import type { WritableStream } from 'node:stream/web';
 import type { BaseStreamInfo, ByteStreamInfo, TextStreamInfo } from './types.js';
 
 class BaseStreamWriter<T, InfoType extends BaseStreamInfo> {
-  protected writableStream: WritableStream<[T, number?]>;
+  protected writableStream: WritableStream<T>;
 
-  protected defaultWriter: WritableStreamDefaultWriter<[T, number?]>;
+  protected defaultWriter: WritableStreamDefaultWriter<T>;
 
   protected onClose?: () => void;
 
   readonly info: InfoType;
 
-  constructor(writableStream: WritableStream<[T, number?]>, info: InfoType, onClose?: () => void) {
+  constructor(writableStream: WritableStream<T>, info: InfoType, onClose?: () => void) {
     this.writableStream = writableStream;
     this.defaultWriter = writableStream.getWriter();
     this.onClose = onClose;
     this.info = info;
   }
 
-  write(chunk: T, chunkIndex?: number): Promise<void> {
-    return this.defaultWriter.write([chunk, chunkIndex]);
+  write(chunk: T): Promise<void> {
+    return this.defaultWriter.write(chunk);
   }
 
   async close() {

package/src/data_streams/types.ts

@@ -27,12 +27,6 @@ export type ByteStreamInfo = BaseStreamInfo & {
 
 export type TextStreamInfo = BaseStreamInfo;
 
-export type TextStreamChunk = {
-  index: number;
-  current: string;
-  collected: string;
-};
-
 export interface DataStreamOptions {
   topic?: string;
   destinationIdentities?: Array<string>;

package/src/participant.ts

@@ -293,40 +293,29 @@ export class LocalParticipant extends Participant {
 
     await this.sendStreamHeader(headerReq);
 
-    let chunkId = 0;
+    let nextChunkId = 0;
     const localHandle = this.ffi_handle.handle;
     const sendTrailer = this.sendStreamTrailer;
     const sendChunk = this.sendStreamChunk;
 
-    const writableStream = new WritableStream<[string, number?]>({
+    const writableStream = new WritableStream<string>({
       // Implement the sink
-      write([textChunk, overrideChunkId]) {
-        const textInBytes = new TextEncoder().encode(textChunk);
-
-        if (textInBytes.byteLength > STREAM_CHUNK_SIZE) {
-          this.abort?.();
-          throw new Error('chunk size too large');
-        }
-
-        return new Promise(async (resolve) => {
+      async write(text) {
+        for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE)) {
           const chunkRequest = new SendStreamChunkRequest({
             senderIdentity,
             localParticipantHandle: localHandle,
             destinationIdentities,
             chunk: new DataStream_Chunk({
-              content: textInBytes,
+              content: textByteChunk,
               streamId,
-              chunkIndex: numberToBigInt(overrideChunkId ?? chunkId),
+              chunkIndex: numberToBigInt(nextChunkId),
             }),
           });
 
           await sendChunk(chunkRequest);
-
-          if (!overrideChunkId) {
-            chunkId += 1;
-            resolve();
-          }
-        });
+          nextChunkId += 1;
+        }
       },
       async close() {
         const trailerReq = new SendStreamTrailerRequest({
@@ -360,10 +349,7 @@ export class LocalParticipant extends Participant {
     },
   ) {
     const writer = await this.streamText(options);
-
-    for (const chunk in splitUtf8(text, STREAM_CHUNK_SIZE)) {
-      await writer.write(chunk);
-    }
+    await writer.write(text);
     await writer.close();
     return writer.info;
   }
@@ -419,8 +405,8 @@ export class LocalParticipant extends Participant {
     const sendChunk = this.sendStreamChunk;
     const writeMutex = new Mutex();
 
-    const writableStream = new WritableStream<[Uint8Array, number?]>({
-      async write([chunk]) {
+    const writableStream = new WritableStream<Uint8Array>({
+      async write(chunk) {
         const unlock = await writeMutex.lock();
 
         let byteOffset = 0;

package/src/utils.test.ts

@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { splitUtf8 } from './utils.js';
+
+describe('splitUtf8', () => {
+  it('splits a string into chunks of the given size', () => {
+    expect(splitUtf8('hello world', 5)).toEqual([
+      new TextEncoder().encode('hello'),
+      new TextEncoder().encode(' worl'),
+      new TextEncoder().encode('d'),
+    ]);
+  });
+
+  it('splits a string with special characters into chunks of the given size', () => {
+    expect(splitUtf8('héllo wörld', 5)).toEqual([
+      new TextEncoder().encode('héll'),
+      new TextEncoder().encode('o wö'),
+      new TextEncoder().encode('rld'),
+    ]);
+  });
+
+  it('splits a string with multi-byte utf8 characters correctly', () => {
+    expect(splitUtf8('こんにちは世界', 5)).toEqual([
+      new TextEncoder().encode('こ'),
+      new TextEncoder().encode('ん'),
+      new TextEncoder().encode('に'),
+      new TextEncoder().encode('ち'),
+      new TextEncoder().encode('は'),
+      new TextEncoder().encode('世'),
+      new TextEncoder().encode('界'),
+    ]);
+  });
+
+  it('handles a string with a single multi-byte utf8 character', () => {
+    expect(splitUtf8('😊', 5)).toEqual([new TextEncoder().encode('😊')]);
+  });
+
+  it('handles a string with mixed single and multi-byte utf8 characters', () => {
+    expect(splitUtf8('a😊b', 4)).toEqual([
+      new TextEncoder().encode('a'),
+      new TextEncoder().encode('😊'),
+      new TextEncoder().encode('b'),
+    ]);
+  });
+
+  it('handles an empty string', () => {
+    expect(splitUtf8('', 5)).toEqual([]);
+  });
+});

package/src/utils.ts

@@ -16,18 +16,27 @@ export function numberToBigInt<T extends number | undefined>(
   return (value !== undefined ? BigInt(value) : undefined) as T extends number ? bigint : undefined;
 }
 
-export function splitUtf8(s: string, n: number): string[] {
+export function splitUtf8(s: string, n: number): Uint8Array[] {
+  if (n < 4) {
+    throw new Error('n must be at least 4 due to utf8 encoding rules');
+  }
   // adapted from https://stackoverflow.com/a/6043797
-  const result: string[] = [];
-  while (s.length > n) {
+  const result: Uint8Array[] = [];
+  let encoded = new TextEncoder().encode(s);
+  while (encoded.length > n) {
     let k = n;
-    // Move back to find the start of a UTF-8 character
-    while ((s.charCodeAt(k) & 0xc0) === 0x80) {
+    while (k > 0) {
+      const byte = encoded[k];
+      if (byte !== undefined && (byte & 0xc0) !== 0x80) {
+        break;
+      }
       k--;
     }
-    result.push(s.slice(0, k));
-    s = s.slice(k);
+    result.push(encoded.slice(0, k));
+    encoded = encoded.slice(k);
+  }
+  if (encoded.length > 0) {
+    result.push(encoded);
   }
-  result.push(s);
   return result;
 }

package/src/version.ts

@@ -1 +1 @@
-export const SDK_VERSION = "0.13.4";
+export const SDK_VERSION = "0.13.5";