@livekit/rtc-node 0.13.3 → 0.13.5

package/src/proto/video_frame_pb.ts

@@ -87,6 +87,9 @@ proto2.util.setEnumType(VideoRotation, "livekit.proto.VideoRotation", [
 ]);
 
 /**
+ * Values of this enum must not be changed
+ * It is used to serialize a rtc.VideoFrame on Python
+ *
  * @generated from enum livekit.proto.VideoBufferType
  */
 export enum VideoBufferType {

package/src/room.ts

@@ -3,9 +3,19 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import EventEmitter from 'events';
+import { ReadableStream } from 'node:stream/web';
+import { ByteStreamReader, TextStreamReader } from './data_streams/stream_reader.js';
+import type {
+  ByteStreamHandler,
+  ByteStreamInfo,
+  StreamController,
+  TextStreamHandler,
+  TextStreamInfo,
+} from './data_streams/types.js';
 import type { E2EEOptions } from './e2ee.js';
 import { E2EEManager, defaultE2EEOptions } from './e2ee.js';
 import { FfiClient, FfiClientEvent, FfiHandle } from './ffi_client.js';
+import { log } from './log.js';
 import type { Participant } from './participant.js';
 import { LocalParticipant, RemoteParticipant } from './participant.js';
 import { EncryptionState } from './proto/e2ee_pb.js';
@@ -16,6 +26,9 @@ import {
   type ConnectResponse,
   type ConnectionQuality,
   type DataPacketKind,
+  type DataStream_Chunk,
+  type DataStream_Header,
+  DataStream_Trailer,
   type DisconnectResponse,
   RoomOptions as FfiRoomOptions,
   type IceServer,
@@ -33,6 +46,7 @@ import { RemoteAudioTrack, RemoteVideoTrack } from './track.js';
 import type { LocalTrackPublication, TrackPublication } from './track_publication.js';
 import { RemoteTrackPublication } from './track_publication.js';
 import type { ChatMessage } from './types.js';
+import { bigIntToNumber } from './utils.js';
 
 export interface RtcConfiguration {
   iceTransportType: IceTransportType;
@@ -66,6 +80,11 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
   private info?: RoomInfo;
   private ffiHandle?: FfiHandle;
 
+  private byteStreamControllers = new Map<string, StreamController<DataStream_Chunk>>();
+  private textStreamControllers = new Map<string, StreamController<DataStream_Chunk>>();
+  private byteStreamHandlers = new Map<string, ByteStreamHandler>();
+  private textStreamHandlers = new Map<string, TextStreamHandler>();
+
   e2eeManager?: E2EEManager;
   connectionState: ConnectionState = ConnectionState.CONN_DISCONNECTED;
 
@@ -175,6 +194,28 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
     this.removeAllListeners();
   }
 
+  registerTextStreamHandler(topic: string, callback: TextStreamHandler) {
+    if (this.textStreamHandlers.has(topic)) {
+      throw new Error(`A text stream handler for topic "${topic}" has already been set.`);
+    }
+    this.textStreamHandlers.set(topic, callback);
+  }
+
+  unregisterTextStreamHandler(topic: string) {
+    this.textStreamHandlers.delete(topic);
+  }
+
+  registerByteStreamHandler(topic: string, callback: ByteStreamHandler) {
+    if (this.byteStreamHandlers.has(topic)) {
+      throw new Error(`A byte stream handler for topic "${topic}" has already been set.`);
+    }
+    this.byteStreamHandlers.set(topic, callback);
+  }
+
+  unregisterByteStreamHandler(topic: string) {
+    this.byteStreamHandlers.delete(topic);
+  }
+
   private onFfiEvent = (ffiEvent: FfiEvent) => {
     if (!this.localParticipant || !this.ffiHandle || !this.info) {
       throw TypeError('cannot handle ffi events before connectCallback');
@@ -373,6 +414,12 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
       this.emit(RoomEvent.Reconnected);
     } else if (ev.case == 'roomSidChanged') {
       this.emit(RoomEvent.RoomSidChanged, ev.value.sid!);
+    } else if (ev.case === 'streamHeaderReceived' && ev.value.header) {
+      this.handleStreamHeader(ev.value.header, ev.value.participantIdentity!);
+    } else if (ev.case === 'streamChunkReceived' && ev.value.chunk) {
+      this.handleStreamChunk(ev.value.chunk);
+    } else if (ev.case === 'streamTrailerReceived' && ev.value.trailer) {
+      this.handleStreamTrailer(ev.value.trailer);
     }
   };
 
@@ -429,6 +476,100 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
     this.remoteParticipants.set(ownedInfo.info!.identity!, participant);
     return participant;
   }
+
+  private handleStreamHeader(streamHeader: DataStream_Header, participantIdentity: string) {
+    if (streamHeader.contentHeader.case === 'byteHeader') {
+      const streamHandlerCallback = this.byteStreamHandlers.get(streamHeader.topic ?? '');
+
+      if (!streamHandlerCallback) {
+        log.debug('ignoring incoming byte stream due to no handler for topic', streamHeader.topic);
+        return;
+      }
+      let streamController: ReadableStreamDefaultController<DataStream_Chunk>;
+      const stream = new ReadableStream({
+        start: (controller) => {
+          streamController = controller;
+          this.byteStreamControllers.set(streamHeader.streamId!, {
+            header: streamHeader,
+            controller: streamController,
+            startTime: Date.now(),
+          });
+        },
+      });
+      const info: ByteStreamInfo = {
+        streamId: streamHeader.streamId!,
+        name: streamHeader.contentHeader.value.name ?? 'unknown',
+        mimeType: streamHeader.mimeType!,
+        totalSize: streamHeader.totalLength ? Number(streamHeader.totalLength) : undefined,
+        topic: streamHeader.topic!,
+        timestamp: bigIntToNumber(streamHeader.timestamp!),
+        attributes: streamHeader.attributes,
+      };
+      streamHandlerCallback(
+        new ByteStreamReader(info, stream, bigIntToNumber(streamHeader.totalLength)),
+        { identity: participantIdentity },
+      );
+    } else if (streamHeader.contentHeader.case === 'textHeader') {
+      const streamHandlerCallback = this.textStreamHandlers.get(streamHeader.topic ?? '');
+
+      if (!streamHandlerCallback) {
+        log.debug('ignoring incoming text stream due to no handler for topic', streamHeader.topic);
+        return;
+      }
+      let streamController: ReadableStreamDefaultController<DataStream_Chunk>;
+      const stream = new ReadableStream<DataStream_Chunk>({
+        start: (controller) => {
+          streamController = controller;
+          this.textStreamControllers.set(streamHeader.streamId!, {
+            header: streamHeader,
+            controller: streamController,
+            startTime: Date.now(),
+          });
+        },
+      });
+      const info: TextStreamInfo = {
+        streamId: streamHeader.streamId!,
+        mimeType: streamHeader.mimeType!,
+        totalSize: streamHeader.totalLength ? Number(streamHeader.totalLength) : undefined,
+        topic: streamHeader.topic!,
+        timestamp: Number(streamHeader.timestamp),
+        attributes: streamHeader.attributes,
+      };
+      streamHandlerCallback(
+        new TextStreamReader(info, stream, bigIntToNumber(streamHeader.totalLength)),
+        { identity: participantIdentity },
+      );
+    }
+  }
+
+  private handleStreamChunk(chunk: DataStream_Chunk) {
+    const fileBuffer = this.byteStreamControllers.get(chunk.streamId!);
+    if (fileBuffer) {
+      if (chunk.content!.length > 0) {
+        fileBuffer.controller.enqueue(chunk);
+      }
+    }
+    const textBuffer = this.textStreamControllers.get(chunk.streamId!);
+    if (textBuffer) {
+      if (chunk.content!.length > 0) {
+        textBuffer.controller.enqueue(chunk);
+      }
+    }
+  }
+
+  private handleStreamTrailer(trailer: DataStream_Trailer) {
+    const streamId = trailer.streamId!;
+    const fileBuffer = this.byteStreamControllers.get(streamId);
+    if (fileBuffer) {
+      fileBuffer.controller.close();
+      this.byteStreamControllers.delete(streamId);
+    }
+    const textBuffer = this.textStreamControllers.get(streamId);
+    if (textBuffer) {
+      textBuffer.controller.close();
+      this.byteStreamControllers.delete(streamId);
+    }
+  }
 }
 
 export class ConnectError extends Error {

package/src/utils.test.ts

@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { splitUtf8 } from './utils.js';
+
+describe('splitUtf8', () => {
+  it('splits a string into chunks of the given size', () => {
+    expect(splitUtf8('hello world', 5)).toEqual([
+      new TextEncoder().encode('hello'),
+      new TextEncoder().encode(' worl'),
+      new TextEncoder().encode('d'),
+    ]);
+  });
+
+  it('splits a string with special characters into chunks of the given size', () => {
+    expect(splitUtf8('héllo wörld', 5)).toEqual([
+      new TextEncoder().encode('héll'),
+      new TextEncoder().encode('o wö'),
+      new TextEncoder().encode('rld'),
+    ]);
+  });
+
+  it('splits a string with multi-byte utf8 characters correctly', () => {
+    expect(splitUtf8('こんにちは世界', 5)).toEqual([
+      new TextEncoder().encode('こ'),
+      new TextEncoder().encode('ん'),
+      new TextEncoder().encode('に'),
+      new TextEncoder().encode('ち'),
+      new TextEncoder().encode('は'),
+      new TextEncoder().encode('世'),
+      new TextEncoder().encode('界'),
+    ]);
+  });
+
+  it('handles a string with a single multi-byte utf8 character', () => {
+    expect(splitUtf8('😊', 5)).toEqual([new TextEncoder().encode('😊')]);
+  });
+
+  it('handles a string with mixed single and multi-byte utf8 characters', () => {
+    expect(splitUtf8('a😊b', 4)).toEqual([
+      new TextEncoder().encode('a'),
+      new TextEncoder().encode('😊'),
+      new TextEncoder().encode('b'),
+    ]);
+  });
+
+  it('handles an empty string', () => {
+    expect(splitUtf8('', 5)).toEqual([]);
+  });
+});

package/src/utils.ts

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/** convert bigints to numbers preserving undefined values */
+export function bigIntToNumber<T extends bigint | undefined>(
+  value: T,
+): T extends bigint ? number : undefined {
+  return (value !== undefined ? Number(value) : undefined) as T extends bigint ? number : undefined;
+}
+
+/** convert numbers to bigints preserving undefined values */
+export function numberToBigInt<T extends number | undefined>(
+  value: T,
+): T extends number ? bigint : undefined {
+  return (value !== undefined ? BigInt(value) : undefined) as T extends number ? bigint : undefined;
+}
+
+export function splitUtf8(s: string, n: number): Uint8Array[] {
+  if (n < 4) {
+    throw new Error('n must be at least 4 due to utf8 encoding rules');
+  }
+  // adapted from https://stackoverflow.com/a/6043797
+  const result: Uint8Array[] = [];
+  let encoded = new TextEncoder().encode(s);
+  while (encoded.length > n) {
+    let k = n;
+    while (k > 0) {
+      const byte = encoded[k];
+      if (byte !== undefined && (byte & 0xc0) !== 0x80) {
+        break;
+      }
+      k--;
+    }
+    result.push(encoded.slice(0, k));
+    encoded = encoded.slice(k);
+  }
+  if (encoded.length > 0) {
+    result.push(encoded);
+  }
+  return result;
+}

package/src/version.ts

@@ -1 +1 @@
-export const SDK_VERSION = "0.13.3";
+export const SDK_VERSION = "0.13.5";

package/src/video_frame.ts

@@ -46,6 +46,8 @@ export class VideoFrame {
       case VideoBufferType.RGB24:
         info.stride = this.width * 3;
         break;
+      default:
+        info.stride = 0;
     }
 
     info.components.push(...getPlaneInfos(this.dataPtr, this.type, this.width, this.height));