@livekit/rtc-node 0.13.2 → 0.13.4

package/src/proto/video_frame_pb.ts

@@ -87,6 +87,9 @@ proto2.util.setEnumType(VideoRotation, "livekit.proto.VideoRotation", [
 ]);
 
 /**
+ * Values of this enum must not be changed
+ * It is used to serialize a rtc.VideoFrame on Python
+ *
  * @generated from enum livekit.proto.VideoBufferType
  */
 export enum VideoBufferType {

package/src/room.ts

@@ -3,9 +3,19 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import EventEmitter from 'events';
+import { ReadableStream } from 'node:stream/web';
+import { ByteStreamReader, TextStreamReader } from './data_streams/stream_reader.js';
+import type {
+  ByteStreamHandler,
+  ByteStreamInfo,
+  StreamController,
+  TextStreamHandler,
+  TextStreamInfo,
+} from './data_streams/types.js';
 import type { E2EEOptions } from './e2ee.js';
 import { E2EEManager, defaultE2EEOptions } from './e2ee.js';
 import { FfiClient, FfiClientEvent, FfiHandle } from './ffi_client.js';
+import { log } from './log.js';
 import type { Participant } from './participant.js';
 import { LocalParticipant, RemoteParticipant } from './participant.js';
 import { EncryptionState } from './proto/e2ee_pb.js';
@@ -16,6 +26,9 @@ import {
   type ConnectResponse,
   type ConnectionQuality,
   type DataPacketKind,
+  type DataStream_Chunk,
+  type DataStream_Header,
+  DataStream_Trailer,
   type DisconnectResponse,
   RoomOptions as FfiRoomOptions,
   type IceServer,
@@ -33,6 +46,7 @@ import { RemoteAudioTrack, RemoteVideoTrack } from './track.js';
 import type { LocalTrackPublication, TrackPublication } from './track_publication.js';
 import { RemoteTrackPublication } from './track_publication.js';
 import type { ChatMessage } from './types.js';
+import { bigIntToNumber } from './utils.js';
 
 export interface RtcConfiguration {
   iceTransportType: IceTransportType;
@@ -66,6 +80,11 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
   private info?: RoomInfo;
   private ffiHandle?: FfiHandle;
 
+  private byteStreamControllers = new Map<string, StreamController<DataStream_Chunk>>();
+  private textStreamControllers = new Map<string, StreamController<DataStream_Chunk>>();
+  private byteStreamHandlers = new Map<string, ByteStreamHandler>();
+  private textStreamHandlers = new Map<string, TextStreamHandler>();
+
   e2eeManager?: E2EEManager;
   connectionState: ConnectionState = ConnectionState.CONN_DISCONNECTED;
 
@@ -88,8 +107,26 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
     return this.ffiHandle != undefined && this.connectionState != ConnectionState.CONN_DISCONNECTED;
   }
 
-  async getSid(): Promise<string | undefined> {
-    return this.info?.sid; // TODO update this to handle async room updates once rust protocol has been updated
+  async getSid(): Promise<string> {
+    if (!this.isConnected) {
+      return '';
+    }
+    if (this.info?.sid && this.info.sid !== '') {
+      return this.info.sid;
+    }
+    return new Promise((resolve, reject) => {
+      const handleRoomUpdate = (sid: string) => {
+        if (sid !== '') {
+          this.off(RoomEvent.RoomSidChanged, handleRoomUpdate);
+          resolve(sid);
+        }
+      };
+      this.on(RoomEvent.RoomSidChanged, handleRoomUpdate);
+      this.once(RoomEvent.Disconnected, () => {
+        this.off(RoomEvent.RoomSidChanged, handleRoomUpdate);
+        reject('Room disconnected before room server id was available');
+      });
+    });
   }
 
   async connect(url: string, token: string, opts?: RoomOptions) {
@@ -157,10 +194,50 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
     this.removeAllListeners();
   }
 
+  registerTextStreamHandler(topic: string, callback: TextStreamHandler) {
+    if (this.textStreamHandlers.has(topic)) {
+      throw new Error(`A text stream handler for topic "${topic}" has already been set.`);
+    }
+    this.textStreamHandlers.set(topic, callback);
+  }
+
+  unregisterTextStreamHandler(topic: string) {
+    this.textStreamHandlers.delete(topic);
+  }
+
+  registerByteStreamHandler(topic: string, callback: ByteStreamHandler) {
+    if (this.byteStreamHandlers.has(topic)) {
+      throw new Error(`A byte stream handler for topic "${topic}" has already been set.`);
+    }
+    this.byteStreamHandlers.set(topic, callback);
+  }
+
+  unregisterByteStreamHandler(topic: string) {
+    this.byteStreamHandlers.delete(topic);
+  }
+
   private onFfiEvent = (ffiEvent: FfiEvent) => {
-    if (
+    if (!this.localParticipant || !this.ffiHandle || !this.info) {
+      throw TypeError('cannot handle ffi events before connectCallback');
+    }
+
+    if (ffiEvent.message.case == 'rpcMethodInvocation') {
+      if (
+        ffiEvent.message.value.localParticipantHandle == this.localParticipant.ffi_handle.handle
+      ) {
+        this.localParticipant.handleRpcMethodInvocation(
+          ffiEvent.message.value.invocationId!,
+          ffiEvent.message.value.method!,
+          ffiEvent.message.value.requestId!,
+          ffiEvent.message.value.callerIdentity!,
+          ffiEvent.message.value.payload!,
+          ffiEvent.message.value.responseTimeoutMs!,
+        );
+      }
+      return;
+    } else if (
       ffiEvent.message.case != 'roomEvent' ||
-      ffiEvent.message.value.roomHandle != this.ffiHandle?.handle
+      ffiEvent.message.value.roomHandle != this.ffiHandle.handle
     ) {
       return;
     }
@@ -191,96 +268,97 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
       participant!.trackPublications.set(publication.sid!, publication);
       this.emit(RoomEvent.TrackPublished, publication, participant!);
     } else if (ev.case == 'trackUnpublished') {
-      const participant = this.remoteParticipants.get(ev.value.participantIdentity!);
-      const publication = participant!.trackPublications.get(ev.value.publicationSid!);
-      participant!.trackPublications.delete(ev.value.publicationSid!);
-      this.emit(RoomEvent.TrackUnpublished, publication!, participant!);
+      const participant = this.requireRemoteParticipant(ev.value.participantIdentity!);
+      const publication = participant.trackPublications.get(ev.value.publicationSid!);
+      participant.trackPublications.delete(ev.value.publicationSid!);
+      if (publication) {
+        this.emit(RoomEvent.TrackUnpublished, publication, participant);
+      }
     } else if (ev.case == 'trackSubscribed') {
-      const ownedTrack = ev.value.track;
-      const participant = this.remoteParticipants.get(ev.value.participantIdentity!);
-      const publication = participant!.trackPublications.get(ownedTrack!.info!.sid!);
-      publication!.subscribed = true;
-      if (ownedTrack!.info!.kind == TrackKind.KIND_VIDEO) {
-        publication!.track = new RemoteVideoTrack(ownedTrack!);
-      } else if (ownedTrack!.info!.kind == TrackKind.KIND_AUDIO) {
-        publication!.track = new RemoteAudioTrack(ownedTrack!);
+      const ownedTrack = ev.value.track!;
+      const trackInfo = ownedTrack.info!;
+      const { participant, publication } = this.requirePublicationOfRemoteParticipant(
+        ev.value.participantIdentity!,
+        trackInfo.sid!,
+      );
+      publication.subscribed = true;
+      if (trackInfo.kind == TrackKind.KIND_VIDEO) {
+        publication.track = new RemoteVideoTrack(ownedTrack);
+      } else if (trackInfo.kind == TrackKind.KIND_AUDIO) {
+        publication.track = new RemoteAudioTrack(ownedTrack);
       }
 
-      this.emit(RoomEvent.TrackSubscribed, publication!.track!, publication!, participant!);
+      this.emit(RoomEvent.TrackSubscribed, publication.track!, publication, participant);
     } else if (ev.case == 'trackUnsubscribed') {
-      const participant = this.remoteParticipants.get(ev.value.participantIdentity!);
-      const publication = participant!.trackPublications.get(ev.value.trackSid!);
-      publication!.track = undefined;
-      publication!.subscribed = false;
-      this.emit(RoomEvent.TrackUnsubscribed, publication!.track!, publication!, participant!);
-    } else if (ev.case == 'trackSubscriptionFailed') {
-      const participant = this.remoteParticipants.get(ev.value.participantIdentity!);
-      this.emit(
-        RoomEvent.TrackSubscriptionFailed,
+      const { participant, publication } = this.requirePublicationOfRemoteParticipant(
+        ev.value.participantIdentity!,
         ev.value.trackSid!,
-        participant!,
-        ev.value.error,
       );
+      const track = publication.track!;
+      publication.track = undefined;
+      publication.subscribed = false;
+      this.emit(RoomEvent.TrackUnsubscribed, track, publication, participant);
+    } else if (ev.case == 'trackSubscriptionFailed') {
+      const participant = this.requireRemoteParticipant(ev.value.participantIdentity!);
+      this.emit(RoomEvent.TrackSubscriptionFailed, ev.value.trackSid!, participant, ev.value.error);
     } else if (ev.case == 'trackMuted') {
-      const participant = this.remoteParticipants.get(ev.value.participantIdentity!);
-      const publication = participant!.trackPublications.get(ev.value.trackSid!);
-      publication!.info!.muted = true;
-      if (publication!.track) {
-        publication!.track.info!.muted = true;
+      const { participant, publication } = this.requirePublicationOfParticipant(
+        ev.value.participantIdentity!,
+        ev.value.trackSid!,
+      );
+      publication.info!.muted = true;
+      if (publication.track) {
+        publication.track.info!.muted = true;
       }
-      this.emit(RoomEvent.TrackMuted, publication!, participant!);
+      this.emit(RoomEvent.TrackMuted, publication, participant);
     } else if (ev.case == 'trackUnmuted') {
-      const participant = this.retrieveParticipantByIdentity(ev.value.participantIdentity!);
-      const publication = participant!.trackPublications.get(ev.value.trackSid!);
-      publication!.info!.muted = false;
-      if (publication!.track) {
-        publication!.track.info!.muted = false;
+      const { participant, publication } = this.requirePublicationOfParticipant(
+        ev.value.participantIdentity!,
+        ev.value.trackSid!,
+      );
+      publication.info!.muted = false;
+      if (publication.track) {
+        publication.track.info!.muted = false;
       }
-      this.emit(RoomEvent.TrackUnmuted, publication!, participant!);
+      this.emit(RoomEvent.TrackUnmuted, publication, participant);
     } else if (ev.case == 'activeSpeakersChanged') {
       const activeSpeakers = ev.value.participantIdentities.map((identity) =>
-        this.retrieveParticipantByIdentity(identity),
-      );
-      this.emit(
-        RoomEvent.ActiveSpeakersChanged,
-        activeSpeakers.map((s) => s!),
+        this.requireParticipantByIdentity(identity),
       );
+      this.emit(RoomEvent.ActiveSpeakersChanged, activeSpeakers);
     } else if (ev.case == 'roomMetadataChanged') {
       this.info!.metadata = ev.value.metadata;
       this.emit(RoomEvent.RoomMetadataChanged, this.info!.metadata!);
     } else if (ev.case == 'participantMetadataChanged') {
-      const participant = this.retrieveParticipantByIdentity(ev.value.participantIdentity!);
-      participant!.info!.metadata = ev.value.metadata;
-      this.emit(RoomEvent.ParticipantMetadataChanged, participant!.metadata, participant!);
+      const participant = this.requireParticipantByIdentity(ev.value.participantIdentity!);
+      participant.info.metadata = ev.value.metadata;
+      this.emit(RoomEvent.ParticipantMetadataChanged, participant.metadata, participant);
     } else if (ev.case == 'participantNameChanged') {
-      const participant = this.retrieveParticipantByIdentity(ev.value.participantIdentity!);
-      participant!.info!.name = ev.value.name;
-      this.emit(RoomEvent.ParticipantNameChanged, participant!.name!, participant!);
+      const participant = this.requireParticipantByIdentity(ev.value.participantIdentity!);
+      participant.info.name = ev.value.name;
+      this.emit(RoomEvent.ParticipantNameChanged, participant.name!, participant);
     } else if (ev.case == 'participantAttributesChanged') {
-      const participant = this.retrieveParticipantByIdentity(ev.value.participantIdentity!);
-      participant!.info!.attributes = ev.value.attributes.reduce(
-        (obj, item) => {
-          obj[item.key!] = item.value!;
-          return obj;
+      const participant = this.requireParticipantByIdentity(ev.value.participantIdentity!);
+      participant.info.attributes = ev.value.attributes.reduce(
+        (acc, value) => {
+          acc[value.key!] = value.value!;
+          return acc;
         },
         {} as Record<string, string>,
       );
       if (Object.keys(ev.value.changedAttributes).length > 0) {
-        this.emit(
-          RoomEvent.ParticipantAttributesChanged,
-          ev.value.changedAttributes.reduce(
-            (obj, item) => {
-              obj[item.key!] = item.value!;
-              return obj;
-            },
-            {} as Record<string, string>,
-          ),
-          participant!,
+        const changedAttributes = ev.value.changedAttributes.reduce(
+          (acc, value) => {
+            acc[value.key!] = value.value!;
+            return acc;
+          },
+          {} as Record<string, string>,
         );
+        this.emit(RoomEvent.ParticipantAttributesChanged, changedAttributes, participant);
       }
     } else if (ev.case == 'connectionQualityChanged') {
-      const participant = this.retrieveParticipantByIdentity(ev.value.participantIdentity!);
-      this.emit(RoomEvent.ConnectionQualityChanged, ev.value.quality!, participant!);
+      const participant = this.requireParticipantByIdentity(ev.value.participantIdentity!);
+      this.emit(RoomEvent.ConnectionQualityChanged, ev.value.quality!, participant);
     } else if (ev.case == 'chatMessage') {
       const participant = this.retrieveParticipantByIdentity(ev.value.participantIdentity!);
       const { id, message: messageText, timestamp, editTimestamp, generated } = ev.value.message!;
@@ -334,6 +412,14 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
       this.emit(RoomEvent.Reconnecting);
     } else if (ev.case == 'reconnected') {
       this.emit(RoomEvent.Reconnected);
+    } else if (ev.case == 'roomSidChanged') {
+      this.emit(RoomEvent.RoomSidChanged, ev.value.sid!);
+    } else if (ev.case === 'streamHeaderReceived' && ev.value.header) {
+      this.handleStreamHeader(ev.value.header, ev.value.participantIdentity!);
+    } else if (ev.case === 'streamChunkReceived' && ev.value.chunk) {
+      this.handleStreamChunk(ev.value.chunk);
+    } else if (ev.case === 'streamTrailerReceived' && ev.value.trailer) {
+      this.handleStreamTrailer(ev.value.trailer);
     }
   };
 
@@ -390,6 +476,100 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
     this.remoteParticipants.set(ownedInfo.info!.identity!, participant);
     return participant;
   }
+
+  private handleStreamHeader(streamHeader: DataStream_Header, participantIdentity: string) {
+    if (streamHeader.contentHeader.case === 'byteHeader') {
+      const streamHandlerCallback = this.byteStreamHandlers.get(streamHeader.topic ?? '');
+
+      if (!streamHandlerCallback) {
+        log.debug('ignoring incoming byte stream due to no handler for topic', streamHeader.topic);
+        return;
+      }
+      let streamController: ReadableStreamDefaultController<DataStream_Chunk>;
+      const stream = new ReadableStream({
+        start: (controller) => {
+          streamController = controller;
+          this.byteStreamControllers.set(streamHeader.streamId!, {
+            header: streamHeader,
+            controller: streamController,
+            startTime: Date.now(),
+          });
+        },
+      });
+      const info: ByteStreamInfo = {
+        streamId: streamHeader.streamId!,
+        name: streamHeader.contentHeader.value.name ?? 'unknown',
+        mimeType: streamHeader.mimeType!,
+        totalSize: streamHeader.totalLength ? Number(streamHeader.totalLength) : undefined,
+        topic: streamHeader.topic!,
+        timestamp: bigIntToNumber(streamHeader.timestamp!),
+        attributes: streamHeader.attributes,
+      };
+      streamHandlerCallback(
+        new ByteStreamReader(info, stream, bigIntToNumber(streamHeader.totalLength)),
+        { identity: participantIdentity },
+      );
+    } else if (streamHeader.contentHeader.case === 'textHeader') {
+      const streamHandlerCallback = this.textStreamHandlers.get(streamHeader.topic ?? '');
+
+      if (!streamHandlerCallback) {
+        log.debug('ignoring incoming text stream due to no handler for topic', streamHeader.topic);
+        return;
+      }
+      let streamController: ReadableStreamDefaultController<DataStream_Chunk>;
+      const stream = new ReadableStream<DataStream_Chunk>({
+        start: (controller) => {
+          streamController = controller;
+          this.textStreamControllers.set(streamHeader.streamId!, {
+            header: streamHeader,
+            controller: streamController,
+            startTime: Date.now(),
+          });
+        },
+      });
+      const info: TextStreamInfo = {
+        streamId: streamHeader.streamId!,
+        mimeType: streamHeader.mimeType!,
+        totalSize: streamHeader.totalLength ? Number(streamHeader.totalLength) : undefined,
+        topic: streamHeader.topic!,
+        timestamp: Number(streamHeader.timestamp),
+        attributes: streamHeader.attributes,
+      };
+      streamHandlerCallback(
+        new TextStreamReader(info, stream, bigIntToNumber(streamHeader.totalLength)),
+        { identity: participantIdentity },
+      );
+    }
+  }
+
+  private handleStreamChunk(chunk: DataStream_Chunk) {
+    const fileBuffer = this.byteStreamControllers.get(chunk.streamId!);
+    if (fileBuffer) {
+      if (chunk.content!.length > 0) {
+        fileBuffer.controller.enqueue(chunk);
+      }
+    }
+    const textBuffer = this.textStreamControllers.get(chunk.streamId!);
+    if (textBuffer) {
+      if (chunk.content!.length > 0) {
+        textBuffer.controller.enqueue(chunk);
+      }
+    }
+  }
+
+  private handleStreamTrailer(trailer: DataStream_Trailer) {
+    const streamId = trailer.streamId!;
+    const fileBuffer = this.byteStreamControllers.get(streamId);
+    if (fileBuffer) {
+      fileBuffer.controller.close();
+      this.byteStreamControllers.delete(streamId);
+    }
+    const textBuffer = this.textStreamControllers.get(streamId);
+    if (textBuffer) {
+      textBuffer.controller.close();
+      this.byteStreamControllers.delete(streamId);
+    }
+  }
 }
 
 export class ConnectError extends Error {
@@ -428,6 +608,7 @@ export type RoomCallbacks = {
   trackUnmuted: (publication: TrackPublication, participant: Participant) => void;
   activeSpeakersChanged: (speakers: Participant[]) => void;
   roomMetadataChanged: (metadata: string) => void;
+  roomInfoUpdated: (info: RoomInfo) => void;
   participantMetadataChanged: (metadata: string | undefined, participant: Participant) => void;
   participantNameChanged: (name: string, participant: Participant) => void;
   participantAttributesChanged: (
@@ -449,6 +630,7 @@ export type RoomCallbacks = {
   disconnected: (reason: DisconnectReason) => void;
   reconnecting: () => void;
   reconnected: () => void;
+  roomSidChanged: (sid: string) => void;
 };
 
 export enum RoomEvent {
@@ -466,6 +648,7 @@ export enum RoomEvent {
   TrackUnmuted = 'trackUnmuted',
   ActiveSpeakersChanged = 'activeSpeakersChanged',
   RoomMetadataChanged = 'roomMetadataChanged',
+  RoomSidChanged = 'roomSidChanged',
   ParticipantMetadataChanged = 'participantMetadataChanged',
   ParticipantNameChanged = 'participantNameChanged',
   ParticipantAttributesChanged = 'participantAttributesChanged',

package/src/utils.ts

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/** convert bigints to numbers preserving undefined values */
+export function bigIntToNumber<T extends bigint | undefined>(
+  value: T,
+): T extends bigint ? number : undefined {
+  return (value !== undefined ? Number(value) : undefined) as T extends bigint ? number : undefined;
+}
+
+/** convert numbers to bigints preserving undefined values */
+export function numberToBigInt<T extends number | undefined>(
+  value: T,
+): T extends number ? bigint : undefined {
+  return (value !== undefined ? BigInt(value) : undefined) as T extends number ? bigint : undefined;
+}
+
+export function splitUtf8(s: string, n: number): string[] {
+  // adapted from https://stackoverflow.com/a/6043797
+  const result: string[] = [];
+  while (s.length > n) {
+    let k = n;
+    // Move back to find the start of a UTF-8 character
+    while ((s.charCodeAt(k) & 0xc0) === 0x80) {
+      k--;
+    }
+    result.push(s.slice(0, k));
+    s = s.slice(k);
+  }
+  result.push(s);
+  return result;
+}

package/src/version.ts

@@ -1 +1 @@
-export const SDK_VERSION = "0.13.2";
+export const SDK_VERSION = "0.13.4";

package/src/video_frame.ts

@@ -46,6 +46,8 @@ export class VideoFrame {
       case VideoBufferType.RGB24:
         info.stride = this.width * 3;
         break;
+      default:
+        info.stride = 0;
     }
 
     info.components.push(...getPlaneInfos(this.dataPtr, this.type, this.width, this.height));