livekit-client 2.8.0 → 2.9.0

package/src/room/Room.ts

@@ -4,6 +4,9 @@ import {
   ConnectionQualityUpdate,
   type DataPacket,
   DataPacket_Kind,
+  DataStream_Chunk,
+  DataStream_Header,
+  DataStream_Trailer,
   DisconnectReason,
   JoinResponse,
   LeaveRequest,
@@ -45,6 +48,12 @@ import { getBrowser } from '../utils/browserParser';
 import DeviceManager from './DeviceManager';
 import RTCEngine from './RTCEngine';
 import { RegionUrlProvider } from './RegionUrlProvider';
+import {
+  type ByteStreamHandler,
+  ByteStreamReader,
+  type TextStreamHandler,
+  TextStreamReader,
+} from './StreamReader';
 import {
   audioDefaults,
   publishDefaults,
@@ -58,6 +67,7 @@ import LocalParticipant from './participant/LocalParticipant';
 import type Participant from './participant/Participant';
 import type { ConnectionQuality } from './participant/Participant';
 import RemoteParticipant from './participant/RemoteParticipant';
+import { MAX_PAYLOAD_BYTES, RpcError, type RpcInvocationData, byteLength } from './rpc';
 import CriticalTimers from './timers';
 import LocalAudioTrack from './track/LocalAudioTrack';
 import type LocalTrack from './track/LocalTrack';
@@ -70,14 +80,18 @@ import type { TrackPublication } from './track/TrackPublication';
 import type { TrackProcessor } from './track/processor/types';
 import type { AdaptiveStreamSettings } from './track/types';
 import { getNewAudioContext, sourceToKind } from './track/utils';
-import type {
-  ChatMessage,
-  SimulationOptions,
-  SimulationScenario,
-  TranscriptionSegment,
+import {
+  type ByteStreamInfo,
+  type ChatMessage,
+  type SimulationOptions,
+  type SimulationScenario,
+  type StreamController,
+  type TextStreamInfo,
+  type TranscriptionSegment,
 } from './types';
 import {
   Future,
+  bigIntToNumber,
   createDummyVideoStreamTrack,
   extractChatMessage,
   extractTranscriptionSegments,
@@ -85,9 +99,14 @@ import {
   getEmptyAudioStreamTrack,
   isBrowserSupported,
   isCloud,
+  isLocalAudioTrack,
+  isLocalParticipant,
   isReactNative,
+  isRemotePub,
   isSafari,
   isWeb,
+  numberToBigInt,
+  sleep,
   supportsSetSinkId,
   toHttpUrl,
   unpackStreamId,
@@ -180,6 +199,16 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
    */
   private transcriptionReceivedTimes: Map<string, number>;
 
+  private byteStreamControllers = new Map<string, StreamController<DataStream_Chunk>>();
+
+  private textStreamControllers = new Map<string, StreamController<DataStream_Chunk>>();
+
+  private byteStreamHandlers = new Map<string, ByteStreamHandler>();
+
+  private textStreamHandlers = new Map<string, TextStreamHandler>();
+
+  private rpcHandlers: Map<string, (data: RpcInvocationData) => Promise<string>> = new Map();
+
   /**
    * Creates a new Room, the primary construct for a LiveKit session.
    * @param options
@@ -211,7 +240,13 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
     this.disconnectLock = new Mutex();
 
-    this.localParticipant = new LocalParticipant('', '', this.engine, this.options);
+    this.localParticipant = new LocalParticipant(
+      '',
+      '',
+      this.engine,
+      this.options,
+      this.rpcHandlers,
+    );
 
     if (this.options.videoCaptureDefaults.deviceId) {
       this.localParticipant.activeDeviceMap.set(
@@ -252,6 +287,135 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
   }
 
+  registerTextStreamHandler(topic: string, callback: TextStreamHandler) {
+    if (this.textStreamHandlers.has(topic)) {
+      throw new TypeError(`A text stream handler for topic "${topic}" has already been set.`);
+    }
+    this.textStreamHandlers.set(topic, callback);
+  }
+
+  unregisterTextStreamHandler(topic: string) {
+    this.textStreamHandlers.delete(topic);
+  }
+
+  registerByteStreamHandler(topic: string, callback: ByteStreamHandler) {
+    if (this.byteStreamHandlers.has(topic)) {
+      throw new TypeError(`A byte stream handler for topic "${topic}" has already been set.`);
+    }
+    this.byteStreamHandlers.set(topic, callback);
+  }
+
+  unregisterByteStreamHandler(topic: string) {
+    this.byteStreamHandlers.delete(topic);
+  }
+
+  /**
+   * Establishes the participant as a receiver for calls of the specified RPC method.
+   * Will overwrite any existing callback for the same method.
+   *
+   * @param method - The name of the indicated RPC method
+   * @param handler - Will be invoked when an RPC request for this method is received
+   * @returns A promise that resolves when the method is successfully registered
+   * @throws {Error} if the handler for a specific method has already been registered already
+   *
+   * @example
+   * ```typescript
+   * room.localParticipant?.registerRpcMethod(
+   *   'greet',
+   *   async (data: RpcInvocationData) => {
+   *     console.log(`Received greeting from ${data.callerIdentity}: ${data.payload}`);
+   *     return `Hello, ${data.callerIdentity}!`;
+   *   }
+   * );
+   * ```
+   *
+   * The handler should return a Promise that resolves to a string.
+   * If unable to respond within `responseTimeout`, the request will result in an error on the caller's side.
+   *
+   * You may throw errors of type `RpcError` with a string `message` in the handler,
+   * and they will be received on the caller's side with the message intact.
+   * Other errors thrown in your handler will not be transmitted as-is, and will instead arrive to the caller as `1500` ("Application Error").
+   */
+  registerRpcMethod(method: string, handler: (data: RpcInvocationData) => Promise<string>) {
+    if (this.rpcHandlers.has(method)) {
+      throw Error(
+        `RPC handler already registered for method ${method}, unregisterRpcMethod before trying to register again`,
+      );
+    }
+    this.rpcHandlers.set(method, handler);
+  }
+
+  /**
+   * Unregisters a previously registered RPC method.
+   *
+   * @param method - The name of the RPC method to unregister
+   */
+  unregisterRpcMethod(method: string) {
+    this.rpcHandlers.delete(method);
+  }
+
+  private async handleIncomingRpcRequest(
+    callerIdentity: string,
+    requestId: string,
+    method: string,
+    payload: string,
+    responseTimeout: number,
+    version: number,
+  ) {
+    await this.engine.publishRpcAck(callerIdentity, requestId);
+
+    if (version !== 1) {
+      await this.engine.publishRpcResponse(
+        callerIdentity,
+        requestId,
+        null,
+        RpcError.builtIn('UNSUPPORTED_VERSION'),
+      );
+      return;
+    }
+
+    const handler = this.rpcHandlers.get(method);
+
+    if (!handler) {
+      await this.engine.publishRpcResponse(
+        callerIdentity,
+        requestId,
+        null,
+        RpcError.builtIn('UNSUPPORTED_METHOD'),
+      );
+      return;
+    }
+
+    let responseError: RpcError | null = null;
+    let responsePayload: string | null = null;
+
+    try {
+      const response = await handler({
+        requestId,
+        callerIdentity,
+        payload,
+        responseTimeout,
+      });
+      if (byteLength(response) > MAX_PAYLOAD_BYTES) {
+        responseError = RpcError.builtIn('RESPONSE_PAYLOAD_TOO_LARGE');
+        console.warn(`RPC Response payload too large for ${method}`);
+      } else {
+        responsePayload = response;
+      }
+    } catch (error) {
+      if (error instanceof RpcError) {
+        responseError = error;
+      } else {
+        console.warn(
+          `Uncaught error returned by RPC handler for ${method}. Returning APPLICATION_ERROR instead.`,
+          error,
+        );
+        responseError = RpcError.builtIn('APPLICATION_ERROR');
+      }
+    }
+    await this.engine.publishRpcResponse(callerIdentity, requestId, responsePayload, responseError);
+  }
+
   /**
    * @experimental
    */
@@ -276,7 +440,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.e2eeManager.on(
         EncryptionEvent.ParticipantEncryptionStatusChanged,
         (enabled, participant) => {
-          if (participant instanceof LocalParticipant) {
+          if (isLocalParticipant(participant)) {
             this.isE2EEEnabled = enabled;
           }
           this.emit(RoomEvent.ParticipantEncryptionStatusChanged, enabled, participant);
@@ -960,7 +1124,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         req = new SimulateScenario({
           scenario: {
             case: 'subscriberBandwidth',
-            value: BigInt(arg),
+            value: numberToBigInt(arg),
           },
         });
         break;
@@ -1184,8 +1348,12 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         throw e;
       }
     }
-    if (needsUpdateWithoutTracks) {
-      this.localParticipant.activeDeviceMap.set(kind, deviceId);
+    if (needsUpdateWithoutTracks || kind === 'audiooutput') {
+      // if there are not active tracks yet or we're switching audiooutput, we need to manually update the active device map here as changing audio output won't result in a track restart
+      this.localParticipant.activeDeviceMap.set(
+        kind,
+        (kind === 'audiooutput' && this.options.audioOutput?.deviceId) || deviceId,
+      );
       this.emit(RoomEvent.ActiveDeviceChanged, kind, deviceId);
     }
 
@@ -1589,9 +1757,136 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.handleChatMessage(participant, packet.value.value);
     } else if (packet.value.case === 'metrics') {
       this.handleMetrics(packet.value.value, participant);
+    } else if (packet.value.case === 'streamHeader') {
+      this.handleStreamHeader(packet.value.value, packet.participantIdentity);
+    } else if (packet.value.case === 'streamChunk') {
+      this.handleStreamChunk(packet.value.value);
+    } else if (packet.value.case === 'streamTrailer') {
+      this.handleStreamTrailer(packet.value.value);
+    } else if (packet.value.case === 'rpcRequest') {
+      const rpc = packet.value.value;
+      this.handleIncomingRpcRequest(
+        packet.participantIdentity,
+        rpc.id,
+        rpc.method,
+        rpc.payload,
+        rpc.responseTimeoutMs,
+        rpc.version,
+      );
     }
   };
 
+  private async handleStreamHeader(streamHeader: DataStream_Header, participantIdentity: string) {
+    if (streamHeader.contentHeader.case === 'byteHeader') {
+      const streamHandlerCallback = this.byteStreamHandlers.get(streamHeader.topic);
+
+      if (!streamHandlerCallback) {
+        this.log.debug(
+          'ignoring incoming byte stream due to no handler for topic',
+          streamHeader.topic,
+        );
+        return;
+      }
+      let streamController: ReadableStreamDefaultController<DataStream_Chunk>;
+      const info: ByteStreamInfo = {
+        id: streamHeader.streamId,
+        name: streamHeader.contentHeader.value.name ?? 'unknown',
+        mimeType: streamHeader.mimeType,
+        size: streamHeader.totalLength ? Number(streamHeader.totalLength) : undefined,
+        topic: streamHeader.topic,
+        timestamp: bigIntToNumber(streamHeader.timestamp),
+        attributes: streamHeader.attributes,
+      };
+      const stream = new ReadableStream({
+        start: (controller) => {
+          streamController = controller;
+          this.byteStreamControllers.set(streamHeader.streamId, {
+            info,
+            controller: streamController,
+            startTime: Date.now(),
+          });
+        },
+      });
+      streamHandlerCallback(
+        new ByteStreamReader(info, stream, bigIntToNumber(streamHeader.totalLength)),
+        {
+          identity: participantIdentity,
+        },
+      );
+    } else if (streamHeader.contentHeader.case === 'textHeader') {
+      const streamHandlerCallback = this.textStreamHandlers.get(streamHeader.topic);
+
+      if (!streamHandlerCallback) {
+        this.log.debug(
+          'ignoring incoming text stream due to no handler for topic',
+          streamHeader.topic,
+        );
+        return;
+      }
+      let streamController: ReadableStreamDefaultController<DataStream_Chunk>;
+      const info: TextStreamInfo = {
+        id: streamHeader.streamId,
+        mimeType: streamHeader.mimeType,
+        size: streamHeader.totalLength ? Number(streamHeader.totalLength) : undefined,
+        topic: streamHeader.topic,
+        timestamp: Number(streamHeader.timestamp),
+        attributes: streamHeader.attributes,
+      };
+
+      const stream = new ReadableStream<DataStream_Chunk>({
+        start: (controller) => {
+          streamController = controller;
+          this.textStreamControllers.set(streamHeader.streamId, {
+            info,
+            controller: streamController,
+            startTime: Date.now(),
+          });
+        },
+      });
+      streamHandlerCallback(
+        new TextStreamReader(info, stream, bigIntToNumber(streamHeader.totalLength)),
+        { identity: participantIdentity },
+      );
+    }
+  }
+
+  private handleStreamChunk(chunk: DataStream_Chunk) {
+    const fileBuffer = this.byteStreamControllers.get(chunk.streamId);
+    if (fileBuffer) {
+      if (chunk.content.length > 0) {
+        fileBuffer.controller.enqueue(chunk);
+      }
+    }
+    const textBuffer = this.textStreamControllers.get(chunk.streamId);
+    if (textBuffer) {
+      if (chunk.content.length > 0) {
+        textBuffer.controller.enqueue(chunk);
+      }
+    }
+  }
+
+  private handleStreamTrailer(trailer: DataStream_Trailer) {
+    const textBuffer = this.textStreamControllers.get(trailer.streamId);
+
+    if (textBuffer) {
+      textBuffer.info.attributes = {
+        ...textBuffer.info.attributes,
+        ...trailer.attributes,
+      };
+      textBuffer.controller.close();
+      this.byteStreamControllers.delete(trailer.streamId);
+    }
+
+    const fileBuffer = this.byteStreamControllers.get(trailer.streamId);
+    if (fileBuffer) {
+      {
+        fileBuffer.info.attributes = { ...fileBuffer.info.attributes, ...trailer.attributes };
+        fileBuffer.controller.close();
+        this.byteStreamControllers.delete(trailer.streamId);
+      }
+    }
+  }
+
   private handleUserPacket = (
     participant: RemoteParticipant | undefined,
     userPacket: UserPacket,
@@ -1767,24 +2062,24 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.audioContext = getNewAudioContext() ?? undefined;
     }
 
+    if (this.options.webAudioMix) {
+      this.remoteParticipants.forEach((participant) =>
+        participant.setAudioContext(this.audioContext),
+      );
+    }
+
+    this.localParticipant.setAudioContext(this.audioContext);
+
     if (this.audioContext && this.audioContext.state === 'suspended') {
       // for iOS a newly created AudioContext is always in `suspended` state.
       // we try our best to resume the context here, if that doesn't work, we just continue with regular processing
       try {
-        await this.audioContext.resume();
+        await Promise.race([this.audioContext.resume(), sleep(200)]);
       } catch (e: any) {
         this.log.warn('Could not resume audio context', { ...this.logContext, error: e });
       }
     }
 
-    if (this.options.webAudioMix) {
-      this.remoteParticipants.forEach((participant) =>
-        participant.setAudioContext(this.audioContext),
-      );
-    }
-
-    this.localParticipant.setAudioContext(this.audioContext);
-
     const newContextIsRunning = this.audioContext?.state === 'running';
     if (newContextIsRunning !== this.canPlaybackAudio) {
       this.audioEnabled = newContextIsRunning;
@@ -1873,9 +2168,6 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           this.emit(RoomEvent.TrackUnsubscribed, track, publication, participant);
         },
       )
-      .on(ParticipantEvent.TrackSubscriptionFailed, (sid: string) => {
-        this.emit(RoomEvent.TrackSubscriptionFailed, sid, participant);
-      })
       .on(ParticipantEvent.TrackMuted, (pub: TrackPublication) => {
         this.emitWhenConnected(RoomEvent.TrackMuted, pub, participant);
       })
@@ -1946,7 +2238,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   private updateSubscriptions() {
     for (const p of this.remoteParticipants.values()) {
       for (const pub of p.videoTrackPublications.values()) {
-        if (pub.isSubscribed && pub instanceof RemoteTrackPublication) {
+        if (pub.isSubscribed && isRemotePub(pub)) {
           pub.emitTrackUpdate();
         }
       }
@@ -2068,7 +2360,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
     this.emit(RoomEvent.LocalTrackPublished, pub, this.localParticipant);
 
-    if (pub.track instanceof LocalAudioTrack) {
+    if (isLocalAudioTrack(pub.track)) {
       const trackIsSilent = await pub.track.checkForSilence();
       if (trackIsSilent) {
         this.emit(RoomEvent.LocalAudioSilenceDetected, pub);
@@ -2296,7 +2588,7 @@ function mapArgs(args: unknown[]): any {
       return mapArgs(arg);
     }
     if (typeof arg === 'object') {
-      return 'logContext' in arg && arg.logContext;
+      return 'logContext' in arg ? arg.logContext : undefined;
     }
     return arg;
   });

package/src/room/StreamReader.ts

@@ -0,0 +1,177 @@
+import type { DataStream_Chunk } from '@livekit/protocol';
+import type { BaseStreamInfo, ByteStreamInfo, TextStreamChunk, TextStreamInfo } from './types';
+import { bigIntToNumber } from './utils';
+
+abstract class BaseStreamReader<T extends BaseStreamInfo> {
+  protected reader: ReadableStream<DataStream_Chunk>;
+
+  protected totalByteSize?: number;
+
+  protected _info: T;
+
+  protected bytesReceived: number;
+
+  get info() {
+    return this._info;
+  }
+
+  constructor(info: T, stream: ReadableStream<DataStream_Chunk>, totalByteSize?: number) {
+    this.reader = stream;
+    this.totalByteSize = totalByteSize;
+    this._info = info;
+    this.bytesReceived = 0;
+  }
+
+  protected abstract handleChunkReceived(chunk: DataStream_Chunk): void;
+
+  onProgress?: (progress: number | undefined) => void;
+
+  abstract readAll(): Promise<string | Array<Uint8Array>>;
+}
+
+export class ByteStreamReader extends BaseStreamReader<ByteStreamInfo> {
+  protected handleChunkReceived(chunk: DataStream_Chunk) {
+    this.bytesReceived += chunk.content.byteLength;
+    const currentProgress = this.totalByteSize
+      ? this.bytesReceived / this.totalByteSize
+      : undefined;
+    this.onProgress?.(currentProgress);
+  }
+
+  onProgress?: (progress: number | undefined) => void;
+
+  [Symbol.asyncIterator]() {
+    const reader = this.reader.getReader();
+
+    return {
+      next: async (): Promise<IteratorResult<Uint8Array>> => {
+        try {
+          const { done, value } = await reader.read();
+          if (done) {
+            return { done: true, value: undefined as any };
+          } else {
+            this.handleChunkReceived(value);
+            return { done: false, value: value.content };
+          }
+        } catch (error) {
+          // TODO handle errors
+          return { done: true, value: undefined };
+        }
+      },
+
+      return(): IteratorResult<Uint8Array> {
+        reader.releaseLock();
+        return { done: true, value: undefined };
+      },
+    };
+  }
+
+  async readAll(): Promise<Array<Uint8Array>> {
+    let chunks: Set<Uint8Array> = new Set();
+    for await (const chunk of this) {
+      chunks.add(chunk);
+    }
+    return Array.from(chunks);
+  }
+}
+
+/**
+ * A class to read chunks from a ReadableStream and provide them in a structured format.
+ */
+export class TextStreamReader extends BaseStreamReader<TextStreamInfo> {
+  private receivedChunks: Map<number, DataStream_Chunk>;
+
+  /**
+   * A TextStreamReader instance can be used as an AsyncIterator that returns the entire string
+   * that has been received up to the current point in time.
+   */
+  constructor(
+    info: TextStreamInfo,
+    stream: ReadableStream<DataStream_Chunk>,
+    totalChunkCount?: number,
+  ) {
+    super(info, stream, totalChunkCount);
+    this.receivedChunks = new Map();
+  }
+
+  protected handleChunkReceived(chunk: DataStream_Chunk) {
+    const index = bigIntToNumber(chunk.chunkIndex);
+    const previousChunkAtIndex = this.receivedChunks.get(index);
+    if (previousChunkAtIndex && previousChunkAtIndex.version > chunk.version) {
+      // we have a newer version already, dropping the old one
+      return;
+    }
+    this.receivedChunks.set(index, chunk);
+    this.bytesReceived += chunk.content.byteLength;
+    const currentProgress = this.totalByteSize
+      ? this.bytesReceived / this.totalByteSize
+      : undefined;
+    this.onProgress?.(currentProgress);
+  }
+
+  /**
+   * @param progress - progress of the stream between 0 and 1. Undefined for streams of unknown size
+   */
+  onProgress?: (progress: number | undefined) => void;
+
+  /**
+   * Async iterator implementation to allow usage of `for await...of` syntax.
+   * Yields structured chunks from the stream.
+   *
+   */
+  [Symbol.asyncIterator]() {
+    const reader = this.reader.getReader();
+    const decoder = new TextDecoder();
+
+    return {
+      next: async (): Promise<IteratorResult<TextStreamChunk>> => {
+        try {
+          const { done, value } = await reader.read();
+          if (done) {
+            return { done: true, value: undefined };
+          } else {
+            this.handleChunkReceived(value);
+
+            return {
+              done: false,
+              value: {
+                index: bigIntToNumber(value.chunkIndex),
+                current: decoder.decode(value.content),
+                collected: Array.from(this.receivedChunks.values())
+                  .sort((a, b) => bigIntToNumber(a.chunkIndex) - bigIntToNumber(b.chunkIndex))
+                  .map((chunk) => decoder.decode(chunk.content))
+                  .join(''),
+              },
+            };
+          }
+        } catch (error) {
+          // TODO handle errors
+          return { done: true, value: undefined };
+        }
+      },
+
+      return(): IteratorResult<TextStreamChunk> {
+        reader.releaseLock();
+        return { done: true, value: undefined };
+      },
+    };
+  }
+
+  async readAll(): Promise<string> {
+    let latestString: string = '';
+    for await (const { collected } of this) {
+      latestString = collected;
+    }
+    return latestString;
+  }
+}
+
+export type ByteStreamHandler = (
+  reader: ByteStreamReader,
+  participantInfo: { identity: string },
+) => void;
+
+export type TextStreamHandler = (
+  reader: TextStreamReader,
+  participantInfo: { identity: string },
+) => void;

package/src/room/StreamWriter.ts

@@ -0,0 +1,32 @@
+import type { BaseStreamInfo, ByteStreamInfo, TextStreamInfo } from './types';
+
+class BaseStreamWriter<T, InfoType extends BaseStreamInfo> {
+  protected writableStream: WritableStream<[T, number?]>;
+
+  protected defaultWriter: WritableStreamDefaultWriter<[T, number?]>;
+
+  protected onClose?: () => void;
+
+  readonly info: InfoType;
+
+  constructor(writableStream: WritableStream<[T, number?]>, info: InfoType, onClose?: () => void) {
+    this.writableStream = writableStream;
+    this.defaultWriter = writableStream.getWriter();
+    this.onClose = onClose;
+    this.info = info;
+  }
+
+  write(chunk: T): Promise<void> {
+    return this.defaultWriter.write([chunk]);
+  }
+
+  async close() {
+    await this.defaultWriter.close();
+    this.defaultWriter.releaseLock();
+    this.onClose?.();
+  }
+}
+
+export class TextStreamWriter extends BaseStreamWriter<string, TextStreamInfo> {}
+
+export class BinaryStreamWriter extends BaseStreamWriter<Uint8Array, ByteStreamInfo> {}