@livekit/rtc-node 0.8.0 → 0.9.0

package/src/proto/audio_frame_pb.ts

@@ -276,9 +276,9 @@ export class NewAudioSourceRequest extends Message<NewAudioSourceRequest> {
   numChannels = 0;
 
   /**
-   * @generated from field: optional bool enable_queue = 5;
+   * @generated from field: uint32 queue_size_ms = 5;
    */
-  enableQueue?: boolean;
+  queueSizeMs = 0;
 
   constructor(data?: PartialMessage<NewAudioSourceRequest>) {
     super();
@@ -292,7 +292,7 @@ export class NewAudioSourceRequest extends Message<NewAudioSourceRequest> {
     { no: 2, name: "options", kind: "message", T: AudioSourceOptions, opt: true },
     { no: 3, name: "sample_rate", kind: "scalar", T: 13 /* ScalarType.UINT32 */ },
     { no: 4, name: "num_channels", kind: "scalar", T: 13 /* ScalarType.UINT32 */ },
-    { no: 5, name: "enable_queue", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true },
+    { no: 5, name: "queue_size_ms", kind: "scalar", T: 13 /* ScalarType.UINT32 */ },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): NewAudioSourceRequest {
@@ -475,6 +475,74 @@ export class CaptureAudioFrameCallback extends Message<CaptureAudioFrameCallback
   }
 }
 
+/**
+ * @generated from message livekit.proto.ClearAudioBufferRequest
+ */
+export class ClearAudioBufferRequest extends Message<ClearAudioBufferRequest> {
+  /**
+   * @generated from field: uint64 source_handle = 1;
+   */
+  sourceHandle = protoInt64.zero;
+
+  constructor(data?: PartialMessage<ClearAudioBufferRequest>) {
+    super();
+    proto3.util.initPartial(data, this);
+  }
+
+  static readonly runtime: typeof proto3 = proto3;
+  static readonly typeName = "livekit.proto.ClearAudioBufferRequest";
+  static readonly fields: FieldList = proto3.util.newFieldList(() => [
+    { no: 1, name: "source_handle", kind: "scalar", T: 4 /* ScalarType.UINT64 */ },
+  ]);
+
+  static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): ClearAudioBufferRequest {
+    return new ClearAudioBufferRequest().fromBinary(bytes, options);
+  }
+
+  static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): ClearAudioBufferRequest {
+    return new ClearAudioBufferRequest().fromJson(jsonValue, options);
+  }
+
+  static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): ClearAudioBufferRequest {
+    return new ClearAudioBufferRequest().fromJsonString(jsonString, options);
+  }
+
+  static equals(a: ClearAudioBufferRequest | PlainMessage<ClearAudioBufferRequest> | undefined, b: ClearAudioBufferRequest | PlainMessage<ClearAudioBufferRequest> | undefined): boolean {
+    return proto3.util.equals(ClearAudioBufferRequest, a, b);
+  }
+}
+
+/**
+ * @generated from message livekit.proto.ClearAudioBufferResponse
+ */
+export class ClearAudioBufferResponse extends Message<ClearAudioBufferResponse> {
+  constructor(data?: PartialMessage<ClearAudioBufferResponse>) {
+    super();
+    proto3.util.initPartial(data, this);
+  }
+
+  static readonly runtime: typeof proto3 = proto3;
+  static readonly typeName = "livekit.proto.ClearAudioBufferResponse";
+  static readonly fields: FieldList = proto3.util.newFieldList(() => [
+  ]);
+
+  static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): ClearAudioBufferResponse {
+    return new ClearAudioBufferResponse().fromBinary(bytes, options);
+  }
+
+  static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): ClearAudioBufferResponse {
+    return new ClearAudioBufferResponse().fromJson(jsonValue, options);
+  }
+
+  static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): ClearAudioBufferResponse {
+    return new ClearAudioBufferResponse().fromJsonString(jsonString, options);
+  }
+
+  static equals(a: ClearAudioBufferResponse | PlainMessage<ClearAudioBufferResponse> | undefined, b: ClearAudioBufferResponse | PlainMessage<ClearAudioBufferResponse> | undefined): boolean {
+    return proto3.util.equals(ClearAudioBufferResponse, a, b);
+  }
+}
+
 /**
  * Create a new AudioResampler
  *

package/src/proto/ffi_pb.ts

@@ -22,7 +22,7 @@ import { Message, proto3, protoInt64 } from "@bufbuild/protobuf";
 import { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, RoomEvent, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } from "./room_pb.js";
 import { CreateAudioTrackRequest, CreateAudioTrackResponse, CreateVideoTrackRequest, CreateVideoTrackResponse, EnableRemoteTrackRequest, EnableRemoteTrackResponse, GetStatsCallback, GetStatsRequest, GetStatsResponse, LocalTrackMuteRequest, LocalTrackMuteResponse, TrackEvent } from "./track_pb.js";
 import { CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } from "./video_frame_pb.js";
-import { AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, RemixAndResampleRequest, RemixAndResampleResponse } from "./audio_frame_pb.js";
+import { AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, RemixAndResampleRequest, RemixAndResampleResponse } from "./audio_frame_pb.js";
 import { E2eeRequest, E2eeResponse } from "./e2ee_pb.js";
 
 /**
@@ -239,25 +239,31 @@ export class FfiRequest extends Message<FfiRequest> {
     case: "captureAudioFrame";
   } | {
     /**
-     * @generated from field: livekit.proto.NewAudioResamplerRequest new_audio_resampler = 28;
+     * @generated from field: livekit.proto.ClearAudioBufferRequest clear_audio_buffer = 28;
+     */
+    value: ClearAudioBufferRequest;
+    case: "clearAudioBuffer";
+  } | {
+    /**
+     * @generated from field: livekit.proto.NewAudioResamplerRequest new_audio_resampler = 29;
      */
     value: NewAudioResamplerRequest;
     case: "newAudioResampler";
   } | {
     /**
-     * @generated from field: livekit.proto.RemixAndResampleRequest remix_and_resample = 29;
+     * @generated from field: livekit.proto.RemixAndResampleRequest remix_and_resample = 30;
      */
     value: RemixAndResampleRequest;
     case: "remixAndResample";
   } | {
     /**
-     * @generated from field: livekit.proto.E2eeRequest e2ee = 30;
+     * @generated from field: livekit.proto.E2eeRequest e2ee = 31;
      */
     value: E2eeRequest;
     case: "e2ee";
   } | {
     /**
-     * @generated from field: livekit.proto.AudioStreamFromParticipantRequest audio_stream_from_participant = 31;
+     * @generated from field: livekit.proto.AudioStreamFromParticipantRequest audio_stream_from_participant = 32;
      */
     value: AudioStreamFromParticipantRequest;
     case: "audioStreamFromParticipant";
@@ -297,10 +303,11 @@ export class FfiRequest extends Message<FfiRequest> {
     { no: 25, name: "new_audio_stream", kind: "message", T: NewAudioStreamRequest, oneof: "message" },
     { no: 26, name: "new_audio_source", kind: "message", T: NewAudioSourceRequest, oneof: "message" },
     { no: 27, name: "capture_audio_frame", kind: "message", T: CaptureAudioFrameRequest, oneof: "message" },
-    { no: 28, name: "new_audio_resampler", kind: "message", T: NewAudioResamplerRequest, oneof: "message" },
-    { no: 29, name: "remix_and_resample", kind: "message", T: RemixAndResampleRequest, oneof: "message" },
-    { no: 30, name: "e2ee", kind: "message", T: E2eeRequest, oneof: "message" },
-    { no: 31, name: "audio_stream_from_participant", kind: "message", T: AudioStreamFromParticipantRequest, oneof: "message" },
+    { no: 28, name: "clear_audio_buffer", kind: "message", T: ClearAudioBufferRequest, oneof: "message" },
+    { no: 29, name: "new_audio_resampler", kind: "message", T: NewAudioResamplerRequest, oneof: "message" },
+    { no: 30, name: "remix_and_resample", kind: "message", T: RemixAndResampleRequest, oneof: "message" },
+    { no: 31, name: "e2ee", kind: "message", T: E2eeRequest, oneof: "message" },
+    { no: 32, name: "audio_stream_from_participant", kind: "message", T: AudioStreamFromParticipantRequest, oneof: "message" },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): FfiRequest {
@@ -495,25 +502,31 @@ export class FfiResponse extends Message<FfiResponse> {
     case: "captureAudioFrame";
   } | {
     /**
-     * @generated from field: livekit.proto.NewAudioResamplerResponse new_audio_resampler = 28;
+     * @generated from field: livekit.proto.ClearAudioBufferResponse clear_audio_buffer = 28;
+     */
+    value: ClearAudioBufferResponse;
+    case: "clearAudioBuffer";
+  } | {
+    /**
+     * @generated from field: livekit.proto.NewAudioResamplerResponse new_audio_resampler = 29;
      */
     value: NewAudioResamplerResponse;
     case: "newAudioResampler";
   } | {
     /**
-     * @generated from field: livekit.proto.RemixAndResampleResponse remix_and_resample = 29;
+     * @generated from field: livekit.proto.RemixAndResampleResponse remix_and_resample = 30;
      */
     value: RemixAndResampleResponse;
     case: "remixAndResample";
   } | {
     /**
-     * @generated from field: livekit.proto.AudioStreamFromParticipantResponse audio_stream_from_participant = 30;
+     * @generated from field: livekit.proto.AudioStreamFromParticipantResponse audio_stream_from_participant = 31;
      */
     value: AudioStreamFromParticipantResponse;
     case: "audioStreamFromParticipant";
   } | {
     /**
-     * @generated from field: livekit.proto.E2eeResponse e2ee = 31;
+     * @generated from field: livekit.proto.E2eeResponse e2ee = 32;
      */
     value: E2eeResponse;
     case: "e2ee";
@@ -553,10 +566,11 @@ export class FfiResponse extends Message<FfiResponse> {
     { no: 25, name: "new_audio_stream", kind: "message", T: NewAudioStreamResponse, oneof: "message" },
     { no: 26, name: "new_audio_source", kind: "message", T: NewAudioSourceResponse, oneof: "message" },
     { no: 27, name: "capture_audio_frame", kind: "message", T: CaptureAudioFrameResponse, oneof: "message" },
-    { no: 28, name: "new_audio_resampler", kind: "message", T: NewAudioResamplerResponse, oneof: "message" },
-    { no: 29, name: "remix_and_resample", kind: "message", T: RemixAndResampleResponse, oneof: "message" },
-    { no: 30, name: "audio_stream_from_participant", kind: "message", T: AudioStreamFromParticipantResponse, oneof: "message" },
-    { no: 31, name: "e2ee", kind: "message", T: E2eeResponse, oneof: "message" },
+    { no: 28, name: "clear_audio_buffer", kind: "message", T: ClearAudioBufferResponse, oneof: "message" },
+    { no: 29, name: "new_audio_resampler", kind: "message", T: NewAudioResamplerResponse, oneof: "message" },
+    { no: 30, name: "remix_and_resample", kind: "message", T: RemixAndResampleResponse, oneof: "message" },
+    { no: 31, name: "audio_stream_from_participant", kind: "message", T: AudioStreamFromParticipantResponse, oneof: "message" },
+    { no: 32, name: "e2ee", kind: "message", T: E2eeResponse, oneof: "message" },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): FfiResponse {

package/src/room.ts

@@ -1,8 +1,8 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import EventEmitter from 'events';
-import type TypedEmitter from 'typed-emitter';
 import type { E2EEOptions } from './e2ee.js';
 import { E2EEManager } from './e2ee.js';
 import { FfiClient, FfiClientEvent, FfiHandle } from './ffi_client.js';
@@ -188,6 +188,7 @@ export class Room extends (EventEmitter as new () => TypedEmitter<RoomCallbacks>
       this.emit(RoomEvent.LocalTrackUnpublished, publication, this.localParticipant);
     } else if (ev.case == 'localTrackSubscribed') {
       const publication = this.localParticipant.trackPublications.get(ev.value.trackSid);
+      publication.resolveFirstSubscription();
       this.emit(RoomEvent.LocalTrackSubscribed, publication.track);
     } else if (ev.case == 'trackPublished') {
       const participant = this.remoteParticipants.get(ev.value.participantIdentity);

package/src/track_publication.ts

@@ -69,8 +69,26 @@ export abstract class TrackPublication {
 }
 
 export class LocalTrackPublication extends TrackPublication {
+  private firstSubscription: Promise<void>;
+  private firstSubscriptionResolver: (() => void) | null = null;
+
   constructor(ownedInfo: OwnedTrackPublication) {
     super(ownedInfo);
+    this.firstSubscription = new Promise<void>((resolve) => {
+      this.firstSubscriptionResolver = resolve;
+    });
+  }
+
+  async waitForSubscription(): Promise<void> {
+    await this.firstSubscription;
+  }
+
+  /** @internal */
+  resolveFirstSubscription(): void {
+    if (this.firstSubscriptionResolver) {
+      this.firstSubscriptionResolver();
+      this.firstSubscriptionResolver = null;
+    }
   }
 }
 

package/src/utils.ts

@@ -0,0 +1,36 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export class Mutex {
+  #locking: Promise<void>;
+  #locks: number;
+  #limit: number;
+
+  constructor(limit = 1) {
+    this.#locking = Promise.resolve();
+    this.#locks = 0;
+    this.#limit = limit;
+  }
+
+  isLocked(): boolean {
+    return this.#locks >= this.#limit;
+  }
+
+  async lock(): Promise<() => void> {
+    this.#locks += 1;
+
+    let unlockNext: () => void;
+
+    const willLock = new Promise<void>(
+      (resolve) =>
+        (unlockNext = () => {
+          this.#locks -= 1;
+          resolve();
+        }),
+    );
+
+    const willUnlock = this.#locking.then(() => unlockNext);
+    this.#locking = this.#locking.then(() => willLock);
+    return willUnlock;
+  }
+}

package/src/video_stream.ts

@@ -1,8 +1,6 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import EventEmitter from 'events';
-import type TypedEmitter from 'typed-emitter';
 import type { FfiEvent } from './ffi_client.js';
 import { FfiClient, FfiClientEvent, FfiHandle } from './ffi_client.js';
 import type {
@@ -12,6 +10,7 @@ import type {
 } from './proto/video_frame_pb.js';
 import { NewVideoStreamRequest, VideoStreamType } from './proto/video_frame_pb.js';
 import type { Track } from './track.js';
+import { Mutex } from './utils.js';
 import { VideoFrame } from './video_frame.js';
 
 export type VideoFrameEvent = {
@@ -20,24 +19,21 @@ export type VideoFrameEvent = {
   rotation: VideoRotation;
 };
 
-export type VideoStreamCallbacks = {
-  frameReceived: (evt: VideoFrameEvent) => void;
-};
-
-export enum VideoStreamEvent {
-  FrameReceived = 'frameReceived',
-}
-
-export class VideoStream extends (EventEmitter as new () => TypedEmitter<VideoStreamCallbacks>) {
+export class VideoStream implements AsyncIterableIterator<VideoFrameEvent> {
   /** @internal */
   info: VideoStreamInfo;
   /** @internal */
   ffiHandle: FfiHandle;
+  /** @internal */
+  eventQueue: (VideoFrameEvent | null)[] = [];
+  /** @internal */
+  queueResolve: ((value: IteratorResult<VideoFrameEvent>) => void) | null = null;
+  /** @internal */
+  mutex = new Mutex();
 
   track: Track;
 
   constructor(track: Track) {
-    super();
     this.track = track;
 
     const req = new NewVideoStreamRequest({
@@ -72,7 +68,12 @@ export class VideoStream extends (EventEmitter as new () => TypedEmitter<VideoSt
         const rotation = streamEvent.value.rotation;
         const timestampUs = streamEvent.value.timestampUs;
         const frame = VideoFrame.fromOwnedInfo(streamEvent.value.buffer);
-        this.emit(VideoStreamEvent.FrameReceived, { frame, timestampUs, rotation });
+        const value = { rotation, timestampUs, frame };
+        if (this.queueResolve) {
+          this.queueResolve({ done: false, value });
+        } else {
+          this.eventQueue.push(value);
+        }
         break;
       case 'eos':
         FfiClient.instance.off(FfiClientEvent.FfiEvent, this.onEvent);
@@ -80,7 +81,30 @@ export class VideoStream extends (EventEmitter as new () => TypedEmitter<VideoSt
     }
   };
 
+  async next(): Promise<IteratorResult<VideoFrameEvent>> {
+    const unlock = await this.mutex.lock();
+    if (this.eventQueue.length > 0) {
+      unlock();
+      const value = this.eventQueue.shift();
+      if (value) {
+        return { done: false, value };
+      } else {
+        return { done: true, value: undefined };
+      }
+    }
+    const promise = new Promise<IteratorResult<VideoFrameEvent>>(
+      (resolve) => (this.queueResolve = resolve),
+    );
+    unlock();
+    return promise;
+  }
+
   close() {
+    this.eventQueue.push(null);
     this.ffiHandle.dispose();
   }
+
+  [Symbol.asyncIterator](): VideoStream {
+    return this;
+  }
 }