livekit-client 2.4.2 → 2.5.1

package/src/room/events.ts

@@ -323,6 +323,11 @@ export enum RoomEvent {
    * args: (kind: MediaDeviceKind, deviceId: string)
    */
   ActiveDeviceChanged = 'activeDeviceChanged',
+
+  /**
+   * fired when the first remote participant has subscribed to the localParticipant's track
+   */
+  LocalTrackSubscribed = 'localTrackSubscribed',
 }
 
 export enum ParticipantEvent {
@@ -509,6 +514,11 @@ export enum ParticipantEvent {
    * When a participant's attributes changed, this event will be emitted with the changed attributes
    */
   AttributesChanged = 'attributesChanged',
+
+  /**
+   * fired on local participant only, when the first remote participant has subscribed to the track specified in the payload
+   */
+  LocalTrackSubscribed = 'localTrackSubscribed',
 }
 
 /** @internal */
@@ -538,8 +548,9 @@ export enum EngineEvent {
   RemoteMute = 'remoteMute',
   SubscribedQualityUpdate = 'subscribedQualityUpdate',
   LocalTrackUnpublished = 'localTrackUnpublished',
+  LocalTrackSubscribed = 'localTrackSubscribed',
   Offline = 'offline',
-  SignalRequestError = 'signalRequestError',
+  SignalRequestResponse = 'signalRequestResponse',
 }
 
 export enum TrackEvent {

package/src/room/participant/LocalParticipant.ts

@@ -1,13 +1,16 @@
 import {
   AddTrackRequest,
+  Codec,
   DataPacket,
   DataPacket_Kind,
   Encryption_Type,
-  ErrorResponse,
   ParticipantInfo,
   ParticipantPermission,
+  RequestResponse,
+  RequestResponse_Reason,
   SimulcastCodec,
   SubscribedQualityUpdate,
+  TrackInfo,
   TrackUnpublishedResponse,
   UserPacket,
 } from '@livekit/protocol';
@@ -109,6 +112,8 @@ export default class LocalParticipant extends Participant {
     }
   >;
 
+  private enabledPublishVideoCodecs: Codec[] = [];
+
   /** @internal */
   constructor(sid: string, identity: string, engine: RTCEngine, options: InternalRoomOptions) {
     super(sid, identity, undefined, undefined, {
@@ -177,7 +182,7 @@ export default class LocalParticipant extends Participant {
       .on(EngineEvent.LocalTrackUnpublished, this.handleLocalTrackUnpublished)
       .on(EngineEvent.SubscribedQualityUpdate, this.handleSubscribedQualityUpdate)
       .on(EngineEvent.Disconnected, this.handleDisconnected)
-      .on(EngineEvent.SignalRequestError, this.handleSignalRequestError);
+      .on(EngineEvent.SignalRequestResponse, this.handleSignalRequestResponse);
   }
 
   private handleReconnecting = () => {
@@ -200,11 +205,13 @@ export default class LocalParticipant extends Participant {
     }
   };
 
-  private handleSignalRequestError = (error: ErrorResponse) => {
-    const { requestId, reason, message } = error;
-    const failedRequest = this.pendingSignalRequests.get(requestId);
-    if (failedRequest) {
-      failedRequest.reject(new SignalRequestError(message, reason));
+  private handleSignalRequestResponse = (response: RequestResponse) => {
+    const { requestId, reason, message } = response;
+    const targetRequest = this.pendingSignalRequests.get(requestId);
+    if (targetRequest) {
+      if (reason !== RequestResponse_Reason.OK) {
+        targetRequest.reject(new SignalRequestError(message, reason));
+      }
       this.pendingSignalRequests.delete(requestId);
     }
   };
@@ -278,7 +285,9 @@ export default class LocalParticipant extends Participant {
           }
           await sleep(50);
         }
-        reject(new SignalRequestError('Request to update local metadata timed out'));
+        reject(
+          new SignalRequestError('Request to update local metadata timed out', 'TimeoutError'),
+        );
       } catch (e: any) {
         if (e instanceof Error) reject(e);
       }
@@ -770,6 +779,17 @@ export default class LocalParticipant extends Participant {
     if (opts.videoCodec === undefined) {
       opts.videoCodec = defaultVideoCodec;
     }
+    if (this.enabledPublishVideoCodecs.length > 0) {
+      // fallback to a supported codec if it is not supported
+      if (
+        !this.enabledPublishVideoCodecs.some(
+          (c) => opts.videoCodec === mimeTypeToVideoCodecString(c.mime),
+        )
+      ) {
+        opts.videoCodec = mimeTypeToVideoCodecString(this.enabledPublishVideoCodecs[0].mime);
+      }
+    }
+
     const videoCodec = opts.videoCodec;
 
     // handle track actions
@@ -903,33 +923,87 @@ export default class LocalParticipant extends Participant {
       throw new UnexpectedConnectionState('cannot publish track when not connected');
     }
 
-    const ti = await this.engine.addTrack(req);
-    // server might not support the codec the client has requested, in that case, fallback
-    // to a supported codec
-    let primaryCodecMime: string | undefined;
-    ti.codecs.forEach((codec) => {
-      if (primaryCodecMime === undefined) {
-        primaryCodecMime = codec.mimeType;
+    const negotiate = async () => {
+      if (!this.engine.pcManager) {
+        throw new UnexpectedConnectionState('pcManager is not ready');
       }
-    });
-    if (primaryCodecMime && track.kind === Track.Kind.Video) {
-      const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime);
-      if (updatedCodec !== videoCodec) {
-        this.log.debug('falling back to server selected codec', {
-          ...this.logContext,
-          ...getLogContextFromTrack(track),
-          codec: updatedCodec,
-        });
-        opts.videoCodec = updatedCodec;
-
-        // recompute encodings since bitrates/etc could have changed
-        encodings = computeVideoEncodings(
-          track.source === Track.Source.ScreenShare,
-          req.width,
-          req.height,
-          opts,
-        );
+
+      track.sender = await this.engine.createSender(track, opts, encodings);
+
+      if (track instanceof LocalVideoTrack) {
+        opts.degradationPreference ??= getDefaultDegradationPreference(track);
+        track.setDegradationPreference(opts.degradationPreference);
+      }
+
+      if (encodings) {
+        if (isFireFox() && track.kind === Track.Kind.Audio) {
+          /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
+             livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to
+             publish high quality audio track. But firefox always uses this value as the actual
+             bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
+             So the client need to modify maxaverragebitrates in answer sdp to user provided value to
+             fix the issue.
+           */
+          let trackTransceiver: RTCRtpTransceiver | undefined = undefined;
+          for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
+            if (transceiver.sender === track.sender) {
+              trackTransceiver = transceiver;
+              break;
+            }
+          }
+          if (trackTransceiver) {
+            this.engine.pcManager.publisher.setTrackCodecBitrate({
+              transceiver: trackTransceiver,
+              codec: 'opus',
+              maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
+            });
+          }
+        } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) {
+          this.engine.pcManager.publisher.setTrackCodecBitrate({
+            cid: req.cid,
+            codec: track.codec,
+            maxbr: encodings[0].maxBitrate / 1000,
+          });
+        }
+      }
+
+      await this.engine.negotiate();
+    };
+
+    let ti: TrackInfo;
+    if (this.enabledPublishVideoCodecs.length > 0) {
+      const rets = await Promise.all([this.engine.addTrack(req), negotiate()]);
+      ti = rets[0];
+    } else {
+      ti = await this.engine.addTrack(req);
+      // server might not support the codec the client has requested, in that case, fallback
+      // to a supported codec
+      let primaryCodecMime: string | undefined;
+      ti.codecs.forEach((codec) => {
+        if (primaryCodecMime === undefined) {
+          primaryCodecMime = codec.mimeType;
+        }
+      });
+      if (primaryCodecMime && track.kind === Track.Kind.Video) {
+        const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime);
+        if (updatedCodec !== videoCodec) {
+          this.log.debug('falling back to server selected codec', {
+            ...this.logContext,
+            ...getLogContextFromTrack(track),
+            codec: updatedCodec,
+          });
+          opts.videoCodec = updatedCodec;
+
+          // recompute encodings since bitrates/etc could have changed
+          encodings = computeVideoEncodings(
+            track.source === Track.Source.ScreenShare,
+            req.width,
+            req.height,
+            opts,
+          );
+        }
       }
+      await negotiate();
     }
 
     const publication = new LocalTrackPublication(track.kind, ti, track, {
@@ -940,56 +1014,12 @@ export default class LocalParticipant extends Participant {
     publication.options = opts;
     track.sid = ti.sid;
 
-    if (!this.engine.pcManager) {
-      throw new UnexpectedConnectionState('pcManager is not ready');
-    }
     this.log.debug(`publishing ${track.kind} with encodings`, {
       ...this.logContext,
       encodings,
       trackInfo: ti,
     });
 
-    track.sender = await this.engine.createSender(track, opts, encodings);
-
-    if (track instanceof LocalVideoTrack) {
-      opts.degradationPreference ??= getDefaultDegradationPreference(track);
-      track.setDegradationPreference(opts.degradationPreference);
-    }
-
-    if (encodings) {
-      if (isFireFox() && track.kind === Track.Kind.Audio) {
-        /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
-           livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to
-           publish high quality audio track. But firefox always uses this value as the actual
-           bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
-           So the client need to modify maxaverragebitrates in answer sdp to user provided value to
-           fix the issue.
-         */
-        let trackTransceiver: RTCRtpTransceiver | undefined = undefined;
-        for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
-          if (transceiver.sender === track.sender) {
-            trackTransceiver = transceiver;
-            break;
-          }
-        }
-        if (trackTransceiver) {
-          this.engine.pcManager.publisher.setTrackCodecBitrate({
-            transceiver: trackTransceiver,
-            codec: 'opus',
-            maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
-          });
-        }
-      } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) {
-        this.engine.pcManager.publisher.setTrackCodecBitrate({
-          cid: req.cid,
-          codec: track.codec,
-          maxbr: encodings[0].maxBitrate / 1000,
-        });
-      }
-    }
-
-    await this.engine.negotiate();
-
     if (track instanceof LocalVideoTrack) {
       track.startMonitor(this.engine.client);
     } else if (track instanceof LocalAudioTrack) {
@@ -1076,15 +1106,19 @@ export default class LocalParticipant extends Participant {
       throw new UnexpectedConnectionState('cannot publish track when not connected');
     }
 
-    const ti = await this.engine.addTrack(req);
+    const negotiate = async () => {
+      const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+      if (encodings) {
+        transceiverInit.sendEncodings = encodings;
+      }
+      await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
 
-    const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
-    if (encodings) {
-      transceiverInit.sendEncodings = encodings;
-    }
-    await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
+      await this.engine.negotiate();
+    };
+
+    const rets = await Promise.all([this.engine.addTrack(req), negotiate()]);
+    const ti = rets[0];
 
-    await this.engine.negotiate();
     this.log.debug(`published ${videoCodec} for track ${track.sid}`, {
       ...this.logContext,
       encodings,
@@ -1304,6 +1338,13 @@ export default class LocalParticipant extends Participant {
     }
   }
 
+  /** @internal */
+  setEnabledPublishCodecs(codecs: Codec[]) {
+    this.enabledPublishVideoCodecs = codecs.filter(
+      (c) => c.mime.split('/')[0].toLowerCase() === 'video',
+    );
+  }
+
   /** @internal */
   updateInfo(info: ParticipantInfo): boolean {
     if (info.sid !== this.sid) {

package/src/room/participant/Participant.ts

@@ -386,4 +386,5 @@ export type ParticipantEventCallbacks = {
     status: TrackPublication.SubscriptionStatus,
   ) => void;
   attributesChanged: (changedAttributes: Record<string, string>) => void;
+  localTrackSubscribed: (trackPublication: LocalTrackPublication) => void;
 };

package/src/room/participant/RemoteParticipant.ts

@@ -164,7 +164,7 @@ export default class RemoteParticipant extends Participant {
     mediaTrack: MediaStreamTrack,
     sid: Track.SID,
     mediaStream: MediaStream,
-    receiver?: RTCRtpReceiver,
+    receiver: RTCRtpReceiver,
     adaptiveStreamSettings?: AdaptiveStreamSettings,
     triesLeft?: number,
   ) {

package/src/room/timers.ts

@@ -4,13 +4,22 @@
  * that the timer fires on time.
  */
 export default class CriticalTimers {
-  // eslint-disable-next-line @typescript-eslint/no-implied-eval
-  static setTimeout = (...args: Parameters<typeof setTimeout>) => setTimeout(...args);
+  static setTimeout: (...args: Parameters<typeof setTimeout>) => ReturnType<typeof setTimeout> = (
+    ...args: Parameters<typeof setTimeout>
+    // eslint-disable-next-line @typescript-eslint/no-implied-eval
+  ) => setTimeout(...args);
 
-  // eslint-disable-next-line @typescript-eslint/no-implied-eval
-  static setInterval = (...args: Parameters<typeof setInterval>) => setInterval(...args);
+  static setInterval: (...args: Parameters<typeof setInterval>) => ReturnType<typeof setInterval> =
+    // eslint-disable-next-line @typescript-eslint/no-implied-eval
+    (...args: Parameters<typeof setInterval>) => setInterval(...args);
 
-  static clearTimeout = (...args: Parameters<typeof clearTimeout>) => clearTimeout(...args);
+  static clearTimeout: (
+    ...args: Parameters<typeof clearTimeout>
+  ) => ReturnType<typeof clearTimeout> = (...args: Parameters<typeof clearTimeout>) =>
+    clearTimeout(...args);
 
-  static clearInterval = (...args: Parameters<typeof clearInterval>) => clearInterval(...args);
+  static clearInterval: (
+    ...args: Parameters<typeof clearInterval>
+  ) => ReturnType<typeof clearInterval> = (...args: Parameters<typeof clearInterval>) =>
+    clearInterval(...args);
 }

package/src/room/track/LocalVideoTrack.test.ts

@@ -47,6 +47,66 @@ describe('videoLayersFromEncodings', () => {
     expect(layers[2].height).toBe(720);
   });
 
+  it('returns qualities starting from lowest for SVC', () => {
+    const layers = videoLayersFromEncodings(
+      1280,
+      720,
+      [
+        {
+          /** @ts-ignore */
+          scalabilityMode: 'L2T2',
+        },
+      ],
+      true,
+    );
+
+    expect(layers).toHaveLength(2);
+    expect(layers[0].quality).toBe(VideoQuality.MEDIUM);
+    expect(layers[0].width).toBe(1280);
+    expect(layers[1].quality).toBe(VideoQuality.LOW);
+    expect(layers[1].width).toBe(640);
+  });
+
+  it('returns qualities starting from lowest for SVC (three layers)', () => {
+    const layers = videoLayersFromEncodings(
+      1280,
+      720,
+      [
+        {
+          /** @ts-ignore */
+          scalabilityMode: 'L3T3',
+        },
+      ],
+      true,
+    );
+
+    expect(layers).toHaveLength(3);
+    expect(layers[0].quality).toBe(VideoQuality.HIGH);
+    expect(layers[0].width).toBe(1280);
+    expect(layers[1].quality).toBe(VideoQuality.MEDIUM);
+    expect(layers[1].width).toBe(640);
+    expect(layers[2].quality).toBe(VideoQuality.LOW);
+    expect(layers[2].width).toBe(320);
+  });
+
+  it('returns qualities starting from lowest for SVC (single layer)', () => {
+    const layers = videoLayersFromEncodings(
+      1280,
+      720,
+      [
+        {
+          /** @ts-ignore */
+          scalabilityMode: 'L1T2',
+        },
+      ],
+      true,
+    );
+
+    expect(layers).toHaveLength(1);
+    expect(layers[0].quality).toBe(VideoQuality.LOW);
+    expect(layers[0].width).toBe(1280);
+  });
+
   it('handles portrait', () => {
     const layers = videoLayersFromEncodings(720, 1280, [
       {

package/src/room/track/LocalVideoTrack.ts

@@ -607,7 +607,7 @@ export function videoLayersFromEncodings(
     for (let i = 0; i < sm.spatial; i += 1) {
       layers.push(
         new VideoLayer({
-          quality: VideoQuality.HIGH - i,
+          quality: Math.min(VideoQuality.HIGH, sm.spatial - 1) - i,
           width: Math.ceil(width / resRatio ** i),
           height: Math.ceil(height / resRatio ** i),
           bitrate: encodings[0].maxBitrate

package/src/room/track/RemoteAudioTrack.ts

@@ -25,7 +25,7 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
   constructor(
     mediaTrack: MediaStreamTrack,
     sid: string,
-    receiver?: RTCRtpReceiver,
+    receiver: RTCRtpReceiver,
     audioContext?: AudioContext,
     audioOutput?: AudioOutputOptions,
     loggerOptions?: LoggerOptions,

package/src/room/track/RemoteTrack.ts

@@ -8,13 +8,13 @@ export default abstract class RemoteTrack<
   TrackKind extends Track.Kind = Track.Kind,
 > extends Track<TrackKind> {
   /** @internal */
-  receiver?: RTCRtpReceiver;
+  receiver: RTCRtpReceiver | undefined;
 
   constructor(
     mediaTrack: MediaStreamTrack,
     sid: string,
     kind: TrackKind,
-    receiver?: RTCRtpReceiver,
+    receiver: RTCRtpReceiver,
     loggerOptions?: LoggerOptions,
   ) {
     super(mediaTrack, kind, loggerOptions);
@@ -39,6 +39,9 @@ export default abstract class RemoteTrack<
     const onRemoveTrack = (event: MediaStreamTrackEvent) => {
       if (event.track === this._mediaStreamTrack) {
         stream.removeEventListener('removetrack', onRemoveTrack);
+        if (this.receiver && 'playoutDelayHint' in this.receiver) {
+          this.receiver.playoutDelayHint = undefined;
+        }
         this.receiver = undefined;
         this._currentBitrate = 0;
         this.emit(TrackEvent.Ended, this);
@@ -73,6 +76,39 @@ export default abstract class RemoteTrack<
     return statsReport;
   }
 
+  /**
+   * Allows to set a playout delay (in seconds) for this track.
+   * A higher value allows for more buffering of the track in the browser
+   * and will result in a delay of media being played back of `delayInSeconds`
+   */
+  setPlayoutDelay(delayInSeconds: number): void {
+    if (this.receiver) {
+      if ('playoutDelayHint' in this.receiver) {
+        this.receiver.playoutDelayHint = delayInSeconds;
+      } else {
+        this.log.warn('Playout delay not supported in this browser');
+      }
+    } else {
+      this.log.warn('Cannot set playout delay, track already ended');
+    }
+  }
+
+  /**
+   * Returns the current playout delay (in seconds) of this track.
+   */
+  getPlayoutDelay(): number {
+    if (this.receiver) {
+      if ('playoutDelayHint' in this.receiver) {
+        return this.receiver.playoutDelayHint as number;
+      } else {
+        this.log.warn('Playout delay not supported in this browser');
+      }
+    } else {
+      this.log.warn('Cannot get playout delay, track already ended');
+    }
+    return 0;
+  }
+
   /* @internal */
   startMonitor() {
     if (!this.monitorInterval) {

package/src/room/track/RemoteVideoTrack.ts

@@ -26,7 +26,7 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
   constructor(
     mediaTrack: MediaStreamTrack,
     sid: string,
-    receiver?: RTCRtpReceiver,
+    receiver: RTCRtpReceiver,
     adaptiveStreamSettings?: AdaptiveStreamSettings,
     loggerOptions?: LoggerOptions,
   ) {
@@ -226,7 +226,7 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
     );
 
     const backgroundPause =
-      this.adaptiveStreamSettings?.pauseVideoInBackground ?? true // default to true
+      (this.adaptiveStreamSettings?.pauseVideoInBackground ?? true) // default to true
         ? this.isInBackground
         : false;
     const isPiPMode = this.elementInfos.some((info) => info.pictureInPicture);

package/src/room/track/options.ts

@@ -64,7 +64,7 @@ export interface TrackPublishDefaults {
   simulcast?: boolean;
 
   /**
-   * scalability mode for svc codecs, defaults to 'L3T3'.
+   * scalability mode for svc codecs, defaults to 'L3T3_KEY'.
    * for svc codecs, simulcast is disabled.
    */
   scalabilityMode?: ScalabilityMode;

package/src/room/types.ts

@@ -65,4 +65,6 @@ export interface TranscriptionSegment {
   startTime: number;
   endTime: number;
   final: boolean;
+  firstReceivedTime: number;
+  lastReceivedTime: number;
 }

package/src/room/utils.ts

@@ -532,8 +532,16 @@ export function toHttpUrl(url: string): string {
 
 export function extractTranscriptionSegments(
   transcription: TranscriptionModel,
+  firstReceivedTimesMap: Map<string, number>,
 ): TranscriptionSegment[] {
   return transcription.segments.map(({ id, text, language, startTime, endTime, final }) => {
+    const firstReceivedTime = firstReceivedTimesMap.get(id) ?? Date.now();
+    const lastReceivedTime = Date.now();
+    if (final) {
+      firstReceivedTimesMap.delete(id);
+    } else {
+      firstReceivedTimesMap.set(id, firstReceivedTime);
+    }
     return {
       id,
       text,
@@ -541,6 +549,8 @@ export function extractTranscriptionSegments(
       endTime: Number.parseInt(endTime.toString()),
       final,
       language,
+      firstReceivedTime,
+      lastReceivedTime,
     };
   });
 }

package/src/version.ts

@@ -1,4 +1,4 @@
 import { version as v } from '../package.json';
 
 export const version = v;
-export const protocolVersion = 13;
+export const protocolVersion = 15;