livekit-client 2.5.0 → 2.5.1

package/src/room/participant/LocalParticipant.ts

@@ -1,5 +1,6 @@
 import {
   AddTrackRequest,
+  Codec,
   DataPacket,
   DataPacket_Kind,
   Encryption_Type,
@@ -9,6 +10,7 @@ import {
   RequestResponse_Reason,
   SimulcastCodec,
   SubscribedQualityUpdate,
+  TrackInfo,
   TrackUnpublishedResponse,
   UserPacket,
 } from '@livekit/protocol';
@@ -110,6 +112,8 @@ export default class LocalParticipant extends Participant {
     }
   >;
 
+  private enabledPublishVideoCodecs: Codec[] = [];
+
   /** @internal */
   constructor(sid: string, identity: string, engine: RTCEngine, options: InternalRoomOptions) {
     super(sid, identity, undefined, undefined, {
@@ -775,6 +779,17 @@ export default class LocalParticipant extends Participant {
     if (opts.videoCodec === undefined) {
       opts.videoCodec = defaultVideoCodec;
     }
+    if (this.enabledPublishVideoCodecs.length > 0) {
+      // fallback to a supported codec if it is not supported
+      if (
+        !this.enabledPublishVideoCodecs.some(
+          (c) => opts.videoCodec === mimeTypeToVideoCodecString(c.mime),
+        )
+      ) {
+        opts.videoCodec = mimeTypeToVideoCodecString(this.enabledPublishVideoCodecs[0].mime);
+      }
+    }
+
     const videoCodec = opts.videoCodec;
 
     // handle track actions
@@ -908,33 +923,87 @@ export default class LocalParticipant extends Participant {
       throw new UnexpectedConnectionState('cannot publish track when not connected');
     }
 
-    const ti = await this.engine.addTrack(req);
-    // server might not support the codec the client has requested, in that case, fallback
-    // to a supported codec
-    let primaryCodecMime: string | undefined;
-    ti.codecs.forEach((codec) => {
-      if (primaryCodecMime === undefined) {
-        primaryCodecMime = codec.mimeType;
+    const negotiate = async () => {
+      if (!this.engine.pcManager) {
+        throw new UnexpectedConnectionState('pcManager is not ready');
       }
-    });
-    if (primaryCodecMime && track.kind === Track.Kind.Video) {
-      const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime);
-      if (updatedCodec !== videoCodec) {
-        this.log.debug('falling back to server selected codec', {
-          ...this.logContext,
-          ...getLogContextFromTrack(track),
-          codec: updatedCodec,
-        });
-        opts.videoCodec = updatedCodec;
-
-        // recompute encodings since bitrates/etc could have changed
-        encodings = computeVideoEncodings(
-          track.source === Track.Source.ScreenShare,
-          req.width,
-          req.height,
-          opts,
-        );
+
+      track.sender = await this.engine.createSender(track, opts, encodings);
+
+      if (track instanceof LocalVideoTrack) {
+        opts.degradationPreference ??= getDefaultDegradationPreference(track);
+        track.setDegradationPreference(opts.degradationPreference);
+      }
+
+      if (encodings) {
+        if (isFireFox() && track.kind === Track.Kind.Audio) {
+          /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
+             livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to
+             publish high quality audio track. But firefox always uses this value as the actual
+             bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
+             So the client need to modify maxaverragebitrates in answer sdp to user provided value to
+             fix the issue.
+           */
+          let trackTransceiver: RTCRtpTransceiver | undefined = undefined;
+          for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
+            if (transceiver.sender === track.sender) {
+              trackTransceiver = transceiver;
+              break;
+            }
+          }
+          if (trackTransceiver) {
+            this.engine.pcManager.publisher.setTrackCodecBitrate({
+              transceiver: trackTransceiver,
+              codec: 'opus',
+              maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
+            });
+          }
+        } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) {
+          this.engine.pcManager.publisher.setTrackCodecBitrate({
+            cid: req.cid,
+            codec: track.codec,
+            maxbr: encodings[0].maxBitrate / 1000,
+          });
+        }
+      }
+
+      await this.engine.negotiate();
+    };
+
+    let ti: TrackInfo;
+    if (this.enabledPublishVideoCodecs.length > 0) {
+      const rets = await Promise.all([this.engine.addTrack(req), negotiate()]);
+      ti = rets[0];
+    } else {
+      ti = await this.engine.addTrack(req);
+      // server might not support the codec the client has requested, in that case, fallback
+      // to a supported codec
+      let primaryCodecMime: string | undefined;
+      ti.codecs.forEach((codec) => {
+        if (primaryCodecMime === undefined) {
+          primaryCodecMime = codec.mimeType;
+        }
+      });
+      if (primaryCodecMime && track.kind === Track.Kind.Video) {
+        const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime);
+        if (updatedCodec !== videoCodec) {
+          this.log.debug('falling back to server selected codec', {
+            ...this.logContext,
+            ...getLogContextFromTrack(track),
+            codec: updatedCodec,
+          });
+          opts.videoCodec = updatedCodec;
+
+          // recompute encodings since bitrates/etc could have changed
+          encodings = computeVideoEncodings(
+            track.source === Track.Source.ScreenShare,
+            req.width,
+            req.height,
+            opts,
+          );
+        }
       }
+      await negotiate();
     }
 
     const publication = new LocalTrackPublication(track.kind, ti, track, {
@@ -945,56 +1014,12 @@ export default class LocalParticipant extends Participant {
     publication.options = opts;
     track.sid = ti.sid;
 
-    if (!this.engine.pcManager) {
-      throw new UnexpectedConnectionState('pcManager is not ready');
-    }
     this.log.debug(`publishing ${track.kind} with encodings`, {
       ...this.logContext,
       encodings,
       trackInfo: ti,
     });
 
-    track.sender = await this.engine.createSender(track, opts, encodings);
-
-    if (track instanceof LocalVideoTrack) {
-      opts.degradationPreference ??= getDefaultDegradationPreference(track);
-      track.setDegradationPreference(opts.degradationPreference);
-    }
-
-    if (encodings) {
-      if (isFireFox() && track.kind === Track.Kind.Audio) {
-        /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
-           livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to
-           publish high quality audio track. But firefox always uses this value as the actual
-           bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
-           So the client need to modify maxaverragebitrates in answer sdp to user provided value to
-           fix the issue.
-         */
-        let trackTransceiver: RTCRtpTransceiver | undefined = undefined;
-        for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
-          if (transceiver.sender === track.sender) {
-            trackTransceiver = transceiver;
-            break;
-          }
-        }
-        if (trackTransceiver) {
-          this.engine.pcManager.publisher.setTrackCodecBitrate({
-            transceiver: trackTransceiver,
-            codec: 'opus',
-            maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
-          });
-        }
-      } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) {
-        this.engine.pcManager.publisher.setTrackCodecBitrate({
-          cid: req.cid,
-          codec: track.codec,
-          maxbr: encodings[0].maxBitrate / 1000,
-        });
-      }
-    }
-
-    await this.engine.negotiate();
-
     if (track instanceof LocalVideoTrack) {
       track.startMonitor(this.engine.client);
     } else if (track instanceof LocalAudioTrack) {
@@ -1081,15 +1106,19 @@ export default class LocalParticipant extends Participant {
       throw new UnexpectedConnectionState('cannot publish track when not connected');
     }
 
-    const ti = await this.engine.addTrack(req);
+    const negotiate = async () => {
+      const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+      if (encodings) {
+        transceiverInit.sendEncodings = encodings;
+      }
+      await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
 
-    const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
-    if (encodings) {
-      transceiverInit.sendEncodings = encodings;
-    }
-    await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
+      await this.engine.negotiate();
+    };
+
+    const rets = await Promise.all([this.engine.addTrack(req), negotiate()]);
+    const ti = rets[0];
 
-    await this.engine.negotiate();
     this.log.debug(`published ${videoCodec} for track ${track.sid}`, {
       ...this.logContext,
       encodings,
@@ -1309,6 +1338,13 @@ export default class LocalParticipant extends Participant {
     }
   }
 
+  /** @internal */
+  setEnabledPublishCodecs(codecs: Codec[]) {
+    this.enabledPublishVideoCodecs = codecs.filter(
+      (c) => c.mime.split('/')[0].toLowerCase() === 'video',
+    );
+  }
+
   /** @internal */
   updateInfo(info: ParticipantInfo): boolean {
     if (info.sid !== this.sid) {

package/src/room/participant/Participant.ts

@@ -386,4 +386,5 @@ export type ParticipantEventCallbacks = {
     status: TrackPublication.SubscriptionStatus,
   ) => void;
   attributesChanged: (changedAttributes: Record<string, string>) => void;
+  localTrackSubscribed: (trackPublication: LocalTrackPublication) => void;
 };

package/src/room/timers.ts

@@ -4,13 +4,22 @@
  * that the timer fires on time.
  */
 export default class CriticalTimers {
-  // eslint-disable-next-line @typescript-eslint/no-implied-eval
-  static setTimeout = (...args: Parameters<typeof setTimeout>) => setTimeout(...args);
+  static setTimeout: (...args: Parameters<typeof setTimeout>) => ReturnType<typeof setTimeout> = (
+    ...args: Parameters<typeof setTimeout>
+    // eslint-disable-next-line @typescript-eslint/no-implied-eval
+  ) => setTimeout(...args);
 
-  // eslint-disable-next-line @typescript-eslint/no-implied-eval
-  static setInterval = (...args: Parameters<typeof setInterval>) => setInterval(...args);
+  static setInterval: (...args: Parameters<typeof setInterval>) => ReturnType<typeof setInterval> =
+    // eslint-disable-next-line @typescript-eslint/no-implied-eval
+    (...args: Parameters<typeof setInterval>) => setInterval(...args);
 
-  static clearTimeout = (...args: Parameters<typeof clearTimeout>) => clearTimeout(...args);
+  static clearTimeout: (
+    ...args: Parameters<typeof clearTimeout>
+  ) => ReturnType<typeof clearTimeout> = (...args: Parameters<typeof clearTimeout>) =>
+    clearTimeout(...args);
 
-  static clearInterval = (...args: Parameters<typeof clearInterval>) => clearInterval(...args);
+  static clearInterval: (
+    ...args: Parameters<typeof clearInterval>
+  ) => ReturnType<typeof clearInterval> = (...args: Parameters<typeof clearInterval>) =>
+    clearInterval(...args);
 }

package/src/room/track/LocalVideoTrack.test.ts

@@ -47,6 +47,66 @@ describe('videoLayersFromEncodings', () => {
     expect(layers[2].height).toBe(720);
   });
 
+  it('returns qualities starting from lowest for SVC', () => {
+    const layers = videoLayersFromEncodings(
+      1280,
+      720,
+      [
+        {
+          /** @ts-ignore */
+          scalabilityMode: 'L2T2',
+        },
+      ],
+      true,
+    );
+
+    expect(layers).toHaveLength(2);
+    expect(layers[0].quality).toBe(VideoQuality.MEDIUM);
+    expect(layers[0].width).toBe(1280);
+    expect(layers[1].quality).toBe(VideoQuality.LOW);
+    expect(layers[1].width).toBe(640);
+  });
+
+  it('returns qualities starting from lowest for SVC (three layers)', () => {
+    const layers = videoLayersFromEncodings(
+      1280,
+      720,
+      [
+        {
+          /** @ts-ignore */
+          scalabilityMode: 'L3T3',
+        },
+      ],
+      true,
+    );
+
+    expect(layers).toHaveLength(3);
+    expect(layers[0].quality).toBe(VideoQuality.HIGH);
+    expect(layers[0].width).toBe(1280);
+    expect(layers[1].quality).toBe(VideoQuality.MEDIUM);
+    expect(layers[1].width).toBe(640);
+    expect(layers[2].quality).toBe(VideoQuality.LOW);
+    expect(layers[2].width).toBe(320);
+  });
+
+  it('returns qualities starting from lowest for SVC (single layer)', () => {
+    const layers = videoLayersFromEncodings(
+      1280,
+      720,
+      [
+        {
+          /** @ts-ignore */
+          scalabilityMode: 'L1T2',
+        },
+      ],
+      true,
+    );
+
+    expect(layers).toHaveLength(1);
+    expect(layers[0].quality).toBe(VideoQuality.LOW);
+    expect(layers[0].width).toBe(1280);
+  });
+
   it('handles portrait', () => {
     const layers = videoLayersFromEncodings(720, 1280, [
       {

package/src/room/track/LocalVideoTrack.ts

@@ -607,7 +607,7 @@ export function videoLayersFromEncodings(
     for (let i = 0; i < sm.spatial; i += 1) {
       layers.push(
         new VideoLayer({
-          quality: VideoQuality.HIGH - i,
+          quality: Math.min(VideoQuality.HIGH, sm.spatial - 1) - i,
           width: Math.ceil(width / resRatio ** i),
           height: Math.ceil(height / resRatio ** i),
           bitrate: encodings[0].maxBitrate

package/src/room/track/options.ts

@@ -64,7 +64,7 @@ export interface TrackPublishDefaults {
   simulcast?: boolean;
 
   /**
-   * scalability mode for svc codecs, defaults to 'L3T3'.
+   * scalability mode for svc codecs, defaults to 'L3T3_KEY'.
    * for svc codecs, simulcast is disabled.
    */
   scalabilityMode?: ScalabilityMode;

package/src/room/types.ts

@@ -65,4 +65,6 @@ export interface TranscriptionSegment {
   startTime: number;
   endTime: number;
   final: boolean;
+  firstReceivedTime: number;
+  lastReceivedTime: number;
 }

package/src/room/utils.ts

@@ -532,8 +532,16 @@ export function toHttpUrl(url: string): string {
 
 export function extractTranscriptionSegments(
   transcription: TranscriptionModel,
+  firstReceivedTimesMap: Map<string, number>,
 ): TranscriptionSegment[] {
   return transcription.segments.map(({ id, text, language, startTime, endTime, final }) => {
+    const firstReceivedTime = firstReceivedTimesMap.get(id) ?? Date.now();
+    const lastReceivedTime = Date.now();
+    if (final) {
+      firstReceivedTimesMap.delete(id);
+    } else {
+      firstReceivedTimesMap.set(id, firstReceivedTime);
+    }
     return {
       id,
       text,
@@ -541,6 +549,8 @@ export function extractTranscriptionSegments(
       endTime: Number.parseInt(endTime.toString()),
       final,
       language,
+      firstReceivedTime,
+      lastReceivedTime,
     };
   });
 }