livekit-client 1.14.1 → 1.14.3

package/src/room/participant/LocalParticipant.ts

@@ -18,6 +18,7 @@ import {
   TrackUnpublishedResponse,
 } from '../../proto/livekit_rtc_pb';
 import type RTCEngine from '../RTCEngine';
+import { defaultVideoCodec } from '../defaults';
 import { DeviceUnsupportedError, TrackInvalidError, UnexpectedConnectionState } from '../errors';
 import { EngineEvent, ParticipantEvent, TrackEvent } from '../events';
 import LocalAudioTrack from '../track/LocalAudioTrack';
@@ -33,10 +34,11 @@ import type {
   TrackPublishOptions,
   VideoCaptureOptions,
 } from '../track/options';
-import { VideoPresets, isBackupCodec, isCodecEqual } from '../track/options';
+import { VideoPresets, isBackupCodec } from '../track/options';
 import {
   constraintsForOptions,
   mergeDefaultOptions,
+  mimeTypeToVideoCodecString,
   screenCaptureToDisplayMediaStreamOptions,
 } from '../track/utils';
 import type { DataPublishOptions } from '../types';
@@ -408,6 +410,7 @@ export default class LocalParticipant extends Participant {
 
     if (constraints.audio) {
       this.microphoneError = undefined;
+      this.emit(ParticipantEvent.AudioStreamAcquired);
     }
     if (constraints.video) {
       this.cameraError = undefined;
@@ -460,6 +463,7 @@ export default class LocalParticipant extends Participant {
     screenVideo.source = Track.Source.ScreenShare;
     const localTracks: Array<LocalTrack> = [screenVideo];
     if (stream.getAudioTracks().length > 0) {
+      this.emit(ParticipantEvent.AudioStreamAcquired);
       const screenAudio = new LocalAudioTrack(
         stream.getAudioTracks()[0],
         undefined,
@@ -599,18 +603,7 @@ export default class LocalParticipant extends Participant {
       (publishedTrack) => track instanceof LocalTrack && publishedTrack.source === track.source,
     );
     if (existingTrackOfSource && track.source !== Track.Source.Unknown) {
-      try {
-        // throw an Error in order to capture the stack trace
-        throw Error(`publishing a second track with the same source: ${track.source}`);
-      } catch (e: unknown) {
-        if (e instanceof Error) {
-          log.warn(e.message, {
-            oldTrack: existingTrackOfSource,
-            newTrack: track,
-            trace: e.stack,
-          });
-        }
-      }
+      log.info(`publishing a second track with the same source: ${track.source}`);
     }
     if (opts.stopMicTrackOnMute && track instanceof LocalAudioTrack) {
       track.stopOnMute = true;
@@ -629,6 +622,10 @@ export default class LocalParticipant extends Participant {
     if (opts.videoCodec === 'vp9' && !supportsVP9()) {
       opts.videoCodec = undefined;
     }
+    if (opts.videoCodec === undefined) {
+      opts.videoCodec = defaultVideoCodec;
+    }
+    const videoCodec = opts.videoCodec;
 
     // handle track actions
     track.on(TrackEvent.Muted, this.onTrackMuted);
@@ -654,7 +651,6 @@ export default class LocalParticipant extends Participant {
 
     // compute encodings and layers for video
     let encodings: RTCRtpEncodingParameters[] | undefined;
-    let simEncodings: RTCRtpEncodingParameters[] | undefined;
     if (track.kind === Track.Kind.Video) {
       let dims: Track.Dimensions = {
         width: 0,
@@ -679,53 +675,40 @@ export default class LocalParticipant extends Participant {
       req.height = dims.height;
       // for svc codecs, disable simulcast and use vp8 for backup codec
       if (track instanceof LocalVideoTrack) {
-        if (isSVCCodec(opts.videoCodec)) {
+        if (isSVCCodec(videoCodec)) {
           // vp9 svc with screenshare has problem to encode, always use L1T3 here
-          if (track.source === Track.Source.ScreenShare && opts.videoCodec === 'vp9') {
+          if (track.source === Track.Source.ScreenShare && videoCodec === 'vp9') {
             opts.scalabilityMode = 'L1T3';
           }
           // set scalabilityMode to 'L3T3_KEY' by default
           opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY';
         }
 
+        req.simulcastCodecs = [
+          new SimulcastCodec({
+            codec: videoCodec,
+            cid: track.mediaStreamTrack.id,
+          }),
+        ];
+
         // set up backup
-        if (opts.videoCodec && opts.backupCodec && opts.videoCodec !== opts.backupCodec.codec) {
+        if (opts.backupCodec && videoCodec !== opts.backupCodec.codec) {
           if (!this.roomOptions.dynacast) {
             this.roomOptions.dynacast = true;
           }
-          const simOpts = { ...opts };
-          simOpts.simulcast = true;
-          simEncodings = computeTrackBackupEncodings(track, opts.backupCodec.codec, simOpts);
-
-          req.simulcastCodecs = [
-            new SimulcastCodec({
-              codec: opts.videoCodec,
-              cid: track.mediaStreamTrack.id,
-              enableSimulcastLayers: true,
-            }),
+          req.simulcastCodecs.push(
             new SimulcastCodec({
               codec: opts.backupCodec.codec,
               cid: '',
-              enableSimulcastLayers: true,
-            }),
-          ];
-        } else if (opts.videoCodec) {
-          // pass codec info to sfu so it can prefer codec for the client which don't support
-          // setCodecPreferences
-          req.simulcastCodecs = [
-            new SimulcastCodec({
-              codec: opts.videoCodec,
-              cid: track.mediaStreamTrack.id,
-              enableSimulcastLayers: opts.simulcast ?? false,
             }),
-          ];
+          );
         }
       }
 
       encodings = computeVideoEncodings(
         track.source === Track.Source.ScreenShare,
-        dims.width,
-        dims.height,
+        req.width,
+        req.height,
         opts,
       );
       req.layers = videoLayersFromEncodings(
@@ -749,30 +732,28 @@ export default class LocalParticipant extends Participant {
     }
 
     const ti = await this.engine.addTrack(req);
-    let primaryCodecSupported = false;
-    let backupCodecSupported = false;
-    ti.codecs.forEach((c) => {
-      if (isCodecEqual(c.mimeType, opts.videoCodec)) {
-        primaryCodecSupported = true;
-      } else if (opts.backupCodec && isCodecEqual(c.mimeType, opts.backupCodec.codec)) {
-        backupCodecSupported = true;
+    // server might not support the codec the client has requested, in that case, fallback
+    // to a supported codec
+    let primaryCodecMime: string | undefined;
+    ti.codecs.forEach((codec) => {
+      if (primaryCodecMime === undefined) {
+        primaryCodecMime = codec.mimeType;
       }
     });
-
-    if (req.simulcastCodecs.length > 0) {
-      if (!primaryCodecSupported && !backupCodecSupported) {
-        throw Error('cannot publish track, codec not supported by server');
-      }
-
-      if (!primaryCodecSupported && opts.backupCodec) {
-        const backupCodec = opts.backupCodec;
-        opts = { ...opts };
-        log.debug(
-          `primary codec ${opts.videoCodec} not supported, fallback to ${backupCodec.codec}`,
+    if (primaryCodecMime && track.kind === Track.Kind.Video) {
+      const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime);
+      if (updatedCodec !== videoCodec) {
+        log.debug('falling back to server selected codec', { codec: updatedCodec });
+        /* @ts-ignore */
+        opts.videoCodec = updatedCodec;
+
+        // recompute encodings since bitrates/etc could have changed
+        encodings = computeVideoEncodings(
+          track.source === Track.Source.ScreenShare,
+          req.width,
+          req.height,
+          opts,
         );
-        opts.videoCodec = backupCodec.codec;
-        opts.videoEncoding = backupCodec.encoding;
-        encodings = simEncodings;
       }
     }
 
@@ -786,20 +767,19 @@ export default class LocalParticipant extends Participant {
     }
     log.debug(`publishing ${track.kind} with encodings`, { encodings, trackInfo: ti });
 
-    // store RTPSender
     track.sender = await this.engine.createSender(track, opts, encodings);
 
     if (encodings) {
       if (isFireFox() && track.kind === Track.Kind.Audio) {
         /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
-           livekit-server uses maxaveragebitrate=510000in the answer sdp to permit client to
+           livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to
            publish high quality audio track. But firefox always uses this value as the actual
            bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
            So the client need to modify maxaverragebitrates in answer sdp to user provided value to
            fix the issue.
          */
         let trackTransceiver: RTCRtpTransceiver | undefined = undefined;
-        for (const transceiver of this.engine.publisher.pc.getTransceivers()) {
+        for (const transceiver of this.engine.publisher.getTransceivers()) {
           if (transceiver.sender === track.sender) {
             trackTransceiver = transceiver;
             break;
@@ -889,7 +869,6 @@ export default class LocalParticipant extends Participant {
         {
           codec: opts.videoCodec,
           cid: simulcastTrack.mediaStreamTrack.id,
-          enableSimulcastLayers: opts.simulcast,
         },
       ],
     });
@@ -947,11 +926,11 @@ export default class LocalParticipant extends Participant {
     track.sender = undefined;
     if (
       this.engine.publisher &&
-      this.engine.publisher.pc.connectionState !== 'closed' &&
+      this.engine.publisher.getConnectionState() !== 'closed' &&
       trackSender
     ) {
       try {
-        for (const transceiver of this.engine.publisher.pc.getTransceivers()) {
+        for (const transceiver of this.engine.publisher.getTransceivers()) {
           // if sender is not currently sending (after replaceTrack(null))
           // removeTrack would have no effect.
           // to ensure we end up successfully removing the track, manually set

package/src/room/participant/Participant.ts

@@ -309,6 +309,7 @@ export type ParticipantEventCallbacks = {
     status: TrackPublication.PermissionStatus,
   ) => void;
   mediaDevicesError: (error: Error) => void;
+  audioStreamAcquired: () => void;
   participantPermissionsChanged: (prevPermissions?: ParticipantPermission) => void;
   trackSubscriptionStatusChanged: (
     publication: RemoteTrackPublication,

package/src/room/track/LocalAudioTrack.ts

@@ -138,7 +138,7 @@ export default class LocalAudioTrack extends LocalTrack {
     this.prevStats = stats;
   };
 
-  async setProcessor(processor: TrackProcessor<typeof this.kind>) {
+  async setProcessor(processor: TrackProcessor<this['kind']>) {
     const unlock = await this.processorLock.lock();
     try {
       if (!this.audioContext) {

package/src/room/track/LocalTrack.ts

@@ -34,7 +34,7 @@ export default abstract class LocalTrack extends Track {
 
   protected processorElement?: HTMLMediaElement;
 
-  protected processor?: TrackProcessor<typeof this.kind>;
+  protected processor?: TrackProcessor<this['kind']>;
 
   protected processorLock: Mutex;
 
@@ -163,6 +163,12 @@ export default abstract class LocalTrack extends Track {
       throw new Error('cannot get dimensions for audio tracks');
     }
 
+    if (getBrowser()?.os === 'iOS') {
+      // browsers report wrong initial resolution on iOS.
+      // when slightly delaying the call to .getSettings(), the correct resolution is being reported
+      await sleep(10);
+    }
+
     const started = Date.now();
     while (Date.now() - started < timeout) {
       const dims = this.dimensions;
@@ -396,10 +402,7 @@ export default abstract class LocalTrack extends Track {
    * @param showProcessedStreamLocally
    * @returns
    */
-  async setProcessor(
-    processor: TrackProcessor<typeof this.kind>,
-    showProcessedStreamLocally = true,
-  ) {
+  async setProcessor(processor: TrackProcessor<this['kind']>, showProcessedStreamLocally = true) {
     const unlock = await this.processorLock.lock();
     try {
       log.debug('setting up processor');

package/src/room/track/LocalVideoTrack.ts

@@ -284,7 +284,8 @@ export default class LocalVideoTrack extends LocalTrack {
 
   /**
    * @internal
-   * Sets codecs that should be publishing
+   * Sets codecs that should be publishing, returns new codecs that have not yet
+   * been published
    */
   async setPublishingCodecs(codecs: SubscribedCodec[]): Promise<VideoCodec[]> {
     log.debug('setting publishing codecs', {

package/src/room/track/Track.ts

@@ -293,7 +293,12 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
     mediaStream.addTrack(track);
   }
 
-  element.autoplay = true;
+  if (!isSafari() || !(element instanceof HTMLVideoElement)) {
+    // when in low power mode (applies to both macOS and iOS), Safari will show a play/pause overlay
+    // when a video starts that has the `autoplay` attribute is set.
+    // we work around this by _not_ setting the autoplay attribute on safari and instead call `setTimeout(() => el.play(),0)` further down
+    element.autoplay = true;
+  }
   // In case there are no audio tracks present on the mediastream, we set the element as muted to ensure autoplay works
   element.muted = mediaStream.getAudioTracks().length === 0;
   if (element instanceof HTMLVideoElement) {

package/src/room/track/options.ts

@@ -301,13 +301,6 @@ export function isBackupCodec(codec: string): codec is BackupVideoCodec {
   return !!backupCodecs.find((backup) => backup === codec);
 }
 
-export function isCodecEqual(c1: string | undefined, c2: string | undefined): boolean {
-  return (
-    c1?.toLowerCase().replace(/audio\/|video\//y, '') ===
-    c2?.toLowerCase().replace(/audio\/|video\//y, '')
-  );
-}
-
 /**
  * scalability modes for svc.
  */

package/src/room/track/utils.ts

@@ -1,10 +1,13 @@
+import { cloneDeep } from '../../utils/cloneDeep';
 import { isSafari, sleep } from '../utils';
 import { Track } from './Track';
-import type {
-  AudioCaptureOptions,
-  CreateLocalTracksOptions,
-  ScreenShareCaptureOptions,
-  VideoCaptureOptions,
+import {
+  type AudioCaptureOptions,
+  type CreateLocalTracksOptions,
+  type ScreenShareCaptureOptions,
+  type VideoCaptureOptions,
+  VideoCodec,
+  videoCodecs,
 } from './options';
 import type { AudioTrack } from './types';
 
@@ -13,9 +16,7 @@ export function mergeDefaultOptions(
   audioDefaults?: AudioCaptureOptions,
   videoDefaults?: VideoCaptureOptions,
 ): CreateLocalTracksOptions {
-  const opts: CreateLocalTracksOptions = {
-    ...options,
-  };
+  const opts: CreateLocalTracksOptions = cloneDeep(options) ?? {};
   if (opts.audio === true) opts.audio = {};
   if (opts.video === true) opts.video = {};
 
@@ -181,3 +182,11 @@ export function screenCaptureToDisplayMediaStreamOptions(
     systemAudio: options.systemAudio,
   };
 }
+
+export function mimeTypeToVideoCodecString(mimeType: string) {
+  const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec;
+  if (!videoCodecs.includes(codec)) {
+    throw Error(`Video codec not supported: ${codec}`);
+  }
+  return codec;
+}

package/src/utils/cloneDeep.test.ts

@@ -0,0 +1,54 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { cloneDeep } from './cloneDeep';
+
+describe('cloneDeep', () => {
+  beforeEach(() => {
+    global.structuredClone = vi.fn((val) => {
+      return JSON.parse(JSON.stringify(val));
+    });
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should clone a simple object', () => {
+    const structuredCloneSpy = vi.spyOn(global, 'structuredClone');
+
+    const original = { name: 'John', age: 30 };
+    const cloned = cloneDeep(original);
+
+    expect(cloned).toEqual(original);
+    expect(cloned).not.toBe(original);
+    expect(structuredCloneSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('should clone an object with nested properties', () => {
+    const structuredCloneSpy = vi.spyOn(global, 'structuredClone');
+
+    const original = { name: 'John', age: 30, children: [{ name: 'Mark', age: 7 }] };
+    const cloned = cloneDeep(original);
+
+    expect(cloned).toEqual(original);
+    expect(cloned).not.toBe(original);
+    expect(cloned?.children).not.toBe(original.children);
+    expect(structuredCloneSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('should use JSON namespace as a fallback', () => {
+    const structuredCloneSpy = vi.spyOn(global, 'structuredClone');
+    const serializeSpy = vi.spyOn(JSON, 'stringify');
+    const deserializeSpy = vi.spyOn(JSON, 'parse');
+
+    global.structuredClone = undefined as any;
+
+    const original = { name: 'John', age: 30 };
+    const cloned = cloneDeep(original);
+
+    expect(cloned).toEqual(original);
+    expect(cloned).not.toBe(original);
+    expect(structuredCloneSpy).not.toHaveBeenCalled();
+    expect(serializeSpy).toHaveBeenCalledTimes(1);
+    expect(deserializeSpy).toHaveBeenCalledTimes(1);
+  });
+});

package/src/utils/cloneDeep.ts

@@ -0,0 +1,11 @@
+export function cloneDeep<T>(value: T) {
+  if (typeof value === 'undefined') {
+    return;
+  }
+
+  if (typeof structuredClone === 'function') {
+    return structuredClone(value);
+  } else {
+    return JSON.parse(JSON.stringify(value)) as T;
+  }
+}