livekit-client 1.0.4 → 1.1.2

package/src/room/participant/LocalParticipant.ts

@@ -15,15 +15,14 @@ import RTCEngine from '../RTCEngine';
 import LocalAudioTrack from '../track/LocalAudioTrack';
 import LocalTrack from '../track/LocalTrack';
 import LocalTrackPublication from '../track/LocalTrackPublication';
-import LocalVideoTrack, {
-  SimulcastTrackInfo,
-  videoLayersFromEncodings,
-} from '../track/LocalVideoTrack';
+import LocalVideoTrack, { videoLayersFromEncodings } from '../track/LocalVideoTrack';
 import {
+  AudioCaptureOptions,
   CreateLocalTracksOptions,
   ScreenShareCaptureOptions,
   ScreenSharePresets,
   TrackPublishOptions,
+  VideoCaptureOptions,
   VideoCodec,
 } from '../track/options';
 import { Track } from '../track/Track';
@@ -34,7 +33,7 @@ import { ParticipantTrackPermission, trackPermissionToProto } from './Participan
 import { computeVideoEncodings, mediaTrackToLocalTrack } from './publishUtils';
 import RemoteParticipant from './RemoteParticipant';
 
-const compatibleCodecForSVC = 'vp8';
+const compatibleCodec = 'vp8';
 export default class LocalParticipant extends Participant {
   audioTracks: Map<string, LocalTrackPublication>;
 
@@ -117,8 +116,11 @@ export default class LocalParticipant extends Participant {
    * If a track has already published, it'll mute or unmute the track.
    * Resolves with a `LocalTrackPublication` instance if successful and `undefined` otherwise
    */
-  setCameraEnabled(enabled: boolean): Promise<LocalTrackPublication | undefined> {
-    return this.setTrackEnabled(Track.Source.Camera, enabled);
+  setCameraEnabled(
+    enabled: boolean,
+    options?: VideoCaptureOptions,
+  ): Promise<LocalTrackPublication | undefined> {
+    return this.setTrackEnabled(Track.Source.Camera, enabled, options);
   }
 
   /**
@@ -127,16 +129,22 @@ export default class LocalParticipant extends Participant {
    * If a track has already published, it'll mute or unmute the track.
    * Resolves with a `LocalTrackPublication` instance if successful and `undefined` otherwise
    */
-  setMicrophoneEnabled(enabled: boolean): Promise<LocalTrackPublication | undefined> {
-    return this.setTrackEnabled(Track.Source.Microphone, enabled);
+  setMicrophoneEnabled(
+    enabled: boolean,
+    options?: AudioCaptureOptions,
+  ): Promise<LocalTrackPublication | undefined> {
+    return this.setTrackEnabled(Track.Source.Microphone, enabled, options);
   }
 
   /**
    * Start or stop sharing a participant's screen
    * Resolves with a `LocalTrackPublication` instance if successful and `undefined` otherwise
    */
-  setScreenShareEnabled(enabled: boolean): Promise<LocalTrackPublication | undefined> {
-    return this.setTrackEnabled(Track.Source.ScreenShare, enabled);
+  setScreenShareEnabled(
+    enabled: boolean,
+    options?: ScreenShareCaptureOptions,
+  ): Promise<LocalTrackPublication | undefined> {
+    return this.setTrackEnabled(Track.Source.ScreenShare, enabled, options);
   }
 
   /** @internal */
@@ -155,16 +163,32 @@ export default class LocalParticipant extends Participant {
    * Resolves with LocalTrackPublication if successful and void otherwise
    */
   private async setTrackEnabled(
-    source: Track.Source,
+    source: Extract<Track.Source, Track.Source.Camera>,
     enabled: boolean,
-  ): Promise<LocalTrackPublication | undefined> {
+    options?: VideoCaptureOptions,
+  ): Promise<LocalTrackPublication | undefined>;
+  private async setTrackEnabled(
+    source: Extract<Track.Source, Track.Source.Microphone>,
+    enabled: boolean,
+    options?: AudioCaptureOptions,
+  ): Promise<LocalTrackPublication | undefined>;
+  private async setTrackEnabled(
+    source: Extract<Track.Source, Track.Source.ScreenShare>,
+    enabled: boolean,
+    options?: ScreenShareCaptureOptions,
+  ): Promise<LocalTrackPublication | undefined>;
+  private async setTrackEnabled(
+    source: Track.Source,
+    enabled: true,
+    options?: VideoCaptureOptions | AudioCaptureOptions | ScreenShareCaptureOptions,
+  ) {
     log.debug('setTrackEnabled', { source, enabled });
     let track = this.getTrack(source);
     if (enabled) {
       if (track) {
         await track.unmute();
       } else {
-        let localTrack: LocalTrack | undefined;
+        let localTracks: Array<LocalTrack> | undefined;
         if (this.pendingPublishing.has(source)) {
           log.info('skipping duplicate published source', { source });
           // no-op it's already been requested
@@ -174,23 +198,32 @@ export default class LocalParticipant extends Participant {
         try {
           switch (source) {
             case Track.Source.Camera:
-              [localTrack] = await this.createTracks({
-                video: true,
+              localTracks = await this.createTracks({
+                video: (options as VideoCaptureOptions | undefined) ?? true,
               });
+
               break;
             case Track.Source.Microphone:
-              [localTrack] = await this.createTracks({
-                audio: true,
+              localTracks = await this.createTracks({
+                audio: (options as AudioCaptureOptions | undefined) ?? true,
               });
               break;
             case Track.Source.ScreenShare:
-              [localTrack] = await this.createScreenTracks({ audio: false });
+              localTracks = await this.createScreenTracks({
+                ...(options as ScreenShareCaptureOptions | undefined),
+              });
               break;
             default:
               throw new TrackInvalidError(source);
           }
-
-          track = await this.publishTrack(localTrack);
+          const publishPromises: Array<Promise<LocalTrackPublication>> = [];
+          for (const localTrack of localTracks) {
+            publishPromises.push(this.publishTrack(localTrack));
+          }
+          const publishedTracks = await Promise.all(publishPromises);
+          // for screen share publications including audio, this will only return the screen share publication, not the screen share audio one
+          // revisit if we want to return an array of tracks instead for v2
+          [track] = publishedTracks;
         } catch (e) {
           if (e instanceof Error && !(e instanceof TrackInvalidError)) {
             this.emit(ParticipantEvent.MediaDevicesError, e);
@@ -204,6 +237,10 @@ export default class LocalParticipant extends Participant {
       // screenshare cannot be muted, unpublish instead
       if (source === Track.Source.ScreenShare) {
         track = this.unpublishTrack(track.track);
+        const screenAudioTrack = this.getTrack(Track.Source.ScreenShareAudio);
+        if (screenAudioTrack && screenAudioTrack.track) {
+          this.unpublishTrack(screenAudioTrack.track);
+        }
       } else {
         await track.mute();
       }
@@ -415,7 +452,6 @@ export default class LocalParticipant extends Participant {
     // compute encodings and layers for video
     let encodings: RTCRtpEncodingParameters[] | undefined;
     let simEncodings: RTCRtpEncodingParameters[] | undefined;
-    let simulcastTracks: SimulcastTrackInfo[] | undefined;
     if (track.kind === Track.Kind.Video) {
       // TODO: support react native, which doesn't expose getSettings
       const settings = track.mediaStreamTrack.getSettings();
@@ -425,37 +461,38 @@ export default class LocalParticipant extends Participant {
       req.width = width ?? 0;
       req.height = height ?? 0;
       // for svc codecs, disable simulcast and use vp8 for backup codec
-      if (
-        track instanceof LocalVideoTrack &&
-        (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1')
-      ) {
-        // set scalabilityMode to 'L3T3' by default
-        opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
-
-        // add backup codec track
-        const simOpts = { ...opts };
-        simOpts.simulcast = true;
-        simOpts.scalabilityMode = undefined;
-        simEncodings = computeVideoEncodings(
-          track.source === Track.Source.ScreenShare,
-          width,
-          height,
-          simOpts,
-        );
-        const simulcastTrack = track.addSimulcastTrack(compatibleCodecForSVC, simEncodings);
-        simulcastTracks = [simulcastTrack];
-        req.simulcastCodecs = [
-          {
-            codec: opts.videoCodec,
-            cid: track.mediaStreamTrack.id,
-            enableSimulcastLayers: true,
-          },
-          {
-            codec: simulcastTrack.codec,
-            cid: simulcastTrack.mediaStreamTrack.id,
-            enableSimulcastLayers: true,
-          },
-        ];
+      if (track instanceof LocalVideoTrack) {
+        if (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1') {
+          // set scalabilityMode to 'L3T3' by default
+          opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
+
+          // add backup codec track
+          const simOpts = { ...opts };
+          simOpts.simulcast = true;
+          simOpts.scalabilityMode = undefined;
+          simEncodings = computeVideoEncodings(
+            track.source === Track.Source.ScreenShare,
+            width,
+            height,
+            simOpts,
+          );
+        }
+
+        // set vp8 codec as backup for any other codecs
+        if (opts.videoCodec && opts.videoCodec !== 'vp8') {
+          req.simulcastCodecs = [
+            {
+              codec: opts.videoCodec,
+              cid: track.mediaStreamTrack.id,
+              enableSimulcastLayers: true,
+            },
+            {
+              codec: compatibleCodec,
+              cid: '',
+              enableSimulcastLayers: true,
+            },
+          ];
+        }
       }
 
       encodings = computeVideoEncodings(
@@ -501,22 +538,6 @@ export default class LocalParticipant extends Participant {
       track.codec = opts.videoCodec;
     }
 
-    const localTrack = track as LocalVideoTrack;
-    if (simulcastTracks) {
-      for await (const simulcastTrack of simulcastTracks) {
-        const simTransceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
-        if (simulcastTrack.encodings) {
-          simTransceiverInit.sendEncodings = simulcastTrack.encodings;
-        }
-        const simTransceiver = await this.engine.publisher!.pc.addTransceiver(
-          simulcastTrack.mediaStreamTrack,
-          simTransceiverInit,
-        );
-        this.setPreferredCodec(simTransceiver, localTrack.kind, simulcastTrack.codec);
-        localTrack.setSimulcastTrackSender(simulcastTrack.codec, simTransceiver.sender);
-      }
-    }
-
     this.engine.negotiate();
 
     // store RTPSender
@@ -534,6 +555,91 @@ export default class LocalParticipant extends Participant {
     return publication;
   }
 
+  /** @internal
+   * publish additional codec to existing track
+   */
+  async publishAdditionalCodecForTrack(
+    track: LocalTrack | MediaStreamTrack,
+    videoCodec: VideoCodec,
+    options?: TrackPublishOptions,
+  ) {
+    const opts: TrackPublishOptions = {
+      ...this.roomOptions?.publishDefaults,
+      ...options,
+    };
+    // clear scalabilityMode setting for backup codec
+    opts.scalabilityMode = undefined;
+    opts.videoCodec = videoCodec;
+    // is it not published? if so skip
+    let existingPublication: LocalTrackPublication | undefined;
+    this.tracks.forEach((publication) => {
+      if (!publication.track) {
+        return;
+      }
+      if (publication.track === track) {
+        existingPublication = <LocalTrackPublication>publication;
+      }
+    });
+    if (!existingPublication) {
+      throw new TrackInvalidError('track is not published');
+    }
+
+    if (!(track instanceof LocalVideoTrack)) {
+      throw new TrackInvalidError('track is not a video track');
+    }
+
+    const settings = track.mediaStreamTrack.getSettings();
+    const width = settings.width ?? track.dimensions?.width;
+    const height = settings.height ?? track.dimensions?.height;
+
+    const encodings = computeVideoEncodings(
+      track.source === Track.Source.ScreenShare,
+      width,
+      height,
+      opts,
+    );
+    const simulcastTrack = track.addSimulcastTrack(opts.videoCodec, encodings);
+    const req = AddTrackRequest.fromPartial({
+      cid: simulcastTrack.mediaStreamTrack.id,
+      type: Track.kindToProto(track.kind),
+      muted: track.isMuted,
+      source: Track.sourceToProto(track.source),
+      sid: track.sid,
+      simulcastCodecs: [
+        {
+          codec: opts.videoCodec,
+          cid: simulcastTrack.mediaStreamTrack.id,
+          enableSimulcastLayers: opts.simulcast,
+        },
+      ],
+    });
+    req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
+
+    if (!this.engine || this.engine.isClosed) {
+      throw new UnexpectedConnectionState('cannot publish track when not connected');
+    }
+
+    const ti = await this.engine.addTrack(req);
+
+    if (!this.engine.publisher) {
+      throw new UnexpectedConnectionState('publisher is closed');
+    }
+    const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+    if (encodings) {
+      transceiverInit.sendEncodings = encodings;
+    }
+    // addTransceiver for react-native is async. web is synchronous, but await won't effect it.
+    const transceiver = await this.engine.publisher.pc.addTransceiver(
+      simulcastTrack.mediaStreamTrack,
+      transceiverInit,
+    );
+    this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
+    track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
+
+    this.engine.negotiate();
+    log.debug(`published ${opts.videoCodec} for track ${track.sid}`, { encodings, trackInfo: ti });
+  }
+
   unpublishTrack(
     track: LocalTrack | MediaStreamTrack,
     stopOnUnpublish?: boolean,
@@ -721,7 +827,7 @@ export default class LocalParticipant extends Participant {
     this.onTrackMuted(track, track.isMuted);
   };
 
-  private handleSubscribedQualityUpdate = (update: SubscribedQualityUpdate) => {
+  private handleSubscribedQualityUpdate = async (update: SubscribedQualityUpdate) => {
     if (!this.roomOptions?.dynacast) {
       return;
     }
@@ -734,7 +840,14 @@ export default class LocalParticipant extends Participant {
       return;
     }
     if (update.subscribedCodecs.length > 0) {
-      pub.videoTrack?.setPublishingCodecs(update.subscribedCodecs);
+      if (!pub.videoTrack) {
+        return;
+      }
+      const newCodecs = await pub.videoTrack.setPublishingCodecs(update.subscribedCodecs);
+      for await (const codec of newCodecs) {
+        log.debug(`publish ${codec} for ${pub.videoTrack.sid}`);
+        await this.publishAdditionalCodecForTrack(pub.videoTrack, codec, pub.options);
+      }
     } else if (update.subscribedQualities.length > 0) {
       pub.videoTrack?.setPublishingLayers(update.subscribedQualities);
     }
@@ -794,35 +907,36 @@ export default class LocalParticipant extends Participant {
     const cap = RTCRtpSender.getCapabilities(kind);
     if (!cap) return;
     log.debug('get capabilities', cap);
-    let selected: RTCRtpCodecCapability | undefined;
-    const codecs: RTCRtpCodecCapability[] = [];
+    const matched: RTCRtpCodecCapability[] = [];
+    const partialMatched: RTCRtpCodecCapability[] = [];
+    const unmatched: RTCRtpCodecCapability[] = [];
     cap.codecs.forEach((c) => {
       const codec = c.mimeType.toLowerCase();
+      if (codec === 'audio/opus') {
+        matched.push(c);
+        return;
+      }
       const matchesVideoCodec = codec === `video/${videoCodec}`;
-
-      if (selected !== undefined) {
-        codecs.push(c);
+      if (!matchesVideoCodec) {
+        unmatched.push(c);
         return;
       }
       // for h264 codecs that have sdpFmtpLine available, use only if the
       // profile-level-id is 42e01f for cross-browser compatibility
-      if (videoCodec === 'h264' && c.sdpFmtpLine) {
-        if (matchesVideoCodec && c.sdpFmtpLine.includes('profile-level-id=42e01f')) {
-          selected = c;
-          return;
+      if (videoCodec === 'h264') {
+        if (c.sdpFmtpLine && c.sdpFmtpLine.includes('profile-level-id=42e01f')) {
+          matched.push(c);
+        } else {
+          partialMatched.push(c);
         }
-      }
-      if (matchesVideoCodec || codec === 'audio/opus') {
-        selected = c;
         return;
       }
-      codecs.push(c);
+
+      matched.push(c);
     });
 
-    if (selected && 'setCodecPreferences' in transceiver) {
-      // @ts-ignore
-      codecs.unshift(selected);
-      transceiver.setCodecPreferences(codecs);
+    if ('setCodecPreferences' in transceiver) {
+      transceiver.setCodecPreferences(matched.concat(partialMatched, unmatched));
     }
   }
 

package/src/room/track/LocalVideoTrack.ts

@@ -24,6 +24,8 @@ export class SimulcastTrackInfo {
   }
 }
 
+const refreshSubscribedCodecAfterNewCodec = 5000;
+
 export default class LocalVideoTrack extends LocalTrack {
   /* internal */
   signalClient?: SignalClient;
@@ -37,6 +39,8 @@ export default class LocalVideoTrack extends LocalTrack {
     SimulcastTrackInfo
   >();
 
+  private subscribedCodecs?: SubscribedCodec[];
+
   constructor(mediaTrack: MediaStreamTrack, constraints?: MediaTrackConstraints) {
     super(mediaTrack, Track.Kind.Video, constraints);
   }
@@ -194,26 +198,48 @@ export default class LocalVideoTrack extends LocalTrack {
       return;
     }
     simulcastCodecInfo.sender = sender;
+
+    // browser will reenable disabled codec/layers after new codec has been published,
+    // so refresh subscribedCodecs after publish a new codec
+    setTimeout(() => {
+      if (this.subscribedCodecs) {
+        this.setPublishingCodecs(this.subscribedCodecs);
+      }
+    }, refreshSubscribedCodecAfterNewCodec);
   }
 
   /**
    * @internal
    * Sets codecs that should be publishing
    */
-  async setPublishingCodecs(codecs: SubscribedCodec[]) {
-    log.debug('setting publishing codecs', codecs);
+  async setPublishingCodecs(codecs: SubscribedCodec[]): Promise<VideoCodec[]> {
+    log.debug('setting publishing codecs', {
+      codecs,
+      currentCodec: this.codec,
+    });
+    // only enable simulcast codec for preference codec setted
+    if (!this.codec && codecs.length > 0) {
+      await this.setPublishingLayers(codecs[0].qualities);
+      return [];
+    }
+
+    this.subscribedCodecs = codecs;
 
+    const newCodecs: VideoCodec[] = [];
     for await (const codec of codecs) {
-      if (this.codec === codec.codec) {
+      if (!this.codec || this.codec === codec.codec) {
         await this.setPublishingLayers(codec.qualities);
       } else {
         const simulcastCodecInfo = this.simulcastCodecs.get(codec.codec as VideoCodec);
         log.debug(`try setPublishingCodec for ${codec.codec}`, simulcastCodecInfo);
         if (!simulcastCodecInfo || !simulcastCodecInfo.sender) {
-          return;
-        }
-
-        if (simulcastCodecInfo.encodings) {
+          for (const q of codec.qualities) {
+            if (q.enabled) {
+              newCodecs.push(codec.codec as VideoCodec);
+              break;
+            }
+          }
+        } else if (simulcastCodecInfo.encodings) {
           log.debug(`try setPublishingLayersForSender ${codec.codec}`);
           await setPublishingLayersForSender(
             simulcastCodecInfo.sender,
@@ -223,6 +249,7 @@ export default class LocalVideoTrack extends LocalTrack {
         }
       }
     }
+    return newCodecs;
   }
 
   /**

package/src/room/track/RemoteVideoTrack.ts

@@ -1,11 +1,11 @@
 import { debounce } from 'ts-debounce';
+import log from '../../logger';
 import { TrackEvent } from '../events';
 import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
 import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { attachToElement, detachTrack, Track } from './Track';
 import { AdaptiveStreamSettings } from './types';
-import log from '../../logger';
 
 const REACTION_DELAY = 100;
 
@@ -305,7 +305,7 @@ class HTMLElementInfo implements ElementInfo {
   }
 
   height(): number {
-    return this.element.clientWidth;
+    return this.element.clientHeight;
   }
 
   observe() {

package/src/room/utils.ts

@@ -81,8 +81,9 @@ let emptyVideoStreamTrack: MediaStreamTrack | undefined;
 export function getEmptyVideoStreamTrack() {
   if (!emptyVideoStreamTrack) {
     const canvas = document.createElement('canvas');
-    canvas.width = 2;
-    canvas.height = 2;
+    // the canvas size is set to 16, because electron apps seem to fail with smaller values
+    canvas.width = 16;
+    canvas.height = 16;
     canvas.getContext('2d')?.fillRect(0, 0, canvas.width, canvas.height);
     // @ts-ignore
     const emptyStream = canvas.captureStream();