livekit-client 1.1.0 → 1.1.3

package/src/room/participant/LocalParticipant.ts

@@ -15,10 +15,7 @@ import RTCEngine from '../RTCEngine';
 import LocalAudioTrack from '../track/LocalAudioTrack';
 import LocalTrack from '../track/LocalTrack';
 import LocalTrackPublication from '../track/LocalTrackPublication';
-import LocalVideoTrack, {
-  SimulcastTrackInfo,
-  videoLayersFromEncodings,
-} from '../track/LocalVideoTrack';
+import LocalVideoTrack, { videoLayersFromEncodings } from '../track/LocalVideoTrack';
 import {
   AudioCaptureOptions,
   CreateLocalTracksOptions,
@@ -36,7 +33,7 @@ import { ParticipantTrackPermission, trackPermissionToProto } from './Participan
 import { computeVideoEncodings, mediaTrackToLocalTrack } from './publishUtils';
 import RemoteParticipant from './RemoteParticipant';
 
-const compatibleCodecForSVC = 'vp8';
+const compatibleCodec = 'vp8';
 export default class LocalParticipant extends Participant {
   audioTracks: Map<string, LocalTrackPublication>;
 
@@ -455,7 +452,6 @@ export default class LocalParticipant extends Participant {
     // compute encodings and layers for video
     let encodings: RTCRtpEncodingParameters[] | undefined;
     let simEncodings: RTCRtpEncodingParameters[] | undefined;
-    let simulcastTracks: SimulcastTrackInfo[] | undefined;
     if (track.kind === Track.Kind.Video) {
       // TODO: support react native, which doesn't expose getSettings
       const settings = track.mediaStreamTrack.getSettings();
@@ -465,37 +461,38 @@ export default class LocalParticipant extends Participant {
       req.width = width ?? 0;
       req.height = height ?? 0;
       // for svc codecs, disable simulcast and use vp8 for backup codec
-      if (
-        track instanceof LocalVideoTrack &&
-        (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1')
-      ) {
-        // set scalabilityMode to 'L3T3' by default
-        opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
-
-        // add backup codec track
-        const simOpts = { ...opts };
-        simOpts.simulcast = true;
-        simOpts.scalabilityMode = undefined;
-        simEncodings = computeVideoEncodings(
-          track.source === Track.Source.ScreenShare,
-          width,
-          height,
-          simOpts,
-        );
-        const simulcastTrack = track.addSimulcastTrack(compatibleCodecForSVC, simEncodings);
-        simulcastTracks = [simulcastTrack];
-        req.simulcastCodecs = [
-          {
-            codec: opts.videoCodec,
-            cid: track.mediaStreamTrack.id,
-            enableSimulcastLayers: true,
-          },
-          {
-            codec: simulcastTrack.codec,
-            cid: simulcastTrack.mediaStreamTrack.id,
-            enableSimulcastLayers: true,
-          },
-        ];
+      if (track instanceof LocalVideoTrack) {
+        if (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1') {
+          // set scalabilityMode to 'L3T3' by default
+          opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
+
+          // add backup codec track
+          const simOpts = { ...opts };
+          simOpts.simulcast = true;
+          simOpts.scalabilityMode = undefined;
+          simEncodings = computeVideoEncodings(
+            track.source === Track.Source.ScreenShare,
+            width,
+            height,
+            simOpts,
+          );
+        }
+
+        // set vp8 codec as backup for any other codecs
+        if (opts.videoCodec && opts.videoCodec !== 'vp8') {
+          req.simulcastCodecs = [
+            {
+              codec: opts.videoCodec,
+              cid: track.mediaStreamTrack.id,
+              enableSimulcastLayers: true,
+            },
+            {
+              codec: compatibleCodec,
+              cid: '',
+              enableSimulcastLayers: true,
+            },
+          ];
+        }
       }
 
       encodings = computeVideoEncodings(
@@ -541,22 +538,6 @@ export default class LocalParticipant extends Participant {
       track.codec = opts.videoCodec;
     }
 
-    const localTrack = track as LocalVideoTrack;
-    if (simulcastTracks) {
-      for await (const simulcastTrack of simulcastTracks) {
-        const simTransceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
-        if (simulcastTrack.encodings) {
-          simTransceiverInit.sendEncodings = simulcastTrack.encodings;
-        }
-        const simTransceiver = await this.engine.publisher!.pc.addTransceiver(
-          simulcastTrack.mediaStreamTrack,
-          simTransceiverInit,
-        );
-        this.setPreferredCodec(simTransceiver, localTrack.kind, simulcastTrack.codec);
-        localTrack.setSimulcastTrackSender(simulcastTrack.codec, simTransceiver.sender);
-      }
-    }
-
     this.engine.negotiate();
 
     // store RTPSender
@@ -574,6 +555,91 @@ export default class LocalParticipant extends Participant {
     return publication;
   }
 
+  /** @internal
+   * publish additional codec to existing track
+   */
+  async publishAdditionalCodecForTrack(
+    track: LocalTrack | MediaStreamTrack,
+    videoCodec: VideoCodec,
+    options?: TrackPublishOptions,
+  ) {
+    const opts: TrackPublishOptions = {
+      ...this.roomOptions?.publishDefaults,
+      ...options,
+    };
+    // clear scalabilityMode setting for backup codec
+    opts.scalabilityMode = undefined;
+    opts.videoCodec = videoCodec;
+    // is it not published? if so skip
+    let existingPublication: LocalTrackPublication | undefined;
+    this.tracks.forEach((publication) => {
+      if (!publication.track) {
+        return;
+      }
+      if (publication.track === track) {
+        existingPublication = <LocalTrackPublication>publication;
+      }
+    });
+    if (!existingPublication) {
+      throw new TrackInvalidError('track is not published');
+    }
+
+    if (!(track instanceof LocalVideoTrack)) {
+      throw new TrackInvalidError('track is not a video track');
+    }
+
+    const settings = track.mediaStreamTrack.getSettings();
+    const width = settings.width ?? track.dimensions?.width;
+    const height = settings.height ?? track.dimensions?.height;
+
+    const encodings = computeVideoEncodings(
+      track.source === Track.Source.ScreenShare,
+      width,
+      height,
+      opts,
+    );
+    const simulcastTrack = track.addSimulcastTrack(opts.videoCodec, encodings);
+    const req = AddTrackRequest.fromPartial({
+      cid: simulcastTrack.mediaStreamTrack.id,
+      type: Track.kindToProto(track.kind),
+      muted: track.isMuted,
+      source: Track.sourceToProto(track.source),
+      sid: track.sid,
+      simulcastCodecs: [
+        {
+          codec: opts.videoCodec,
+          cid: simulcastTrack.mediaStreamTrack.id,
+          enableSimulcastLayers: opts.simulcast,
+        },
+      ],
+    });
+    req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
+
+    if (!this.engine || this.engine.isClosed) {
+      throw new UnexpectedConnectionState('cannot publish track when not connected');
+    }
+
+    const ti = await this.engine.addTrack(req);
+
+    if (!this.engine.publisher) {
+      throw new UnexpectedConnectionState('publisher is closed');
+    }
+    const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+    if (encodings) {
+      transceiverInit.sendEncodings = encodings;
+    }
+    // addTransceiver for react-native is async. web is synchronous, but await won't effect it.
+    const transceiver = await this.engine.publisher.pc.addTransceiver(
+      simulcastTrack.mediaStreamTrack,
+      transceiverInit,
+    );
+    this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
+    track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
+
+    this.engine.negotiate();
+    log.debug(`published ${opts.videoCodec} for track ${track.sid}`, { encodings, trackInfo: ti });
+  }
+
   unpublishTrack(
     track: LocalTrack | MediaStreamTrack,
     stopOnUnpublish?: boolean,
@@ -761,7 +827,7 @@ export default class LocalParticipant extends Participant {
     this.onTrackMuted(track, track.isMuted);
   };
 
-  private handleSubscribedQualityUpdate = (update: SubscribedQualityUpdate) => {
+  private handleSubscribedQualityUpdate = async (update: SubscribedQualityUpdate) => {
     if (!this.roomOptions?.dynacast) {
       return;
     }
@@ -774,7 +840,14 @@ export default class LocalParticipant extends Participant {
       return;
     }
     if (update.subscribedCodecs.length > 0) {
-      pub.videoTrack?.setPublishingCodecs(update.subscribedCodecs);
+      if (!pub.videoTrack) {
+        return;
+      }
+      const newCodecs = await pub.videoTrack.setPublishingCodecs(update.subscribedCodecs);
+      for await (const codec of newCodecs) {
+        log.debug(`publish ${codec} for ${pub.videoTrack.sid}`);
+        await this.publishAdditionalCodecForTrack(pub.videoTrack, codec, pub.options);
+      }
     } else if (update.subscribedQualities.length > 0) {
       pub.videoTrack?.setPublishingLayers(update.subscribedQualities);
     }

package/src/room/participant/RemoteParticipant.ts

@@ -23,9 +23,7 @@ export default class RemoteParticipant extends Participant {
 
   /** @internal */
   static fromParticipantInfo(signalClient: SignalClient, pi: ParticipantInfo): RemoteParticipant {
-    const rp = new RemoteParticipant(signalClient, pi.sid, pi.identity);
-    rp.updateInfo(pi);
-    return rp;
+    return new RemoteParticipant(signalClient, pi.sid, pi.identity);
   }
 
   /** @internal */
@@ -182,8 +180,6 @@ export default class RemoteParticipant extends Participant {
 
   /** @internal */
   updateInfo(info: ParticipantInfo) {
-    const alreadyHasMetadata = this.hasMetadata;
-
     super.updateInfo(info);
 
     // we are getting a list of all available tracks, reconcile in here
@@ -212,12 +208,10 @@ export default class RemoteParticipant extends Participant {
       validTracks.set(ti.sid, publication);
     });
 
-    // send new tracks
-    if (alreadyHasMetadata) {
-      newTracks.forEach((publication) => {
-        this.emit(ParticipantEvent.TrackPublished, publication);
-      });
-    }
+    // always emit events for new publications, Room will not forward them unless it's ready
+    newTracks.forEach((publication) => {
+      this.emit(ParticipantEvent.TrackPublished, publication);
+    });
 
     // detect removed tracks
     this.tracks.forEach((publication) => {

package/src/room/track/LocalVideoTrack.ts

@@ -24,6 +24,8 @@ export class SimulcastTrackInfo {
   }
 }
 
+const refreshSubscribedCodecAfterNewCodec = 5000;
+
 export default class LocalVideoTrack extends LocalTrack {
   /* internal */
   signalClient?: SignalClient;
@@ -37,6 +39,8 @@ export default class LocalVideoTrack extends LocalTrack {
     SimulcastTrackInfo
   >();
 
+  private subscribedCodecs?: SubscribedCodec[];
+
   constructor(mediaTrack: MediaStreamTrack, constraints?: MediaTrackConstraints) {
     super(mediaTrack, Track.Kind.Video, constraints);
   }
@@ -194,26 +198,48 @@ export default class LocalVideoTrack extends LocalTrack {
       return;
     }
     simulcastCodecInfo.sender = sender;
+
+    // browser will reenable disabled codec/layers after new codec has been published,
+    // so refresh subscribedCodecs after publish a new codec
+    setTimeout(() => {
+      if (this.subscribedCodecs) {
+        this.setPublishingCodecs(this.subscribedCodecs);
+      }
+    }, refreshSubscribedCodecAfterNewCodec);
   }
 
   /**
    * @internal
    * Sets codecs that should be publishing
    */
-  async setPublishingCodecs(codecs: SubscribedCodec[]) {
-    log.debug('setting publishing codecs', codecs);
+  async setPublishingCodecs(codecs: SubscribedCodec[]): Promise<VideoCodec[]> {
+    log.debug('setting publishing codecs', {
+      codecs,
+      currentCodec: this.codec,
+    });
+    // only enable simulcast codec for preference codec setted
+    if (!this.codec && codecs.length > 0) {
+      await this.setPublishingLayers(codecs[0].qualities);
+      return [];
+    }
+
+    this.subscribedCodecs = codecs;
 
+    const newCodecs: VideoCodec[] = [];
     for await (const codec of codecs) {
-      if (this.codec === codec.codec) {
+      if (!this.codec || this.codec === codec.codec) {
         await this.setPublishingLayers(codec.qualities);
       } else {
         const simulcastCodecInfo = this.simulcastCodecs.get(codec.codec as VideoCodec);
         log.debug(`try setPublishingCodec for ${codec.codec}`, simulcastCodecInfo);
         if (!simulcastCodecInfo || !simulcastCodecInfo.sender) {
-          return;
-        }
-
-        if (simulcastCodecInfo.encodings) {
+          for (const q of codec.qualities) {
+            if (q.enabled) {
+              newCodecs.push(codec.codec as VideoCodec);
+              break;
+            }
+          }
+        } else if (simulcastCodecInfo.encodings) {
           log.debug(`try setPublishingLayersForSender ${codec.codec}`);
           await setPublishingLayersForSender(
             simulcastCodecInfo.sender,
@@ -223,6 +249,7 @@ export default class LocalVideoTrack extends LocalTrack {
         }
       }
     }
+    return newCodecs;
   }
 
   /**

package/src/room/track/RemoteVideoTrack.ts

@@ -1,11 +1,11 @@
 import { debounce } from 'ts-debounce';
+import log from '../../logger';
 import { TrackEvent } from '../events';
 import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
 import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { attachToElement, detachTrack, Track } from './Track';
 import { AdaptiveStreamSettings } from './types';
-import log from '../../logger';
 
 const REACTION_DELAY = 100;
 
@@ -110,6 +110,8 @@ export default class RemoteVideoTrack extends RemoteTrack {
       // the tab comes into focus for the first time.
       this.debouncedHandleResize();
       this.updateVisibility();
+    } else {
+      log.warn('visibility resize observer not triggered');
     }
   }
 
@@ -294,9 +296,9 @@ class HTMLElementInfo implements ElementInfo {
 
   handleVisibilityChanged?: () => void;
 
-  constructor(element: HTMLMediaElement, visible: boolean = false) {
+  constructor(element: HTMLMediaElement, visible?: boolean) {
     this.element = element;
-    this.visible = visible;
+    this.visible = visible ?? isElementInViewport(element);
     this.visibilityChangedAt = 0;
   }
 
@@ -305,7 +307,7 @@ class HTMLElementInfo implements ElementInfo {
   }
 
   height(): number {
-    return this.element.clientWidth;
+    return this.element.clientHeight;
   }
 
   observe() {
@@ -332,3 +334,29 @@ class HTMLElementInfo implements ElementInfo {
     getResizeObserver()?.unobserve(this.element);
   }
 }
+
+// does not account for occlusion by other elements
+function isElementInViewport(el: HTMLElement) {
+  let top = el.offsetTop;
+  let left = el.offsetLeft;
+  const width = el.offsetWidth;
+  const height = el.offsetHeight;
+  const { hidden } = el;
+  const { opacity, display } = getComputedStyle(el);
+
+  while (el.offsetParent) {
+    el = el.offsetParent as HTMLElement;
+    top += el.offsetTop;
+    left += el.offsetLeft;
+  }
+
+  return (
+    top < window.pageYOffset + window.innerHeight &&
+    left < window.pageXOffset + window.innerWidth &&
+    top + height > window.pageYOffset &&
+    left + width > window.pageXOffset &&
+    !hidden &&
+    (opacity !== '' ? parseFloat(opacity) > 0 : true) &&
+    display !== 'none'
+  );
+}

package/src/room/utils.ts

@@ -81,8 +81,9 @@ let emptyVideoStreamTrack: MediaStreamTrack | undefined;
 export function getEmptyVideoStreamTrack() {
   if (!emptyVideoStreamTrack) {
     const canvas = document.createElement('canvas');
-    canvas.width = 2;
-    canvas.height = 2;
+    // the canvas size is set to 16, because electron apps seem to fail with smaller values
+    canvas.width = 16;
+    canvas.height = 16;
     canvas.getContext('2d')?.fillRect(0, 0, canvas.width, canvas.height);
     // @ts-ignore
     const emptyStream = canvas.captureStream();