livekit-client 1.1.1 → 1.1.4

package/src/room/track/LocalVideoTrack.ts

@@ -24,6 +24,8 @@ export class SimulcastTrackInfo {
   }
 }
 
+const refreshSubscribedCodecAfterNewCodec = 5000;
+
 export default class LocalVideoTrack extends LocalTrack {
   /* internal */
   signalClient?: SignalClient;
@@ -37,8 +39,14 @@ export default class LocalVideoTrack extends LocalTrack {
     SimulcastTrackInfo
   >();
 
-  constructor(mediaTrack: MediaStreamTrack, constraints?: MediaTrackConstraints) {
-    super(mediaTrack, Track.Kind.Video, constraints);
+  private subscribedCodecs?: SubscribedCodec[];
+
+  constructor(
+    mediaTrack: MediaStreamTrack,
+    constraints?: MediaTrackConstraints,
+    userProvidedTrack = true,
+  ) {
+    super(mediaTrack, Track.Kind.Video, constraints, userProvidedTrack);
   }
 
   get isSimulcast(): boolean {
@@ -88,7 +96,7 @@ export default class LocalVideoTrack extends LocalTrack {
   }
 
   async unmute(): Promise<LocalVideoTrack> {
-    if (this.source === Track.Source.Camera) {
+    if (this.source === Track.Source.Camera && !this.isUserProvided) {
       log.debug('reacquiring camera track');
       await this.restartTrack();
     }
@@ -194,26 +202,48 @@ export default class LocalVideoTrack extends LocalTrack {
       return;
     }
     simulcastCodecInfo.sender = sender;
+
+    // browser will reenable disabled codec/layers after new codec has been published,
+    // so refresh subscribedCodecs after publish a new codec
+    setTimeout(() => {
+      if (this.subscribedCodecs) {
+        this.setPublishingCodecs(this.subscribedCodecs);
+      }
+    }, refreshSubscribedCodecAfterNewCodec);
   }
 
   /**
    * @internal
    * Sets codecs that should be publishing
    */
-  async setPublishingCodecs(codecs: SubscribedCodec[]) {
-    log.debug('setting publishing codecs', codecs);
+  async setPublishingCodecs(codecs: SubscribedCodec[]): Promise<VideoCodec[]> {
+    log.debug('setting publishing codecs', {
+      codecs,
+      currentCodec: this.codec,
+    });
+    // only enable simulcast codec for preference codec setted
+    if (!this.codec && codecs.length > 0) {
+      await this.setPublishingLayers(codecs[0].qualities);
+      return [];
+    }
 
+    this.subscribedCodecs = codecs;
+
+    const newCodecs: VideoCodec[] = [];
     for await (const codec of codecs) {
-      if (this.codec === codec.codec) {
+      if (!this.codec || this.codec === codec.codec) {
         await this.setPublishingLayers(codec.qualities);
       } else {
         const simulcastCodecInfo = this.simulcastCodecs.get(codec.codec as VideoCodec);
         log.debug(`try setPublishingCodec for ${codec.codec}`, simulcastCodecInfo);
         if (!simulcastCodecInfo || !simulcastCodecInfo.sender) {
-          return;
-        }
-
-        if (simulcastCodecInfo.encodings) {
+          for (const q of codec.qualities) {
+            if (q.enabled) {
+              newCodecs.push(codec.codec as VideoCodec);
+              break;
+            }
+          }
+        } else if (simulcastCodecInfo.encodings) {
           log.debug(`try setPublishingLayersForSender ${codec.codec}`);
           await setPublishingLayersForSender(
             simulcastCodecInfo.sender,
@@ -223,6 +253,7 @@ export default class LocalVideoTrack extends LocalTrack {
         }
       }
     }
+    return newCodecs;
   }
 
   /**

package/src/room/track/RemoteVideoTrack.ts

@@ -110,6 +110,8 @@ export default class RemoteVideoTrack extends RemoteTrack {
       // the tab comes into focus for the first time.
       this.debouncedHandleResize();
       this.updateVisibility();
+    } else {
+      log.warn('visibility resize observer not triggered');
     }
   }
 
@@ -294,9 +296,9 @@ class HTMLElementInfo implements ElementInfo {
 
   handleVisibilityChanged?: () => void;
 
-  constructor(element: HTMLMediaElement, visible: boolean = false) {
+  constructor(element: HTMLMediaElement, visible?: boolean) {
     this.element = element;
-    this.visible = visible;
+    this.visible = visible ?? isElementInViewport(element);
     this.visibilityChangedAt = 0;
   }
 
@@ -332,3 +334,29 @@ class HTMLElementInfo implements ElementInfo {
     getResizeObserver()?.unobserve(this.element);
   }
 }
+
+// does not account for occlusion by other elements
+function isElementInViewport(el: HTMLElement) {
+  let top = el.offsetTop;
+  let left = el.offsetLeft;
+  const width = el.offsetWidth;
+  const height = el.offsetHeight;
+  const { hidden } = el;
+  const { opacity, display } = getComputedStyle(el);
+
+  while (el.offsetParent) {
+    el = el.offsetParent as HTMLElement;
+    top += el.offsetTop;
+    left += el.offsetLeft;
+  }
+
+  return (
+    top < window.pageYOffset + window.innerHeight &&
+    left < window.pageXOffset + window.innerWidth &&
+    top + height > window.pageYOffset &&
+    left + width > window.pageXOffset &&
+    !hidden &&
+    (opacity !== '' ? parseFloat(opacity) > 0 : true) &&
+    display !== 'none'
+  );
+}

package/src/room/track/create.ts

@@ -110,11 +110,11 @@ export async function createLocalScreenTracks(
   if (tracks.length === 0) {
     throw new TrackInvalidError('no video track found');
   }
-  const screenVideo = new LocalVideoTrack(tracks[0]);
+  const screenVideo = new LocalVideoTrack(tracks[0], undefined, false);
   screenVideo.source = Track.Source.ScreenShare;
   const localTracks: Array<LocalTrack> = [screenVideo];
   if (stream.getAudioTracks().length > 0) {
-    const screenAudio = new LocalAudioTrack(stream.getAudioTracks()[0]);
+    const screenAudio = new LocalAudioTrack(stream.getAudioTracks()[0], undefined, false);
     screenAudio.source = Track.Source.ScreenShareAudio;
     localTracks.push(screenAudio);
   }

package/src/room/utils.ts

@@ -81,8 +81,9 @@ let emptyVideoStreamTrack: MediaStreamTrack | undefined;
 export function getEmptyVideoStreamTrack() {
   if (!emptyVideoStreamTrack) {
     const canvas = document.createElement('canvas');
-    canvas.width = 2;
-    canvas.height = 2;
+    // the canvas size is set to 16, because electron apps seem to fail with smaller values
+    canvas.width = 16;
+    canvas.height = 16;
     canvas.getContext('2d')?.fillRect(0, 0, canvas.width, canvas.height);
     // @ts-ignore
     const emptyStream = canvas.captureStream();