livekit-client 0.18.6 → 1.0.2

package/src/room/track/LocalVideoTrack.ts

@@ -1,14 +1,29 @@
 import { SignalClient } from '../../api/SignalClient';
 import log from '../../logger';
 import { VideoLayer, VideoQuality } from '../../proto/livekit_models';
-import { SubscribedQuality } from '../../proto/livekit_rtc';
+import { SubscribedCodec, SubscribedQuality } from '../../proto/livekit_rtc';
 import { computeBitrate, monitorFrequency, VideoSenderStats } from '../stats';
-import { isFireFox, isMobile } from '../utils';
+import { isFireFox, isMobile, isWeb } from '../utils';
 import LocalTrack from './LocalTrack';
-import { VideoCaptureOptions } from './options';
+import { VideoCaptureOptions, VideoCodec } from './options';
 import { Track } from './Track';
 import { constraintsForOptions } from './utils';
 
+export class SimulcastTrackInfo {
+  codec: VideoCodec;
+
+  mediaStreamTrack: MediaStreamTrack;
+
+  sender?: RTCRtpSender;
+
+  encodings?: RTCRtpEncodingParameters[];
+
+  constructor(codec: VideoCodec, mediaStreamTrack: MediaStreamTrack) {
+    this.codec = codec;
+    this.mediaStreamTrack = mediaStreamTrack;
+  }
+}
+
 export default class LocalVideoTrack extends LocalTrack {
   /* internal */
   signalClient?: SignalClient;
@@ -17,6 +32,11 @@ export default class LocalVideoTrack extends LocalTrack {
 
   private encodings?: RTCRtpEncodingParameters[];
 
+  private simulcastCodecs: Map<VideoCodec, SimulcastTrackInfo> = new Map<
+    VideoCodec,
+    SimulcastTrackInfo
+  >();
+
   constructor(mediaTrack: MediaStreamTrack, constraints?: MediaTrackConstraints) {
     super(mediaTrack, Track.Kind.Video, constraints);
   }
@@ -31,7 +51,11 @@ export default class LocalVideoTrack extends LocalTrack {
   /* @internal */
   startMonitor(signalClient: SignalClient) {
     this.signalClient = signalClient;
+    if (!isWeb()) {
+      return;
+    }
     // save original encodings
+    // TODO : merge simulcast tracks stats
     const params = this.sender?.getParameters();
     if (params) {
       this.encodings = params.encodings;
@@ -45,6 +69,11 @@ export default class LocalVideoTrack extends LocalTrack {
   stop() {
     this.sender = undefined;
     this._mediaStreamTrack.getConstraints();
+    this.simulcastCodecs.forEach((trackInfo) => {
+      trackInfo.mediaStreamTrack.stop();
+      trackInfo.sender = undefined;
+    });
+    this.simulcastCodecs.clear();
     super.stop();
   }
 
@@ -89,7 +118,7 @@ export default class LocalVideoTrack extends LocalTrack {
           bytesSent: v.bytesSent,
           framesSent: v.framesSent,
           timestamp: v.timestamp,
-          rid: v.rid ?? '',
+          rid: v.rid ?? v.id,
           retransmittedPacketsSent: v.retransmittedPacketsSent,
           qualityLimitationReason: v.qualityLimitationReason,
           qualityLimitationResolutionChanges: v.qualityLimitationResolutionChanges,
@@ -145,6 +174,57 @@ export default class LocalVideoTrack extends LocalTrack {
     await this.restart(constraints);
   }
 
+  addSimulcastTrack(codec: VideoCodec, encodings?: RTCRtpEncodingParameters[]): SimulcastTrackInfo {
+    if (this.simulcastCodecs.has(codec)) {
+      throw new Error(`${codec} already added`);
+    }
+    const simulcastCodecInfo: SimulcastTrackInfo = {
+      codec,
+      mediaStreamTrack: this.mediaStreamTrack.clone(),
+      sender: undefined,
+      encodings,
+    };
+    this.simulcastCodecs.set(codec, simulcastCodecInfo);
+    return simulcastCodecInfo;
+  }
+
+  setSimulcastTrackSender(codec: VideoCodec, sender: RTCRtpSender) {
+    const simulcastCodecInfo = this.simulcastCodecs.get(codec);
+    if (!simulcastCodecInfo) {
+      return;
+    }
+    simulcastCodecInfo.sender = sender;
+  }
+
+  /**
+   * @internal
+   * Sets codecs that should be publishing
+   */
+  async setPublishingCodecs(codecs: SubscribedCodec[]) {
+    log.debug('setting publishing codecs', codecs);
+
+    for await (const codec of codecs) {
+      if (this.codec === codec.codec) {
+        await this.setPublishingLayers(codec.qualities);
+      } else {
+        const simulcastCodecInfo = this.simulcastCodecs.get(codec.codec as VideoCodec);
+        log.debug(`try setPublishingCodec for ${codec.codec}`, simulcastCodecInfo);
+        if (!simulcastCodecInfo || !simulcastCodecInfo.sender) {
+          return;
+        }
+
+        if (simulcastCodecInfo.encodings) {
+          log.debug(`try setPublishingLayersForSender ${codec.codec}`);
+          await setPublishingLayersForSender(
+            simulcastCodecInfo.sender,
+            simulcastCodecInfo.encodings!,
+            codec.qualities,
+          );
+        }
+      }
+    }
+  }
+
   /**
    * @internal
    * Sets layers that should be publishing
@@ -154,59 +234,8 @@ export default class LocalVideoTrack extends LocalTrack {
     if (!this.sender || !this.encodings) {
       return;
     }
-    const params = this.sender.getParameters();
-    const { encodings } = params;
-    if (!encodings) {
-      return;
-    }
-
-    if (encodings.length !== this.encodings.length) {
-      log.warn('cannot set publishing layers, encodings mismatch');
-      return;
-    }
 
-    let hasChanged = false;
-    encodings.forEach((encoding, idx) => {
-      let rid = encoding.rid ?? '';
-      if (rid === '') {
-        rid = 'q';
-      }
-      const quality = videoQualityForRid(rid);
-      const subscribedQuality = qualities.find((q) => q.quality === quality);
-      if (!subscribedQuality) {
-        return;
-      }
-      if (encoding.active !== subscribedQuality.enabled) {
-        hasChanged = true;
-        encoding.active = subscribedQuality.enabled;
-        log.debug(
-          `setting layer ${subscribedQuality.quality} to ${
-            encoding.active ? 'enabled' : 'disabled'
-          }`,
-        );
-
-        // FireFox does not support setting encoding.active to false, so we
-        // have a workaround of lowering its bitrate and resolution to the min.
-        if (isFireFox()) {
-          if (subscribedQuality.enabled) {
-            encoding.scaleResolutionDownBy = this.encodings![idx].scaleResolutionDownBy;
-            encoding.maxBitrate = this.encodings![idx].maxBitrate;
-            /* @ts-ignore */
-            encoding.maxFrameRate = this.encodings![idx].maxFrameRate;
-          } else {
-            encoding.scaleResolutionDownBy = 4;
-            encoding.maxBitrate = 10;
-            /* @ts-ignore */
-            encoding.maxFrameRate = 2;
-          }
-        }
-      }
-    });
-
-    if (hasChanged) {
-      params.encodings = encodings;
-      await this.sender.setParameters(params);
-    }
+    await setPublishingLayersForSender(this.sender, this.encodings, qualities);
   }
 
   private monitorSender = async () => {
@@ -248,6 +277,65 @@ export default class LocalVideoTrack extends LocalTrack {
   }
 }
 
+async function setPublishingLayersForSender(
+  sender: RTCRtpSender,
+  senderEncodings: RTCRtpEncodingParameters[],
+  qualities: SubscribedQuality[],
+) {
+  log.debug('setPublishingLayersForSender', { sender, qualities, senderEncodings });
+  const params = sender.getParameters();
+  const { encodings } = params;
+  if (!encodings) {
+    return;
+  }
+
+  if (encodings.length !== senderEncodings.length) {
+    log.warn('cannot set publishing layers, encodings mismatch');
+    return;
+  }
+
+  let hasChanged = false;
+  encodings.forEach((encoding, idx) => {
+    let rid = encoding.rid ?? '';
+    if (rid === '') {
+      rid = 'q';
+    }
+    const quality = videoQualityForRid(rid);
+    const subscribedQuality = qualities.find((q) => q.quality === quality);
+    if (!subscribedQuality) {
+      return;
+    }
+    if (encoding.active !== subscribedQuality.enabled) {
+      hasChanged = true;
+      encoding.active = subscribedQuality.enabled;
+      log.debug(
+        `setting layer ${subscribedQuality.quality} to ${encoding.active ? 'enabled' : 'disabled'}`,
+      );
+
+      // FireFox does not support setting encoding.active to false, so we
+      // have a workaround of lowering its bitrate and resolution to the min.
+      if (isFireFox()) {
+        if (subscribedQuality.enabled) {
+          encoding.scaleResolutionDownBy = senderEncodings[idx].scaleResolutionDownBy;
+          encoding.maxBitrate = senderEncodings[idx].maxBitrate;
+          /* @ts-ignore */
+          encoding.maxFrameRate = senderEncodings[idx].maxFrameRate;
+        } else {
+          encoding.scaleResolutionDownBy = 4;
+          encoding.maxBitrate = 10;
+          /* @ts-ignore */
+          encoding.maxFrameRate = 2;
+        }
+      }
+    }
+  });
+
+  if (hasChanged) {
+    params.encodings = encodings;
+    await sender.setParameters(params);
+  }
+}
+
 export function videoQualityForRid(rid: string): VideoQuality {
   switch (rid) {
     case 'f':

package/src/room/track/RemoteTrack.ts

@@ -6,8 +6,6 @@ export default abstract class RemoteTrack extends Track {
   /** @internal */
   receiver?: RTCRtpReceiver;
 
-  streamState: Track.StreamState = Track.StreamState.Active;
-
   constructor(
     mediaTrack: MediaStreamTrack,
     sid: string,

package/src/room/track/RemoteVideoTrack.test.ts

@@ -0,0 +1,149 @@
+import { TrackEvent } from '../events';
+import RemoteVideoTrack, { ElementInfo } from './RemoteVideoTrack';
+import MockMediaStreamTrack from '../../test/MockMediaStreamTrack';
+import { Track } from './Track';
+
+jest.useFakeTimers();
+
+describe('RemoteVideoTrack', () => {
+  let track: RemoteVideoTrack;
+
+  beforeEach(() => {
+    track = new RemoteVideoTrack(new MockMediaStreamTrack(), 'sid', undefined, {});
+  });
+  describe('element visibility', () => {
+    let events: boolean[] = [];
+
+    beforeEach(() => {
+      track.on(TrackEvent.VisibilityChanged, (visible) => {
+        events.push(visible);
+      });
+    });
+    afterEach(() => {
+      events = [];
+    });
+
+    it('emits a visibility event upon observing visible element', () => {
+      const elementInfo = new MockElementInfo();
+      elementInfo.visible = true;
+
+      track.observeElementInfo(elementInfo);
+
+      expect(events).toHaveLength(1);
+      expect(events[0]).toBeTruthy();
+    });
+
+    it('emits a visibility event upon element becoming visible', () => {
+      const elementInfo = new MockElementInfo();
+      track.observeElementInfo(elementInfo);
+
+      elementInfo.setVisible(true);
+
+      expect(events).toHaveLength(2);
+      expect(events[1]).toBeTruthy();
+    });
+
+    it('emits a visibility event upon removing only visible element', () => {
+      const elementInfo = new MockElementInfo();
+      elementInfo.visible = true;
+
+      track.observeElementInfo(elementInfo);
+      track.stopObservingElementInfo(elementInfo);
+
+      expect(events).toHaveLength(2);
+      expect(events[1]).toBeFalsy();
+    });
+  });
+
+  describe('element dimensions', () => {
+    let events: Track.Dimensions[] = [];
+
+    beforeEach(() => {
+      track.on(TrackEvent.VideoDimensionsChanged, (dimensions) => {
+        events.push(dimensions);
+      });
+    });
+
+    afterEach(() => {
+      events = [];
+    });
+
+    it('emits a dimensions event upon observing element', () => {
+      const elementInfo = new MockElementInfo();
+      elementInfo.setDimensions(100, 100);
+
+      track.observeElementInfo(elementInfo);
+      jest.runAllTimers();
+
+      expect(events).toHaveLength(1);
+      expect(events[0].width).toBe(100);
+      expect(events[0].height).toBe(100);
+    });
+
+    it('emits a dimensions event upon element resize', () => {
+      const elementInfo = new MockElementInfo();
+      elementInfo.setDimensions(100, 100);
+
+      track.observeElementInfo(elementInfo);
+      jest.runAllTimers();
+
+      elementInfo.setDimensions(200, 200);
+      jest.runAllTimers();
+
+      expect(events).toHaveLength(2);
+      expect(events[1].width).toBe(200);
+      expect(events[1].height).toBe(200);
+    });
+  });
+});
+
+class MockElementInfo implements ElementInfo {
+  element: object = {};
+
+  private _width = 0;
+
+  private _height = 0;
+
+  setDimensions(width: number, height: number) {
+    let shouldEmit = false;
+    if (this._width !== width) {
+      this._width = width;
+      shouldEmit = true;
+    }
+    if (this._height !== height) {
+      this._height = height;
+      shouldEmit = true;
+    }
+
+    if (shouldEmit) {
+      this.handleResize?.();
+    }
+  }
+
+  width(): number {
+    return this._width;
+  }
+
+  height(): number {
+    return this._height;
+  }
+
+  visible = false;
+
+  setVisible = (visible: boolean) => {
+    if (this.visible !== visible) {
+      this.visible = visible;
+      this.handleVisibilityChanged?.();
+    }
+  };
+
+  visibilityChangedAt = 0;
+
+  handleResize?: () => void;
+
+  handleVisibilityChanged?: () => void;
+
+  observe(): void {}
+
+  stopObserving(): void {}
+}

package/src/room/track/RemoteVideoTrack.ts

@@ -1,12 +1,7 @@
 import { debounce } from 'ts-debounce';
 import { TrackEvent } from '../events';
 import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
-import {
-  getIntersectionObserver,
-  getResizeObserver,
-  isMobile,
-  ObservableMediaElement,
-} from '../utils';
+import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { attachToElement, detachTrack, Track } from './Track';
 import { AdaptiveStreamSettings } from './types';
@@ -85,24 +80,51 @@ export default class RemoteVideoTrack extends RemoteTrack {
       this.adaptiveStreamSettings &&
       this.elementInfos.find((info) => info.element === element) === undefined
     ) {
-      this.elementInfos.push({
-        element,
-        visible: true, // default visible
-      });
-
-      (element as ObservableMediaElement).handleResize = this.debouncedHandleResize;
-      (element as ObservableMediaElement).handleVisibilityChanged = this.handleVisibilityChanged;
-
-      getIntersectionObserver().observe(element);
-      getResizeObserver().observe(element);
+      const elementInfo = new HTMLElementInfo(element);
+      this.observeElementInfo(elementInfo);
+    }
+    this.hasUsedAttach = true;
+    return element;
+  }
 
+  /**
+   * Observe an ElementInfo for changes when adaptive streaming.
+   * @param elementInfo
+   * @internal
+   */
+  observeElementInfo(elementInfo: ElementInfo) {
+    if (
+      this.adaptiveStreamSettings &&
+      this.elementInfos.find((info) => info === elementInfo) === undefined
+    ) {
+      elementInfo.handleResize = () => {
+        this.debouncedHandleResize();
+      };
+      elementInfo.handleVisibilityChanged = () => {
+        this.updateVisibility();
+      };
+      this.elementInfos.push(elementInfo);
+      elementInfo.observe();
       // trigger the first resize update cycle
       // if the tab is backgrounded, the initial resize event does not fire until
       // the tab comes into focus for the first time.
       this.debouncedHandleResize();
+      this.updateVisibility();
     }
-    this.hasUsedAttach = true;
-    return element;
+  }
+
+  /**
+   * Stop observing an ElementInfo for changes.
+   * @param elementInfo
+   * @internal
+   */
+  stopObservingElementInfo(elementInfo: ElementInfo) {
+    const stopElementInfos = this.elementInfos.filter((info) => info === elementInfo);
+    for (const info of stopElementInfos) {
+      info.stopObserving();
+    }
+    this.elementInfos = this.elementInfos.filter((info) => info !== elementInfo);
+    this.updateVisibility();
   }
 
   detach(): HTMLMediaElement[];
@@ -122,6 +144,11 @@ export default class RemoteVideoTrack extends RemoteTrack {
     return detachedElements;
   }
 
+  /** @internal */
+  getDecoderImplementation(): string | undefined {
+    return this.prevStats?.decoderImplementation;
+  }
+
   protected monitorReceiver = async () => {
     if (!this.receiver) {
       this._currentBitrate = 0;
@@ -163,6 +190,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
           jitter: v.jitter,
           timestamp: v.timestamp,
           bytesReceived: v.bytesReceived,
+          decoderImplementation: v.decoderImplementation,
         };
       }
     });
@@ -170,26 +198,16 @@ export default class RemoteVideoTrack extends RemoteTrack {
   }
 
   private stopObservingElement(element: HTMLMediaElement) {
-    getIntersectionObserver()?.unobserve(element);
-    getResizeObserver()?.unobserve(element);
+    const stopElementInfos = this.elementInfos.filter((info) => info.element === element);
+    for (const info of stopElementInfos) {
+      info.stopObserving();
+    }
     this.elementInfos = this.elementInfos.filter((info) => info.element !== element);
   }
 
-  private handleVisibilityChanged = (entry: IntersectionObserverEntry) => {
-    const { target, isIntersecting } = entry;
-    const elementInfo = this.elementInfos.find((info) => info.element === target);
-    if (elementInfo) {
-      elementInfo.visible = isIntersecting;
-      elementInfo.visibilityChangedAt = Date.now();
-    }
-    this.updateVisibility();
-  };
-
   protected async handleAppVisibilityChanged() {
     await super.handleAppVisibilityChanged();
     if (!this.isAdaptiveStream) return;
-    // on desktop don't pause when tab is backgrounded
-    if (!isMobile()) return;
     this.updateVisibility();
   }
 
@@ -202,7 +220,12 @@ export default class RemoteVideoTrack extends RemoteTrack {
       (prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
       0,
     );
-    const isVisible = this.elementInfos.some((info) => info.visible) && !this.isInBackground;
+
+    const backgroundPause =
+      this.adaptiveStreamSettings?.pauseVideoInBackground ?? true // default to true
+        ? this.isInBackground
+        : false;
+    const isVisible = this.elementInfos.some((info) => info.visible) && !backgroundPause;
 
     if (this.lastVisible === isVisible) {
       return;
@@ -226,8 +249,8 @@ export default class RemoteVideoTrack extends RemoteTrack {
     for (const info of this.elementInfos) {
       const pixelDensity = this.adaptiveStreamSettings?.pixelDensity ?? 1;
       const pixelDensityValue = pixelDensity === 'screen' ? window.devicePixelRatio : pixelDensity;
-      const currentElementWidth = info.element.clientWidth * pixelDensityValue;
-      const currentElementHeight = info.element.clientHeight * pixelDensityValue;
+      const currentElementWidth = info.width() * pixelDensityValue;
+      const currentElementHeight = info.height() * pixelDensityValue;
       if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {
         maxWidth = currentElementWidth;
         maxHeight = currentElementHeight;
@@ -242,12 +265,70 @@ export default class RemoteVideoTrack extends RemoteTrack {
       width: maxWidth,
       height: maxHeight,
     };
+
     this.emit(TrackEvent.VideoDimensionsChanged, this.lastDimensions, this);
   }
 }
 
-interface ElementInfo {
+export interface ElementInfo {
+  element: object;
+  width(): number;
+  height(): number;
+  visible: boolean;
+  visibilityChangedAt: number | undefined;
+
+  handleResize?: () => void;
+  handleVisibilityChanged?: () => void;
+  observe(): void;
+  stopObserving(): void;
+}
+
+class HTMLElementInfo implements ElementInfo {
   element: HTMLMediaElement;
+
   visible: boolean;
-  visibilityChangedAt?: number;
+
+  visibilityChangedAt: number | undefined;
+
+  handleResize?: () => void;
+
+  handleVisibilityChanged?: () => void;
+
+  constructor(element: HTMLMediaElement, visible: boolean = false) {
+    this.element = element;
+    this.visible = visible;
+    this.visibilityChangedAt = 0;
+  }
+
+  width(): number {
+    return this.element.clientWidth;
+  }
+
+  height(): number {
+    return this.element.clientWidth;
+  }
+
+  observe() {
+    (this.element as ObservableMediaElement).handleResize = () => {
+      this.handleResize?.();
+    };
+    (this.element as ObservableMediaElement).handleVisibilityChanged = this.onVisibilityChanged;
+
+    getIntersectionObserver().observe(this.element);
+    getResizeObserver().observe(this.element);
+  }
+
+  private onVisibilityChanged = (entry: IntersectionObserverEntry) => {
+    const { target, isIntersecting } = entry;
+    if (target === this.element) {
+      this.visible = isIntersecting;
+      this.visibilityChangedAt = Date.now();
+      this.handleVisibilityChanged?.();
+    }
+  };
+
+  stopObserving() {
+    getIntersectionObserver()?.unobserve(this.element);
+    getResizeObserver()?.unobserve(this.element);
+  }
 }

package/src/room/track/Track.ts

@@ -5,6 +5,8 @@ import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc';
 import { TrackEvent } from '../events';
 import { isFireFox, isSafari, isWeb } from '../utils';
 
+const BACKGROUND_REACTION_DELAY = 5000;
+
 // keep old audio elements when detached, we would re-use them since on iOS
 // Safari tracks which audio elements have been "blessed" by the user.
 const recycledElements: Array<HTMLAudioElement> = [];
@@ -28,10 +30,17 @@ export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEve
    */
   mediaStream?: MediaStream;
 
+  /**
+   * indicates current state of stream
+   */
+  streamState: Track.StreamState = Track.StreamState.Active;
+
   protected _mediaStreamTrack: MediaStreamTrack;
 
   protected isInBackground: boolean;
 
+  private backgroundTimeout: ReturnType<typeof setTimeout> | undefined;
+
   protected _currentBitrate: number = 0;
 
   protected constructor(mediaTrack: MediaStreamTrack, kind: Track.Kind) {
@@ -179,8 +188,20 @@ export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEve
     }
   }
 
-  appVisibilityChangedListener = () => {
-    this.handleAppVisibilityChanged();
+  protected appVisibilityChangedListener = () => {
+    if (this.backgroundTimeout) {
+      clearTimeout(this.backgroundTimeout);
+    }
+    // delay app visibility update if it goes to hidden
+    // update immediately if it comes back to focus
+    if (document.visibilityState === 'hidden') {
+      this.backgroundTimeout = setTimeout(
+        () => this.handleAppVisibilityChanged(),
+        BACKGROUND_REACTION_DELAY,
+      );
+    } else {
+      this.handleAppVisibilityChanged();
+    }
   };
 
   protected async handleAppVisibilityChanged() {

package/src/room/track/create.ts

@@ -89,7 +89,7 @@ export async function createLocalScreenTracks(
     options = {};
   }
   if (options.resolution === undefined) {
-    options.resolution = VideoPresets.fhd.resolution;
+    options.resolution = VideoPresets.h1080.resolution;
   }
 
   let videoConstraints: MediaTrackConstraints | boolean = true;