livekit-client 1.1.0 → 1.1.3

package/dist/livekit-client.esm.mjs

@@ -7049,7 +7049,8 @@ function createBaseAddTrackRequest() {
     disableDtx: false,
     source: 0,
     layers: [],
-    simulcastCodecs: []
+    simulcastCodecs: [],
+    sid: ''
   };
 }
 
@@ -7097,6 +7098,10 @@ const AddTrackRequest = {
       SimulcastCodec.encode(v, writer.uint32(82).fork()).ldelim();
     }
 
+    if (message.sid !== '') {
+      writer.uint32(90).string(message.sid);
+    }
+
     return writer;
   },
 
@@ -7149,6 +7154,10 @@ const AddTrackRequest = {
           message.simulcastCodecs.push(SimulcastCodec.decode(reader, reader.uint32()));
           break;
 
+        case 11:
+          message.sid = reader.string();
+          break;
+
         default:
           reader.skipType(tag & 7);
           break;
@@ -7169,7 +7178,8 @@ const AddTrackRequest = {
       disableDtx: isSet(object.disableDtx) ? Boolean(object.disableDtx) : false,
       source: isSet(object.source) ? trackSourceFromJSON(object.source) : 0,
       layers: Array.isArray(object === null || object === void 0 ? void 0 : object.layers) ? object.layers.map(e => VideoLayer.fromJSON(e)) : [],
-      simulcastCodecs: Array.isArray(object === null || object === void 0 ? void 0 : object.simulcastCodecs) ? object.simulcastCodecs.map(e => SimulcastCodec.fromJSON(e)) : []
+      simulcastCodecs: Array.isArray(object === null || object === void 0 ? void 0 : object.simulcastCodecs) ? object.simulcastCodecs.map(e => SimulcastCodec.fromJSON(e)) : [],
+      sid: isSet(object.sid) ? String(object.sid) : ''
     };
   },
 
@@ -7196,11 +7206,12 @@ const AddTrackRequest = {
       obj.simulcastCodecs = [];
     }
 
+    message.sid !== undefined && (obj.sid = message.sid);
     return obj;
   },
 
   fromPartial(object) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l;
 
     const message = createBaseAddTrackRequest();
     message.cid = (_a = object.cid) !== null && _a !== void 0 ? _a : '';
@@ -7213,6 +7224,7 @@ const AddTrackRequest = {
     message.source = (_h = object.source) !== null && _h !== void 0 ? _h : 0;
     message.layers = ((_j = object.layers) === null || _j === void 0 ? void 0 : _j.map(e => VideoLayer.fromPartial(e))) || [];
     message.simulcastCodecs = ((_k = object.simulcastCodecs) === null || _k === void 0 ? void 0 : _k.map(e => SimulcastCodec.fromPartial(e))) || [];
+    message.sid = (_l = object.sid) !== null && _l !== void 0 ? _l : '';
     return message;
   }
 
@@ -10058,7 +10070,7 @@ function computeBitrate(currentStats, prevStats) {
   return (bytesNow - bytesPrev) * 8 * 1000 / (currentStats.timestamp - prevStats.timestamp);
 }
 
-var version$1 = "1.1.0";
+var version$1 = "1.1.3";
 
 const version = version$1;
 const protocolVersion = 8;
@@ -10130,9 +10142,10 @@ function getEmptyVideoStreamTrack() {
   var _a;
 
   if (!emptyVideoStreamTrack) {
-    const canvas = document.createElement('canvas');
-    canvas.width = 2;
-    canvas.height = 2;
+    const canvas = document.createElement('canvas'); // the canvas size is set to 16, because electron apps seem to fail with smaller values
+
+    canvas.width = 16;
+    canvas.height = 16;
     (_a = canvas.getContext('2d')) === null || _a === void 0 ? void 0 : _a.fillRect(0, 0, canvas.width, canvas.height); // @ts-ignore
 
     const emptyStream = canvas.captureStream();
@@ -11514,6 +11527,7 @@ class LocalAudioTrack extends LocalTrack {
 
 }
 
+const refreshSubscribedCodecAfterNewCodec = 5000;
 class LocalVideoTrack extends LocalTrack {
   constructor(mediaTrack, constraints) {
     super(mediaTrack, Track.Kind.Video, constraints);
@@ -11730,7 +11744,14 @@ class LocalVideoTrack extends LocalTrack {
       return;
     }
 
-    simulcastCodecInfo.sender = sender;
+    simulcastCodecInfo.sender = sender; // browser will reenable disabled codec/layers after new codec has been published,
+    // so refresh subscribedCodecs after publish a new codec
+
+    setTimeout(() => {
+      if (this.subscribedCodecs) {
+        this.setPublishingCodecs(this.subscribedCodecs);
+      }
+    }, refreshSubscribedCodecAfterNewCodec);
   }
   /**
    * @internal
@@ -11739,25 +11760,41 @@ class LocalVideoTrack extends LocalTrack {
 
 
   async setPublishingCodecs(codecs) {
-    livekitLogger.debug('setting publishing codecs', codecs);
+    livekitLogger.debug('setting publishing codecs', {
+      codecs,
+      currentCodec: this.codec
+    }); // only enable simulcast codec for preference codec setted
+
+    if (!this.codec && codecs.length > 0) {
+      await this.setPublishingLayers(codecs[0].qualities);
+      return [];
+    }
+
+    this.subscribedCodecs = codecs;
+    const newCodecs = [];
 
     for await (const codec of codecs) {
-      if (this.codec === codec.codec) {
+      if (!this.codec || this.codec === codec.codec) {
         await this.setPublishingLayers(codec.qualities);
       } else {
         const simulcastCodecInfo = this.simulcastCodecs.get(codec.codec);
         livekitLogger.debug("try setPublishingCodec for ".concat(codec.codec), simulcastCodecInfo);
 
         if (!simulcastCodecInfo || !simulcastCodecInfo.sender) {
-          return;
-        }
-
-        if (simulcastCodecInfo.encodings) {
+          for (const q of codec.qualities) {
+            if (q.enabled) {
+              newCodecs.push(codec.codec);
+              break;
+            }
+          }
+        } else if (simulcastCodecInfo.encodings) {
           livekitLogger.debug("try setPublishingLayersForSender ".concat(codec.codec));
           await setPublishingLayersForSender(simulcastCodecInfo.sender, simulcastCodecInfo.encodings, codec.qualities);
         }
       }
     }
+
+    return newCodecs;
   }
   /**
    * @internal
@@ -12188,6 +12225,8 @@ class RemoteVideoTrack extends RemoteTrack {
 
       this.debouncedHandleResize();
       this.updateVisibility();
+    } else {
+      livekitLogger.warn('visibility resize observer not triggered');
     }
   }
   /**
@@ -12336,9 +12375,7 @@ class RemoteVideoTrack extends RemoteTrack {
 }
 
 class HTMLElementInfo {
-  constructor(element) {
-    let visible = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;
-
+  constructor(element, visible) {
     this.onVisibilityChanged = entry => {
       var _a;
 
@@ -12355,7 +12392,7 @@ class HTMLElementInfo {
     };
 
     this.element = element;
-    this.visible = visible;
+    this.visible = visible !== null && visible !== void 0 ? visible : isElementInViewport(element);
     this.visibilityChangedAt = 0;
   }
 
@@ -12364,7 +12401,7 @@ class HTMLElementInfo {
   }
 
   height() {
-    return this.element.clientWidth;
+    return this.element.clientHeight;
   }
 
   observe() {
@@ -12386,6 +12423,29 @@ class HTMLElementInfo {
     (_b = getResizeObserver()) === null || _b === void 0 ? void 0 : _b.unobserve(this.element);
   }
 
+} // does not account for occlusion by other elements
+
+
+function isElementInViewport(el) {
+  let top = el.offsetTop;
+  let left = el.offsetLeft;
+  const width = el.offsetWidth;
+  const height = el.offsetHeight;
+  const {
+    hidden
+  } = el;
+  const {
+    opacity,
+    display
+  } = getComputedStyle(el);
+
+  while (el.offsetParent) {
+    el = el.offsetParent;
+    top += el.offsetTop;
+    left += el.offsetLeft;
+  }
+
+  return top < window.pageYOffset + window.innerHeight && left < window.pageXOffset + window.innerWidth && top + height > window.pageYOffset && left + width > window.pageXOffset && !hidden && (opacity !== '' ? parseFloat(opacity) > 0 : true) && display !== 'none';
 }
 
 class TrackPublication extends events.exports.EventEmitter {
@@ -13358,9 +13418,7 @@ class RemoteParticipant extends Participant {
 
 
   static fromParticipantInfo(signalClient, pi) {
-    const rp = new RemoteParticipant(signalClient, pi.sid, pi.identity);
-    rp.updateInfo(pi);
-    return rp;
+    return new RemoteParticipant(signalClient, pi.sid, pi.identity);
   }
 
   addTrackPublication(publication) {
@@ -13501,7 +13559,6 @@ class RemoteParticipant extends Participant {
 
 
   updateInfo(info) {
-    const alreadyHasMetadata = this.hasMetadata;
     super.updateInfo(info); // we are getting a list of all available tracks, reconcile in here
     // and send out events for changes
     // reconcile track publications, publish events only if metadata is already there
@@ -13529,14 +13586,11 @@ class RemoteParticipant extends Participant {
       }
 
       validTracks.set(ti.sid, publication);
-    }); // send new tracks
-
-    if (alreadyHasMetadata) {
-      newTracks.forEach(publication => {
-        this.emit(ParticipantEvent.TrackPublished, publication);
-      });
-    } // detect removed tracks
+    }); // always emit events for new publications, Room will not forward them unless it's ready
 
+    newTracks.forEach(publication => {
+      this.emit(ParticipantEvent.TrackPublished, publication);
+    }); // detect removed tracks
 
     this.tracks.forEach(publication => {
       if (!validTracks.has(publication.trackSid)) {
@@ -13605,7 +13659,7 @@ class RemoteParticipant extends Participant {
 
 }
 
-const compatibleCodecForSVC = 'vp8';
+const compatibleCodec = 'vp8';
 class LocalParticipant extends Participant {
   /** @internal */
   constructor(sid, identity, engine, options) {
@@ -13654,8 +13708,8 @@ class LocalParticipant extends Participant {
       this.onTrackMuted(track, track.isMuted);
     };
 
-    this.handleSubscribedQualityUpdate = update => {
-      var _a, _b, _c;
+    this.handleSubscribedQualityUpdate = async update => {
+      var _a, _b;
 
       if (!((_a = this.roomOptions) === null || _a === void 0 ? void 0 : _a.dynacast)) {
         return;
@@ -13672,9 +13726,18 @@ class LocalParticipant extends Participant {
       }
 
       if (update.subscribedCodecs.length > 0) {
-        (_b = pub.videoTrack) === null || _b === void 0 ? void 0 : _b.setPublishingCodecs(update.subscribedCodecs);
+        if (!pub.videoTrack) {
+          return;
+        }
+
+        const newCodecs = await pub.videoTrack.setPublishingCodecs(update.subscribedCodecs);
+
+        for await (const codec of newCodecs) {
+          livekitLogger.debug("publish ".concat(codec, " for ").concat(pub.videoTrack.sid));
+          await this.publishAdditionalCodecForTrack(pub.videoTrack, codec, pub.options);
+        }
       } else if (update.subscribedQualities.length > 0) {
-        (_c = pub.videoTrack) === null || _c === void 0 ? void 0 : _c.setPublishingLayers(update.subscribedQualities);
+        (_b = pub.videoTrack) === null || _b === void 0 ? void 0 : _b.setPublishingLayers(update.subscribedQualities);
       }
     };
 
@@ -14088,7 +14151,6 @@ class LocalParticipant extends Participant {
 
     let encodings;
     let simEncodings;
-    let simulcastTracks;
 
     if (track.kind === Track.Kind.Video) {
       // TODO: support react native, which doesn't expose getSettings
@@ -14099,26 +14161,30 @@ class LocalParticipant extends Participant {
       req.width = width !== null && width !== void 0 ? width : 0;
       req.height = height !== null && height !== void 0 ? height : 0; // for svc codecs, disable simulcast and use vp8 for backup codec
 
-      if (track instanceof LocalVideoTrack && ((opts === null || opts === void 0 ? void 0 : opts.videoCodec) === 'vp9' || (opts === null || opts === void 0 ? void 0 : opts.videoCodec) === 'av1')) {
-        // set scalabilityMode to 'L3T3' by default
-        opts.scalabilityMode = (_g = opts.scalabilityMode) !== null && _g !== void 0 ? _g : 'L3T3'; // add backup codec track
-
-        const simOpts = _objectSpread2({}, opts);
-
-        simOpts.simulcast = true;
-        simOpts.scalabilityMode = undefined;
-        simEncodings = computeVideoEncodings(track.source === Track.Source.ScreenShare, width, height, simOpts);
-        const simulcastTrack = track.addSimulcastTrack(compatibleCodecForSVC, simEncodings);
-        simulcastTracks = [simulcastTrack];
-        req.simulcastCodecs = [{
-          codec: opts.videoCodec,
-          cid: track.mediaStreamTrack.id,
-          enableSimulcastLayers: true
-        }, {
-          codec: simulcastTrack.codec,
-          cid: simulcastTrack.mediaStreamTrack.id,
-          enableSimulcastLayers: true
-        }];
+      if (track instanceof LocalVideoTrack) {
+        if ((opts === null || opts === void 0 ? void 0 : opts.videoCodec) === 'vp9' || (opts === null || opts === void 0 ? void 0 : opts.videoCodec) === 'av1') {
+          // set scalabilityMode to 'L3T3' by default
+          opts.scalabilityMode = (_g = opts.scalabilityMode) !== null && _g !== void 0 ? _g : 'L3T3'; // add backup codec track
+
+          const simOpts = _objectSpread2({}, opts);
+
+          simOpts.simulcast = true;
+          simOpts.scalabilityMode = undefined;
+          simEncodings = computeVideoEncodings(track.source === Track.Source.ScreenShare, width, height, simOpts);
+        } // set vp8 codec as backup for any other codecs
+
+
+        if (opts.videoCodec && opts.videoCodec !== 'vp8') {
+          req.simulcastCodecs = [{
+            codec: opts.videoCodec,
+            cid: track.mediaStreamTrack.id,
+            enableSimulcastLayers: true
+          }, {
+            codec: compatibleCodec,
+            cid: '',
+            enableSimulcastLayers: true
+          }];
+        }
       }
 
       encodings = computeVideoEncodings(track.source === Track.Source.ScreenShare, width, height, opts);
@@ -14163,24 +14229,6 @@ class LocalParticipant extends Participant {
       track.codec = opts.videoCodec;
     }
 
-    const localTrack = track;
-
-    if (simulcastTracks) {
-      for await (const simulcastTrack of simulcastTracks) {
-        const simTransceiverInit = {
-          direction: 'sendonly'
-        };
-
-        if (simulcastTrack.encodings) {
-          simTransceiverInit.sendEncodings = simulcastTrack.encodings;
-        }
-
-        const simTransceiver = await this.engine.publisher.pc.addTransceiver(simulcastTrack.mediaStreamTrack, simTransceiverInit);
-        this.setPreferredCodec(simTransceiver, localTrack.kind, simulcastTrack.codec);
-        localTrack.setSimulcastTrackSender(simulcastTrack.codec, simTransceiver.sender);
-      }
-    }
-
     this.engine.negotiate(); // store RTPSender
 
     track.sender = transceiver.sender;
@@ -14196,6 +14244,86 @@ class LocalParticipant extends Participant {
     this.emit(ParticipantEvent.LocalTrackPublished, publication);
     return publication;
   }
+  /** @internal
+   * publish additional codec to existing track
+   */
+
+
+  async publishAdditionalCodecForTrack(track, videoCodec, options) {
+    var _a, _b, _c, _d, _e;
+
+    const opts = _objectSpread2(_objectSpread2({}, (_a = this.roomOptions) === null || _a === void 0 ? void 0 : _a.publishDefaults), options); // clear scalabilityMode setting for backup codec
+
+
+    opts.scalabilityMode = undefined;
+    opts.videoCodec = videoCodec; // is it not published? if so skip
+
+    let existingPublication;
+    this.tracks.forEach(publication => {
+      if (!publication.track) {
+        return;
+      }
+
+      if (publication.track === track) {
+        existingPublication = publication;
+      }
+    });
+
+    if (!existingPublication) {
+      throw new TrackInvalidError('track is not published');
+    }
+
+    if (!(track instanceof LocalVideoTrack)) {
+      throw new TrackInvalidError('track is not a video track');
+    }
+
+    const settings = track.mediaStreamTrack.getSettings();
+    const width = (_b = settings.width) !== null && _b !== void 0 ? _b : (_c = track.dimensions) === null || _c === void 0 ? void 0 : _c.width;
+    const height = (_d = settings.height) !== null && _d !== void 0 ? _d : (_e = track.dimensions) === null || _e === void 0 ? void 0 : _e.height;
+    const encodings = computeVideoEncodings(track.source === Track.Source.ScreenShare, width, height, opts);
+    const simulcastTrack = track.addSimulcastTrack(opts.videoCodec, encodings);
+    const req = AddTrackRequest.fromPartial({
+      cid: simulcastTrack.mediaStreamTrack.id,
+      type: Track.kindToProto(track.kind),
+      muted: track.isMuted,
+      source: Track.sourceToProto(track.source),
+      sid: track.sid,
+      simulcastCodecs: [{
+        codec: opts.videoCodec,
+        cid: simulcastTrack.mediaStreamTrack.id,
+        enableSimulcastLayers: opts.simulcast
+      }]
+    });
+    req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
+
+    if (!this.engine || this.engine.isClosed) {
+      throw new UnexpectedConnectionState('cannot publish track when not connected');
+    }
+
+    const ti = await this.engine.addTrack(req);
+
+    if (!this.engine.publisher) {
+      throw new UnexpectedConnectionState('publisher is closed');
+    }
+
+    const transceiverInit = {
+      direction: 'sendonly'
+    };
+
+    if (encodings) {
+      transceiverInit.sendEncodings = encodings;
+    } // addTransceiver for react-native is async. web is synchronous, but await won't effect it.
+
+
+    const transceiver = await this.engine.publisher.pc.addTransceiver(simulcastTrack.mediaStreamTrack, transceiverInit);
+    this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
+    track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
+    this.engine.negotiate();
+    livekitLogger.debug("published ".concat(opts.videoCodec, " for track ").concat(track.sid), {
+      encodings,
+      trackInfo: ti
+    });
+  }
 
   unpublishTrack(track, stopOnUnpublish) {
     var _a, _b; // look through all published tracks to find the right ones
@@ -19266,8 +19394,6 @@ class Room extends events.exports.EventEmitter {
   constructor(options) {
     var _this;
 
-    var _a, _b;
-
     super();
     _this = this;
     this.state = ConnectionState.Disconnected;
@@ -19692,14 +19818,6 @@ class Room extends events.exports.EventEmitter {
     this.participants = new Map();
     this.identityToSid = new Map();
     this.options = options || {};
-
-    switch ((_b = (_a = this.options) === null || _a === void 0 ? void 0 : _a.publishDefaults) === null || _b === void 0 ? void 0 : _b.videoCodec) {
-      case 'av1':
-      case 'vp9':
-        this.options.publishDefaults.simulcast = undefined;
-        break;
-    }
-
     this.options.audioCaptureDefaults = _objectSpread2(_objectSpread2({}, audioDefaults), options === null || options === void 0 ? void 0 : options.audioCaptureDefaults);
     this.options.videoCaptureDefaults = _objectSpread2(_objectSpread2({}, videoDefaults), options === null || options === void 0 ? void 0 : options.videoCaptureDefaults);
     this.options.publishDefaults = _objectSpread2(_objectSpread2({}, publishDefaults), options === null || options === void 0 ? void 0 : options.publishDefaults);
@@ -19914,6 +20032,22 @@ class Room extends events.exports.EventEmitter {
   }
 
   onTrackAdded(mediaTrack, stream, receiver) {
+    // don't fire onSubscribed when connecting
+    // WebRTC fires onTrack as soon as setRemoteDescription is called on the offer
+    // at that time, ICE connectivity has not been established so the track is not
+    // technically subscribed.
+    // We'll defer these events until when the room is connected or eventually disconnected.
+    if (this.state === ConnectionState.Connecting || this.state === ConnectionState.Reconnecting) {
+      setTimeout(() => {
+        this.onTrackAdded(mediaTrack, stream, receiver);
+      }, 10);
+      return;
+    }
+
+    if (this.state === ConnectionState.Disconnected) {
+      livekitLogger.warn('skipping incoming track after Room disconnected');
+    }
+
     const parts = unpackStreamId(stream.id);
     const participantId = parts[0];
     let trackId = parts[1];
@@ -20030,7 +20164,7 @@ class Room extends events.exports.EventEmitter {
 
 
     participant.on(ParticipantEvent.TrackPublished, trackPublication => {
-      this.emit(RoomEvent.TrackPublished, trackPublication, participant);
+      this.emitWhenConnected(RoomEvent.TrackPublished, trackPublication, participant);
     }).on(ParticipantEvent.TrackSubscribed, (track, publication) => {
       // monitor playback status
       if (track.kind === Track.Kind.Audio) {
@@ -20040,22 +20174,27 @@ class Room extends events.exports.EventEmitter {
 
       this.emit(RoomEvent.TrackSubscribed, track, publication, participant);
     }).on(ParticipantEvent.TrackUnpublished, publication => {
-      this.emit(RoomEvent.TrackUnpublished, publication, participant);
+      this.emitWhenConnected(RoomEvent.TrackUnpublished, publication, participant);
     }).on(ParticipantEvent.TrackUnsubscribed, (track, publication) => {
       this.emit(RoomEvent.TrackUnsubscribed, track, publication, participant);
     }).on(ParticipantEvent.TrackSubscriptionFailed, sid => {
       this.emit(RoomEvent.TrackSubscriptionFailed, sid, participant);
     }).on(ParticipantEvent.TrackMuted, pub => {
-      this.emit(RoomEvent.TrackMuted, pub, participant);
+      this.emitWhenConnected(RoomEvent.TrackMuted, pub, participant);
     }).on(ParticipantEvent.TrackUnmuted, pub => {
-      this.emit(RoomEvent.TrackUnmuted, pub, participant);
+      this.emitWhenConnected(RoomEvent.TrackUnmuted, pub, participant);
     }).on(ParticipantEvent.ParticipantMetadataChanged, metadata => {
-      this.emit(RoomEvent.ParticipantMetadataChanged, metadata, participant);
+      this.emitWhenConnected(RoomEvent.ParticipantMetadataChanged, metadata, participant);
     }).on(ParticipantEvent.ConnectionQualityChanged, quality => {
-      this.emit(RoomEvent.ConnectionQualityChanged, quality, participant);
+      this.emitWhenConnected(RoomEvent.ConnectionQualityChanged, quality, participant);
     }).on(ParticipantEvent.ParticipantPermissionsChanged, prevPermissions => {
-      this.emit(RoomEvent.ParticipantPermissionsChanged, prevPermissions, participant);
-    });
+      this.emitWhenConnected(RoomEvent.ParticipantPermissionsChanged, prevPermissions, participant);
+    }); // update info at the end after callbacks have been set up
+
+    if (info) {
+      participant.updateInfo(info);
+    }
+
     return participant;
   }
 
@@ -20121,12 +20260,24 @@ class Room extends events.exports.EventEmitter {
     this.state = state;
     this.emit(RoomEvent.ConnectionStateChanged, this.state);
     return true;
+  }
+
+  emitWhenConnected(event) {
+    if (this.state === ConnectionState.Connected) {
+      for (var _len = arguments.length, args = new Array(_len > 1 ? _len - 1 : 0), _key = 1; _key < _len; _key++) {
+        args[_key - 1] = arguments[_key];
+      }
+
+      return this.emit(event, ...args);
+    }
+
+    return false;
   } // /** @internal */
 
 
   emit(event) {
-    for (var _len = arguments.length, args = new Array(_len > 1 ? _len - 1 : 0), _key = 1; _key < _len; _key++) {
-      args[_key - 1] = arguments[_key];
+    for (var _len2 = arguments.length, args = new Array(_len2 > 1 ? _len2 - 1 : 0), _key2 = 1; _key2 < _len2; _key2++) {
+      args[_key2 - 1] = arguments[_key2];
     }
 
     livekitLogger.debug('room event', {