livekit-client 2.5.0 → 2.5.1

package/README.md

@@ -75,6 +75,10 @@ import {
   RemoteTrackPublication,
   Room,
   RoomEvent,
+  VideoPresets,
+  Track,
+  LocalTrackPublication,
+  LocalParticipant
 } from 'livekit-client';
 
 // creates a new room with options

package/dist/livekit-client.esm.mjs

@@ -3397,6 +3397,122 @@ target => {
   }
 });
 
+// @generated by protoc-gen-es v1.10.0 with parameter "target=js+dts"
+// @generated from file livekit_metrics.proto (package livekit, syntax proto3)
+/* eslint-disable */
+// @ts-nocheck
+
+
+/**
+ * @generated from message livekit.MetricsBatch
+ */
+const MetricsBatch = /*@__PURE__*/proto3.makeMessageType("livekit.MetricsBatch", () => [{
+  no: 1,
+  name: "str_data",
+  kind: "scalar",
+  T: 9 /* ScalarType.STRING */,
+  repeated: true
+}, {
+  no: 2,
+  name: "time_series",
+  kind: "message",
+  T: TimeSeriesMetric,
+  repeated: true
+}, {
+  no: 3,
+  name: "events",
+  kind: "message",
+  T: EventMetric,
+  repeated: true
+}]);
+
+/**
+ * @generated from message livekit.TimeSeriesMetric
+ */
+const TimeSeriesMetric = /*@__PURE__*/proto3.makeMessageType("livekit.TimeSeriesMetric", () => [{
+  no: 1,
+  name: "label",
+  kind: "scalar",
+  T: 13 /* ScalarType.UINT32 */
+}, {
+  no: 2,
+  name: "participant_identity",
+  kind: "scalar",
+  T: 13 /* ScalarType.UINT32 */
+}, {
+  no: 3,
+  name: "track_sid",
+  kind: "scalar",
+  T: 13 /* ScalarType.UINT32 */
+}, {
+  no: 4,
+  name: "start_timestamp",
+  kind: "scalar",
+  T: 3 /* ScalarType.INT64 */
+}, {
+  no: 5,
+  name: "end_timestamp",
+  kind: "scalar",
+  T: 3 /* ScalarType.INT64 */
+}, {
+  no: 6,
+  name: "samples",
+  kind: "message",
+  T: MetricSample,
+  repeated: true
+}]);
+
+/**
+ * @generated from message livekit.MetricSample
+ */
+const MetricSample = /*@__PURE__*/proto3.makeMessageType("livekit.MetricSample", () => [{
+  no: 1,
+  name: "timestamp",
+  kind: "scalar",
+  T: 3 /* ScalarType.INT64 */
+}, {
+  no: 2,
+  name: "value",
+  kind: "scalar",
+  T: 2 /* ScalarType.FLOAT */
+}]);
+
+/**
+ * @generated from message livekit.EventMetric
+ */
+const EventMetric = /*@__PURE__*/proto3.makeMessageType("livekit.EventMetric", () => [{
+  no: 1,
+  name: "label",
+  kind: "scalar",
+  T: 13 /* ScalarType.UINT32 */
+}, {
+  no: 2,
+  name: "participant_identity",
+  kind: "scalar",
+  T: 13 /* ScalarType.UINT32 */
+}, {
+  no: 3,
+  name: "track_sid",
+  kind: "scalar",
+  T: 13 /* ScalarType.UINT32 */
+}, {
+  no: 4,
+  name: "start_timestamp",
+  kind: "scalar",
+  T: 3 /* ScalarType.INT64 */
+}, {
+  no: 5,
+  name: "end_timestamp",
+  kind: "scalar",
+  T: 3 /* ScalarType.INT64 */,
+  opt: true
+}, {
+  no: 6,
+  name: "metadata",
+  kind: "scalar",
+  T: 9 /* ScalarType.STRING */
+}]);
+
 // Copyright 2023 LiveKit, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -3719,6 +3835,11 @@ const ParticipantPermission = /*@__PURE__*/proto3.makeMessageType("livekit.Parti
   name: "agent",
   kind: "scalar",
   T: 8 /* ScalarType.BOOL */
+}, {
+  no: 12,
+  name: "can_subscribe_metrics",
+  kind: "scalar",
+  T: 8 /* ScalarType.BOOL */
 }]);
 
 /**
@@ -4058,6 +4179,12 @@ const DataPacket = /*@__PURE__*/proto3.makeMessageType("livekit.DataPacket", ()
   kind: "message",
   T: Transcription,
   oneof: "value"
+}, {
+  no: 8,
+  name: "metrics",
+  kind: "message",
+  T: MetricsBatch,
+  oneof: "value"
 }]);
 
 /**
@@ -4964,6 +5091,17 @@ const JoinResponse = /*@__PURE__*/proto3.makeMessageType("livekit.JoinResponse",
   name: "sif_trailer",
   kind: "scalar",
   T: 12 /* ScalarType.BYTES */
+}, {
+  no: 14,
+  name: "enabled_publish_codecs",
+  kind: "message",
+  T: Codec,
+  repeated: true
+}, {
+  no: 15,
+  name: "fast_publish",
+  kind: "scalar",
+  T: 8 /* ScalarType.BOOL */
 }]);
 
 /**
@@ -10182,6 +10320,10 @@ var RoomEvent;
    * args: (kind: MediaDeviceKind, deviceId: string)
    */
   RoomEvent["ActiveDeviceChanged"] = "activeDeviceChanged";
+  /**
+   * fired when the first remote participant has subscribed to the localParticipant's track
+   */
+  RoomEvent["LocalTrackSubscribed"] = "localTrackSubscribed";
 })(RoomEvent || (RoomEvent = {}));
 var ParticipantEvent;
 (function (ParticipantEvent) {
@@ -10345,6 +10487,10 @@ var ParticipantEvent;
    * When a participant's attributes changed, this event will be emitted with the changed attributes
    */
   ParticipantEvent["AttributesChanged"] = "attributesChanged";
+  /**
+   * fired on local participant only, when the first remote participant has subscribed to the track specified in the payload
+   */
+  ParticipantEvent["LocalTrackSubscribed"] = "localTrackSubscribed";
 })(ParticipantEvent || (ParticipantEvent = {}));
 /** @internal */
 var EngineEvent;
@@ -10568,7 +10714,7 @@ function getOSVersion(ua) {
   return ua.includes('mac os') ? getMatch(/\(.+?(\d+_\d+(:?_\d+)?)/, ua, 1).replace(/_/g, '.') : undefined;
 }
 
-var version$1 = "2.5.0";
+var version$1 = "2.5.1";
 
 const version = version$1;
 const protocolVersion = 15;
@@ -10579,12 +10725,12 @@ const protocolVersion = 15;
  * that the timer fires on time.
  */
 class CriticalTimers {}
-// eslint-disable-next-line @typescript-eslint/no-implied-eval
 CriticalTimers.setTimeout = function () {
   return setTimeout(...arguments);
 };
+CriticalTimers.setInterval =
 // eslint-disable-next-line @typescript-eslint/no-implied-eval
-CriticalTimers.setInterval = function () {
+function () {
   return setInterval(...arguments);
 };
 CriticalTimers.clearTimeout = function () {
@@ -11696,7 +11842,7 @@ function toHttpUrl(url) {
   }
   return url;
 }
-function extractTranscriptionSegments(transcription) {
+function extractTranscriptionSegments(transcription, firstReceivedTimesMap) {
   return transcription.segments.map(_ref => {
     let {
       id,
@@ -11706,13 +11852,23 @@ function extractTranscriptionSegments(transcription) {
       endTime,
       final
     } = _ref;
+    var _a;
+    const firstReceivedTime = (_a = firstReceivedTimesMap.get(id)) !== null && _a !== void 0 ? _a : Date.now();
+    const lastReceivedTime = Date.now();
+    if (final) {
+      firstReceivedTimesMap.delete(id);
+    } else {
+      firstReceivedTimesMap.set(id, firstReceivedTime);
+    }
     return {
       id,
       text,
       startTime: Number.parseInt(startTime.toString()),
       endTime: Number.parseInt(endTime.toString()),
       final,
-      language
+      language,
+      firstReceivedTime,
+      lastReceivedTime
     };
   });
 }
@@ -14013,6 +14169,7 @@ the first few seconds. So we use a 70% of target bitrate here as the start bitra
 eliminate this issue.
 */
 const startBitrateForSVC = 0.7;
+const debounceInterval = 20;
 const PCEvents = {
   NegotiationStarted: 'negotiationStarted',
   NegotiationComplete: 'negotiationComplete',
@@ -14049,7 +14206,7 @@ class PCTransport extends eventsExports.EventEmitter {
           throw e;
         }
       }
-    }), 100);
+    }), debounceInterval);
     this.close = () => {
       if (!this._pc) {
         return;
@@ -16017,7 +16174,7 @@ function videoLayersFromEncodings(width, height, encodings, svc) {
     const bitratesRatio = sm.suffix == 'h' ? 2 : 3;
     for (let i = 0; i < sm.spatial; i += 1) {
       layers.push(new VideoLayer({
-        quality: VideoQuality.HIGH - i,
+        quality: Math.min(VideoQuality.HIGH, sm.spatial - 1) - i,
         width: Math.ceil(width / Math.pow(resRatio, i)),
         height: Math.ceil(height / Math.pow(resRatio, i)),
         bitrate: encodings[0].maxBitrate ? Math.ceil(encodings[0].maxBitrate / Math.pow(bitratesRatio, i)) : 0,
@@ -16276,7 +16433,7 @@ class RTCEngine extends eventsExports.EventEmitter {
           yield this.configure(joinResponse);
         }
         // create offer
-        if (!this.subscriberPrimary) {
+        if (!this.subscriberPrimary || joinResponse.fastPublish) {
           this.negotiate();
         }
         this.clientConfiguration = joinResponse.clientConfiguration;
@@ -18315,6 +18472,7 @@ class LocalParticipant extends Participant {
     this.participantTrackPermissions = [];
     this.allParticipantsAllowedToSubscribe = true;
     this.encryptionType = Encryption_Type.NONE;
+    this.enabledPublishVideoCodecs = [];
     this.handleReconnecting = () => {
       if (!this.reconnectFuture) {
         this.reconnectFuture = new Future();
@@ -18979,7 +19137,7 @@ class LocalParticipant extends Participant {
   }
   publish(track, opts, isStereo) {
     return __awaiter(this, void 0, void 0, function* () {
-      var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
+      var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
       const existingTrackOfSource = Array.from(this.trackPublications.values()).find(publishedTrack => track instanceof LocalTrack && publishedTrack.source === track.source);
       if (existingTrackOfSource && track.source !== Track.Source.Unknown) {
         this.log.info("publishing a second track with the same source: ".concat(track.source), Object.assign(Object.assign({}, this.logContext), getLogContextFromTrack(track)));
@@ -19002,6 +19160,12 @@ class LocalParticipant extends Participant {
       if (opts.videoCodec === undefined) {
         opts.videoCodec = defaultVideoCodec;
       }
+      if (this.enabledPublishVideoCodecs.length > 0) {
+        // fallback to a supported codec if it is not supported
+        if (!this.enabledPublishVideoCodecs.some(c => opts.videoCodec === mimeTypeToVideoCodecString(c.mime))) {
+          opts.videoCodec = mimeTypeToVideoCodecString(this.enabledPublishVideoCodecs[0].mime);
+        }
+      }
       const videoCodec = opts.videoCodec;
       // handle track actions
       track.on(TrackEvent.Muted, this.onTrackMuted);
@@ -19104,25 +19268,75 @@ class LocalParticipant extends Participant {
       if (!this.engine || this.engine.isClosed) {
         throw new UnexpectedConnectionState('cannot publish track when not connected');
       }
-      const ti = yield this.engine.addTrack(req);
-      // server might not support the codec the client has requested, in that case, fallback
-      // to a supported codec
-      let primaryCodecMime;
-      ti.codecs.forEach(codec => {
-        if (primaryCodecMime === undefined) {
-          primaryCodecMime = codec.mimeType;
+      const negotiate = () => __awaiter(this, void 0, void 0, function* () {
+        var _a, _b, _c;
+        if (!this.engine.pcManager) {
+          throw new UnexpectedConnectionState('pcManager is not ready');
+        }
+        track.sender = yield this.engine.createSender(track, opts, encodings);
+        if (track instanceof LocalVideoTrack) {
+          (_a = opts.degradationPreference) !== null && _a !== void 0 ? _a : opts.degradationPreference = getDefaultDegradationPreference(track);
+          track.setDegradationPreference(opts.degradationPreference);
+        }
+        if (encodings) {
+          if (isFireFox() && track.kind === Track.Kind.Audio) {
+            /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
+               livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to
+               publish high quality audio track. But firefox always uses this value as the actual
+               bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
+               So the client need to modify maxaverragebitrates in answer sdp to user provided value to
+               fix the issue.
+             */
+            let trackTransceiver = undefined;
+            for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
+              if (transceiver.sender === track.sender) {
+                trackTransceiver = transceiver;
+                break;
+              }
+            }
+            if (trackTransceiver) {
+              this.engine.pcManager.publisher.setTrackCodecBitrate({
+                transceiver: trackTransceiver,
+                codec: 'opus',
+                maxbr: ((_b = encodings[0]) === null || _b === void 0 ? void 0 : _b.maxBitrate) ? encodings[0].maxBitrate / 1000 : 0
+              });
+            }
+          } else if (track.codec && isSVCCodec(track.codec) && ((_c = encodings[0]) === null || _c === void 0 ? void 0 : _c.maxBitrate)) {
+            this.engine.pcManager.publisher.setTrackCodecBitrate({
+              cid: req.cid,
+              codec: track.codec,
+              maxbr: encodings[0].maxBitrate / 1000
+            });
+          }
         }
+        yield this.engine.negotiate();
       });
-      if (primaryCodecMime && track.kind === Track.Kind.Video) {
-        const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime);
-        if (updatedCodec !== videoCodec) {
-          this.log.debug('falling back to server selected codec', Object.assign(Object.assign(Object.assign({}, this.logContext), getLogContextFromTrack(track)), {
-            codec: updatedCodec
-          }));
-          opts.videoCodec = updatedCodec;
-          // recompute encodings since bitrates/etc could have changed
-          encodings = computeVideoEncodings(track.source === Track.Source.ScreenShare, req.width, req.height, opts);
+      let ti;
+      if (this.enabledPublishVideoCodecs.length > 0) {
+        const rets = yield Promise.all([this.engine.addTrack(req), negotiate()]);
+        ti = rets[0];
+      } else {
+        ti = yield this.engine.addTrack(req);
+        // server might not support the codec the client has requested, in that case, fallback
+        // to a supported codec
+        let primaryCodecMime;
+        ti.codecs.forEach(codec => {
+          if (primaryCodecMime === undefined) {
+            primaryCodecMime = codec.mimeType;
+          }
+        });
+        if (primaryCodecMime && track.kind === Track.Kind.Video) {
+          const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime);
+          if (updatedCodec !== videoCodec) {
+            this.log.debug('falling back to server selected codec', Object.assign(Object.assign(Object.assign({}, this.logContext), getLogContextFromTrack(track)), {
+              codec: updatedCodec
+            }));
+            opts.videoCodec = updatedCodec;
+            // recompute encodings since bitrates/etc could have changed
+            encodings = computeVideoEncodings(track.source === Track.Source.ScreenShare, req.width, req.height, opts);
+          }
         }
+        yield negotiate();
       }
       const publication = new LocalTrackPublication(track.kind, ti, track, {
         loggerName: this.roomOptions.loggerName,
@@ -19131,50 +19345,10 @@ class LocalParticipant extends Participant {
       // save options for when it needs to be republished again
       publication.options = opts;
       track.sid = ti.sid;
-      if (!this.engine.pcManager) {
-        throw new UnexpectedConnectionState('pcManager is not ready');
-      }
       this.log.debug("publishing ".concat(track.kind, " with encodings"), Object.assign(Object.assign({}, this.logContext), {
         encodings,
         trackInfo: ti
       }));
-      track.sender = yield this.engine.createSender(track, opts, encodings);
-      if (track instanceof LocalVideoTrack) {
-        (_l = opts.degradationPreference) !== null && _l !== void 0 ? _l : opts.degradationPreference = getDefaultDegradationPreference(track);
-        track.setDegradationPreference(opts.degradationPreference);
-      }
-      if (encodings) {
-        if (isFireFox() && track.kind === Track.Kind.Audio) {
-          /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
-             livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to
-             publish high quality audio track. But firefox always uses this value as the actual
-             bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
-             So the client need to modify maxaverragebitrates in answer sdp to user provided value to
-             fix the issue.
-           */
-          let trackTransceiver = undefined;
-          for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
-            if (transceiver.sender === track.sender) {
-              trackTransceiver = transceiver;
-              break;
-            }
-          }
-          if (trackTransceiver) {
-            this.engine.pcManager.publisher.setTrackCodecBitrate({
-              transceiver: trackTransceiver,
-              codec: 'opus',
-              maxbr: ((_m = encodings[0]) === null || _m === void 0 ? void 0 : _m.maxBitrate) ? encodings[0].maxBitrate / 1000 : 0
-            });
-          }
-        } else if (track.codec && isSVCCodec(track.codec) && ((_o = encodings[0]) === null || _o === void 0 ? void 0 : _o.maxBitrate)) {
-          this.engine.pcManager.publisher.setTrackCodecBitrate({
-            cid: req.cid,
-            codec: track.codec,
-            maxbr: encodings[0].maxBitrate / 1000
-          });
-        }
-      }
-      yield this.engine.negotiate();
       if (track instanceof LocalVideoTrack) {
         track.startMonitor(this.engine.client);
       } else if (track instanceof LocalAudioTrack) {
@@ -19240,9 +19414,12 @@ class LocalParticipant extends Participant {
       if (!this.engine || this.engine.isClosed) {
         throw new UnexpectedConnectionState('cannot publish track when not connected');
       }
-      const ti = yield this.engine.addTrack(req);
-      yield this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
-      yield this.engine.negotiate();
+      const negotiate = () => __awaiter(this, void 0, void 0, function* () {
+        yield this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
+        yield this.engine.negotiate();
+      });
+      const rets = yield Promise.all([this.engine.addTrack(req), negotiate()]);
+      const ti = rets[0];
       this.log.debug("published ".concat(videoCodec, " for track ").concat(track.sid), Object.assign(Object.assign({}, this.logContext), {
         encodings,
         trackInfo: ti
@@ -19426,6 +19603,10 @@ class LocalParticipant extends Participant {
     }
   }
   /** @internal */
+  setEnabledPublishCodecs(codecs) {
+    this.enabledPublishVideoCodecs = codecs.filter(c => c.mime.split('/')[0].toLowerCase() === 'video');
+  }
+  /** @internal */
   updateInfo(info) {
     if (info.sid !== this.sid) {
       // drop updates that specify a wrong sid.
@@ -20147,6 +20328,7 @@ class Room extends eventsExports.EventEmitter {
       const pi = joinResponse.participant;
       this.localParticipant.sid = pi.sid;
       this.localParticipant.identity = pi.identity;
+      this.localParticipant.setEnabledPublishCodecs(joinResponse.enabledPublishCodecs);
       if (this.options.e2ee && this.e2eeManager) {
         try {
           this.e2eeManager.setSifTrailer(joinResponse.sifTrailer);
@@ -20559,7 +20741,7 @@ class Room extends eventsExports.EventEmitter {
       // find the participant
       const participant = transcription.transcribedParticipantIdentity === this.localParticipant.identity ? this.localParticipant : this.getParticipantByIdentity(transcription.transcribedParticipantIdentity);
       const publication = participant === null || participant === void 0 ? void 0 : participant.trackPublications.get(transcription.trackId);
-      const segments = extractTranscriptionSegments(transcription);
+      const segments = extractTranscriptionSegments(transcription, this.transcriptionReceivedTimes);
       publication === null || publication === void 0 ? void 0 : publication.emit(TrackEvent.TranscriptionReceived, segments);
       participant === null || participant === void 0 ? void 0 : participant.emit(ParticipantEvent.TranscriptionReceived, segments, publication);
       this.emit(RoomEvent.TranscriptionReceived, segments, participant, publication);
@@ -20674,6 +20856,7 @@ class Room extends eventsExports.EventEmitter {
     this.sidToIdentity = new Map();
     this.options = Object.assign(Object.assign({}, roomOptionDefaults), options);
     this.log = getLogger((_a = this.options.loggerName) !== null && _a !== void 0 ? _a : LoggerNames.Room);
+    this.transcriptionReceivedTimes = new Map();
     this.options.audioCaptureDefaults = Object.assign(Object.assign({}, audioDefaults), options === null || options === void 0 ? void 0 : options.audioCaptureDefaults);
     this.options.videoCaptureDefaults = Object.assign(Object.assign({}, videoDefaults), options === null || options === void 0 ? void 0 : options.videoCaptureDefaults);
     this.options.publishDefaults = Object.assign(Object.assign({}, publishDefaults), options === null || options === void 0 ? void 0 : options.publishDefaults);
@@ -20819,6 +21002,19 @@ class Room extends eventsExports.EventEmitter {
       }
     }).on(EngineEvent.DCBufferStatusChanged, (status, kind) => {
       this.emit(RoomEvent.DCBufferStatusChanged, status, kind);
+    }).on(EngineEvent.LocalTrackSubscribed, subscribedSid => {
+      const trackPublication = this.localParticipant.getTrackPublications().find(_ref2 => {
+        let {
+          trackSid
+        } = _ref2;
+        return trackSid === subscribedSid;
+      });
+      if (!trackPublication) {
+        this.log.warn('could not find local track subscription for subscribed event', this.logContext);
+        return;
+      }
+      this.localParticipant.emit(ParticipantEvent.LocalTrackSubscribed, trackPublication);
+      this.emitWhenConnected(RoomEvent.LocalTrackSubscribed, trackPublication, this.localParticipant);
     });
     if (this.localParticipant) {
       this.localParticipant.setupEngine(this.engine);
@@ -21188,6 +21384,7 @@ class Room extends eventsExports.EventEmitter {
     this.clearConnectionReconcile();
     this.isResuming = false;
     this.bufferedEvents = [];
+    this.transcriptionReceivedTimes.clear();
     if (this.state === ConnectionState.Disconnected) {
       return;
     }
@@ -21431,8 +21628,8 @@ class Room extends eventsExports.EventEmitter {
     return true;
   }
   emitBufferedEvents() {
-    this.bufferedEvents.forEach(_ref2 => {
-      let [ev, args] = _ref2;
+    this.bufferedEvents.forEach(_ref3 => {
+      let [ev, args] = _ref3;
       this.emit(ev, ...args);
     });
     this.bufferedEvents = [];