livekit-client 2.12.0 → 2.13.1

package/src/room/Room.ts

@@ -103,7 +103,7 @@ import {
   isLocalParticipant,
   isReactNative,
   isRemotePub,
-  isSafari,
+  isSafariBased,
   isWeb,
   numberToBigInt,
   sleep,
@@ -1356,6 +1356,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           // @ts-expect-error setSinkId is not yet in the typescript type of AudioContext
           this.audioContext?.setSinkId(deviceId);
         }
+
         // also set audio output on all audio elements, even if webAudioMix is enabled in order to workaround echo cancellation not working on chrome with non-default output devices
         // see https://issues.chromium.org/issues/40252911#comment7
         await Promise.all(
@@ -1644,6 +1645,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       participant.unpublishTrack(publication.trackSid, true);
     });
     this.emit(RoomEvent.ParticipantDisconnected, participant);
+    participant.setDisconnected();
     this.localParticipant?.handleParticipantDisconnected(participant.identity);
   }
 
@@ -2039,14 +2041,16 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         }
       }
 
-      if ((kind === 'audioinput' && !isSafari()) || kind === 'videoinput') {
+      if ((kind === 'audioinput' && !isSafariBased()) || kind === 'videoinput') {
         // airpods on Safari need special handling for audioinput as the track doesn't end as soon as you take them out
         continue;
       }
       // switch to first available device if previously active device is not available any more
       if (
         devicesOfKind.length > 0 &&
-        !devicesOfKind.find((deviceInfo) => deviceInfo.deviceId === this.getActiveDevice(kind))
+        !devicesOfKind.find((deviceInfo) => deviceInfo.deviceId === this.getActiveDevice(kind)) &&
+        // avoid switching audio output on safari without explicit user action as it leads to slowed down audio playback
+        (kind !== 'audiooutput' || !isSafariBased())
       ) {
         await this.switchActiveDevice(kind, devicesOfKind[0].deviceId);
       }
@@ -2240,6 +2244,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           status,
           participant,
         );
+      })
+      .on(ParticipantEvent.Active, () => {
+        this.emitWhenConnected(RoomEvent.ParticipantActive, participant);
       });
 
     // update info at the end after callbacks have been set up
@@ -2432,8 +2439,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     this.emit(RoomEvent.ConnectionQualityChanged, quality, this.localParticipant);
   };
 
-  private onMediaDevicesError = (e: Error) => {
-    this.emit(RoomEvent.MediaDevicesError, e);
+  private onMediaDevicesError = (e: Error, kind?: MediaDeviceKind) => {
+    this.emit(RoomEvent.MediaDevicesError, e, kind);
   };
 
   private onLocalParticipantPermissionsChanged = (prevPermissions?: ParticipantPermission) => {
@@ -2687,7 +2694,7 @@ export type RoomEventCallbacks = {
     publication?: TrackPublication,
   ) => void;
   connectionQualityChanged: (quality: ConnectionQuality, participant: Participant) => void;
-  mediaDevicesError: (error: Error) => void;
+  mediaDevicesError: (error: Error, kind?: MediaDeviceKind) => void;
   trackStreamStateChanged: (
     publication: RemoteTrackPublication,
     streamState: Track.StreamState,
@@ -2714,4 +2721,5 @@ export type RoomEventCallbacks = {
   chatMessage: (message: ChatMessage, participant?: RemoteParticipant | LocalParticipant) => void;
   localTrackSubscribed: (publication: LocalTrackPublication, participant: LocalParticipant) => void;
   metricsReceived: (metrics: MetricsBatch, participant?: Participant) => void;
+  participantActive: (participant: Participant) => void;
 };

package/src/room/errors.ts

@@ -16,6 +16,7 @@ export enum ConnectionErrorReason {
   InternalError,
   Cancelled,
   LeaveRequest,
+  Timeout,
 }
 
 export class ConnectionError extends LivekitError {

package/src/room/events.ts

@@ -203,6 +203,13 @@ export enum RoomEvent {
    */
   ParticipantAttributesChanged = 'participantAttributesChanged',
 
+  /**
+   * Emitted when the participant's state changes to ACTIVE and is ready to send/receive data messages
+   *
+   * args: (participant: [[Participant]])
+   */
+  ParticipantActive = 'participantActive',
+
   /**
    * Room metadata is a simple way for app-specific state to be pushed to
    * all users.
@@ -540,6 +547,11 @@ export enum ParticipantEvent {
 
   /** only emitted on local participant */
   ChatMessage = 'chatMessage',
+
+  /**
+   * Emitted when the participant's state changes to ACTIVE and is ready to send/receive data messages
+   */
+  Active = 'active',
 }
 
 /** @internal */

package/src/room/participant/LocalParticipant.ts

@@ -71,6 +71,7 @@ import {
   mergeDefaultOptions,
   mimeTypeToVideoCodecString,
   screenCaptureToDisplayMediaStreamOptions,
+  sourceToKind,
 } from '../track/utils';
 import {
   type ByteStreamInfo,
@@ -517,7 +518,7 @@ export default class LocalParticipant extends Participant {
             tr.stop();
           });
           if (e instanceof Error) {
-            this.emit(ParticipantEvent.MediaDevicesError, e);
+            this.emit(ParticipantEvent.MediaDevicesError, e, sourceToKind(source));
           }
           this.pendingPublishing.delete(source);
           throw e;
@@ -1142,11 +1143,32 @@ export default class LocalParticipant extends Participant {
     };
 
     let ti: TrackInfo;
+    const addTrackPromise = new Promise<TrackInfo>(async (resolve, reject) => {
+      try {
+        ti = await this.engine.addTrack(req);
+        resolve(ti);
+      } catch (err) {
+        if (track.sender && this.engine.pcManager?.publisher) {
+          this.engine.pcManager.publisher.removeTrack(track.sender);
+          await this.engine.negotiate().catch((negotiateErr) => {
+            this.log.error(
+              'failed to negotiate after removing track due to failed add track request',
+              {
+                ...this.logContext,
+                ...getLogContextFromTrack(track),
+                error: negotiateErr,
+              },
+            );
+          });
+        }
+        reject(err);
+      }
+    });
     if (this.enabledPublishVideoCodecs.length > 0) {
-      const rets = await Promise.all([this.engine.addTrack(req), negotiate()]);
+      const rets = await Promise.all([addTrackPromise, negotiate()]);
       ti = rets[0];
     } else {
-      ti = await this.engine.addTrack(req);
+      ti = await addTrackPromise;
       // server might not support the codec the client has requested, in that case, fallback
       // to a supported codec
       let primaryCodecMime: string | undefined;
@@ -1780,6 +1802,7 @@ export default class LocalParticipant extends Participant {
       streamId,
       topic: info.topic,
       timestamp: numberToBigInt(Date.now()),
+      attributes: info.attributes,
       contentHeader: {
         case: 'byteHeader',
         value: new DataStream_ByteHeader({
@@ -2153,22 +2176,18 @@ export default class LocalParticipant extends Participant {
       });
       return;
     }
-    if (update.subscribedCodecs.length > 0) {
-      if (!pub.videoTrack) {
-        return;
-      }
-      const newCodecs = await pub.videoTrack.setPublishingCodecs(update.subscribedCodecs);
-      for await (const codec of newCodecs) {
-        if (isBackupCodec(codec)) {
-          this.log.debug(`publish ${codec} for ${pub.videoTrack.sid}`, {
-            ...this.logContext,
-            ...getLogContextFromTrack(pub),
-          });
-          await this.publishAdditionalCodecForTrack(pub.videoTrack, codec, pub.options);
-        }
+    if (!pub.videoTrack) {
+      return;
+    }
+    const newCodecs = await pub.videoTrack.setPublishingCodecs(update.subscribedCodecs);
+    for await (const codec of newCodecs) {
+      if (isBackupCodec(codec)) {
+        this.log.debug(`publish ${codec} for ${pub.videoTrack.sid}`, {
+          ...this.logContext,
+          ...getLogContextFromTrack(pub),
+        });
+        await this.publishAdditionalCodecForTrack(pub.videoTrack, codec, pub.options);
       }
-    } else if (update.subscribedQualities.length > 0) {
-      await pub.videoTrack?.setPublishingLayers(update.subscribedQualities);
     }
   };
 

package/src/room/participant/Participant.ts

@@ -1,6 +1,7 @@
 import {
   DataPacket_Kind,
   ParticipantInfo,
+  ParticipantInfo_State,
   ParticipantInfo_Kind as ParticipantKind,
   ParticipantPermission,
   ConnectionQuality as ProtoQuality,
@@ -18,7 +19,7 @@ import { Track } from '../track/Track';
 import type { TrackPublication } from '../track/TrackPublication';
 import { diffAttributes } from '../track/utils';
 import type { ChatMessage, LoggerOptions, TranscriptionSegment } from '../types';
-import { isAudioTrack } from '../utils';
+import { Future, isAudioTrack } from '../utils';
 
 export enum ConnectionQuality {
   Excellent = 'excellent',
@@ -93,6 +94,8 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
 
   protected loggerOptions?: LoggerOptions;
 
+  protected activeFuture?: Future<void>;
+
   protected get logContext() {
     return {
       ...this.loggerOptions?.loggerContextCb?.(),
@@ -110,6 +113,10 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
     return this.permissions?.agent || this.kind === ParticipantKind.AGENT;
   }
 
+  get isActive() {
+    return this.participantInfo?.state === ParticipantInfo_State.ACTIVE;
+  }
+
   get kind() {
     return this._kind;
   }
@@ -173,6 +180,28 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
     }
   }
 
+  /**
+   * Waits until the participant is active and ready to receive data messages
+   * @returns a promise that resolves when the participant is active
+   */
+  waitUntilActive(): Promise<void> {
+    if (this.isActive) {
+      return Promise.resolve();
+    }
+
+    if (this.activeFuture) {
+      return this.activeFuture.promise;
+    }
+
+    this.activeFuture = new Future<void>();
+
+    this.once(ParticipantEvent.Active, () => {
+      this.activeFuture?.resolve?.();
+      this.activeFuture = undefined;
+    });
+    return this.activeFuture.promise;
+  }
+
   get connectionQuality(): ConnectionQuality {
     return this._connectionQuality;
   }
@@ -224,12 +253,17 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
     this._setName(info.name);
     this._setMetadata(info.metadata);
     this._setAttributes(info.attributes);
+    if (
+      info.state === ParticipantInfo_State.ACTIVE &&
+      this.participantInfo?.state !== ParticipantInfo_State.ACTIVE
+    ) {
+      this.emit(ParticipantEvent.Active);
+    }
     if (info.permission) {
       this.setPermissions(info.permission);
     }
     // set this last so setMetadata can detect changes
     this.participantInfo = info;
-    this.log.trace('update participant info', { ...this.logContext, info });
     return true;
   }
 
@@ -310,6 +344,16 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
     }
   }
 
+  /**
+   * @internal
+   */
+  setDisconnected() {
+    if (this.activeFuture) {
+      this.activeFuture.reject?.(new Error('Participant disconnected'));
+      this.activeFuture = undefined;
+    }
+  }
+
   /**
    * @internal
    */
@@ -377,7 +421,7 @@ export type ParticipantEventCallbacks = {
     publication: RemoteTrackPublication,
     status: TrackPublication.PermissionStatus,
   ) => void;
-  mediaDevicesError: (error: Error) => void;
+  mediaDevicesError: (error: Error, kind?: MediaDeviceKind) => void;
   audioStreamAcquired: () => void;
   participantPermissionsChanged: (prevPermissions?: ParticipantPermission) => void;
   trackSubscriptionStatusChanged: (
@@ -387,4 +431,5 @@ export type ParticipantEventCallbacks = {
   attributesChanged: (changedAttributes: Record<string, string>) => void;
   localTrackSubscribed: (trackPublication: LocalTrackPublication) => void;
   chatMessage: (msg: ChatMessage) => void;
+  active: () => void;
 };

package/src/room/participant/publishUtils.ts

@@ -19,6 +19,7 @@ import {
   isReactNative,
   isSVCCodec,
   isSafari,
+  isSafariSvcApi,
   unwrapConstraint,
 } from '../utils';
 
@@ -158,12 +159,15 @@ export function computeVideoEncodings(
       (browser?.name === 'Chrome' && compareVersions(browser?.version, '113') < 0)
     ) {
       const bitratesRatio = sm.suffix == 'h' ? 2 : 3;
+      // safari 18.4 uses a different svc API that requires scaleResolutionDownBy to be set.
+      const requireScale = isSafariSvcApi(browser);
       for (let i = 0; i < sm.spatial; i += 1) {
         // in legacy SVC, scaleResolutionDownBy cannot be set
         encodings.push({
           rid: videoRids[2 - i],
           maxBitrate: videoEncoding.maxBitrate / bitratesRatio ** i,
           maxFramerate: original.encoding.maxFramerate,
+          scaleResolutionDownBy: requireScale ? 2 ** i : undefined,
         });
       }
       // legacy SVC, scalabilityMode is set only on the first encoding

package/src/room/track/LocalVideoTrack.ts

@@ -12,7 +12,7 @@ import { ScalabilityMode } from '../participant/publishUtils';
 import type { VideoSenderStats } from '../stats';
 import { computeBitrate, monitorFrequency } from '../stats';
 import type { LoggerOptions } from '../types';
-import { compareVersions, isFireFox, isMobile, isWeb } from '../utils';
+import { compareVersions, isFireFox, isMobile, isSVCCodec, isWeb } from '../utils';
 import LocalTrack from './LocalTrack';
 import { Track, VideoQuality } from './Track';
 import type { VideoCaptureOptions, VideoCodec } from './options';
@@ -239,7 +239,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
       );
     }
     this.log.debug(`setting publishing quality. max quality ${maxQuality}`, this.logContext);
-    this.setPublishingLayers(qualities);
+    this.setPublishingLayers(isSVCCodec(this.codec), qualities);
   }
 
   async restartTrack(options?: VideoCaptureOptions) {
@@ -334,7 +334,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     });
     // only enable simulcast codec for preference codec setted
     if (!this.codec && codecs.length > 0) {
-      await this.setPublishingLayers(codecs[0].qualities);
+      await this.setPublishingLayers(isSVCCodec(codecs[0].codec), codecs[0].qualities);
       return [];
     }
 
@@ -343,7 +343,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     const newCodecs: VideoCodec[] = [];
     for await (const codec of codecs) {
       if (!this.codec || this.codec === codec.codec) {
-        await this.setPublishingLayers(codec.qualities);
+        await this.setPublishingLayers(isSVCCodec(codec.codec), codec.qualities);
       } else {
         const simulcastCodecInfo = this.simulcastCodecs.get(codec.codec as VideoCodec);
         this.log.debug(`try setPublishingCodec for ${codec.codec}`, {
@@ -364,6 +364,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
             simulcastCodecInfo.encodings!,
             codec.qualities,
             this.senderLock,
+            isSVCCodec(codec.codec),
             this.log,
             this.logContext,
           );
@@ -377,7 +378,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
    * @internal
    * Sets layers that should be publishing
    */
-  async setPublishingLayers(qualities: SubscribedQuality[]) {
+  async setPublishingLayers(isSvc: boolean, qualities: SubscribedQuality[]) {
     this.log.debug('setting publishing layers', { ...this.logContext, qualities });
     if (!this.sender || !this.encodings) {
       return;
@@ -388,6 +389,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
       this.encodings,
       qualities,
       this.senderLock,
+      isSvc,
       this.log,
       this.logContext,
     );
@@ -434,6 +436,7 @@ async function setPublishingLayersForSender(
   senderEncodings: RTCRtpEncodingParameters[],
   qualities: SubscribedQuality[],
   senderLock: Mutex,
+  isSVC: boolean,
   log: StructuredLogger,
   logContext: Record<string, unknown>,
 ) {
@@ -498,6 +501,12 @@ async function setPublishingLayersForSender(
         }
       }
     } else {
+      if (isSVC) {
+        const hasEnabledEncoding = qualities.some((q) => q.enabled);
+        if (hasEnabledEncoding) {
+          qualities.forEach((q) => (q.enabled = true));
+        }
+      }
       // simulcast dynacast encodings
       encodings.forEach((encoding, idx) => {
         let rid = encoding.rid ?? '';

package/src/room/track/create.ts

@@ -78,18 +78,16 @@ export async function createLocalTracks(
       deviceId: { ideal: deviceId },
     };
   }
-  // TODO if internal options don't have device Id specified, set it to 'default'
   if (
     internalOptions.audio === true ||
     (typeof internalOptions.audio === 'object' && !internalOptions.audio.deviceId)
   ) {
     internalOptions.audio = { deviceId: 'default' };
   }
-  if (
-    internalOptions.video === true ||
-    (typeof internalOptions.video === 'object' && !internalOptions.video.deviceId)
-  ) {
+  if (internalOptions.video === true) {
     internalOptions.video = { deviceId: 'default' };
+  } else if (typeof internalOptions.video === 'object' && !internalOptions.video.deviceId) {
+    internalOptions.video.deviceId = 'default';
   }
   const opts = mergeDefaultOptions(internalOptions, audioDefaults, videoDefaults);
   const constraints = constraintsForOptions(opts);

package/src/room/utils.ts

@@ -5,7 +5,7 @@ import {
   DisconnectReason,
   Transcription as TranscriptionModel,
 } from '@livekit/protocol';
-import { getBrowser } from '../utils/browserParser';
+import { type BrowserDetails, getBrowser } from '../utils/browserParser';
 import { protocolVersion, version } from '../version';
 import { type ConnectionError, ConnectionErrorReason } from './errors';
 import type LocalParticipant from './participant/LocalParticipant';
@@ -143,11 +143,24 @@ export function isSafari(): boolean {
   return getBrowser()?.name === 'Safari';
 }
 
+export function isSafariBased(): boolean {
+  const b = getBrowser();
+  return b?.name === 'Safari' || b?.os === 'iOS';
+}
+
 export function isSafari17(): boolean {
   const b = getBrowser();
   return b?.name === 'Safari' && b.version.startsWith('17.');
 }
 
+export function isSafariSvcApi(browser?: BrowserDetails): boolean {
+  if (!browser) {
+    browser = getBrowser();
+  }
+  // Safari 18.4 requires legacy svc api and scaleResolutionDown to be set
+  return browser?.name === 'Safari' && compareVersions(browser.version, '18.3') > 0;
+}
+
 export function isMobile(): boolean {
   if (!isWeb()) return false;