livekit-client 1.11.4 → 1.12.1

package/src/room/participant/LocalParticipant.ts

@@ -1,6 +1,12 @@
 import log from '../../logger';
 import type { InternalRoomOptions } from '../../options';
-import { DataPacket, DataPacket_Kind, ParticipantInfo } from '../../proto/livekit_models';
+import {
+  DataPacket,
+  DataPacket_Kind,
+  Encryption_Type,
+  ParticipantInfo,
+  ParticipantPermission,
+} from '../../proto/livekit_models';
 import {
   AddTrackRequest,
   DataChannelInfo,
@@ -68,6 +74,8 @@ export default class LocalParticipant extends Participant {
   // keep a pointer to room options
   private roomOptions: InternalRoomOptions;
 
+  private encryptionType: Encryption_Type = Encryption_Type.NONE;
+
   private reconnectFuture?: Future<void>;
 
   /** @internal */
@@ -214,6 +222,22 @@ export default class LocalParticipant extends Participant {
     return this.setTrackEnabled(Track.Source.ScreenShare, enabled, options, publishOptions);
   }
 
+  /** @internal */
+  setPermissions(permissions: ParticipantPermission): boolean {
+    const prevPermissions = this.permissions;
+    const changed = super.setPermissions(permissions);
+    if (changed && prevPermissions) {
+      this.emit(ParticipantEvent.ParticipantPermissionsChanged, prevPermissions);
+    }
+    return changed;
+  }
+
+  /** @internal */
+  async setE2EEEnabled(enabled: boolean) {
+    this.encryptionType = enabled ? Encryption_Type.GCM : Encryption_Type.NONE;
+    await this.republishAllTracks(undefined, false);
+  }
+
   /**
    * Enable or disable publishing for a track by source. This serves as a simple
    * way to manage the common tracks (camera, mic, or screen share).
@@ -552,6 +576,12 @@ export default class LocalParticipant extends Participant {
       ...options,
     };
 
+    // disable simulcast if e2ee is set on safari
+    if (isSafari() && this.roomOptions.e2ee) {
+      log.info(`End-to-end encryption is set up, simulcast publishing will be disabled on Safari`);
+      opts.simulcast = false;
+    }
+
     if (opts.source) {
       track.source = opts.source;
     }
@@ -624,8 +654,9 @@ export default class LocalParticipant extends Participant {
       muted: track.isMuted,
       source: Track.sourceToProto(track.source),
       disableDtx: !(opts.dtx ?? true),
+      encryption: this.encryptionType,
       stereo: isStereo,
-      disableRed: !(opts.red ?? true),
+      // disableRed: !(opts.red ?? true),
     });
 
     // compute encodings and layers for video
@@ -760,11 +791,11 @@ export default class LocalParticipant extends Participant {
 
     if (encodings) {
       if (isFireFox() && track.kind === Track.Kind.Audio) {
-        /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1, 
+        /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
            livekit-server uses maxaveragebitrate=510000in the answer sdp to permit client to
-           publish high quality audio track. But firefox always uses this value as the actual 
+           publish high quality audio track. But firefox always uses this value as the actual
            bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
-           So the client need to modify maxaverragebitrates in answer sdp to user provided value to 
+           So the client need to modify maxaverragebitrates in answer sdp to user provided value to
            fix the issue.
          */
         let trackTransceiver: RTCRtpTransceiver | undefined = undefined;
@@ -790,7 +821,7 @@ export default class LocalParticipant extends Participant {
       }
     }
 
-    this.engine.negotiate();
+    await this.engine.negotiate();
 
     if (track instanceof LocalVideoTrack) {
       track.startMonitor(this.engine.client);
@@ -876,7 +907,7 @@ export default class LocalParticipant extends Participant {
     }
     await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
 
-    this.engine.negotiate();
+    await this.engine.negotiate();
     log.debug(`published ${videoCodec} for track ${track.sid}`, { encodings, trackInfo: ti });
   }
 
@@ -980,7 +1011,7 @@ export default class LocalParticipant extends Participant {
     ) as LocalTrackPublication[];
   }
 
-  async republishAllTracks(options?: TrackPublishOptions) {
+  async republishAllTracks(options?: TrackPublishOptions, restartTracks: boolean = true) {
     const localPubs: LocalTrackPublication[] = [];
     this.tracks.forEach((pub) => {
       if (pub.track) {
@@ -995,6 +1026,19 @@ export default class LocalParticipant extends Participant {
       localPubs.map(async (pub) => {
         const track = pub.track!;
         await this.unpublishTrack(track, false);
+        if (
+          restartTracks &&
+          !track.isMuted &&
+          (track instanceof LocalAudioTrack || track instanceof LocalVideoTrack) &&
+          !track.isUserProvided
+        ) {
+          // generally we need to restart the track before publishing, often a full reconnect
+          // is necessary because computer had gone to sleep.
+          log.debug('restarting existing track', {
+            track: pub.trackSid,
+          });
+          await track.restartTrack();
+        }
         await this.publishTrack(track, pub.options);
       }),
     );

package/src/room/participant/Participant.ts

@@ -68,6 +68,10 @@ export default class Participant extends EventEmitter<ParticipantEventCallbacks>
 
   private _connectionQuality: ConnectionQuality = ConnectionQuality.Unknown;
 
+  get isEncrypted() {
+    return this.tracks.size > 0 && Array.from(this.tracks.values()).every((tr) => tr.isEncrypted);
+  }
+
   /** @internal */
   constructor(sid: string, identity: string, name?: string, metadata?: string) {
     super();

package/src/room/participant/RemoteParticipant.ts

@@ -24,7 +24,7 @@ export default class RemoteParticipant extends Participant {
 
   signalClient: SignalClient;
 
-  private volume?: number;
+  private volumeMap: Map<Track.Source, number>;
 
   private audioContext?: AudioContext;
 
@@ -48,6 +48,7 @@ export default class RemoteParticipant extends Participant {
     this.tracks = new Map();
     this.audioTracks = new Map();
     this.videoTracks = new Map();
+    this.volumeMap = new Map();
   }
 
   protected addTrackPublication(publication: RemoteTrackPublication) {
@@ -102,12 +103,17 @@ export default class RemoteParticipant extends Participant {
   }
 
   /**
-   * sets the volume on the participant's microphone track
+   * sets the volume on the participant's audio track
+   * by default, this affects the microphone publication
+   * a different source can be passed in as a second argument
    * if no track exists the volume will be applied when the microphone track is added
    */
-  setVolume(volume: number) {
-    this.volume = volume;
-    const audioPublication = this.getTrack(Track.Source.Microphone);
+  setVolume(
+    volume: number,
+    source: Track.Source.Microphone | Track.Source.ScreenShareAudio = Track.Source.Microphone,
+  ) {
+    this.volumeMap.set(source, volume);
+    const audioPublication = this.getTrack(source);
     if (audioPublication && audioPublication.track) {
       (audioPublication.track as RemoteAudioTrack).setVolume(volume);
     }
@@ -116,12 +122,14 @@ export default class RemoteParticipant extends Participant {
   /**
    * gets the volume on the participant's microphone track
    */
-  getVolume() {
-    const audioPublication = this.getTrack(Track.Source.Microphone);
+  getVolume(
+    source: Track.Source.Microphone | Track.Source.ScreenShareAudio = Track.Source.Microphone,
+  ) {
+    const audioPublication = this.getTrack(source);
     if (audioPublication && audioPublication.track) {
       return (audioPublication.track as RemoteAudioTrack).getVolume();
     }
-    return this.volume;
+    return this.volumeMap.get(source);
   }
 
   /** @internal */
@@ -198,13 +206,9 @@ export default class RemoteParticipant extends Participant {
     track.start();
 
     publication.setTrack(track);
-    // set participant volume on new microphone tracks
-    if (
-      this.volume !== undefined &&
-      track instanceof RemoteAudioTrack &&
-      track.source === Track.Source.Microphone
-    ) {
-      track.setVolume(this.volume);
+    // set participant volumes on new audio tracks
+    if (this.volumeMap.has(publication.source) && track instanceof RemoteAudioTrack) {
+      track.setVolume(this.volumeMap.get(publication.source)!);
     }
 
     return publication;

package/src/room/track/LocalTrack.ts

@@ -56,9 +56,10 @@ export default abstract class LocalTrack extends Track {
     this.muteLock = new Mutex();
     this.pauseUpstreamLock = new Mutex();
     this.processorLock = new Mutex();
+    this.setMediaStreamTrack(mediaTrack, true);
+
     // added to satisfy TS compiler, constraints are synced with MediaStreamTrack
     this._constraints = mediaTrack.getConstraints();
-    this.setMediaStreamTrack(mediaTrack);
     if (constraints) {
       this._constraints = constraints;
     }
@@ -97,8 +98,8 @@ export default abstract class LocalTrack extends Track {
     return this.processor?.processedTrack ?? this._mediaStreamTrack;
   }
 
-  private async setMediaStreamTrack(newTrack: MediaStreamTrack) {
-    if (newTrack === this._mediaStreamTrack) {
+  private async setMediaStreamTrack(newTrack: MediaStreamTrack, force?: boolean) {
+    if (newTrack === this._mediaStreamTrack && !force) {
       return;
     }
     if (this._mediaStreamTrack) {
@@ -109,7 +110,7 @@ export default abstract class LocalTrack extends Track {
       this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
       this._mediaStreamTrack.removeEventListener('mute', this.pauseUpstream);
       this._mediaStreamTrack.removeEventListener('unmute', this.resumeUpstream);
-      if (!this.providedByUser) {
+      if (!this.providedByUser && this._mediaStreamTrack !== newTrack) {
         this._mediaStreamTrack.stop();
       }
     }

package/src/room/track/RemoteVideoTrack.ts

@@ -123,6 +123,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
     }
     this.elementInfos = this.elementInfos.filter((info) => info !== elementInfo);
     this.updateVisibility();
+    this.debouncedHandleResize();
   }
 
   detach(): HTMLMediaElement[];
@@ -195,9 +196,8 @@ export default class RemoteVideoTrack extends RemoteTrack {
   private stopObservingElement(element: HTMLMediaElement) {
     const stopElementInfos = this.elementInfos.filter((info) => info.element === element);
     for (const info of stopElementInfos) {
-      info.stopObserving();
+      this.stopObservingElementInfo(info);
     }
-    this.elementInfos = this.elementInfos.filter((info) => info.element !== element);
   }
 
   protected async handleAppVisibilityChanged() {

package/src/room/track/TrackPublication.ts

@@ -1,5 +1,6 @@
 import EventEmitter from 'eventemitter3';
 import log from '../../logger';
+import { Encryption_Type } from '../../proto/livekit_models';
 import type { SubscriptionError, TrackInfo } from '../../proto/livekit_models';
 import type { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
 import { TrackEvent } from '../events';
@@ -35,6 +36,8 @@ export class TrackPublication extends EventEmitter<PublicationEventCallbacks> {
 
   protected metadataMuted: boolean = false;
 
+  protected encryption: Encryption_Type = Encryption_Type.NONE;
+
   constructor(kind: Track.Kind, id: string, name: string) {
     super();
     this.kind = kind;
@@ -71,6 +74,10 @@ export class TrackPublication extends EventEmitter<PublicationEventCallbacks> {
     return this.track !== undefined;
   }
 
+  get isEncrypted(): boolean {
+    return this.encryption !== Encryption_Type.NONE;
+  }
+
   /**
    * an [AudioTrack] if this publication holds an audio track
    */
@@ -110,8 +117,9 @@ export class TrackPublication extends EventEmitter<PublicationEventCallbacks> {
       };
       this.simulcasted = info.simulcast;
     }
+    this.encryption = info.encryption;
     this.trackInfo = info;
-    log.trace('update publication info', { info });
+    log.debug('update publication info', { info });
   }
 }
 

package/src/room/track/create.ts

@@ -57,6 +57,15 @@ export async function createLocalTracks(
     if (typeof conOrBool !== 'boolean') {
       trackConstraints = conOrBool;
     }
+
+    // update the constraints with the device id the user gave permissions to in the permission prompt
+    // otherwise each track restart (e.g. mute - unmute) will try to initialize the device again -> causing additional permission prompts
+    if (trackConstraints) {
+      trackConstraints.deviceId = mediaStreamTrack.getSettings().deviceId;
+    } else {
+      trackConstraints = { deviceId: mediaStreamTrack.getSettings().deviceId };
+    }
+
     const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints);
     if (track.kind === Track.Kind.Video) {
       track.source = Track.Source.Camera;

package/src/room/track/options.ts

@@ -272,10 +272,11 @@ export interface AudioPreset {
   priority?: RTCPriorityType;
 }
 
-const codecs = ['vp8', 'h264', 'vp9', 'av1'] as const;
 const backupCodecs = ['vp8', 'h264'] as const;
 
-export type VideoCodec = (typeof codecs)[number];
+export const videoCodecs = ['vp8', 'h264', 'vp9', 'av1'] as const;
+
+export type VideoCodec = (typeof videoCodecs)[number];
 
 export type BackupVideoCodec = (typeof backupCodecs)[number];
 

package/src/room/track/utils.ts

@@ -1,4 +1,5 @@
 import { sleep } from '../utils';
+import { Track } from './Track';
 import type { AudioCaptureOptions, CreateLocalTracksOptions, VideoCaptureOptions } from './options';
 import type { AudioTrack } from './types';
 
@@ -112,3 +113,29 @@ export function getNewAudioContext(): AudioContext | void {
     return new AudioContext({ latencyHint: 'interactive' });
   }
 }
+
+/**
+ * @internal
+ */
+export function kindToSource(kind: MediaDeviceKind) {
+  if (kind === 'audioinput') {
+    return Track.Source.Microphone;
+  } else if (kind === 'videoinput') {
+    return Track.Source.Camera;
+  } else {
+    return Track.Source.Unknown;
+  }
+}
+
+/**
+ * @internal
+ */
+export function sourceToKind(source: Track.Source): MediaDeviceKind | undefined {
+  if (source === Track.Source.Microphone) {
+    return 'audioinput';
+  } else if (source === Track.Source.Camera) {
+    return 'videoinput';
+  } else {
+    return undefined;
+  }
+}

package/src/room/utils.ts

@@ -4,6 +4,7 @@ import type { DetectableBrowser } from '../utils/browserParser';
 import { protocolVersion, version } from '../version';
 import type LocalAudioTrack from './track/LocalAudioTrack';
 import type RemoteAudioTrack from './track/RemoteAudioTrack';
+import { VideoCodec, videoCodecs } from './track/options';
 import { getNewAudioContext } from './track/utils';
 import type { LiveKitReactNativeInfo } from './types';
 
@@ -456,6 +457,10 @@ export class Mutex {
   }
 }
 
+export function isVideoCodec(maybeCodec: string): maybeCodec is VideoCodec {
+  return videoCodecs.includes(maybeCodec as VideoCodec);
+}
+
 export function unwrapConstraint(constraint: ConstrainDOMString): string {
   if (typeof constraint === 'string') {
     return constraint;

package/src/room/worker.d.ts

@@ -0,0 +1,4 @@
+declare module 'web-worker:*' {
+  const WorkerFactory: new () => Worker;
+  export default WorkerFactory;
+}

package/src/test/MockMediaStreamTrack.ts

@@ -1,4 +1,5 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
+// @ts-ignore
 export default class MockMediaStreamTrack implements MediaStreamTrack {
   contentHint: string = '';
 

package/src/utils/browserParser.ts

@@ -4,10 +4,12 @@
 const commonVersionIdentifier = /version\/(\d+(\.?_?\d+)+)/i;
 
 export type DetectableBrowser = 'Chrome' | 'Firefox' | 'Safari';
+export type DetectableOS = 'iOS' | 'macOS';
 
 export type BrowserDetails = {
   name: DetectableBrowser;
   version: string;
+  os?: DetectableOS;
 };
 
 let browserDetails: BrowserDetails | undefined;
@@ -34,6 +36,7 @@ const browsersList = [
       const browser: BrowserDetails = {
         name: 'Firefox',
         version: getMatch(/(?:firefox|iceweasel|fxios)[\s/](\d+(\.?_?\d+)+)/i, ua),
+        os: ua.toLowerCase().includes('fxios') ? 'iOS' : undefined,
       };
       return browser;
     },
@@ -44,6 +47,7 @@ const browsersList = [
       const browser: BrowserDetails = {
         name: 'Chrome',
         version: getMatch(/(?:chrome|chromium|crios|crmo)\/(\d+(\.?_?\d+)+)/i, ua),
+        os: ua.toLowerCase().includes('crios') ? 'iOS' : undefined,
       };
 
       return browser;
@@ -56,6 +60,7 @@ const browsersList = [
       const browser: BrowserDetails = {
         name: 'Safari',
         version: getMatch(commonVersionIdentifier, ua),
+        os: ua.includes('Mobile/') ? 'iOS' : 'macOS',
       };
 
       return browser;