livekit-client 2.15.5 → 2.15.7

package/src/e2ee/worker/sifPayload.ts

@@ -0,0 +1,75 @@
+import type { VideoCodec } from '../..';
+
+//  Payload definitions taken from https://github.com/livekit/livekit/blob/master/pkg/sfu/downtrack.go#L104
+
+export const VP8KeyFrame8x8 = new Uint8Array([
+  0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x08, 0x00, 0x08, 0x00, 0x00, 0x47, 0x08, 0x85, 0x85, 0x88,
+  0x85, 0x84, 0x88, 0x02, 0x02, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xff, 0xab, 0x50, 0x80,
+]);
+
+export const H264KeyFrame2x2SPS = new Uint8Array([
+  0x67, 0x42, 0xc0, 0x1f, 0x0f, 0xd9, 0x1f, 0x88, 0x88, 0x84, 0x00, 0x00, 0x03, 0x00, 0x04, 0x00,
+  0x00, 0x03, 0x00, 0xc8, 0x3c, 0x60, 0xc9, 0x20,
+]);
+
+export const H264KeyFrame2x2PPS = new Uint8Array([0x68, 0x87, 0xcb, 0x83, 0xcb, 0x20]);
+
+export const H264KeyFrame2x2IDR = new Uint8Array([
+  0x65, 0x88, 0x84, 0x0a, 0xf2, 0x62, 0x80, 0x00, 0xa7, 0xbe,
+]);
+
+export const H264KeyFrame2x2 = [H264KeyFrame2x2SPS, H264KeyFrame2x2PPS, H264KeyFrame2x2IDR];
+
+export const OpusSilenceFrame = new Uint8Array([
+  0xf8, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+]);
+
+/**
+ * Create a crypto hash using Web Crypto API for secure comparison operations
+ */
+async function cryptoHash(data: Uint8Array | ArrayBuffer): Promise<string> {
+  const hashBuffer = await crypto.subtle.digest('SHA-256', data);
+  const hashArray = new Uint8Array(hashBuffer);
+  return Array.from(hashArray)
+    .map((b) => b.toString(16).padStart(2, '0'))
+    .join('');
+}
+
+/**
+ * Pre-computed SHA-256 hashes for secure comparison operations
+ */
+export const CryptoHashes = {
+  VP8KeyFrame8x8: 'ef0161653d8b2b23aad46624b420af1d03ce48950e9fc85718028f91b50f9219',
+  H264KeyFrame2x2SPS: 'f0a0e09647d891d6d50aa898bce7108090375d0d55e50a2bb21147afee558e44',
+  H264KeyFrame2x2PPS: '61d9665eed71b6d424ae9539330a3bdd5cb386d4d781c808219a6e36750493a7',
+  H264KeyFrame2x2IDR: 'faffc26b68a2fc09096fa20f3351e706398b6f838a7500c8063472c2e476e90d',
+  OpusSilenceFrame: 'aad8d31fc56b2802ca500e58c2fb9d0b29ad71bb7cb52cd6530251eade188988',
+} as const;
+
+/**
+ * Check if a byte array matches any of the known SIF payload frame types using secure crypto hashes
+ */
+export async function identifySifPayload(
+  data: Uint8Array | ArrayBuffer,
+): Promise<VideoCodec | 'opus' | null> {
+  const hash = await cryptoHash(data);
+
+  switch (hash) {
+    case CryptoHashes.VP8KeyFrame8x8:
+      return 'vp8';
+    case CryptoHashes.H264KeyFrame2x2SPS:
+      return 'h264';
+    case CryptoHashes.H264KeyFrame2x2PPS:
+      return 'h264';
+    case CryptoHashes.H264KeyFrame2x2IDR:
+      return 'h264';
+    case CryptoHashes.OpusSilenceFrame:
+      return 'opus';
+    default:
+      return null;
+  }
+}

package/src/room/PCTransport.ts

@@ -165,10 +165,11 @@ export default class PCTransport extends EventEmitter {
     } else if (sd.type === 'answer') {
       const sdpParsed = parse(sd.sdp ?? '');
       sdpParsed.media.forEach((media) => {
+        const mid = getMidString(media.mid!);
         if (media.type === 'audio') {
           // mung sdp for opus bitrate settings
           this.trackBitrates.some((trackbr): boolean => {
-            if (!trackbr.transceiver || media.mid != trackbr.transceiver.mid) {
+            if (!trackbr.transceiver || mid != trackbr.transceiver.mid) {
               return false;
             }
 
@@ -593,6 +594,9 @@ function ensureAudioNackAndStereo(
   stereoMids: string[],
   nackMids: string[],
 ) {
+  // sdp-transform types don't include number however the parser outputs mids as numbers in some cases
+  const mid = getMidString(media.mid!);
+
   // found opus codec to add nack fb
   let opusPayload = 0;
   media.rtp.some((rtp): boolean => {
@@ -610,7 +614,7 @@ function ensureAudioNackAndStereo(
     }
 
     if (
-      nackMids.includes(media.mid!) &&
+      nackMids.includes(mid) &&
       !media.rtcpFb.some((fb) => fb.payload === opusPayload && fb.type === 'nack')
     ) {
       media.rtcpFb.push({
@@ -619,7 +623,7 @@ function ensureAudioNackAndStereo(
       });
     }
 
-    if (stereoMids.includes(media.mid!)) {
+    if (stereoMids.includes(mid)) {
       media.fmtp.some((fmtp): boolean => {
         if (fmtp.payload === opusPayload) {
           if (!fmtp.config.includes('stereo=1')) {
@@ -642,6 +646,7 @@ function extractStereoAndNackAudioFromOffer(offer: RTCSessionDescriptionInit): {
   const sdpParsed = parse(offer.sdp ?? '');
   let opusPayload = 0;
   sdpParsed.media.forEach((media) => {
+    const mid = getMidString(media.mid!);
     if (media.type === 'audio') {
       media.rtp.some((rtp): boolean => {
         if (rtp.codec === 'opus') {
@@ -652,13 +657,13 @@ function extractStereoAndNackAudioFromOffer(offer: RTCSessionDescriptionInit): {
       });
 
       if (media.rtcpFb?.some((fb) => fb.payload === opusPayload && fb.type === 'nack')) {
-        nackMids.push(media.mid!);
+        nackMids.push(mid);
       }
 
       media.fmtp.some((fmtp): boolean => {
         if (fmtp.payload === opusPayload) {
           if (fmtp.config.includes('sprop-stereo=1')) {
-            stereoMids.push(media.mid!);
+            stereoMids.push(mid);
           }
           return true;
         }
@@ -682,3 +687,7 @@ function ensureIPAddrMatchVersion(media: MediaDescription) {
     }
   }
 }
+
+function getMidString(mid: string | number) {
+  return typeof mid === 'number' ? mid.toFixed(0) : mid;
+}

package/src/room/Room.ts

@@ -970,7 +970,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
    * @internal for testing
    */
   async simulateScenario(scenario: SimulationScenario, arg?: any) {
-    let postAction = () => {};
+    let postAction = async () => {};
     let req: SimulateScenario | undefined;
     switch (scenario) {
       case 'signal-reconnect':
@@ -1409,13 +1409,22 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       }
     }
 
-    participant.addSubscribedMediaTrack(
+    const publication = participant.addSubscribedMediaTrack(
       mediaTrack,
       trackId,
       stream,
       receiver,
       adaptiveStreamSettings,
     );
+
+    if (publication?.isEncrypted && !this.e2eeManager) {
+      this.emit(
+        RoomEvent.EncryptionError,
+        new Error(
+          `Encrypted ${publication.source} track received from participant ${participant.sid}, but room does not have encryption enabled!`,
+        ),
+      );
+    }
   }
 
   private handleRestarting = () => {
@@ -1661,8 +1670,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         return;
       }
       const newStreamState = Track.streamStateFromProto(streamState.state);
+      pub.track.setStreamState(newStreamState);
       if (newStreamState !== pub.track.streamState) {
-        pub.track.streamState = newStreamState;
         participant.emit(ParticipantEvent.TrackStreamStateChanged, pub, pub.track.streamState);
         this.emitWhenConnected(
           RoomEvent.TrackStreamStateChanged,

package/src/room/participant/LocalParticipant.ts

@@ -9,7 +9,6 @@ import {
   Encryption_Type,
   JoinResponse,
   ParticipantInfo,
-  ParticipantPermission,
   RequestResponse,
   RequestResponse_Reason,
   RpcAck,
@@ -453,16 +452,6 @@ export default class LocalParticipant extends Participant {
     return this.setTrackEnabled(Track.Source.ScreenShare, enabled, options, publishOptions);
   }
 
-  /** @internal */
-  setPermissions(permissions: ParticipantPermission): boolean {
-    const prevPermissions = this.permissions;
-    const changed = super.setPermissions(permissions);
-    if (changed && prevPermissions) {
-      this.emit(ParticipantEvent.ParticipantPermissionsChanged, prevPermissions);
-    }
-    return changed;
-  }
-
   /** @internal */
   async setE2EEEnabled(enabled: boolean) {
     this.encryptionType = enabled ? Encryption_Type.GCM : Encryption_Type.NONE;
@@ -841,19 +830,20 @@ export default class LocalParticipant extends Participant {
       return existingPublication;
     }
 
+    const opts: TrackPublishOptions = {
+      ...this.roomOptions.publishDefaults,
+      ...options,
+    };
     const isStereoInput =
       ('channelCount' in track.mediaStreamTrack.getSettings() &&
         // @ts-ignore `channelCount` on getSettings() is currently only available for Safari, but is generally the best way to determine a stereo track https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings/channelCount
         track.mediaStreamTrack.getSettings().channelCount === 2) ||
       track.mediaStreamTrack.getConstraints().channelCount === 2;
-    const isStereo = options?.forceStereo ?? isStereoInput;
+    const isStereo = opts.forceStereo ?? isStereoInput;
 
     // disable dtx for stereo track if not enabled explicitly
     if (isStereo) {
-      if (!options) {
-        options = {};
-      }
-      if (options.dtx === undefined) {
+      if (opts.dtx === undefined) {
         this.log.info(
           `Opus DTX will be disabled for stereo tracks by default. Enable them explicitly to make it work.`,
           {
@@ -862,18 +852,14 @@ export default class LocalParticipant extends Participant {
           },
         );
       }
-      if (options.red === undefined) {
+      if (opts.red === undefined) {
         this.log.info(
           `Opus RED will be disabled for stereo tracks by default. Enable them explicitly to make it work.`,
         );
       }
-      options.dtx ??= false;
-      options.red ??= false;
+      opts.dtx ??= false;
+      opts.red ??= false;
     }
-    const opts: TrackPublishOptions = {
-      ...this.roomOptions.publishDefaults,
-      ...options,
-    };
 
     if (!isE2EESimulcastSupported() && this.roomOptions.e2ee) {
       this.log.info(

package/src/room/track/LocalTrack.ts

@@ -339,7 +339,7 @@ export default abstract class LocalTrack<
       if (this.kind === Track.Kind.Video) {
         streamConstraints.video = deviceId || facingMode ? { deviceId, facingMode } : true;
       } else {
-        streamConstraints.audio = deviceId ? { deviceId } : true;
+        streamConstraints.audio = deviceId ? { deviceId, ...otherConstraints } : true;
       }
 
       // these steps are duplicated from setMediaStreamTrack because we must stop
@@ -356,7 +356,10 @@ export default abstract class LocalTrack<
       // create new track and attach
       const mediaStream = await navigator.mediaDevices.getUserMedia(streamConstraints);
       const newTrack = mediaStream.getTracks()[0];
-      await newTrack.applyConstraints(otherConstraints);
+      if (this.kind === Track.Kind.Video) {
+        // we already captured the audio track with the constraints, so we only need to apply the video constraints
+        await newTrack.applyConstraints(otherConstraints);
+      }
       newTrack.addEventListener('ended', this.handleEnded);
       this.log.debug('re-acquired MediaStreamTrack', this.logContext);
 

package/src/room/track/RemoteVideoTrack.ts

@@ -38,6 +38,16 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
     return this.adaptiveStreamSettings !== undefined;
   }
 
+  override setStreamState(value: Track.StreamState) {
+    super.setStreamState(value);
+    console.log('setStreamState', value);
+    if (value === Track.StreamState.Active) {
+      // update visibility for adaptive stream tracks when stream state received from server is active
+      // this is needed to ensure the track is stopped when there's no element attached to it at all
+      this.updateVisibility();
+    }
+  }
+
   /**
    * Note: When using adaptiveStream, you need to use remoteVideoTrack.attach() to add the track to a HTMLVideoElement, otherwise your video tracks might never start
    */
@@ -220,7 +230,7 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
     this.updateDimensions();
   }, REACTION_DELAY);
 
-  private updateVisibility() {
+  private updateVisibility(forceEmit?: boolean) {
     const lastVisibilityChange = this.elementInfos.reduce(
       (prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
       0,
@@ -234,7 +244,7 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
     const isVisible =
       (this.elementInfos.some((info) => info.visible) && !backgroundPause) || isPiPMode;
 
-    if (this.lastVisible === isVisible) {
+    if (this.lastVisible === isVisible && !forceEmit) {
       return;
     }
 

package/src/room/track/Track.ts

@@ -37,6 +37,8 @@ export abstract class Track<
 
   source: Track.Source;
 
+  private _streamState: Track.StreamState = Track.StreamState.Active;
+
   /**
    * sid is set after track is published to server, or if it's a remote track
    */
@@ -51,7 +53,14 @@ export abstract class Track<
    * indicates current state of stream, it'll indicate `paused` if the track
    * has been paused by congestion controller
    */
-  streamState: Track.StreamState = Track.StreamState.Active;
+  get streamState(): Track.StreamState {
+    return this._streamState;
+  }
+
+  /** @internal */
+  setStreamState(value: Track.StreamState) {
+    this._streamState = value;
+  }
 
   /** @internal */
   rtpTimestamp: number | undefined;

package/src/room/utils.ts

@@ -64,8 +64,9 @@ export function supportsAV1(): boolean {
   if (!('getCapabilities' in RTCRtpSender)) {
     return false;
   }
-  if (isSafari()) {
+  if (isSafari() || isFireFox()) {
     // Safari 17 on iPhone14 reports AV1 capability, but does not actually support it
+    // Firefox does support AV1, but SVC publishing is not supported
     return false;
   }
   const capabilities = RTCRtpSender.getCapabilities('video');
@@ -214,12 +215,12 @@ export function isE2EESimulcastSupported() {
     } else if (
       browser.os === 'iOS' &&
       browser.osVersion &&
-      compareVersions(supportedSafariVersion, browser.osVersion) >= 0
+      compareVersions(browser.osVersion, supportedSafariVersion) >= 0
     ) {
       return true;
     } else if (
       browser.name === 'Safari' &&
-      compareVersions(supportedSafariVersion, browser.version) >= 0
+      compareVersions(browser.version, supportedSafariVersion) >= 0
     ) {
       return true;
     } else {
@@ -282,6 +283,14 @@ export function getDevicePixelRatio(): number {
   return 1;
 }
 
+/**
+ * @param v1 - The first version string to compare.
+ * @param v2 - The second version string to compare.
+ * @returns A number indicating the order of the versions:
+ *   - 1 if v1 is greater than v2
+ *   - -1 if v1 is less than v2
+ *   - 0 if v1 and v2 are equal
+ */
 export function compareVersions(v1: string, v2: string): number {
   const parts1 = v1.split('.');
   const parts2 = v2.split('.');

package/dist/src/e2ee/worker/SifGuard.d.ts

@@ -1,11 +0,0 @@
-export declare class SifGuard {
-    private consecutiveSifCount;
-    private sifSequenceStartedAt;
-    private lastSifReceivedAt;
-    private userFramesSinceSif;
-    recordSif(): void;
-    recordUserFrame(): void;
-    isSifAllowed(): boolean;
-    reset(): void;
-}
-//# sourceMappingURL=SifGuard.d.ts.map

package/dist/src/e2ee/worker/SifGuard.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"SifGuard.d.ts","sourceRoot":"","sources":["../../../../src/e2ee/worker/SifGuard.ts"],"names":[],"mappings":"AAEA,qBAAa,QAAQ;IACnB,OAAO,CAAC,mBAAmB,CAAK;IAEhC,OAAO,CAAC,oBAAoB,CAAqB;IAEjD,OAAO,CAAC,iBAAiB,CAAa;IAEtC,OAAO,CAAC,kBAAkB,CAAa;IAEvC,SAAS;IAMT,eAAe;IAgBf,YAAY;IAQZ,KAAK;CAKN"}

package/dist/ts4.2/src/e2ee/worker/SifGuard.d.ts

@@ -1,11 +0,0 @@
-export declare class SifGuard {
-    private consecutiveSifCount;
-    private sifSequenceStartedAt;
-    private lastSifReceivedAt;
-    private userFramesSinceSif;
-    recordSif(): void;
-    recordUserFrame(): void;
-    isSifAllowed(): boolean;
-    reset(): void;
-}
-//# sourceMappingURL=SifGuard.d.ts.map

package/src/e2ee/worker/SifGuard.ts

@@ -1,47 +0,0 @@
-import { MAX_SIF_COUNT, MAX_SIF_DURATION } from '../constants';
-
-export class SifGuard {
-  private consecutiveSifCount = 0;
-
-  private sifSequenceStartedAt: number | undefined;
-
-  private lastSifReceivedAt: number = 0;
-
-  private userFramesSinceSif: number = 0;
-
-  recordSif() {
-    this.consecutiveSifCount += 1;
-    this.sifSequenceStartedAt ??= Date.now();
-    this.lastSifReceivedAt = Date.now();
-  }
-
-  recordUserFrame() {
-    if (this.sifSequenceStartedAt === undefined) {
-      return;
-    } else {
-      this.userFramesSinceSif += 1;
-    }
-    if (
-      // reset if we received more user frames than SIFs
-      this.userFramesSinceSif > this.consecutiveSifCount ||
-      // also reset if we got a new user frame and the latest SIF frame hasn't been updated in a while
-      Date.now() - this.lastSifReceivedAt > MAX_SIF_DURATION
-    ) {
-      this.reset();
-    }
-  }
-
-  isSifAllowed() {
-    return (
-      this.consecutiveSifCount < MAX_SIF_COUNT &&
-      (this.sifSequenceStartedAt === undefined ||
-        Date.now() - this.sifSequenceStartedAt < MAX_SIF_DURATION)
-    );
-  }
-
-  reset() {
-    this.userFramesSinceSif = 0;
-    this.consecutiveSifCount = 0;
-    this.sifSequenceStartedAt = undefined;
-  }
-}