livekit-client 1.12.3 → 1.13.1

package/src/e2ee/worker/ParticipantKeyHandler.ts

@@ -2,7 +2,8 @@ import { EventEmitter } from 'events';
 import type TypedEventEmitter from 'typed-emitter';
 import { workerLogger } from '../../logger';
 import { KEYRING_SIZE } from '../constants';
-import type { KeyProviderOptions, KeySet, ParticipantKeyHandlerCallbacks } from '../types';
+import { KeyHandlerEvent, type ParticipantKeyHandlerCallbacks } from '../events';
+import type { KeyProviderOptions, KeySet } from '../types';
 import { deriveKeys, importKey, ratchet } from '../utils';
 
 // TODO ParticipantKeyHandlers currently don't get destroyed on participant disconnect
@@ -19,15 +20,13 @@ import { deriveKeys, importKey, ratchet } from '../utils';
 export class ParticipantKeyHandler extends (EventEmitter as new () => TypedEventEmitter<ParticipantKeyHandlerCallbacks>) {
   private currentKeyIndex: number;
 
-  private cryptoKeyRing: Array<KeySet>;
-
-  private enabled: boolean;
+  private cryptoKeyRing: Array<KeySet | undefined>;
 
   private keyProviderOptions: KeyProviderOptions;
 
   private ratchetPromiseMap: Map<number, Promise<CryptoKey>>;
 
-  private participantId: string | undefined;
+  private participantIdentity: string;
 
   private decryptionFailureCount = 0;
 
@@ -37,25 +36,16 @@ export class ParticipantKeyHandler extends (EventEmitter as new () => TypedEvent
     return this._hasValidKey;
   }
 
-  constructor(
-    participantId: string | undefined,
-    isEnabled: boolean,
-    keyProviderOptions: KeyProviderOptions,
-  ) {
+  constructor(participantIdentity: string, keyProviderOptions: KeyProviderOptions) {
     super();
     this.currentKeyIndex = 0;
-    this.cryptoKeyRing = new Array(KEYRING_SIZE);
-    this.enabled = isEnabled;
+    this.cryptoKeyRing = new Array(KEYRING_SIZE).fill(undefined);
     this.keyProviderOptions = keyProviderOptions;
     this.ratchetPromiseMap = new Map();
-    this.participantId = participantId;
+    this.participantIdentity = participantIdentity;
     this.resetKeyStatus();
   }
 
-  setEnabled(enabled: boolean) {
-    this.enabled = enabled;
-  }
-
   decryptionFailure() {
     if (this.keyProviderOptions.failureTolerance < 0) {
       return;
@@ -63,6 +53,7 @@ export class ParticipantKeyHandler extends (EventEmitter as new () => TypedEvent
     this.decryptionFailureCount += 1;
 
     if (this.decryptionFailureCount > this.keyProviderOptions.failureTolerance) {
+      workerLogger.warn(`key for ${this.participantIdentity} is being marked as invalid`);
       this._hasValidKey = false;
     }
   }
@@ -88,7 +79,7 @@ export class ParticipantKeyHandler extends (EventEmitter as new () => TypedEvent
    * @param setKey
    */
   ratchetKey(keyIndex?: number, setKey = true): Promise<CryptoKey> {
-    const currentKeyIndex = (keyIndex ??= this.getCurrentKeyIndex());
+    const currentKeyIndex = keyIndex ?? this.getCurrentKeyIndex();
 
     const existingPromise = this.ratchetPromiseMap.get(currentKeyIndex);
     if (typeof existingPromise !== 'undefined') {
@@ -96,7 +87,13 @@ export class ParticipantKeyHandler extends (EventEmitter as new () => TypedEvent
     }
     const ratchetPromise = new Promise<CryptoKey>(async (resolve, reject) => {
       try {
-        const currentMaterial = this.getKeySet(currentKeyIndex).material;
+        const keySet = this.getKeySet(currentKeyIndex);
+        if (!keySet) {
+          throw new TypeError(
+            `Cannot ratchet key without a valid keyset of participant ${this.participantIdentity}`,
+          );
+        }
+        const currentMaterial = keySet.material;
         const newMaterial = await importKey(
           await ratchet(currentMaterial, this.keyProviderOptions.ratchetSalt),
           currentMaterial.algorithm.name,
@@ -105,8 +102,13 @@ export class ParticipantKeyHandler extends (EventEmitter as new () => TypedEvent
 
         if (setKey) {
           this.setKeyFromMaterial(newMaterial, currentKeyIndex, true);
+          this.emit(
+            KeyHandlerEvent.KeyRatcheted,
+            newMaterial,
+            this.participantIdentity,
+            currentKeyIndex,
+          );
         }
-        this.emit('keyRatcheted', newMaterial, keyIndex, this.participantId);
         resolve(newMaterial);
       } catch (e) {
         reject(e);
@@ -144,10 +146,11 @@ export class ParticipantKeyHandler extends (EventEmitter as new () => TypedEvent
     this.setKeySet(keySet, this.currentKeyIndex, emitRatchetEvent);
   }
 
-  async setKeySet(keySet: KeySet, keyIndex: number, emitRatchetEvent = false) {
+  setKeySet(keySet: KeySet, keyIndex: number, emitRatchetEvent = false) {
     this.cryptoKeyRing[keyIndex % this.cryptoKeyRing.length] = keySet;
+
     if (emitRatchetEvent) {
-      this.emit('keyRatcheted', keySet.material, keyIndex, this.participantId);
+      this.emit(KeyHandlerEvent.KeyRatcheted, keySet.material, this.participantIdentity, keyIndex);
     }
   }
 
@@ -156,10 +159,6 @@ export class ParticipantKeyHandler extends (EventEmitter as new () => TypedEvent
     this.resetKeyStatus();
   }
 
-  isEnabled() {
-    return this.enabled;
-  }
-
   getCurrentKeyIndex() {
     return this.currentKeyIndex;
   }

package/src/e2ee/worker/SifGuard.ts

@@ -0,0 +1,47 @@
+import { MAX_SIF_COUNT, MAX_SIF_DURATION } from '../constants';
+
+export class SifGuard {
+  private consecutiveSifCount = 0;
+
+  private sifSequenceStartedAt: number | undefined;
+
+  private lastSifReceivedAt: number = 0;
+
+  private userFramesSinceSif: number = 0;
+
+  recordSif() {
+    this.consecutiveSifCount += 1;
+    this.sifSequenceStartedAt ??= Date.now();
+    this.lastSifReceivedAt = Date.now();
+  }
+
+  recordUserFrame() {
+    if (this.sifSequenceStartedAt === undefined) {
+      return;
+    } else {
+      this.userFramesSinceSif += 1;
+    }
+    if (
+      // reset if we received more user frames than SIFs
+      this.userFramesSinceSif > this.consecutiveSifCount ||
+      // also reset if we got a new user frame and the latest SIF frame hasn't been updated in a while
+      Date.now() - this.lastSifReceivedAt > MAX_SIF_DURATION
+    ) {
+      this.reset();
+    }
+  }
+
+  isSifAllowed() {
+    return (
+      this.consecutiveSifCount < MAX_SIF_COUNT &&
+      (this.sifSequenceStartedAt === undefined ||
+        Date.now() - this.sifSequenceStartedAt < MAX_SIF_DURATION)
+    );
+  }
+
+  reset() {
+    this.userFramesSinceSif = 0;
+    this.consecutiveSifCount = 0;
+    this.sifSequenceStartedAt = undefined;
+  }
+}

package/src/e2ee/worker/e2ee.worker.ts

@@ -1,22 +1,21 @@
 import { workerLogger } from '../../logger';
 import { KEY_PROVIDER_DEFAULTS } from '../constants';
 import { CryptorErrorReason } from '../errors';
+import { CryptorEvent, KeyHandlerEvent } from '../events';
 import type {
   E2EEWorkerMessage,
-  EnableMessage,
   ErrorMessage,
+  InitAck,
   KeyProviderOptions,
   RatchetMessage,
   RatchetRequestMessage,
 } from '../types';
-import { FrameCryptor } from './FrameCryptor';
+import { FrameCryptor, encryptionEnabledMap } from './FrameCryptor';
 import { ParticipantKeyHandler } from './ParticipantKeyHandler';
 
 const participantCryptors: FrameCryptor[] = [];
 const participantKeys: Map<string, ParticipantKeyHandler> = new Map();
-
-let publishCryptors: FrameCryptor[] = [];
-let publisherKeys: ParticipantKeyHandler;
+let sharedKeyHandler: ParticipantKeyHandler | undefined;
 
 let isEncryptionEnabled: boolean = false;
 
@@ -24,6 +23,8 @@ let useSharedKey: boolean = false;
 
 let sharedKey: CryptoKey | undefined;
 
+let sifTrailer: Uint8Array | undefined;
+
 let keyProviderOptions: KeyProviderOptions = KEY_PROVIDER_DEFAULTS;
 
 workerLogger.setDefaultLevel('info');
@@ -37,22 +38,20 @@ onmessage = (ev) => {
       keyProviderOptions = data.keyProviderOptions;
       useSharedKey = !!data.keyProviderOptions.sharedKey;
       // acknowledge init successful
-      const enableMsg: EnableMessage = {
-        kind: 'enable',
+      const ackMsg: InitAck = {
+        kind: 'initAck',
         data: { enabled: isEncryptionEnabled },
       };
-      publisherKeys = new ParticipantKeyHandler(undefined, isEncryptionEnabled, keyProviderOptions);
-      publisherKeys.on('keyRatcheted', emitRatchetedKeys);
-      postMessage(enableMsg);
+      postMessage(ackMsg);
       break;
     case 'enable':
-      setEncryptionEnabled(data.enabled, data.participantId);
+      setEncryptionEnabled(data.enabled, data.participantIdentity);
       workerLogger.info('updated e2ee enabled status');
       // acknowledge enable call successful
       postMessage(ev.data);
       break;
     case 'decode':
-      let cryptor = getTrackCryptor(data.participantId, data.trackId);
+      let cryptor = getTrackCryptor(data.participantIdentity, data.trackId);
       cryptor.setupTransform(
         kind,
         data.readableStream,
@@ -62,7 +61,7 @@ onmessage = (ev) => {
       );
       break;
     case 'encode':
-      let pubCryptor = getPublisherCryptor(data.trackId);
+      let pubCryptor = getTrackCryptor(data.participantIdentity, data.trackId);
       pubCryptor.setupTransform(
         kind,
         data.readableStream,
@@ -73,10 +72,11 @@ onmessage = (ev) => {
       break;
     case 'setKey':
       if (useSharedKey) {
-        workerLogger.debug('set shared key');
+        workerLogger.warn('set shared key');
         setSharedKey(data.key, data.keyIndex);
-      } else if (data.participantId) {
-        getParticipantKeyHandler(data.participantId).setKey(data.key, data.keyIndex);
+      } else if (data.participantIdentity) {
+        workerLogger.warn(`set participant sender key ${data.participantIdentity}`);
+        getParticipantKeyHandler(data.participantIdentity).setKey(data.key, data.keyIndex);
       } else {
         workerLogger.error('no participant Id was provided and shared key usage is disabled');
       }
@@ -85,106 +85,107 @@ onmessage = (ev) => {
       unsetCryptorParticipant(data.trackId);
       break;
     case 'updateCodec':
-      getTrackCryptor(data.participantId, data.trackId).setVideoCodec(data.codec);
+      getTrackCryptor(data.participantIdentity, data.trackId).setVideoCodec(data.codec);
       break;
     case 'setRTPMap':
-      publishCryptors.forEach((cr) => {
-        cr.setRtpMap(data.map);
+      // this is only used for the local participant
+      participantCryptors.forEach((cr) => {
+        if (cr.getParticipantIdentity() === data.participantIdentity) {
+          cr.setRtpMap(data.map);
+        }
       });
       break;
     case 'ratchetRequest':
       handleRatchetRequest(data);
+      break;
+    case 'setSifTrailer':
+      handleSifTrailer(data.trailer);
+      break;
     default:
       break;
   }
 };
 
 async function handleRatchetRequest(data: RatchetRequestMessage['data']) {
-  const keyHandler = getParticipantKeyHandler(data.participantId);
-  await keyHandler.ratchetKey(data.keyIndex);
-  keyHandler.resetKeyStatus();
+  if (useSharedKey) {
+    const keyHandler = getSharedKeyHandler();
+    await keyHandler.ratchetKey(data.keyIndex);
+    keyHandler.resetKeyStatus();
+  } else if (data.participantIdentity) {
+    const keyHandler = getParticipantKeyHandler(data.participantIdentity);
+    await keyHandler.ratchetKey(data.keyIndex);
+    keyHandler.resetKeyStatus();
+  } else {
+    workerLogger.error(
+      'no participant Id was provided for ratchet request and shared key usage is disabled',
+    );
+  }
 }
 
-function getTrackCryptor(participantId: string, trackId: string) {
+function getTrackCryptor(participantIdentity: string, trackId: string) {
   let cryptor = participantCryptors.find((c) => c.getTrackId() === trackId);
   if (!cryptor) {
-    workerLogger.info('creating new cryptor for', { participantId });
+    workerLogger.info('creating new cryptor for', { participantIdentity });
     if (!keyProviderOptions) {
       throw Error('Missing keyProvider options');
     }
     cryptor = new FrameCryptor({
-      participantId,
-      keys: getParticipantKeyHandler(participantId),
+      participantIdentity,
+      keys: getParticipantKeyHandler(participantIdentity),
       keyProviderOptions,
+      sifTrailer,
     });
 
     setupCryptorErrorEvents(cryptor);
     participantCryptors.push(cryptor);
-  } else if (participantId !== cryptor.getParticipantId()) {
+  } else if (participantIdentity !== cryptor.getParticipantIdentity()) {
     // assign new participant id to track cryptor and pass in correct key handler
-    cryptor.setParticipant(participantId, getParticipantKeyHandler(participantId));
+    cryptor.setParticipant(participantIdentity, getParticipantKeyHandler(participantIdentity));
   }
   if (sharedKey) {
   }
   return cryptor;
 }
 
-function getParticipantKeyHandler(participantId?: string) {
-  if (!participantId) {
-    return publisherKeys!;
+function getParticipantKeyHandler(participantIdentity: string) {
+  if (useSharedKey) {
+    return getSharedKeyHandler();
   }
-  let keys = participantKeys.get(participantId);
+  let keys = participantKeys.get(participantIdentity);
   if (!keys) {
-    keys = new ParticipantKeyHandler(participantId, true, keyProviderOptions);
+    keys = new ParticipantKeyHandler(participantIdentity, keyProviderOptions);
     if (sharedKey) {
       keys.setKey(sharedKey);
     }
-    participantKeys.set(participantId, keys);
+    keys.on(KeyHandlerEvent.KeyRatcheted, emitRatchetedKeys);
+    participantKeys.set(participantIdentity, keys);
   }
   return keys;
 }
 
-function unsetCryptorParticipant(trackId: string) {
-  participantCryptors.find((c) => c.getTrackId() === trackId)?.unsetParticipant();
+function getSharedKeyHandler() {
+  if (!sharedKeyHandler) {
+    sharedKeyHandler = new ParticipantKeyHandler('shared-key', keyProviderOptions);
+  }
+  return sharedKeyHandler;
 }
 
-function getPublisherCryptor(trackId: string) {
-  let publishCryptor = publishCryptors.find((cryptor) => cryptor.getTrackId() === trackId);
-  if (!publishCryptor) {
-    if (!keyProviderOptions) {
-      throw new TypeError('Missing keyProvider options');
-    }
-    publishCryptor = new FrameCryptor({
-      keys: publisherKeys!,
-      participantId: 'publisher',
-      keyProviderOptions,
-    });
-    setupCryptorErrorEvents(publishCryptor);
-    publishCryptors.push(publishCryptor);
-  }
-  return publishCryptor;
+function unsetCryptorParticipant(trackId: string) {
+  participantCryptors.find((c) => c.getTrackId() === trackId)?.unsetParticipant();
 }
 
-function setEncryptionEnabled(enable: boolean, participantId?: string) {
-  if (!participantId) {
-    isEncryptionEnabled = enable;
-    publisherKeys.setEnabled(enable);
-  } else {
-    getParticipantKeyHandler(participantId).setEnabled(enable);
-  }
+function setEncryptionEnabled(enable: boolean, participantIdentity: string) {
+  encryptionEnabledMap.set(participantIdentity, enable);
 }
 
 function setSharedKey(key: CryptoKey, index?: number) {
   workerLogger.debug('setting shared key');
   sharedKey = key;
-  publisherKeys?.setKey(key, index);
-  for (const [, keyHandler] of participantKeys) {
-    keyHandler.setKey(key, index);
-  }
+  getSharedKeyHandler().setKey(key, index);
 }
 
 function setupCryptorErrorEvents(cryptor: FrameCryptor) {
-  cryptor.on('cryptorError', (error) => {
+  cryptor.on(CryptorEvent.Error, (error) => {
     const msg: ErrorMessage = {
       kind: 'error',
       data: { error: new Error(`${CryptorErrorReason[error.reason]}: ${error.message}`) },
@@ -193,11 +194,11 @@ function setupCryptorErrorEvents(cryptor: FrameCryptor) {
   });
 }
 
-function emitRatchetedKeys(material: CryptoKey, keyIndex?: number) {
+function emitRatchetedKeys(material: CryptoKey, participantIdentity: string, keyIndex?: number) {
   const msg: RatchetMessage = {
     kind: `ratchetKey`,
     data: {
-      // participantId,
+      participantIdentity,
       keyIndex,
       material,
     },
@@ -205,6 +206,13 @@ function emitRatchetedKeys(material: CryptoKey, keyIndex?: number) {
   postMessage(msg);
 }
 
+function handleSifTrailer(trailer: Uint8Array) {
+  sifTrailer = trailer;
+  participantCryptors.forEach((c) => {
+    c.setSifTrailer(trailer);
+  });
+}
+
 // Operations using RTCRtpScriptTransform.
 // @ts-ignore
 if (self.RTCTransformEvent) {
@@ -214,9 +222,8 @@ if (self.RTCTransformEvent) {
     const transformer = event.transformer;
     workerLogger.debug('transformer', transformer);
     transformer.handled = true;
-    const { kind, participantId, trackId, codec } = transformer.options;
-    const cryptor =
-      kind === 'encode' ? getPublisherCryptor(trackId) : getTrackCryptor(participantId, trackId);
+    const { kind, participantIdentity, trackId, codec } = transformer.options;
+    const cryptor = getTrackCryptor(participantIdentity, trackId);
     workerLogger.debug('transform', { codec });
     cryptor.setupTransform(kind, transformer.readable, transformer.writable, trackId, codec);
   };

package/src/options.ts

@@ -31,6 +31,9 @@ export interface InternalRoomOptions {
    * enable Dynacast, off by default. With Dynacast dynamically pauses
    * video layers that are not being consumed by any subscribers, significantly
    * reducing publishing CPU and bandwidth usage.
+   *
+   * Dynacast will be enabled if SVC codecs (VP9/AV1) are used. Multi-codec simulcast
+   * requires dynacast
    */
   dynacast: boolean;
 
@@ -119,6 +122,9 @@ export interface InternalRoomConnectOptions {
 
   /** specifies how often an initial join connection is allowed to retry (only applicable if server is not reachable) */
   maxRetries: number;
+
+  /** amount of time for Websocket connection to be established, defaults to 15s */
+  websocketTimeout: number;
 }
 
 /**

package/src/proto/livekit_models_pb.ts

@@ -1,3 +1,17 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // @generated by protoc-gen-es v1.3.0 with parameter "target=ts"
 // @generated from file livekit_models.proto (package livekit, syntax proto3)
 /* eslint-disable */

package/src/proto/livekit_rtc_pb.ts

@@ -1,3 +1,17 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // @generated by protoc-gen-es v1.3.0 with parameter "target=ts"
 // @generated from file livekit_rtc.proto (package livekit, syntax proto3)
 /* eslint-disable */

package/src/room/DeviceManager.ts

@@ -37,9 +37,8 @@ export default class DeviceManager {
 
     if (
       requestPermissions &&
-      kind &&
       // for safari we need to skip this check, as otherwise it will re-acquire user media and fail on iOS https://bugs.webkit.org/show_bug.cgi?id=179363
-      (!DeviceManager.userMediaPromiseMap.get(kind) || !isSafari())
+      !(isSafari() && this.hasDeviceInUse(kind))
     ) {
       const isDummyDeviceOrEmpty =
         devices.length === 0 ||
@@ -85,4 +84,10 @@ export default class DeviceManager {
 
     return device?.deviceId;
   }
+
+  private hasDeviceInUse(kind?: MediaDeviceKind): boolean {
+    return kind
+      ? DeviceManager.userMediaPromiseMap.has(kind)
+      : DeviceManager.userMediaPromiseMap.size > 0;
+  }
 }

package/src/room/PCTransport.ts

@@ -1,6 +1,6 @@
 import { EventEmitter } from 'events';
-import { parse, write } from 'sdp-transform';
 import type { MediaDescription } from 'sdp-transform';
+import { parse, write } from 'sdp-transform';
 import { debounce } from 'ts-debounce';
 import log from '../logger';
 import { NegotiationError, UnexpectedConnectionState } from './errors';
@@ -16,7 +16,7 @@ interface TrackBitrateInfo {
 
 /* The svc codec (av1/vp9) would use a very low bitrate at the begining and
 increase slowly by the bandwidth estimator until it reach the target bitrate. The
-process commonly cost more than 10 seconds cause subscriber will get blur video at 
+process commonly cost more than 10 seconds cause subscriber will get blur video at
 the first few seconds. So we use a 70% of target bitrate here as the start bitrate to
 eliminate this issue.
 */
@@ -308,6 +308,7 @@ export default class PCTransport extends EventEmitter {
       } catch (e) {
         log.warn(`not able to set ${sd.type}, falling back to unmodified sdp`, {
           error: e,
+          sdp: munged,
         });
         sd.sdp = originalSdp;
       }
@@ -328,6 +329,15 @@ export default class PCTransport extends EventEmitter {
       } else if (typeof e === 'string') {
         msg = e;
       }
+
+      const fields: any = {
+        error: msg,
+        sdp: sd.sdp,
+      };
+      if (!remote && this.pc.remoteDescription) {
+        fields.remoteSdp = this.pc.remoteDescription;
+      }
+      log.error(`unable to set ${sd.type}`, fields);
       throw new NegotiationError(msg);
     }
   }

package/src/room/RTCEngine.ts

@@ -191,8 +191,11 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     this.url = url;
     this.token = token;
     this.signalOpts = opts;
+    this.maxJoinAttempts = opts.maxRetries;
     try {
       this.joinAttempts += 1;
+
+      this.setupSignalClientCallbacks();
       const joinResponse = await this.client.join(url, token, opts, abortSignal);
       this._isClosed = false;
       this.latestJoinResponse = joinResponse;
@@ -206,10 +209,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       if (!this.subscriberPrimary) {
         this.negotiate();
       }
-      this.setupSignalClientCallbacks();
 
       this.clientConfiguration = joinResponse.clientConfiguration;
-
       return joinResponse;
     } catch (e) {
       if (e instanceof ConnectionError) {