livekit-client 1.11.3 → 1.12.0

package/src/e2ee/worker/ParticipantKeyHandler.ts

@@ -0,0 +1,129 @@
+import EventEmitter from 'eventemitter3';
+import { workerLogger } from '../../logger';
+import { KEYRING_SIZE } from '../constants';
+import type { KeyProviderOptions, KeySet, ParticipantKeyHandlerCallbacks } from '../types';
+import { deriveKeys, importKey, ratchet } from '../utils';
+
+// TODO ParticipantKeyHandlers currently don't get destroyed on participant disconnect
+// we could do this by having a separate worker message on participant disconnected.
+
+/**
+ * ParticipantKeyHandler is responsible for providing a cryptor instance with the
+ * en-/decryption key of a participant. It assumes that all tracks of a specific participant
+ * are encrypted with the same key.
+ * Additionally it exposes a method to ratchet a key which can be used by the cryptor either automatically
+ * if decryption fails or can be triggered manually on both sender and receiver side.
+ *
+ */
+export class ParticipantKeyHandler extends EventEmitter<ParticipantKeyHandlerCallbacks> {
+  private currentKeyIndex: number;
+
+  private cryptoKeyRing: Array<KeySet>;
+
+  private enabled: boolean;
+
+  private keyProviderOptions: KeyProviderOptions;
+
+  private ratchetPromiseMap: Map<number, Promise<CryptoKey>>;
+
+  private participantId: string | undefined;
+
+  constructor(
+    participantId: string | undefined,
+    isEnabled: boolean,
+    keyProviderOptions: KeyProviderOptions,
+  ) {
+    super();
+    this.currentKeyIndex = 0;
+    this.cryptoKeyRing = new Array(KEYRING_SIZE);
+    this.enabled = isEnabled;
+    this.keyProviderOptions = keyProviderOptions;
+    this.ratchetPromiseMap = new Map();
+    this.participantId = participantId;
+  }
+
+  setEnabled(enabled: boolean) {
+    this.enabled = enabled;
+  }
+
+  /**
+   * Ratchets the current key (or the one at keyIndex if provided) and
+   * returns the ratcheted material
+   * if `setKey` is true (default), it will also set the ratcheted key directly on the crypto key ring
+   * @param keyIndex
+   * @param setKey
+   */
+  ratchetKey(keyIndex?: number, setKey = true): Promise<CryptoKey> {
+    const currentKeyIndex = (keyIndex ??= this.getCurrentKeyIndex());
+
+    const existingPromise = this.ratchetPromiseMap.get(currentKeyIndex);
+    if (typeof existingPromise !== 'undefined') {
+      return existingPromise;
+    }
+    const ratchetPromise = new Promise<CryptoKey>(async (resolve, reject) => {
+      try {
+        const currentMaterial = this.getKeySet(currentKeyIndex).material;
+        const newMaterial = await importKey(
+          await ratchet(currentMaterial, this.keyProviderOptions.ratchetSalt),
+          currentMaterial.algorithm.name,
+          'derive',
+        );
+
+        if (setKey) {
+          this.setKeyFromMaterial(newMaterial, currentKeyIndex, true);
+        }
+        this.emit('keyRatcheted', newMaterial, keyIndex, this.participantId);
+        resolve(newMaterial);
+      } catch (e) {
+        reject(e);
+      } finally {
+        this.ratchetPromiseMap.delete(currentKeyIndex);
+      }
+    });
+    this.ratchetPromiseMap.set(currentKeyIndex, ratchetPromise);
+    return ratchetPromise;
+  }
+
+  /**
+   * takes in a key material with `deriveBits` and `deriveKey` set as key usages
+   * and derives encryption keys from the material and sets it on the key ring buffer
+   * together with the material
+   * also updates the currentKeyIndex
+   */
+  async setKeyFromMaterial(material: CryptoKey, keyIndex = 0, emitRatchetEvent = false) {
+    workerLogger.debug('setting new key');
+    if (keyIndex >= 0) {
+      this.currentKeyIndex = keyIndex % this.cryptoKeyRing.length;
+    }
+    const keySet = await deriveKeys(material, this.keyProviderOptions.ratchetSalt);
+    this.setKeySet(keySet, this.currentKeyIndex, emitRatchetEvent);
+  }
+
+  async setKeySet(keySet: KeySet, keyIndex: number, emitRatchetEvent = false) {
+    this.cryptoKeyRing[keyIndex % this.cryptoKeyRing.length] = keySet;
+    if (emitRatchetEvent) {
+      this.emit('keyRatcheted', keySet.material, keyIndex, this.participantId);
+    }
+  }
+
+  async setCurrentKeyIndex(index: number) {
+    this.currentKeyIndex = index % this.cryptoKeyRing.length;
+  }
+
+  isEnabled() {
+    return this.enabled;
+  }
+
+  getCurrentKeyIndex() {
+    return this.currentKeyIndex;
+  }
+
+  /**
+   * returns currently used KeySet or the one at `keyIndex` if provided
+   * @param keyIndex
+   * @returns
+   */
+  getKeySet(keyIndex?: number) {
+    return this.cryptoKeyRing[keyIndex ?? this.currentKeyIndex];
+  }
+}

package/src/e2ee/worker/e2ee.worker.ts

@@ -0,0 +1,217 @@
+import { workerLogger } from '../../logger';
+import { KEY_PROVIDER_DEFAULTS } from '../constants';
+import { CryptorErrorReason } from '../errors';
+import type {
+  E2EEWorkerMessage,
+  EnableMessage,
+  ErrorMessage,
+  KeyProviderOptions,
+  RatchetMessage,
+} from '../types';
+import { FrameCryptor } from './FrameCryptor';
+import { ParticipantKeyHandler } from './ParticipantKeyHandler';
+
+const participantCryptors: FrameCryptor[] = [];
+const participantKeys: Map<string, ParticipantKeyHandler> = new Map();
+
+let publishCryptors: FrameCryptor[] = [];
+let publisherKeys: ParticipantKeyHandler;
+
+let isEncryptionEnabled: boolean = false;
+
+let useSharedKey: boolean = false;
+
+let sharedKey: CryptoKey | undefined;
+
+let keyProviderOptions: KeyProviderOptions = KEY_PROVIDER_DEFAULTS;
+
+workerLogger.setDefaultLevel('info');
+
+onmessage = (ev) => {
+  const { kind, data }: E2EEWorkerMessage = ev.data;
+
+  switch (kind) {
+    case 'init':
+      workerLogger.info('worker initialized');
+      keyProviderOptions = data.keyProviderOptions;
+      useSharedKey = !!data.keyProviderOptions.sharedKey;
+      // acknowledge init successful
+      const enableMsg: EnableMessage = {
+        kind: 'enable',
+        data: { enabled: isEncryptionEnabled },
+      };
+      publisherKeys = new ParticipantKeyHandler(undefined, isEncryptionEnabled, keyProviderOptions);
+      publisherKeys.on('keyRatcheted', emitRatchetedKeys);
+      postMessage(enableMsg);
+      break;
+    case 'enable':
+      setEncryptionEnabled(data.enabled, data.participantId);
+      workerLogger.info('updated e2ee enabled status');
+      // acknowledge enable call successful
+      postMessage(ev.data);
+      break;
+    case 'decode':
+      let cryptor = getTrackCryptor(data.participantId, data.trackId);
+      cryptor.setupTransform(
+        kind,
+        data.readableStream,
+        data.writableStream,
+        data.trackId,
+        data.codec,
+      );
+      break;
+    case 'encode':
+      let pubCryptor = getPublisherCryptor(data.trackId);
+      pubCryptor.setupTransform(
+        kind,
+        data.readableStream,
+        data.writableStream,
+        data.trackId,
+        data.codec,
+      );
+      break;
+    case 'setKey':
+      if (useSharedKey) {
+        workerLogger.debug('set shared key');
+        setSharedKey(data.key, data.keyIndex);
+      } else if (data.participantId) {
+        getParticipantKeyHandler(data.participantId).setKeyFromMaterial(data.key, data.keyIndex);
+      } else {
+        workerLogger.error('no participant Id was provided and shared key usage is disabled');
+      }
+      break;
+    case 'removeTransform':
+      unsetCryptorParticipant(data.trackId);
+      break;
+    case 'updateCodec':
+      getTrackCryptor(data.participantId, data.trackId).setVideoCodec(data.codec);
+      break;
+    case 'setRTPMap':
+      publishCryptors.forEach((cr) => {
+        cr.setRtpMap(data.map);
+      });
+      break;
+    case 'ratchetRequest':
+      getParticipantKeyHandler(data.participantId).ratchetKey(data.keyIndex);
+
+    default:
+      break;
+  }
+};
+
+function getTrackCryptor(participantId: string, trackId: string) {
+  let cryptor = participantCryptors.find((c) => c.getTrackId() === trackId);
+  if (!cryptor) {
+    workerLogger.info('creating new cryptor for', { participantId });
+    if (!keyProviderOptions) {
+      throw Error('Missing keyProvider options');
+    }
+    cryptor = new FrameCryptor({
+      participantId,
+      keys: getParticipantKeyHandler(participantId),
+      keyProviderOptions,
+    });
+
+    setupCryptorErrorEvents(cryptor);
+    participantCryptors.push(cryptor);
+  } else if (participantId !== cryptor.getParticipantId()) {
+    // assign new participant id to track cryptor and pass in correct key handler
+    cryptor.setParticipant(participantId, getParticipantKeyHandler(participantId));
+  }
+  if (sharedKey) {
+  }
+  return cryptor;
+}
+
+function getParticipantKeyHandler(participantId?: string) {
+  if (!participantId) {
+    return publisherKeys!;
+  }
+  let keys = participantKeys.get(participantId);
+  if (!keys) {
+    keys = new ParticipantKeyHandler(participantId, true, keyProviderOptions);
+    if (sharedKey) {
+      keys.setKeyFromMaterial(sharedKey);
+    }
+    participantKeys.set(participantId, keys);
+  }
+  return keys;
+}
+
+function unsetCryptorParticipant(trackId: string) {
+  participantCryptors.find((c) => c.getTrackId() === trackId)?.unsetParticipant();
+}
+
+function getPublisherCryptor(trackId: string) {
+  let publishCryptor = publishCryptors.find((cryptor) => cryptor.getTrackId() === trackId);
+  if (!publishCryptor) {
+    if (!keyProviderOptions) {
+      throw new TypeError('Missing keyProvider options');
+    }
+    publishCryptor = new FrameCryptor({
+      keys: publisherKeys!,
+      participantId: 'publisher',
+      keyProviderOptions,
+    });
+    setupCryptorErrorEvents(publishCryptor);
+    publishCryptors.push(publishCryptor);
+  }
+  return publishCryptor;
+}
+
+function setEncryptionEnabled(enable: boolean, participantId?: string) {
+  if (!participantId) {
+    isEncryptionEnabled = enable;
+    publisherKeys.setEnabled(enable);
+  } else {
+    getParticipantKeyHandler(participantId).setEnabled(enable);
+  }
+}
+
+function setSharedKey(key: CryptoKey, index?: number) {
+  workerLogger.debug('setting shared key');
+  sharedKey = key;
+  publisherKeys?.setKeyFromMaterial(key, index);
+  for (const [, keyHandler] of participantKeys) {
+    keyHandler.setKeyFromMaterial(key, index);
+  }
+}
+
+function setupCryptorErrorEvents(cryptor: FrameCryptor) {
+  cryptor.on('cryptorError', (error) => {
+    const msg: ErrorMessage = {
+      kind: 'error',
+      data: { error: new Error(`${CryptorErrorReason[error.reason]}: ${error.message}`) },
+    };
+    postMessage(msg);
+  });
+}
+
+function emitRatchetedKeys(material: CryptoKey, keyIndex?: number) {
+  const msg: RatchetMessage = {
+    kind: `ratchetKey`,
+    data: {
+      // participantId,
+      keyIndex,
+      material,
+    },
+  };
+  postMessage(msg);
+}
+
+// Operations using RTCRtpScriptTransform.
+// @ts-ignore
+if (self.RTCTransformEvent) {
+  workerLogger.debug('setup transform event');
+  // @ts-ignore
+  self.onrtctransform = (event) => {
+    const transformer = event.transformer;
+    workerLogger.debug('transformer', transformer);
+    transformer.handled = true;
+    const { kind, participantId, trackId, codec } = transformer.options;
+    const cryptor =
+      kind === 'encode' ? getPublisherCryptor(trackId) : getTrackCryptor(participantId, trackId);
+    workerLogger.debug('transform', { codec });
+    cryptor.setupTransform(kind, transformer.readable, transformer.writable, trackId, codec);
+  };
+}

package/src/e2ee/worker/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "lib": ["DOM", "DOM.Iterable", "ES2017", "ES2018.Promise", "WebWorker"]
+  }
+}

package/src/index.ts

@@ -37,9 +37,11 @@ export * from './room/events';
 export * from './room/track/Track';
 export * from './room/track/create';
 export * from './room/track/options';
+export { facingModeFromDeviceLabel, facingModeFromLocalTrack } from './room/track/facingMode';
 export * from './room/track/types';
 export type { DataPublishOptions, SimulationScenario } from './room/types';
 export * from './version';
+export * from './e2ee';
 export * from './room/track/processor/types';
 export {
   setLogLevel,

package/src/logger.ts

@@ -17,6 +17,7 @@ type StructuredLogger = {
   info: (msg: string, context?: object) => void;
   warn: (msg: string, context?: object) => void;
   error: (msg: string, context?: object) => void;
+  setDefaultLevel: (level: log.LogLevelDesc) => void;
 };
 
 const livekitLogger = log.getLogger('livekit');
@@ -25,8 +26,13 @@ livekitLogger.setDefaultLevel(LogLevel.info);
 
 export default livekitLogger as StructuredLogger;
 
-export function setLogLevel(level: LogLevel | LogLevelString) {
-  livekitLogger.setLevel(level);
+export function setLogLevel(level: LogLevel | LogLevelString, loggerName?: 'livekit' | 'lk-e2ee') {
+  if (loggerName) {
+    log.getLogger(loggerName).setLevel(level);
+  }
+  for (const logger of Object.values(log.getLoggers())) {
+    logger.setLevel(level);
+  }
 }
 
 export type LogExtension = (level: LogLevel, msg: string, context?: object) => void;
@@ -54,3 +60,5 @@ export function setLogExtension(extension: LogExtension) {
   };
   livekitLogger.setLevel(livekitLogger.getLevel()); // Be sure to call setLevel method in order to apply plugin
 }
+
+export const workerLogger = log.getLogger('lk-e2ee') as StructuredLogger;

package/src/options.ts

@@ -1,3 +1,4 @@
+import type { E2EEOptions } from './e2ee/types';
 import type { ReconnectPolicy } from './room/ReconnectPolicy';
 import type {
   AudioCaptureOptions,
@@ -83,6 +84,11 @@ export interface InternalRoomOptions {
    */
 
   expWebAudioMix: boolean | WebAudioSettings;
+
+  /**
+   * @experimental
+   */
+  e2ee?: E2EEOptions;
 }
 
 /**

package/src/proto/livekit_models.ts

@@ -378,7 +378,7 @@ export function disconnectReasonToJSON(object: DisconnectReason): string {
 }
 
 export enum ReconnectReason {
-  RR_UNKOWN = 0,
+  RR_UNKNOWN = 0,
   RR_SIGNAL_DISCONNECTED = 1,
   RR_PUBLISHER_FAILED = 2,
   RR_SUBSCRIBER_FAILED = 3,
@@ -389,8 +389,8 @@ export enum ReconnectReason {
 export function reconnectReasonFromJSON(object: any): ReconnectReason {
   switch (object) {
     case 0:
-    case "RR_UNKOWN":
-      return ReconnectReason.RR_UNKOWN;
+    case "RR_UNKNOWN":
+      return ReconnectReason.RR_UNKNOWN;
     case 1:
     case "RR_SIGNAL_DISCONNECTED":
       return ReconnectReason.RR_SIGNAL_DISCONNECTED;
@@ -412,8 +412,8 @@ export function reconnectReasonFromJSON(object: any): ReconnectReason {
 
 export function reconnectReasonToJSON(object: ReconnectReason): string {
   switch (object) {
-    case ReconnectReason.RR_UNKOWN:
-      return "RR_UNKOWN";
+    case ReconnectReason.RR_UNKNOWN:
+      return "RR_UNKNOWN";
     case ReconnectReason.RR_SIGNAL_DISCONNECTED:
       return "RR_SIGNAL_DISCONNECTED";
     case ReconnectReason.RR_PUBLISHER_FAILED:
@@ -429,7 +429,7 @@ export function reconnectReasonToJSON(object: ReconnectReason): string {
 }
 
 export enum SubscriptionError {
-  SE_UNKOWN = 0,
+  SE_UNKNOWN = 0,
   SE_CODEC_UNSUPPORTED = 1,
   SE_TRACK_NOTFOUND = 2,
   UNRECOGNIZED = -1,
@@ -438,8 +438,8 @@ export enum SubscriptionError {
 export function subscriptionErrorFromJSON(object: any): SubscriptionError {
   switch (object) {
     case 0:
-    case "SE_UNKOWN":
-      return SubscriptionError.SE_UNKOWN;
+    case "SE_UNKNOWN":
+      return SubscriptionError.SE_UNKNOWN;
     case 1:
     case "SE_CODEC_UNSUPPORTED":
       return SubscriptionError.SE_CODEC_UNSUPPORTED;
@@ -455,8 +455,8 @@ export function subscriptionErrorFromJSON(object: any): SubscriptionError {
 
 export function subscriptionErrorToJSON(object: SubscriptionError): string {
   switch (object) {
-    case SubscriptionError.SE_UNKOWN:
-      return "SE_UNKOWN";
+    case SubscriptionError.SE_UNKNOWN:
+      return "SE_UNKNOWN";
     case SubscriptionError.SE_CODEC_UNSUPPORTED:
       return "SE_CODEC_UNSUPPORTED";
     case SubscriptionError.SE_TRACK_NOTFOUND:
@@ -3970,8 +3970,8 @@ function toTimestamp(date: Date): Timestamp {
 }
 
 function fromTimestamp(t: Timestamp): Date {
-  let millis = t.seconds * 1_000;
-  millis += t.nanos / 1_000_000;
+  let millis = (t.seconds || 0) * 1_000;
+  millis += (t.nanos || 0) / 1_000_000;
   return new Date(millis);
 }
 

package/src/room/PCTransport.ts

@@ -3,7 +3,7 @@ import { parse, write } from 'sdp-transform';
 import type { MediaDescription } from 'sdp-transform';
 import { debounce } from 'ts-debounce';
 import log from '../logger';
-import { NegotiationError } from './errors';
+import { NegotiationError, UnexpectedConnectionState } from './errors';
 import { ddExtensionURI, isChromiumBased, isSVCCodec } from './utils';
 
 /** @internal */
@@ -25,11 +25,17 @@ const startBitrateForSVC = 0.7;
 export const PCEvents = {
   NegotiationStarted: 'negotiationStarted',
   NegotiationComplete: 'negotiationComplete',
+  RTPVideoPayloadTypes: 'rtpVideoPayloadTypes',
 } as const;
 
 /** @internal */
 export default class PCTransport extends EventEmitter {
-  pc: RTCPeerConnection;
+  private _pc: RTCPeerConnection | null;
+
+  public get pc() {
+    if (this._pc) return this._pc;
+    throw new UnexpectedConnectionState('Expected peer connection to be available');
+  }
 
   pendingCandidates: RTCIceCandidateInit[] = [];
 
@@ -47,14 +53,17 @@ export default class PCTransport extends EventEmitter {
 
   constructor(config?: RTCConfiguration, mediaConstraints: Record<string, unknown> = {}) {
     super();
-    this.pc = isChromiumBased()
+    this._pc = isChromiumBased()
       ? // @ts-expect-error chrome allows additional media constraints to be passed into the RTCPeerConnection constructor
         new RTCPeerConnection(config, mediaConstraints)
       : new RTCPeerConnection(config);
   }
 
   get isICEConnected(): boolean {
-    return this.pc.iceConnectionState === 'connected' || this.pc.iceConnectionState === 'completed';
+    return (
+      this._pc !== null &&
+      (this.pc.iceConnectionState === 'connected' || this.pc.iceConnectionState === 'completed')
+    );
   }
 
   async addIceCandidate(candidate: RTCIceCandidateInit): Promise<void> {
@@ -136,6 +145,14 @@ export default class PCTransport extends EventEmitter {
       this.createAndSendOffer();
     } else if (sd.type === 'answer') {
       this.emit(PCEvents.NegotiationComplete);
+      if (sd.sdp) {
+        const sdpParsed = parse(sd.sdp);
+        sdpParsed.media.forEach((media) => {
+          if (media.type === 'video') {
+            this.emit(PCEvents.RTPVideoPayloadTypes, media.rtp);
+          }
+        });
+      }
     }
   }
 
@@ -163,7 +180,7 @@ export default class PCTransport extends EventEmitter {
       this.restartingIce = true;
     }
 
-    if (this.pc.signalingState === 'have-local-offer') {
+    if (this._pc && this._pc.signalingState === 'have-local-offer') {
       // we're waiting for the peer to accept our offer, so we'll just wait
       // the only exception to this is when ICE restart is needed
       const currentSD = this.pc.remoteDescription;
@@ -175,7 +192,7 @@ export default class PCTransport extends EventEmitter {
         this.renegotiate = true;
         return;
       }
-    } else if (this.pc.signalingState === 'closed') {
+    } else if (!this._pc || this._pc.signalingState === 'closed') {
       log.warn('could not createOffer with closed peer connection');
       return;
     }
@@ -258,9 +275,22 @@ export default class PCTransport extends EventEmitter {
   }
 
   close() {
-    this.pc.onconnectionstatechange = null;
-    this.pc.oniceconnectionstatechange = null;
-    this.pc.close();
+    if (!this._pc) {
+      return;
+    }
+    this._pc.close();
+    this._pc.onconnectionstatechange = null;
+    this._pc.oniceconnectionstatechange = null;
+    this._pc.onicegatheringstatechange = null;
+    this._pc.ondatachannel = null;
+    this._pc.onnegotiationneeded = null;
+    this._pc.onsignalingstatechange = null;
+    this._pc.onicecandidate = null;
+    this._pc.ondatachannel = null;
+    this._pc.ontrack = null;
+    this._pc.onconnectionstatechange = null;
+    this._pc.oniceconnectionstatechange = null;
+    this._pc = null;
   }
 
   private async setMungedSDP(sd: RTCSessionDescriptionInit, munged?: string, remote?: boolean) {