livekit-client 1.11.4 → 1.12.0

package/src/e2ee/E2eeManager.ts

@@ -0,0 +1,374 @@
+import EventEmitter from 'eventemitter3';
+import log from '../logger';
+import { Encryption_Type, TrackInfo } from '../proto/livekit_models';
+import type RTCEngine from '../room/RTCEngine';
+import type Room from '../room/Room';
+import { ConnectionState } from '../room/Room';
+import { DeviceUnsupportedError } from '../room/errors';
+import { EngineEvent, ParticipantEvent, RoomEvent } from '../room/events';
+import LocalTrack from '../room/track/LocalTrack';
+import type RemoteTrack from '../room/track/RemoteTrack';
+import type { Track } from '../room/track/Track';
+import type { VideoCodec } from '../room/track/options';
+import type { BaseKeyProvider } from './KeyProvider';
+import { E2EE_FLAG } from './constants';
+import type {
+  E2EEManagerCallbacks,
+  E2EEOptions,
+  E2EEWorkerMessage,
+  EnableMessage,
+  EncodeMessage,
+  InitMessage,
+  KeyInfo,
+  RTPVideoMapMessage,
+  RatchetRequestMessage,
+  RemoveTransformMessage,
+  SetKeyMessage,
+  UpdateCodecMessage,
+} from './types';
+import { EncryptionEvent } from './types';
+import { isE2EESupported, isScriptTransformSupported, mimeTypeToVideoCodecString } from './utils';
+
+/**
+ * @experimental
+ */
+export class E2EEManager extends EventEmitter<E2EEManagerCallbacks> {
+  protected worker: Worker;
+
+  protected room?: Room;
+
+  private encryptionEnabled: boolean;
+
+  private keyProvider: BaseKeyProvider;
+
+  get isEnabled() {
+    return this.encryptionEnabled;
+  }
+
+  constructor(options: E2EEOptions) {
+    super();
+    this.keyProvider = options.keyProvider;
+    this.worker = options.worker;
+    this.encryptionEnabled = false;
+  }
+
+  /**
+   * @internal
+   */
+  setup(room: Room) {
+    if (!isE2EESupported()) {
+      throw new DeviceUnsupportedError(
+        'tried to setup end-to-end encryption on an unsupported browser',
+      );
+    }
+    log.info('setting up e2ee');
+    if (room !== this.room) {
+      this.room = room;
+      this.setupEventListeners(room, this.keyProvider);
+      // this.worker = new Worker('');
+      const msg: InitMessage = {
+        kind: 'init',
+        data: {
+          keyProviderOptions: this.keyProvider.getOptions(),
+        },
+      };
+      if (this.worker) {
+        log.info(`initializing worker`, { worker: this.worker });
+        this.worker.onmessage = this.onWorkerMessage;
+        this.worker.onerror = this.onWorkerError;
+        this.worker.postMessage(msg);
+      }
+    }
+  }
+
+  /**
+   * @internal
+   */
+  async setParticipantCryptorEnabled(enabled: boolean, participantId?: string) {
+    log.info(`set e2ee to ${enabled}`);
+
+    if (this.worker) {
+      const enableMsg: EnableMessage = {
+        kind: 'enable',
+        data: { enabled, participantId },
+      };
+      this.worker.postMessage(enableMsg);
+    } else {
+      throw new ReferenceError('failed to enable e2ee, worker is not ready');
+    }
+  }
+
+  private onWorkerMessage = (ev: MessageEvent<E2EEWorkerMessage>) => {
+    const { kind, data } = ev.data;
+    switch (kind) {
+      case 'error':
+        console.error('error in worker', { data });
+        this.emit(EncryptionEvent.Error, data.error);
+        break;
+      case 'enable':
+        if (this.encryptionEnabled !== data.enabled && !data.participantId) {
+          this.emit(
+            EncryptionEvent.ParticipantEncryptionStatusChanged,
+            data.enabled,
+            this.room?.localParticipant,
+          );
+          this.encryptionEnabled = data.enabled;
+        } else if (data.participantId) {
+          const participant = this.room?.getParticipantByIdentity(data.participantId);
+          this.emit(EncryptionEvent.ParticipantEncryptionStatusChanged, data.enabled, participant);
+        }
+        if (this.encryptionEnabled) {
+          this.keyProvider.getKeys().forEach((keyInfo) => {
+            this.postKey(keyInfo);
+          });
+        }
+        break;
+      case 'ratchetKey':
+        this.keyProvider.emit('keyRatcheted', data.material, data.keyIndex);
+        break;
+      default:
+        break;
+    }
+  };
+
+  private onWorkerError = (ev: ErrorEvent) => {
+    log.error('e2ee worker encountered an error:', { error: ev.error });
+    this.emit(EncryptionEvent.Error, ev.error);
+  };
+
+  public setupEngine(engine: RTCEngine) {
+    engine.on(EngineEvent.RTPVideoMapUpdate, (rtpMap) => {
+      this.postRTPMap(rtpMap);
+    });
+  }
+
+  private setupEventListeners(room: Room, keyProvider: BaseKeyProvider) {
+    room.on(RoomEvent.TrackPublished, (pub, participant) =>
+      this.setParticipantCryptorEnabled(
+        pub.trackInfo!.encryption !== Encryption_Type.NONE,
+        participant.identity,
+      ),
+    );
+    room.on(RoomEvent.ConnectionStateChanged, (state) => {
+      if (state === ConnectionState.Connected) {
+        room.participants.forEach((participant) => {
+          participant.tracks.forEach((pub) => {
+            this.setParticipantCryptorEnabled(
+              pub.trackInfo!.encryption !== Encryption_Type.NONE,
+              participant.identity,
+            );
+          });
+        });
+      }
+    });
+
+    room.on(RoomEvent.TrackUnsubscribed, (track, _, participant) => {
+      const msg: RemoveTransformMessage = {
+        kind: 'removeTransform',
+        data: {
+          participantId: participant.identity,
+          trackId: track.mediaStreamID,
+        },
+      };
+      this.worker?.postMessage(msg);
+    });
+    room.on(RoomEvent.TrackSubscribed, (track, pub, participant) => {
+      this.setupE2EEReceiver(track, participant.identity, pub.trackInfo);
+    });
+    room.localParticipant.on(ParticipantEvent.LocalTrackPublished, async (publication) => {
+      this.setupE2EESender(
+        publication.track!,
+        publication.track!.sender!,
+        room.localParticipant.identity,
+      );
+    });
+
+    keyProvider
+      .on('setKey', (keyInfo) => this.postKey(keyInfo))
+      .on('ratchetRequest', (participantId, keyIndex) =>
+        this.postRatchetRequest(participantId, keyIndex),
+      );
+  }
+
+  private postRatchetRequest(participantId?: string, keyIndex?: number) {
+    if (!this.worker) {
+      throw Error('could not ratchet key, worker is missing');
+    }
+    const msg: RatchetRequestMessage = {
+      kind: 'ratchetRequest',
+      data: {
+        participantId,
+        keyIndex,
+      },
+    };
+    this.worker.postMessage(msg);
+  }
+
+  private postKey({ key, participantId, keyIndex }: KeyInfo) {
+    if (!this.worker) {
+      throw Error('could not set key, worker is missing');
+    }
+    const msg: SetKeyMessage = {
+      kind: 'setKey',
+      data: {
+        participantId,
+        key,
+        keyIndex,
+      },
+    };
+    this.worker.postMessage(msg);
+  }
+
+  private postRTPMap(map: Map<number, VideoCodec>) {
+    if (!this.worker) {
+      throw Error('could not post rtp map, worker is missing');
+    }
+    const msg: RTPVideoMapMessage = {
+      kind: 'setRTPMap',
+      data: {
+        map,
+      },
+    };
+    this.worker.postMessage(msg);
+  }
+
+  private setupE2EEReceiver(track: RemoteTrack, remoteId: string, trackInfo?: TrackInfo) {
+    if (!track.receiver) {
+      return;
+    }
+    if (!trackInfo?.mimeType || trackInfo.mimeType === '') {
+      throw new TypeError('MimeType missing from trackInfo, cannot set up E2EE cryptor');
+    }
+    this.handleReceiver(
+      track.receiver,
+      track.mediaStreamID,
+      remoteId,
+      track.kind === 'video' ? mimeTypeToVideoCodecString(trackInfo.mimeType) : undefined,
+    );
+  }
+
+  private setupE2EESender(track: Track, sender: RTCRtpSender, localId: string) {
+    if (!(track instanceof LocalTrack) || !sender) {
+      if (!sender) log.warn('early return because sender is not ready');
+      return;
+    }
+    this.handleSender(sender, track.mediaStreamID, localId, undefined);
+  }
+
+  /**
+   * Handles the given {@code RTCRtpReceiver} by creating a {@code TransformStream} which will inject
+   * a frame decoder.
+   *
+   */
+  private async handleReceiver(
+    receiver: RTCRtpReceiver,
+    trackId: string,
+    participantId: string,
+    codec?: VideoCodec,
+  ) {
+    if (!this.worker) {
+      return;
+    }
+
+    if (isScriptTransformSupported()) {
+      const options = {
+        kind: 'decode',
+        participantId,
+        trackId,
+        codec,
+      };
+      // @ts-ignore
+      receiver.transform = new RTCRtpScriptTransform(this.worker, options);
+    } else {
+      if (E2EE_FLAG in receiver && codec) {
+        // only update codec
+        const msg: UpdateCodecMessage = {
+          kind: 'updateCodec',
+          data: {
+            trackId,
+            codec,
+            participantId,
+          },
+        };
+        this.worker.postMessage(msg);
+        return;
+      }
+      // @ts-ignore
+      let writable: WritableStream = receiver.writableStream;
+      // @ts-ignore
+      let readable: ReadableStream = receiver.readableStream;
+      if (!writable || !readable) {
+        // @ts-ignore
+        const receiverStreams = receiver.createEncodedStreams();
+        // @ts-ignore
+        receiver.writableStream = receiverStreams.writable;
+        writable = receiverStreams.writable;
+        // @ts-ignore
+        receiver.readableStream = receiverStreams.readable;
+        readable = receiverStreams.readable;
+      }
+
+      const msg: EncodeMessage = {
+        kind: 'decode',
+        data: {
+          readableStream: readable,
+          writableStream: writable,
+          trackId: trackId,
+          codec,
+          participantId,
+        },
+      };
+      this.worker.postMessage(msg, [readable, writable]);
+    }
+
+    // @ts-ignore
+    receiver[E2EE_FLAG] = true;
+  }
+
+  /**
+   * Handles the given {@code RTCRtpSender} by creating a {@code TransformStream} which will inject
+   * a frame encoder.
+   *
+   */
+  private handleSender(
+    sender: RTCRtpSender,
+    trackId: string,
+    participantId: string,
+    codec?: VideoCodec,
+  ) {
+    if (E2EE_FLAG in sender || !this.worker) {
+      return;
+    }
+
+    if (isScriptTransformSupported()) {
+      log.warn('initialize script transform');
+
+      const options = {
+        kind: 'encode',
+        participantId,
+        trackId,
+        codec,
+      };
+      // @ts-ignore
+      sender.transform = new RTCRtpScriptTransform(this.worker, options);
+    } else {
+      log.warn('initialize encoded streams');
+      // @ts-ignore
+      const senderStreams = sender.createEncodedStreams();
+      const msg: EncodeMessage = {
+        kind: 'encode',
+        data: {
+          readableStream: senderStreams.readable,
+          writableStream: senderStreams.writable,
+          codec,
+          trackId,
+          participantId,
+        },
+      };
+      this.worker.postMessage(msg, [senderStreams.readable, senderStreams.writable]);
+    }
+
+    // @ts-ignore
+    sender[E2EE_FLAG] = true;
+  }
+}

package/src/e2ee/KeyProvider.ts

@@ -0,0 +1,77 @@
+import EventEmitter from 'eventemitter3';
+import { KEY_PROVIDER_DEFAULTS } from './constants';
+import type { KeyInfo, KeyProviderCallbacks, KeyProviderOptions } from './types';
+import { createKeyMaterialFromString } from './utils';
+
+/**
+ * @experimental
+ */
+export class BaseKeyProvider extends EventEmitter<KeyProviderCallbacks> {
+  private keyInfoMap: Map<string, KeyInfo>;
+
+  private options: KeyProviderOptions;
+
+  constructor(options: Partial<KeyProviderOptions> = {}) {
+    super();
+    this.keyInfoMap = new Map();
+    this.options = { ...KEY_PROVIDER_DEFAULTS, ...options };
+    this.on('keyRatcheted', this.onKeyRatcheted);
+  }
+
+  /**
+   * callback to invoke once a key has been set for a participant
+   * @param key
+   * @param participantId
+   * @param keyIndex
+   */
+  protected onSetEncryptionKey(key: CryptoKey, participantId?: string, keyIndex?: number) {
+    const keyInfo: KeyInfo = { key, participantId, keyIndex };
+    this.keyInfoMap.set(`${participantId ?? 'shared'}-${keyIndex ?? 0}`, keyInfo);
+    this.emit('setKey', keyInfo);
+  }
+
+  /**
+   * callback being invoked after a ratchet request has been performed on the local participant
+   * that surfaces the new key material.
+   * @param material
+   * @param keyIndex
+   */
+  protected onKeyRatcheted = (material: CryptoKey, keyIndex?: number) => {
+    console.debug('key ratcheted event received', material, keyIndex);
+  };
+
+  getKeys() {
+    return Array.from(this.keyInfoMap.values());
+  }
+
+  getOptions() {
+    return this.options;
+  }
+
+  ratchetKey(participantId?: string, keyIndex?: number) {
+    this.emit('ratchetRequest', participantId, keyIndex);
+  }
+}
+
+/**
+ * A basic KeyProvider implementation intended for a single shared
+ * passphrase between all participants
+ * @experimental
+ */
+export class ExternalE2EEKeyProvider extends BaseKeyProvider {
+  ratchetInterval: number | undefined;
+
+  constructor(options: Partial<Omit<KeyProviderOptions, 'sharedKey'>> = {}) {
+    const opts: Partial<KeyProviderOptions> = { ...options, sharedKey: true };
+    super(opts);
+  }
+
+  /**
+   * Accepts a passphrase that's used to create the crypto keys
+   * @param key
+   */
+  async setKey(key: string) {
+    const derivedKey = await createKeyMaterialFromString(key);
+    this.onSetEncryptionKey(derivedKey);
+  }
+}

package/src/e2ee/constants.ts

@@ -0,0 +1,40 @@
+import type { KeyProviderOptions } from './types';
+
+export const ENCRYPTION_ALGORITHM = 'AES-GCM';
+
+// We use a ringbuffer of keys so we can change them and still decode packets that were
+// encrypted with an old key. We use a size of 16 which corresponds to the four bits
+// in the frame trailer.
+export const KEYRING_SIZE = 16;
+
+// We copy the first bytes of the VP8 payload unencrypted.
+// For keyframes this is 10 bytes, for non-keyframes (delta) 3. See
+//   https://tools.ietf.org/html/rfc6386#section-9.1
+// This allows the bridge to continue detecting keyframes (only one byte needed in the JVB)
+// and is also a bit easier for the VP8 decoder (i.e. it generates funny garbage pictures
+// instead of being unable to decode).
+// This is a bit for show and we might want to reduce to 1 unconditionally in the final version.
+//
+// For audio (where frame.type is not set) we do not encrypt the opus TOC byte:
+//   https://tools.ietf.org/html/rfc6716#section-3.1
+export const UNENCRYPTED_BYTES = {
+  key: 10,
+  delta: 3,
+  audio: 1, // frame.type is not set on audio, so this is set manually
+  empty: 0,
+} as const;
+
+/* We use a 12 byte bit IV. This is signalled in plain together with the
+ packet. See https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/encrypt#parameters */
+export const IV_LENGTH = 12;
+
+// flag set to indicate that e2ee has been setup for sender/receiver;
+export const E2EE_FLAG = 'lk_e2ee';
+
+export const SALT = 'LKFrameEncryptionKey';
+
+export const KEY_PROVIDER_DEFAULTS: KeyProviderOptions = {
+  sharedKey: false,
+  ratchetSalt: SALT,
+  ratchetWindowSize: 8,
+} as const;

package/src/e2ee/errors.ts

@@ -0,0 +1,16 @@
+import { LivekitError } from '../room/errors';
+
+export enum CryptorErrorReason {
+  InvalidKey = 0,
+  MissingKey = 1,
+  InternalError = 2,
+}
+
+export class CryptorError extends LivekitError {
+  reason: CryptorErrorReason;
+
+  constructor(message?: string, reason: CryptorErrorReason = CryptorErrorReason.InternalError) {
+    super(40, message);
+    this.reason = reason;
+  }
+}

package/src/e2ee/index.ts

@@ -0,0 +1,3 @@
+export * from './KeyProvider';
+export * from './utils';
+export * from './types';

package/src/e2ee/types.ts

@@ -0,0 +1,160 @@
+import type Participant from '../room/participant/Participant';
+import type { VideoCodec } from '../room/track/options';
+import type { BaseKeyProvider } from './KeyProvider';
+import type { CryptorError } from './errors';
+
+export interface BaseMessage {
+  kind: string;
+  data?: unknown;
+}
+
+export interface InitMessage extends BaseMessage {
+  kind: 'init';
+  data: {
+    keyProviderOptions: KeyProviderOptions;
+  };
+}
+
+export interface SetKeyMessage extends BaseMessage {
+  kind: 'setKey';
+  data: {
+    participantId?: string;
+    key: CryptoKey;
+    keyIndex?: number;
+  };
+}
+
+export interface RTPVideoMapMessage extends BaseMessage {
+  kind: 'setRTPMap';
+  data: {
+    map: Map<number, VideoCodec>;
+  };
+}
+
+export interface EncodeMessage extends BaseMessage {
+  kind: 'decode' | 'encode';
+  data: {
+    participantId: string;
+    readableStream: ReadableStream;
+    writableStream: WritableStream;
+    trackId: string;
+    codec?: VideoCodec;
+  };
+}
+
+export interface RemoveTransformMessage extends BaseMessage {
+  kind: 'removeTransform';
+  data: {
+    participantId: string;
+    trackId: string;
+  };
+}
+
+export interface UpdateCodecMessage extends BaseMessage {
+  kind: 'updateCodec';
+  data: {
+    participantId: string;
+    trackId: string;
+    codec: VideoCodec;
+  };
+}
+
+export interface RatchetRequestMessage extends BaseMessage {
+  kind: 'ratchetRequest';
+  data: {
+    participantId: string | undefined;
+    keyIndex?: number;
+  };
+}
+
+export interface RatchetMessage extends BaseMessage {
+  kind: 'ratchetKey';
+  data: {
+    // participantId: string | undefined;
+    keyIndex?: number;
+    material: CryptoKey;
+  };
+}
+
+export interface ErrorMessage extends BaseMessage {
+  kind: 'error';
+  data: {
+    error: Error;
+  };
+}
+
+export interface EnableMessage extends BaseMessage {
+  kind: 'enable';
+  data: {
+    // if no participant id is set it indicates publisher encryption enable/disable
+    participantId?: string;
+    enabled: boolean;
+  };
+}
+
+export type E2EEWorkerMessage =
+  | InitMessage
+  | SetKeyMessage
+  | EncodeMessage
+  | ErrorMessage
+  | EnableMessage
+  | RemoveTransformMessage
+  | RTPVideoMapMessage
+  | UpdateCodecMessage
+  | RatchetRequestMessage
+  | RatchetMessage;
+
+export type KeySet = { material: CryptoKey; encryptionKey: CryptoKey };
+
+export type KeyProviderOptions = {
+  sharedKey: boolean;
+  ratchetSalt: string;
+  ratchetWindowSize: number;
+};
+
+export type KeyProviderCallbacks = {
+  setKey: (keyInfo: KeyInfo) => void;
+  ratchetRequest: (participantId?: string, keyIndex?: number) => void;
+  /** currently only emitted for local participant */
+  keyRatcheted: (material: CryptoKey, keyIndex?: number) => void;
+};
+
+export type ParticipantKeyHandlerCallbacks = {
+  keyRatcheted: (material: CryptoKey, keyIndex?: number, participantId?: string) => void;
+};
+
+export type E2EEManagerCallbacks = {
+  participantEncryptionStatusChanged: (enabled: boolean, participant?: Participant) => void;
+  encryptionError: (error: Error) => void;
+};
+
+export const EncryptionEvent = {
+  ParticipantEncryptionStatusChanged: 'participantEncryptionStatusChanged',
+  Error: 'encryptionError',
+} as const;
+
+export type CryptorCallbacks = {
+  cryptorError: (error: CryptorError) => void;
+};
+
+export const CryptorEvent = {
+  Error: 'cryptorError',
+} as const;
+
+export type KeyInfo = {
+  key: CryptoKey;
+  participantId?: string;
+  keyIndex?: number;
+};
+
+export type E2EEOptions = {
+  keyProvider: BaseKeyProvider;
+  worker: Worker;
+};
+
+export type DecodeRatchetOptions = {
+  /** attempts  */
+  ratchetCount: number;
+  /** ratcheted key to try */
+  encryptionKey?: CryptoKey;
+};