livekit-client 1.14.1 → 1.14.2

package/src/e2ee/worker/FrameCryptor.ts

@@ -326,11 +326,11 @@ export class FrameCryptor extends BaseFrameCryptor {
       }
     } else if (!this.keys.getKeySet(keyIndex) && this.keys.hasValidKey) {
       // emit an error in case the key index is out of bounds but the key handler thinks we still have a valid key
-      workerLogger.warn('skipping decryption due to missing key at index');
+      workerLogger.warn(`skipping decryption due to missing key at index ${keyIndex}`);
       this.emit(
         CryptorEvent.Error,
         new CryptorError(
-          `missing key at index for participant ${this.participantIdentity}`,
+          `missing key at index ${keyIndex} for participant ${this.participantIdentity}`,
           CryptorErrorReason.MissingKey,
         ),
       );
@@ -418,7 +418,7 @@ export class FrameCryptor extends BaseFrameCryptor {
           );
 
           let ratchetedKeySet: KeySet | undefined;
-          if (keySet === this.keys.getKeySet(keyIndex)) {
+          if ((initialMaterial ?? keySet) === this.keys.getKeySet(keyIndex)) {
             // only ratchet if the currently set key is still the same as the one used to decrypt this frame
             // if not, it might be that a different frame has already ratcheted and we try with that one first
             const newMaterial = await this.keys.ratchetKey(keyIndex, false);
@@ -431,22 +431,21 @@ export class FrameCryptor extends BaseFrameCryptor {
             encryptionKey: ratchetedKeySet?.encryptionKey,
           });
           if (frame && ratchetedKeySet) {
-            this.keys.setKeySet(ratchetedKeySet, keyIndex, true);
-            // decryption was successful, set the new key index to reflect the ratcheted key set
-            this.keys.setCurrentKeyIndex(keyIndex);
+            // before updating the keys, make sure that the keySet used for this frame is still the same as the currently set key
+            // if it's not, a new key might have been set already, which we don't want to override
+            if ((initialMaterial ?? keySet) === this.keys.getKeySet(keyIndex)) {
+              this.keys.setKeySet(ratchetedKeySet, keyIndex, true);
+              // decryption was successful, set the new key index to reflect the ratcheted key set
+              this.keys.setCurrentKeyIndex(keyIndex);
+            }
           }
           return frame;
         } else {
           /**
-           * Since the key it is first send and only afterwards actually used for encrypting, there were
-           * situations when the decrypting failed due to the fact that the received frame was not encrypted
-           * yet and ratcheting, of course, did not solve the problem. So if we fail RATCHET_WINDOW_SIZE times,
-           * we come back to the initial key.
+           * Because we only set a new key once decryption has been successful,
+           * we can be sure that we don't need to reset the key to the initial material at this point
+           * as the key has not been updated on the keyHandler instance
            */
-          if (initialMaterial) {
-            workerLogger.debug('resetting to initial material');
-            this.keys.setKeyFromMaterial(initialMaterial.material, keyIndex);
-          }
 
           workerLogger.warn('maximum ratchet attempts exceeded');
           throw new CryptorError(

package/src/e2ee/worker/ParticipantKeyHandler.ts

@@ -138,12 +138,11 @@ export class ParticipantKeyHandler extends (EventEmitter as new () => TypedEvent
    * also updates the currentKeyIndex
    */
   async setKeyFromMaterial(material: CryptoKey, keyIndex = 0, emitRatchetEvent = false) {
-    workerLogger.debug('setting new key');
-    if (keyIndex >= 0) {
-      this.currentKeyIndex = keyIndex % this.cryptoKeyRing.length;
-    }
+    const newIndex = keyIndex >= 0 ? keyIndex % this.cryptoKeyRing.length : -1;
+    workerLogger.debug(`setting new key with index ${newIndex}`);
     const keySet = await deriveKeys(material, this.keyProviderOptions.ratchetSalt);
-    this.setKeySet(keySet, this.currentKeyIndex, emitRatchetEvent);
+    this.setKeySet(keySet, newIndex >= 0 ? newIndex : this.currentKeyIndex, emitRatchetEvent);
+    if (newIndex >= 0) this.currentKeyIndex = newIndex;
   }
 
   setKeySet(keySet: KeySet, keyIndex: number, emitRatchetEvent = false) {

package/src/e2ee/worker/e2ee.worker.ts

@@ -75,7 +75,9 @@ onmessage = (ev) => {
         workerLogger.warn('set shared key');
         setSharedKey(data.key, data.keyIndex);
       } else if (data.participantIdentity) {
-        workerLogger.warn(`set participant sender key ${data.participantIdentity}`);
+        workerLogger.warn(
+          `set participant sender key ${data.participantIdentity} index ${data.keyIndex}`,
+        );
         getParticipantKeyHandler(data.participantIdentity).setKey(data.key, data.keyIndex);
       } else {
         workerLogger.error('no participant Id was provided and shared key usage is disabled');

package/src/proto/livekit_models_pb.ts

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// @generated by protoc-gen-es v1.3.0 with parameter "target=ts"
+// @generated by protoc-gen-es v1.3.3 with parameter "target=ts"
 // @generated from file livekit_models.proto (package livekit, syntax proto3)
 /* eslint-disable */
 // @ts-nocheck
@@ -84,6 +84,26 @@ proto3.util.setEnumType(VideoCodec, "livekit.VideoCodec", [
   { no: 4, name: "VP8" },
 ]);
 
+/**
+ * @generated from enum livekit.ImageCodec
+ */
+export enum ImageCodec {
+  /**
+   * @generated from enum value: IC_DEFAULT = 0;
+   */
+  IC_DEFAULT = 0,
+
+  /**
+   * @generated from enum value: IC_JPEG = 1;
+   */
+  IC_JPEG = 1,
+}
+// Retrieve enum metadata with: proto3.getEnumType(ImageCodec)
+proto3.util.setEnumType(ImageCodec, "livekit.ImageCodec", [
+  { no: 0, name: "IC_DEFAULT" },
+  { no: 1, name: "IC_JPEG" },
+]);
+
 /**
  * @generated from enum livekit.TrackType
  */
@@ -411,11 +431,6 @@ export class Room extends Message<Room> {
    */
   activeRecording = false;
 
-  /**
-   * @generated from field: livekit.PlayoutDelay playout_delay = 12;
-   */
-  playoutDelay?: PlayoutDelay;
-
   constructor(data?: PartialMessage<Room>) {
     super();
     proto3.util.initPartial(data, this);
@@ -435,7 +450,6 @@ export class Room extends Message<Room> {
     { no: 9, name: "num_participants", kind: "scalar", T: 13 /* ScalarType.UINT32 */ },
     { no: 11, name: "num_publishers", kind: "scalar", T: 13 /* ScalarType.UINT32 */ },
     { no: 10, name: "active_recording", kind: "scalar", T: 8 /* ScalarType.BOOL */ },
-    { no: 12, name: "playout_delay", kind: "message", T: PlayoutDelay },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): Room {
@@ -1282,6 +1296,11 @@ export class UserPacket extends Message<UserPacket> {
    */
   participantSid = "";
 
+  /**
+   * @generated from field: string participant_identity = 5;
+   */
+  participantIdentity = "";
+
   /**
    * user defined payload
    *
@@ -1290,12 +1309,19 @@ export class UserPacket extends Message<UserPacket> {
   payload = new Uint8Array(0);
 
   /**
-   * the ID of the participants who will receive the message (the message will be sent to all the people in the room if this variable is empty)
+   * the ID of the participants who will receive the message (sent to all by default)
    *
    * @generated from field: repeated string destination_sids = 3;
    */
   destinationSids: string[] = [];
 
+  /**
+   * identities of participants who will receive the message (sent to all by default)
+   *
+   * @generated from field: repeated string destination_identities = 6;
+   */
+  destinationIdentities: string[] = [];
+
   /**
    * topic under which the message was published
    *
@@ -1312,8 +1338,10 @@ export class UserPacket extends Message<UserPacket> {
   static readonly typeName = "livekit.UserPacket";
   static readonly fields: FieldList = proto3.util.newFieldList(() => [
     { no: 1, name: "participant_sid", kind: "scalar", T: 9 /* ScalarType.STRING */ },
+    { no: 5, name: "participant_identity", kind: "scalar", T: 9 /* ScalarType.STRING */ },
     { no: 2, name: "payload", kind: "scalar", T: 12 /* ScalarType.BYTES */ },
     { no: 3, name: "destination_sids", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true },
+    { no: 6, name: "destination_identities", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true },
     { no: 4, name: "topic", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
   ]);
 
@@ -1613,6 +1641,16 @@ export enum ClientInfo_SDK {
    * @generated from enum value: RUST = 8;
    */
   RUST = 8,
+
+  /**
+   * @generated from enum value: PYTHON = 9;
+   */
+  PYTHON = 9,
+
+  /**
+   * @generated from enum value: CPP = 10;
+   */
+  CPP = 10,
 }
 // Retrieve enum metadata with: proto3.getEnumType(ClientInfo_SDK)
 proto3.util.setEnumType(ClientInfo_SDK, "livekit.ClientInfo.SDK", [
@@ -1625,6 +1663,8 @@ proto3.util.setEnumType(ClientInfo_SDK, "livekit.ClientInfo.SDK", [
   { no: 6, name: "UNITY" },
   { no: 7, name: "REACT_NATIVE" },
   { no: 8, name: "RUST" },
+  { no: 9, name: "PYTHON" },
+  { no: 10, name: "CPP" },
 ]);
 
 /**
@@ -1774,6 +1814,91 @@ export class DisabledCodecs extends Message<DisabledCodecs> {
   }
 }
 
+/**
+ * @generated from message livekit.RTPDrift
+ */
+export class RTPDrift extends Message<RTPDrift> {
+  /**
+   * @generated from field: google.protobuf.Timestamp start_time = 1;
+   */
+  startTime?: Timestamp;
+
+  /**
+   * @generated from field: google.protobuf.Timestamp end_time = 2;
+   */
+  endTime?: Timestamp;
+
+  /**
+   * @generated from field: double duration = 3;
+   */
+  duration = 0;
+
+  /**
+   * @generated from field: uint64 start_timestamp = 4;
+   */
+  startTimestamp = protoInt64.zero;
+
+  /**
+   * @generated from field: uint64 end_timestamp = 5;
+   */
+  endTimestamp = protoInt64.zero;
+
+  /**
+   * @generated from field: uint64 rtp_clock_ticks = 6;
+   */
+  rtpClockTicks = protoInt64.zero;
+
+  /**
+   * @generated from field: int64 drift_samples = 7;
+   */
+  driftSamples = protoInt64.zero;
+
+  /**
+   * @generated from field: double drift_ms = 8;
+   */
+  driftMs = 0;
+
+  /**
+   * @generated from field: double clock_rate = 9;
+   */
+  clockRate = 0;
+
+  constructor(data?: PartialMessage<RTPDrift>) {
+    super();
+    proto3.util.initPartial(data, this);
+  }
+
+  static readonly runtime: typeof proto3 = proto3;
+  static readonly typeName = "livekit.RTPDrift";
+  static readonly fields: FieldList = proto3.util.newFieldList(() => [
+    { no: 1, name: "start_time", kind: "message", T: Timestamp },
+    { no: 2, name: "end_time", kind: "message", T: Timestamp },
+    { no: 3, name: "duration", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ },
+    { no: 4, name: "start_timestamp", kind: "scalar", T: 4 /* ScalarType.UINT64 */ },
+    { no: 5, name: "end_timestamp", kind: "scalar", T: 4 /* ScalarType.UINT64 */ },
+    { no: 6, name: "rtp_clock_ticks", kind: "scalar", T: 4 /* ScalarType.UINT64 */ },
+    { no: 7, name: "drift_samples", kind: "scalar", T: 3 /* ScalarType.INT64 */ },
+    { no: 8, name: "drift_ms", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ },
+    { no: 9, name: "clock_rate", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ },
+  ]);
+
+  static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): RTPDrift {
+    return new RTPDrift().fromBinary(bytes, options);
+  }
+
+  static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): RTPDrift {
+    return new RTPDrift().fromJson(jsonValue, options);
+  }
+
+  static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): RTPDrift {
+    return new RTPDrift().fromJsonString(jsonString, options);
+  }
+
+  static equals(a: RTPDrift | PlainMessage<RTPDrift> | undefined, b: RTPDrift | PlainMessage<RTPDrift> | undefined): boolean {
+    return proto3.util.equals(RTPDrift, a, b);
+  }
+}
+
 /**
  * @generated from message livekit.RTPStats
  */
@@ -1984,16 +2109,16 @@ export class RTPStats extends Message<RTPStats> {
   lastLayerLockPli?: Timestamp;
 
   /**
-   * @generated from field: double sample_rate = 42;
+   * @generated from field: livekit.RTPDrift packet_drift = 44;
    */
-  sampleRate = 0;
+  packetDrift?: RTPDrift;
 
   /**
-   * NEXT_ID: 44
+   * NEXT_ID: 46
    *
-   * @generated from field: double drift_ms = 43;
+   * @generated from field: livekit.RTPDrift report_drift = 45;
    */
-  driftMs = 0;
+  reportDrift?: RTPDrift;
 
   constructor(data?: PartialMessage<RTPStats>) {
     super();
@@ -2044,8 +2169,8 @@ export class RTPStats extends Message<RTPStats> {
     { no: 34, name: "last_key_frame", kind: "message", T: Timestamp },
     { no: 35, name: "layer_lock_plis", kind: "scalar", T: 13 /* ScalarType.UINT32 */ },
     { no: 36, name: "last_layer_lock_pli", kind: "message", T: Timestamp },
-    { no: 42, name: "sample_rate", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ },
-    { no: 43, name: "drift_ms", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ },
+    { no: 44, name: "packet_drift", kind: "message", T: RTPDrift },
+    { no: 45, name: "report_drift", kind: "message", T: RTPDrift },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): RTPStats {

package/src/proto/livekit_rtc_pb.ts

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// @generated by protoc-gen-es v1.3.0 with parameter "target=ts"
+// @generated by protoc-gen-es v1.3.3 with parameter "target=ts"
 // @generated from file livekit_rtc.proto (package livekit, syntax proto3)
 /* eslint-disable */
 // @ts-nocheck
@@ -487,11 +487,6 @@ export class SimulcastCodec extends Message<SimulcastCodec> {
    */
   cid = "";
 
-  /**
-   * @generated from field: bool enable_simulcast_layers = 3;
-   */
-  enableSimulcastLayers = false;
-
   constructor(data?: PartialMessage<SimulcastCodec>) {
     super();
     proto3.util.initPartial(data, this);
@@ -502,7 +497,6 @@ export class SimulcastCodec extends Message<SimulcastCodec> {
   static readonly fields: FieldList = proto3.util.newFieldList(() => [
     { no: 1, name: "codec", kind: "scalar", T: 9 /* ScalarType.STRING */ },
     { no: 2, name: "cid", kind: "scalar", T: 9 /* ScalarType.STRING */ },
-    { no: 3, name: "enable_simulcast_layers", kind: "scalar", T: 8 /* ScalarType.BOOL */ },
   ]);
 
   static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): SimulcastCodec {

package/src/room/PCTransport.ts

@@ -32,7 +32,7 @@ export const PCEvents = {
 export default class PCTransport extends EventEmitter {
   private _pc: RTCPeerConnection | null;
 
-  public get pc() {
+  private get pc() {
     if (this._pc) return this._pc;
     throw new UnexpectedConnectionState('Expected peer connection to be available');
   }
@@ -51,12 +51,38 @@ export default class PCTransport extends EventEmitter {
 
   onOffer?: (offer: RTCSessionDescriptionInit) => void;
 
+  onIceCandidate?: (candidate: RTCIceCandidate) => void;
+
+  onIceCandidateError?: (ev: Event) => void;
+
+  onConnectionStateChange?: (state: RTCPeerConnectionState) => void;
+
+  onDataChannel?: (ev: RTCDataChannelEvent) => void;
+
+  onTrack?: (ev: RTCTrackEvent) => void;
+
   constructor(config?: RTCConfiguration, mediaConstraints: Record<string, unknown> = {}) {
     super();
     this._pc = isChromiumBased()
       ? // @ts-expect-error chrome allows additional media constraints to be passed into the RTCPeerConnection constructor
         new RTCPeerConnection(config, mediaConstraints)
       : new RTCPeerConnection(config);
+    this._pc.onicecandidate = (ev) => {
+      if (!ev.candidate) return;
+      this.onIceCandidate?.(ev.candidate);
+    };
+    this._pc.onicecandidateerror = (ev) => {
+      this.onIceCandidateError?.(ev);
+    };
+    this._pc.onconnectionstatechange = () => {
+      this.onConnectionStateChange?.(this._pc?.connectionState ?? 'closed');
+    };
+    this._pc.ondatachannel = (ev) => {
+      this.onDataChannel?.(ev);
+    };
+    this._pc.ontrack = (ev) => {
+      this.onTrack?.(ev);
+    };
   }
 
   get isICEConnected(): boolean {
@@ -270,10 +296,99 @@ export default class PCTransport extends EventEmitter {
     return answer;
   }
 
+  createDataChannel(label: string, dataChannelDict: RTCDataChannelInit) {
+    return this.pc.createDataChannel(label, dataChannelDict);
+  }
+
+  addTransceiver(mediaStreamTrack: MediaStreamTrack, transceiverInit: RTCRtpTransceiverInit) {
+    return this.pc.addTransceiver(mediaStreamTrack, transceiverInit);
+  }
+
+  addTrack(track: MediaStreamTrack) {
+    return this.pc.addTrack(track);
+  }
+
   setTrackCodecBitrate(info: TrackBitrateInfo) {
     this.trackBitrates.push(info);
   }
 
+  setConfiguration(rtcConfig: RTCConfiguration) {
+    return this.pc.setConfiguration(rtcConfig);
+  }
+
+  canRemoveTrack(): boolean {
+    return !!this.pc.removeTrack;
+  }
+
+  removeTrack(sender: RTCRtpSender) {
+    return this.pc.removeTrack(sender);
+  }
+
+  getConnectionState() {
+    return this.pc.connectionState;
+  }
+
+  getICEConnectionState() {
+    return this.pc.iceConnectionState;
+  }
+
+  getSignallingState() {
+    return this.pc.signalingState;
+  }
+
+  getTransceivers() {
+    return this.pc.getTransceivers();
+  }
+
+  getSenders() {
+    return this.pc.getSenders();
+  }
+
+  getLocalDescription() {
+    return this.pc.localDescription;
+  }
+
+  getRemoteDescription() {
+    return this.pc.remoteDescription;
+  }
+
+  async getConnectedAddress(): Promise<string | undefined> {
+    if (!this._pc) {
+      return;
+    }
+    let selectedCandidatePairId = '';
+    const candidatePairs = new Map<string, RTCIceCandidatePairStats>();
+    // id -> candidate ip
+    const candidates = new Map<string, string>();
+    const stats: RTCStatsReport = await this._pc.getStats();
+    stats.forEach((v) => {
+      switch (v.type) {
+        case 'transport':
+          selectedCandidatePairId = v.selectedCandidatePairId;
+          break;
+        case 'candidate-pair':
+          if (selectedCandidatePairId === '' && v.selected) {
+            selectedCandidatePairId = v.id;
+          }
+          candidatePairs.set(v.id, v);
+          break;
+        case 'remote-candidate':
+          candidates.set(v.id, `${v.address}:${v.port}`);
+          break;
+        default:
+      }
+    });
+
+    if (selectedCandidatePairId === '') {
+      return undefined;
+    }
+    const selectedID = candidatePairs.get(selectedCandidatePairId)?.remoteCandidateId;
+    if (selectedID === undefined) {
+      return undefined;
+    }
+    return candidates.get(selectedID);
+  }
+
   close() {
     if (!this._pc) {
       return;