livekit-client 2.22.1 → 2.22.2

package/src/e2ee/types.ts

@@ -53,7 +53,6 @@ export interface EncodeMessage extends BaseMessage {
     writableStream: WritableStream;
     trackId: string;
     codec?: VideoCodec;
-    isReuse: boolean;
     /**
      * Whether the published track advertises packet trailer features.
      * When false, the cryptor skips the per-frame trailer extraction path
@@ -80,7 +79,15 @@ export interface UpdateCodecMessage extends BaseMessage {
   data: {
     participantIdentity: string;
     trackId: string;
-    codec: VideoCodec;
+    /** undefined for audio tracks */
+    codec?: VideoCodec;
+    /**
+     * trackId this receiver's pipeline was previously set up for, set when a
+     * transceiver gets reused for a new track. Lets the worker find the cryptor
+     * that still owns the (already transferred) encoded streams and re-point it,
+     * rather than creating a fresh cryptor with no pipeline at all.
+     */
+    previousTrackId?: string;
     hasPacketTrailer: boolean;
   };
 }

package/src/e2ee/worker/FrameCryptor.race.test.ts

@@ -68,8 +68,8 @@ describe('FrameCryptor Race Conditions', () => {
     vitest.useRealTimers();
   });
 
-  describe('Race Condition 1: setupTransform with isReuse does not update trackId', () => {
-    it('should update trackId even when returning early on reuse', async () => {
+  describe('Race Condition 1: setupTransform on transceiver reuse updates trackId', () => {
+    it('should update trackId when the transform is replaced', async () => {
       const { cryptor, keys } = createCryptor('participant1');
       await keys.setKey(await createKeyMaterialFromString('key1'), 0);
 
@@ -82,7 +82,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input1),
         new WritableStream(output1),
         'track1',
-        false,
         undefined,
       );
 
@@ -91,18 +90,16 @@ describe('FrameCryptor Race Conditions', () => {
       const input2 = new TestUnderlyingSource<RTCEncodedVideoFrame>();
       const output2 = new TestUnderlyingSink<RTCEncodedVideoFrame>();
 
-      // Second setup with isReuse=true and different trackId 'track2'
+      // Second setup with a different trackId 'track2'
       // This simulates transceiver reuse for a new track
       cryptor.setupTransform(
         'encode',
         new ReadableStream(input2),
         new WritableStream(output2),
         'track2',
-        true, // isReuse = true
         undefined,
       );
 
-      // BUG: trackId should be updated to 'track2' but remains 'track1'
       expect(cryptor.getTrackId()).toBe('track2');
     });
   });
@@ -120,7 +117,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input),
         new WritableStream(output),
         'track1',
-        false,
         undefined,
       );
 
@@ -147,7 +143,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input2),
         new WritableStream(output2),
         'track2',
-        false,
         undefined,
       );
 
@@ -198,7 +193,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input1),
         new WritableStream(output1),
         'track1',
-        false,
         undefined,
       );
 
@@ -213,7 +207,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input2),
         new WritableStream(output2),
         'track2',
-        false,
         undefined,
       );
 
@@ -244,7 +237,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input1),
         new WritableStream(output1),
         'track1',
-        false,
         undefined,
       );
 
@@ -257,13 +249,12 @@ describe('FrameCryptor Race Conditions', () => {
         const input2 = new TestUnderlyingSource<RTCEncodedVideoFrame>();
         const output2 = new TestUnderlyingSink<RTCEncodedVideoFrame>();
 
-        // This should create a new transform since isTransformActive should be false
+        // This should create a new transform
         cryptor.setupTransform(
           'encode',
           new ReadableStream(input2),
           new WritableStream(output2),
           'track2',
-          true, // isReuse=true
           undefined,
         );
 
@@ -274,7 +265,7 @@ describe('FrameCryptor Race Conditions', () => {
       expect(cryptor.getTrackId()).toBe('track2');
     });
 
-    it('should handle race between pipe completion and new setupTransform with isReuse', async () => {
+    it('should handle race between pipe completion and a new setupTransform', async () => {
       const { cryptor, keys } = createCryptor('participant1');
       await keys.setKey(await createKeyMaterialFromString('key1'), 0);
 
@@ -286,11 +277,10 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input1),
         new WritableStream(output1),
         'track1',
-        false,
         undefined,
       );
 
-      // Immediately call with isReuse=true (simulating quick reuse detection)
+      // Immediately set up again (simulating a quick resubscribe)
       const input2 = new TestUnderlyingSource<RTCEncodedVideoFrame>();
       const output2 = new TestUnderlyingSink<RTCEncodedVideoFrame>();
 
@@ -298,8 +288,7 @@ describe('FrameCryptor Race Conditions', () => {
         'encode',
         new ReadableStream(input2),
         new WritableStream(output2),
-        'track2',
-        true, // Should return early
+        'track2', // Should return early
         undefined,
       );
 
@@ -308,7 +297,7 @@ describe('FrameCryptor Race Conditions', () => {
 
       await vitest.advanceTimersToNextTimerAsync();
 
-      // Now try to setup with isReuse=true again after first pipe completed
+      // Now set up again after the first pipe completed
       const input3 = new TestUnderlyingSource<RTCEncodedVideoFrame>();
       const output3 = new TestUnderlyingSink<RTCEncodedVideoFrame>();
 
@@ -317,7 +306,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input3),
         new WritableStream(output3),
         'track3',
-        true,
         undefined,
       );
 
@@ -345,7 +333,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input),
         new WritableStream(output),
         'track1',
-        false,
         undefined,
       );
 
@@ -431,7 +418,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input),
         new WritableStream(output),
         'track1',
-        false,
         undefined,
       );
 
@@ -469,7 +455,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input),
         new WritableStream(output),
         'track1',
-        false,
         'vp8',
       );
 
@@ -509,7 +494,6 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input1),
         new WritableStream(output1),
         'track1',
-        false,
         undefined,
       );
 
@@ -534,11 +518,10 @@ describe('FrameCryptor Race Conditions', () => {
         new ReadableStream(input2),
         new WritableStream(output2),
         'track2',
-        true, // isReuse
         undefined,
       );
 
-      // Track ID should be updated even with isReuse
+      // Track ID should be updated
       expect(cryptor.getTrackId()).toBe('track2');
       expect(cryptor.getParticipantIdentity()).toBe('participant2');
     });

package/src/e2ee/worker/FrameCryptor.test.ts

@@ -110,7 +110,6 @@ function prepareParticipantTest(
     new ReadableStream(input),
     new WritableStream(output),
     'testTrack',
-    false,
     undefined,
     packetTrailer,
   );

package/src/e2ee/worker/FrameCryptor.ts

@@ -84,6 +84,22 @@ export class FrameCryptor extends BaseFrameCryptor {
 
   private currentTransform?: TransformerInfo;
 
+  /**
+   * The encoded streams this cryptor was last set up with, retained beyond the
+   * lifetime of {@link currentTransform}.
+   *
+   * The main thread transfers a receiver's encoded streams to the worker once and
+   * cannot transfer them again (a transferred stream is locked), and
+   * `createEncodedStreams()` may only be called once per receiver. So when a
+   * transceiver is reused, or when a pipe dies underneath us, re-establishing the
+   * transform is only possible from here. See {@link ensureTransform}.
+   */
+  private retainedStreams?: {
+    readable: ReadableStream<RTCEncodedVideoFrame | RTCEncodedAudioFrame>;
+    writable: WritableStream<RTCEncodedVideoFrame | RTCEncodedAudioFrame>;
+    operation: 'encode' | 'decode';
+  };
+
   /**
    * Whether the subscribed track advertises packet trailer features.
    * When false, we skip the per-frame trailer extraction path entirely
@@ -108,6 +124,11 @@ export class FrameCryptor extends BaseFrameCryptor {
 
   private readonly ERROR_WINDOW_MS = 60000; // 1 minute window
 
+  private undecryptedTrackTimeout?: ReturnType<typeof setTimeout>;
+
+  /** grace period for a teardown or a resubscribe to land before we report a stalled track */
+  private readonly UNDECRYPTED_TRACK_GRACE_MS = 2000;
+
   /**
    * Tracks (participant, trackId, payloadType) tuples for which we've already logged a NALU
    * fallback, so a persistent bad state doesn't flood the console (Firefox doesn't filter debug).
@@ -167,10 +188,13 @@ export class FrameCryptor extends BaseFrameCryptor {
   unsetParticipant() {
     workerLogger.debug('unsetting participant', this.logContext);
 
-    if (this.currentTransform) {
-      this.currentTransform = undefined;
-    }
-
+    // NOTE: deliberately does not clear `currentTransform`. Nothing here cancels
+    // the pipe, so the transform really is still running and clearing it would
+    // desync our bookkeeping from reality -- which previously let a later
+    // resubscribe skip setup while frames kept flowing through a cryptor with no
+    // participant assigned.
+    clearTimeout(this.undecryptedTrackTimeout);
+    this.undecryptedTrackTimeout = undefined;
     this.participantIdentity = undefined;
     this.lastErrorTimestamp = new Map();
     this.errorCounts = new Map();
@@ -192,6 +216,78 @@ export class FrameCryptor extends BaseFrameCryptor {
     return this.trackId;
   }
 
+  /**
+   * Re-point this cryptor at a new trackId, keeping its (already transferred)
+   * encoded streams. Used when a transceiver is reused for a new track.
+   */
+  setTrackId(trackId: string) {
+    if (this.trackId === trackId) {
+      return;
+    }
+    workerLogger.debug('re-pointing cryptor at new trackId', {
+      ...this.logContext,
+      newTrackId: trackId,
+    });
+    this.trackId = trackId;
+  }
+
+  hasActiveTransform() {
+    return !!this.currentTransform;
+  }
+
+  /**
+   * A track that is subscribed and known to be encrypted, but has no transform to
+   * decrypt it, will never produce a decodable frame again. That state used to be
+   * completely silent (a black tile and endless PLIs), so report it.
+   *
+   * Deferred, because on teardown the encoded streams routinely close before the
+   * 'removeTransform' message arrives -- checking immediately would cry wolf on
+   * every unsubscribe.
+   *
+   * Only meaningful while decoding: a sender's pipe closing on unpublish is
+   * routine and has no 'removeTransform' equivalent to quiet it down.
+   */
+  private scheduleUndecryptedTrackWatchdog(operation: 'encode' | 'decode') {
+    clearTimeout(this.undecryptedTrackTimeout);
+    this.undecryptedTrackTimeout = undefined;
+    if (operation !== 'decode') {
+      return;
+    }
+    this.undecryptedTrackTimeout = setTimeout(() => {
+      if (this.currentTransform || !this.isEnabled()) {
+        return;
+      }
+      workerLogger.warn('encrypted track has no active decrypt transform', this.logContext);
+      this.emitThrottledError(
+        new CryptorError(
+          `no active decrypt transform for encrypted track ${this.trackId}`,
+          CryptorErrorReason.InternalError,
+          this.participantIdentity,
+        ),
+      );
+    }, this.UNDECRYPTED_TRACK_GRACE_MS);
+  }
+
+  /**
+   * Re-establish the transform if it is gone while we still own the encoded
+   * streams, e.g. after a pipe died on its own (the swallowed
+   * 'Destination stream closed') or when a reused transceiver never got a new
+   * pipeline. Without this the frames pile up in a readable nobody reads and the
+   * decoder starves, which shows up as a permanently black tile.
+   */
+  ensureTransform() {
+    if (this.currentTransform) {
+      return true;
+    }
+    if (!this.retainedStreams || this.trackId === undefined) {
+      workerLogger.warn('no streams retained, cannot re-establish transform', this.logContext);
+      return false;
+    }
+    const { readable, writable, operation } = this.retainedStreams;
+    workerLogger.info('re-establishing transform', { ...this.logContext, operation });
+    return this.setupTransform(operation, readable, writable, this.trackId);
+  }
+
   /**
    * Update the video codec used by the mediaStreamTrack
    * @param codec
@@ -227,7 +323,6 @@ export class FrameCryptor extends BaseFrameCryptor {
     readable: ReadableStream<RTCEncodedVideoFrame | RTCEncodedAudioFrame>,
     writable: WritableStream<RTCEncodedVideoFrame | RTCEncodedAudioFrame>,
     trackId: string,
-    isReuse: boolean,
     codec?: VideoCodec,
     frameMetadata?: FrameMetadataPublishOptions,
   ) {
@@ -243,27 +338,17 @@ export class FrameCryptor extends BaseFrameCryptor {
       operation,
       passedTrackId: trackId,
       codec,
-      isReuse,
       hasCurrentTransform: !!this.currentTransform,
       ...this.logContext,
     });
 
-    // Always update trackId, even on reuse
     this.trackId = trackId;
 
-    // If we're reusing and have an active transform skip setup
-    if (
-      isReuse &&
-      this.currentTransform &&
-      readable === this.currentTransform.readable &&
-      writable === this.currentTransform.writable
-    ) {
-      workerLogger.debug('reusing existing transform', {
-        ...this.logContext,
-        trackId,
-      });
-      return;
-    }
+    // Retain the streams so we can rebuild the pipe later even if this cryptor
+    // gets detached from its participant in between (see `ensureTransform`).
+    this.retainedStreams = { readable, writable, operation };
+
+    clearTimeout(this.undecryptedTrackTimeout);
 
     const symbol = Symbol('transform');
 
@@ -281,33 +366,54 @@ export class FrameCryptor extends BaseFrameCryptor {
       symbol,
     };
 
-    readable
-      .pipeThrough(transformStream)
-      .pipeTo(writable)
-      .catch((e) => {
-        if (e instanceof TypeError && e.message === 'Destination stream closed') {
-          // this can happen when subscriptions happen in quick successions, but doesn't influence functionality
-          workerLogger.debug('destination stream closed');
-        } else {
-          workerLogger.warn('transform error', { error: e, ...this.logContext });
-          this.emit(
-            CryptorEvent.Error,
-            e instanceof CryptorError
-              ? e
-              : new CryptorError(e.message, undefined, this.participantIdentity),
-          );
-        }
-      })
-      .finally(() => {
-        // Only clear currentTransform if it's still the same one we started
-        if (this.currentTransform?.symbol === symbol) {
-          workerLogger.debug('transform completed', {
-            ...this.logContext,
-            trackId,
-          });
-          this.currentTransform = undefined;
-        }
-      });
+    // pipeThrough/pipeTo throw synchronously on an already locked stream, which
+    // would bypass the .catch below and leave us with no transform at all.
+    try {
+      readable
+        .pipeThrough(transformStream)
+        .pipeTo(writable)
+        .catch((e) => {
+          if (e instanceof TypeError && e.message === 'Destination stream closed') {
+            // this can happen when subscriptions happen in quick successions, but doesn't influence functionality
+            workerLogger.debug('destination stream closed');
+          } else {
+            workerLogger.warn('transform error', { error: e, ...this.logContext });
+            this.emit(
+              CryptorEvent.Error,
+              e instanceof CryptorError
+                ? e
+                : new CryptorError(e.message, undefined, this.participantIdentity),
+            );
+          }
+        })
+        .finally(() => {
+          // Only clear currentTransform if it's still the same one we started
+          if (this.currentTransform?.symbol === symbol) {
+            workerLogger.debug('transform completed', {
+              ...this.logContext,
+              trackId,
+            });
+            this.currentTransform = undefined;
+          }
+          // A pipe ending while the track is still assigned to a participant that
+          // publishes encrypted media means we have silently stopped decrypting.
+          this.scheduleUndecryptedTrackWatchdog(operation);
+        });
+    } catch (e: any) {
+      if (this.currentTransform?.symbol === symbol) {
+        this.currentTransform = undefined;
+      }
+      workerLogger.error('failed to set up transform', { error: e, ...this.logContext });
+      this.emit(
+        CryptorEvent.Error,
+        e instanceof CryptorError
+          ? e
+          : new CryptorError(e.message, CryptorErrorReason.InternalError, this.participantIdentity),
+      );
+      return false;
+    }
+
+    return true;
   }
 
   setSifTrailer(trailer: NonSharedUint8Array) {
@@ -539,11 +645,38 @@ export class FrameCryptor extends BaseFrameCryptor {
       }
     }
 
-    if (
-      !this.isEnabled() ||
-      // skip for decryption for empty dtx frames
-      encodedFrame.data.byteLength === 0
-    ) {
+    // skip decryption for empty dtx frames
+    if (encodedFrame.data.byteLength === 0) {
+      return controller.enqueue(encodedFrame);
+    }
+
+    const encryptionEnabled = this.isEnabled();
+
+    if (encryptionEnabled === undefined) {
+      // Either way we drop: forwarding would hand ciphertext straight to the
+      // decoder, which renders as a black frame and logs nothing.
+      if (this.participantIdentity === undefined) {
+        // The pipe outlived its subscription. Expected on unsubscribe/disconnect:
+        // we deliberately leave the pipe running so a reused transceiver can be
+        // re-pointed at a new track, so in-flight frames still arrive here for a
+        // moment. Not worth reporting -- the watchdog covers a track that stays
+        // subscribed without a transform.
+        workerLogger.debug('dropping frame for unassigned cryptor', this.logContext);
+        return;
+      }
+      // A live subscription whose encryption state we were never told about.
+      this.emitThrottledError(
+        new CryptorError(
+          `encryption state unknown for track ${this.trackId}, dropping frame`,
+          CryptorErrorReason.InternalError,
+          this.participantIdentity,
+        ),
+      );
+      return;
+    }
+
+    if (!encryptionEnabled) {
+      // participant is known to publish unencrypted media
       return controller.enqueue(encodedFrame);
     }
 

package/src/e2ee/worker/e2ee.worker.ts

@@ -71,7 +71,6 @@ onmessage = (ev) => {
           data.readableStream,
           data.writableStream,
           data.trackId,
-          data.isReuse,
           data.codec,
         );
         break;
@@ -83,7 +82,6 @@ onmessage = (ev) => {
           data.readableStream,
           data.writableStream,
           data.trackId,
-          data.isReuse,
           data.codec,
           data.packetTrailer, // wire format still uses 'packetTrailer' field name
         );
@@ -163,15 +161,32 @@ onmessage = (ev) => {
         unsetCryptorParticipant(data.trackId, data.participantIdentity);
         break;
       case 'updateCodec':
-        const trackCryptor = getTrackCryptor(data.participantIdentity, data.trackId);
-        trackCryptor.setVideoCodec(data.codec);
+        const trackCryptor = getTrackCryptor(
+          data.participantIdentity,
+          data.trackId,
+          data.previousTrackId,
+        );
+        if (data.codec) {
+          trackCryptor.setVideoCodec(data.codec);
+        }
         trackCryptor.setHasFrameMetadata(data.hasPacketTrailer);
         workerLogger.info('updated codec', {
           participantIdentity: data.participantIdentity,
           trackId: data.trackId,
+          previousTrackId: data.previousTrackId,
           codec: data.codec,
           hasPacketTrailer: data.hasPacketTrailer,
         });
+        // Sent on transceiver reuse, where the main thread cannot hand us a new
+        // pipeline. If the previous one died in the meantime we have to rebuild it
+        // here or no frame ever gets decrypted again.
+        if (data.previousTrackId !== undefined && !trackCryptor.ensureTransform()) {
+          workerLogger.error('could not re-establish transform for reused receiver', {
+            participantIdentity: data.participantIdentity,
+            trackId: data.trackId,
+            previousTrackId: data.previousTrackId,
+          });
+        }
         break;
       case 'setRTPMap':
         // this is only used for the local participant
@@ -210,8 +225,26 @@ async function handleRatchetRequest(data: RatchetRequestMessage['data']) {
   }
 }
 
-function getTrackCryptor(participantIdentity: string, trackId: string) {
+function getTrackCryptor(participantIdentity: string, trackId: string, previousTrackId?: string) {
   let cryptors = participantCryptors.filter((c) => c.getTrackId() === trackId);
+
+  if (cryptors.length === 0 && previousTrackId !== undefined && previousTrackId !== trackId) {
+    // A reused transceiver carries a new trackId, but the cryptor that owns this
+    // receiver's encoded streams is still keyed on the old one. Re-point it instead
+    // of creating a fresh cryptor that has no pipeline (and leaving the old one
+    // piping frames with no participant assigned).
+    const previous = participantCryptors.filter((c) => c.getTrackId() === previousTrackId);
+    if (previous.length > 0) {
+      workerLogger.info('reusing cryptor from previous trackId', {
+        participantIdentity,
+        trackId,
+        previousTrackId,
+      });
+      previous[0].setTrackId(trackId);
+      cryptors = previous;
+    }
+  }
+
   if (cryptors.length > 1) {
     const debugInfo = cryptors
       .map((c) => {
@@ -351,7 +384,6 @@ if (self.RTCTransformEvent) {
         transformer.readable,
         transformer.writable,
         trackId,
-        false,
         codec,
         kind === 'encode' ? options.packetTrailer : undefined,
       );

package/src/room/PCTransportManager.test.ts

@@ -279,3 +279,38 @@ describe('PCTransportManager.negotiate', () => {
     await expect(p).resolves.toBeUndefined();
   });
 });
+
+describe('PCTransportManager.triggerIceRestart', () => {
+  let originalRTCPeerConnection: unknown;
+
+  beforeEach(() => {
+    originalRTCPeerConnection = (globalThis as unknown as { RTCPeerConnection?: unknown })
+      .RTCPeerConnection;
+    (globalThis as unknown as { RTCPeerConnection: unknown }).RTCPeerConnection = StubPC;
+  });
+
+  afterEach(() => {
+    (globalThis as unknown as { RTCPeerConnection: unknown }).RTCPeerConnection =
+      originalRTCPeerConnection;
+  });
+
+  /**
+   * The subscriber must keep applying remote candidates across a reconnect.
+   *
+   * Putting it into `restartingIce` would queue them until a new remote description arrives —
+   * but the server only re-offers the subscriber when the reconnect moved us to another node,
+   * so on an ordinary signal-only resume nothing would ever flush that queue, and the
+   * transport would stop adopting new network paths for the rest of the session. The server
+   * also never sends candidates ahead of the offer that introduces them, so queueing buys
+   * nothing in exchange.
+   */
+  it('does not stop the subscriber applying remote candidates', async () => {
+    const manager = new PCTransportManager('subscriber-primary', {});
+    const publisher = new FakePublisher();
+    (manager as unknown as { publisher: FakePublisher }).publisher = publisher;
+
+    await manager.triggerIceRestart();
+
+    expect(manager.subscriber?.restartingIce).toBe(false);
+  });
+});

package/src/room/PCTransportManager.ts

@@ -162,12 +162,20 @@ export class PCTransportManager {
     this.updateState();
   }
 
+  /**
+   * Restarts ICE on the transports that need it. Only the publisher: the server restarts the
+   * subscriber's ICE itself and follows with a fresh offer.
+   *
+   * The subscriber deliberately does NOT enter `restartingIce` here. Queueing its remote
+   * candidates would guard against candidates for a new generation arriving before the offer
+   * that introduces it, but the server does not send them in that order -- on a same-node
+   * resume it buffers them until the offer has gone out, and on a reconnect that lands on
+   * another node it withholds subscriber candidates until immediately before creating the
+   * offer. Setting the flag only risks withholding candidates during the window that decides
+   * whether the reconnect succeeded.
+   */
   async triggerIceRestart() {
     this.iceLog.warn('triggering ICE restart');
-    if (this.subscriber) {
-      this.subscriber.restartingIce = true;
-    }
-    // only restart publisher if it's needed
     if (this.needsPublisher) {
       await this.createAndSendPublisherOffer({ iceRestart: true });
     }