livekit-client 0.15.4 → 0.16.0

package/src/room/RTCEngine.ts

@@ -4,6 +4,7 @@ import log from '../logger';
 import { DataPacket, DataPacket_Kind, TrackInfo } from '../proto/livekit_models';
 import {
   AddTrackRequest, JoinResponse,
+  LeaveRequest,
   SignalTarget,
   TrackPublishedResponse,
 } from '../proto/livekit_rtc';
@@ -14,8 +15,10 @@ import { sleep } from './utils';
 
 const lossyDataChannel = '_lossy';
 const reliableDataChannel = '_reliable';
-const maxReconnectRetries = 5;
-export const maxICEConnectTimeout = 5 * 1000;
+const maxReconnectRetries = 10;
+const minReconnectWait = 1 * 1000;
+const maxReconnectDuration = 60 * 1000;
+export const maxICEConnectTimeout = 15 * 1000;
 
 /** @internal */
 export default class RTCEngine extends EventEmitter {
@@ -39,7 +42,9 @@ export default class RTCEngine extends EventEmitter {
 
   private subscriberPrimary: boolean = false;
 
-  private iceConnected: boolean = false;
+  private primaryPC?: RTCPeerConnection;
+
+  private pcConnected: boolean = false;
 
   private isClosed: boolean = true;
 
@@ -54,8 +59,14 @@ export default class RTCEngine extends EventEmitter {
 
   private token?: string;
 
+  private signalOpts?: SignalOptions;
+
   private reconnectAttempts: number = 0;
 
+  private reconnectStart: number = 0;
+
+  private fullReconnect: boolean = false;
+
   private connectedServerAddr?: string;
 
   constructor() {
@@ -66,9 +77,9 @@ export default class RTCEngine extends EventEmitter {
   async join(url: string, token: string, opts?: SignalOptions): Promise<JoinResponse> {
     this.url = url;
     this.token = token;
+    this.signalOpts = opts;
 
     const joinResponse = await this.client.join(url, token, opts);
-    this.emit(EngineEvent.SignalConnected);
     this.isClosed = false;
 
     this.subscriberPrimary = joinResponse.subscriberPrimary;
@@ -174,21 +185,22 @@ export default class RTCEngine extends EventEmitter {
       // in subscriber primary mode, server side opens sub data channels.
       this.subscriber.pc.ondatachannel = this.handleDataChannel;
     }
-    primaryPC.oniceconnectionstatechange = () => {
-      if (primaryPC.iceConnectionState === 'connected') {
-        log.trace('ICE connected');
-        if (!this.iceConnected) {
-          this.iceConnected = true;
+    this.primaryPC = primaryPC;
+    primaryPC.onconnectionstatechange = () => {
+      if (primaryPC.connectionState === 'connected') {
+        log.trace('pc connected');
+        if (!this.pcConnected) {
+          this.pcConnected = true;
           this.emit(EngineEvent.Connected);
         }
         getConnectedAddress(primaryPC).then((v) => {
           this.connectedServerAddr = v;
         });
-      } else if (primaryPC.iceConnectionState === 'failed') {
+      } else if (primaryPC.connectionState === 'failed') {
         // on Safari, PeerConnection will switch to 'disconnected' during renegotiation
-        log.trace('ICE disconnected');
-        if (this.iceConnected) {
-          this.iceConnected = false;
+        log.trace('pc disconnected');
+        if (this.pcConnected) {
+          this.pcConnected = false;
 
           this.handleDisconnect('peerconnection');
         }
@@ -268,13 +280,22 @@ export default class RTCEngine extends EventEmitter {
       resolve(res.track!);
     };
 
+    this.client.onTokenRefresh = (token: string) => {
+      this.token = token;
+    };
+
     this.client.onClose = () => {
       this.handleDisconnect('signal');
     };
 
-    this.client.onLeave = () => {
-      this.emit(EngineEvent.Disconnected);
-      this.close();
+    this.client.onLeave = (leave?: LeaveRequest) => {
+      if (leave?.canReconnect) {
+        this.fullReconnect = true;
+        this.primaryPC = undefined;
+      } else {
+        this.emit(EngineEvent.Disconnected);
+        this.close();
+      }
     };
   }
 
@@ -320,48 +341,106 @@ export default class RTCEngine extends EventEmitter {
       return;
     }
     log.debug(`${connection} disconnected`);
-    if (this.reconnectAttempts >= maxReconnectRetries) {
-      log.info(
-        'could not connect to signal after',
-        maxReconnectRetries,
-        'attempts. giving up',
-      );
-      this.emit(EngineEvent.Disconnected);
-      this.close();
-      return;
+    if (this.reconnectAttempts === 0) {
+      // only reset start time on the first try
+      this.reconnectStart = Date.now();
     }
 
     const delay = (this.reconnectAttempts * this.reconnectAttempts) * 300;
-    setTimeout(() => {
-      this.reconnect()
-        .then(() => {
-          this.reconnectAttempts = 0;
-        })
-        .catch(this.handleDisconnect);
+    setTimeout(async () => {
+      if (this.isClosed) {
+        return;
+      }
+
+      try {
+        if (this.fullReconnect) {
+          await this.restartConnection();
+        } else {
+          await this.resumeConnection();
+        }
+        this.reconnectAttempts = 0;
+        this.fullReconnect = false;
+      } catch (e) {
+        this.reconnectAttempts += 1;
+        let recoverable = true;
+        if (e instanceof UnexpectedConnectionState) {
+          log.debug('received unrecoverable error', e.message);
+          // unrecoverable
+          recoverable = false;
+        } else if (!(e instanceof SignalReconnectError)) {
+          // cannot resume
+          this.fullReconnect = true;
+        }
+
+        const duration = Date.now() - this.reconnectStart;
+        if (this.reconnectAttempts >= maxReconnectRetries || duration > maxReconnectDuration) {
+          recoverable = false;
+        }
+
+        if (recoverable) {
+          this.handleDisconnect('reconnect');
+        } else {
+          log.info(
+            `could not recover connection after ${maxReconnectRetries} attempts, ${duration}ms. giving up`,
+          );
+          this.emit(EngineEvent.Disconnected);
+          this.close();
+        }
+      }
     }, delay);
   };
 
-  private async reconnect(): Promise<void> {
-    if (this.isClosed) {
-      return;
-    }
+  private async restartConnection() {
     if (!this.url || !this.token) {
-      throw new ConnectionError('could not reconnect, url or token not saved');
+      // permanent failure, don't attempt reconnection
+      throw new UnexpectedConnectionState('could not reconnect, url or token not saved');
     }
-    log.info('reconnecting to signal connection, attempt', this.reconnectAttempts);
 
+    log.info('reconnecting, attempt', this.reconnectAttempts);
     if (this.reconnectAttempts === 0) {
-      this.emit(EngineEvent.Reconnecting);
+      this.emit(EngineEvent.Restarting);
+    }
+
+    this.primaryPC = undefined;
+    this.publisher?.close();
+    this.publisher = undefined;
+    this.subscriber?.close();
+    this.subscriber = undefined;
+
+    let joinResponse: JoinResponse;
+    try {
+      joinResponse = await this.join(this.url, this.token, this.signalOpts);
+    } catch (e) {
+      throw new SignalReconnectError();
     }
-    this.reconnectAttempts += 1;
 
-    await this.client.reconnect(this.url, this.token);
-    this.emit(EngineEvent.SignalConnected);
+    await this.waitForPCConnected();
+
+    // reconnect success
+    this.emit(EngineEvent.Restarted, joinResponse);
+  }
 
+  private async resumeConnection(): Promise<void> {
+    if (!this.url || !this.token) {
+      // permanent failure, don't attempt reconnection
+      throw new UnexpectedConnectionState('could not reconnect, url or token not saved');
+    }
     // trigger publisher reconnect
     if (!this.publisher || !this.subscriber) {
       throw new UnexpectedConnectionState('publisher and subscriber connections unset');
     }
+    log.info('resuming signal connection, attempt', this.reconnectAttempts);
+    if (this.reconnectAttempts === 0) {
+      this.emit(EngineEvent.Resuming);
+    }
+
+    try {
+      await this.client.reconnect(this.url, this.token);
+    } catch (e) {
+      throw new SignalReconnectError();
+    }
+    this.emit(EngineEvent.SignalResumed);
+
     this.subscriber.restartingIce = true;
 
     // only restart publisher if it's needed
@@ -369,19 +448,35 @@ export default class RTCEngine extends EventEmitter {
       await this.publisher.createAndSendOffer({ iceRestart: true });
     }
 
-    const startTime = (new Date()).getTime();
+    await this.waitForPCConnected();
+
+    // resume success
+    this.emit(EngineEvent.Resumed);
+  }
 
-    while ((new Date()).getTime() - startTime < maxICEConnectTimeout * 2) {
-      if (this.iceConnected) {
-        // reconnect success
-        this.emit(EngineEvent.Reconnected);
+  async waitForPCConnected() {
+    const startTime = (new Date()).getTime();
+    let now = startTime;
+    this.pcConnected = false;
+
+    while (now - startTime < maxICEConnectTimeout) {
+      // if there is no connectionstatechange callback fired
+      // check connectionstate after minReconnectWait
+      if (this.primaryPC === undefined) {
+        // we can abort early, connection is hosed
+        break;
+      } else if (now - startTime > minReconnectWait && this.primaryPC?.connectionState === 'connected') {
+        this.pcConnected = true;
+      }
+      if (this.pcConnected) {
         return;
       }
       await sleep(100);
+      now = (new Date()).getTime();
     }
 
     // have not reconnected, throw
-    throw new ConnectionError('could not establish ICE connection');
+    throw new ConnectionError('could not establish PC connection');
   }
 
   /* @internal */
@@ -482,3 +577,6 @@ async function getConnectedAddress(pc: RTCPeerConnection): Promise<string | unde
   }
   return candidates.get(selectedID);
 }
+
+class SignalReconnectError extends Error {
+}

package/src/room/Room.ts

@@ -7,7 +7,11 @@ import {
   ParticipantInfo_State, Room as RoomModel, SpeakerInfo, UserPacket,
 } from '../proto/livekit_models';
 import {
-  ConnectionQualityUpdate, SimulateScenario, StreamStateUpdate, SubscriptionPermissionUpdate,
+  ConnectionQualityUpdate,
+  JoinResponse,
+  SimulateScenario,
+  StreamStateUpdate,
+  SubscriptionPermissionUpdate,
 } from '../proto/livekit_rtc';
 import DeviceManager from './DeviceManager';
 import { ConnectionError, UnsupportedServer } from './errors';
@@ -113,48 +117,47 @@ class Room extends EventEmitter {
     }
 
     this.engine = new RTCEngine();
-    this.engine.client.signalLatency = this.options.expSignalLatency;
-
-    this.engine.on(
-      EngineEvent.MediaTrackAdded,
-      (
-        mediaTrack: MediaStreamTrack,
-        stream: MediaStream,
-        receiver?: RTCRtpReceiver,
-      ) => {
-        this.onTrackAdded(mediaTrack, stream, receiver);
-      },
-    );
-
-    this.engine.on(EngineEvent.Disconnected, () => {
-      this.handleDisconnect();
-    });
 
+    this.engine.client.signalLatency = this.options.expSignalLatency;
     this.engine.client.onParticipantUpdate = this.handleParticipantUpdates;
     this.engine.client.onRoomUpdate = this.handleRoomUpdate;
     this.engine.client.onSpeakersChanged = this.handleSpeakersChanged;
     this.engine.client.onStreamStateUpdate = this.handleStreamStateUpdate;
     this.engine.client.onSubscriptionPermissionUpdate = this.handleSubscriptionPermissionUpdate;
-    this.engine.on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate);
-    this.engine.on(EngineEvent.DataPacketReceived, this.handleDataPacket);
-
-    this.engine.on(EngineEvent.Reconnecting, () => {
-      this.state = RoomState.Reconnecting;
-      this.emit(RoomEvent.Reconnecting);
-    });
-
-    this.engine.on(EngineEvent.Reconnected, () => {
-      this.state = RoomState.Connected;
-      this.emit(RoomEvent.Reconnected);
-    });
-
-    this.engine.on(EngineEvent.SignalConnected, () => {
-      if (this.state === RoomState.Reconnecting) {
-        this.sendSyncState();
-      }
-    });
-
     this.engine.client.onConnectionQuality = this.handleConnectionQualityUpdate;
+
+    this.engine
+      .on(
+        EngineEvent.MediaTrackAdded,
+        (
+          mediaTrack: MediaStreamTrack,
+          stream: MediaStream,
+          receiver?: RTCRtpReceiver,
+        ) => {
+          this.onTrackAdded(mediaTrack, stream, receiver);
+        },
+      )
+      .on(EngineEvent.Disconnected, () => {
+        this.handleDisconnect();
+      })
+      .on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate)
+      .on(EngineEvent.DataPacketReceived, this.handleDataPacket)
+      .on(EngineEvent.Resuming, () => {
+        this.state = RoomState.Reconnecting;
+        this.emit(RoomEvent.Reconnecting);
+      })
+      .on(EngineEvent.Resumed, () => {
+        this.state = RoomState.Connected;
+        this.emit(RoomEvent.Reconnected);
+        this.updateSubscriptions();
+      })
+      .on(EngineEvent.SignalResumed, () => {
+        if (this.state === RoomState.Reconnecting) {
+          this.sendSyncState();
+        }
+      })
+      .on(EngineEvent.Restarting, this.handleRestarting)
+      .on(EngineEvent.Restarted, this.handleRestarted);
   }
 
   /**
@@ -439,6 +442,43 @@ class Room extends EventEmitter {
     );
   }
 
+  private handleRestarting = () => {
+    this.state = RoomState.Reconnecting;
+    this.emit(RoomEvent.Reconnecting);
+
+    // also unwind existing participants & existing subscriptions
+    for (const p of this.participants.values()) {
+      this.handleParticipantDisconnected(p.sid, p);
+    }
+  };
+
+  private handleRestarted = async (joinResponse: JoinResponse) => {
+    this.state = RoomState.Connected;
+    this.emit(RoomEvent.Reconnected);
+
+    // rehydrate participants
+    if (joinResponse.participant) {
+      // with a restart, the sid will have changed, we'll map our understanding to it
+      this.localParticipant.sid = joinResponse.participant.sid;
+      this.handleParticipantUpdates([joinResponse.participant]);
+    }
+    this.handleParticipantUpdates(joinResponse.otherParticipants);
+
+    // unpublish & republish tracks
+    const localPubs: LocalTrackPublication[] = [];
+    this.localParticipant.tracks.forEach((pub) => {
+      if (pub.track) {
+        localPubs.push(pub);
+      }
+    });
+
+    await Promise.all(localPubs.map(async (pub) => {
+      const track = pub.track!;
+      this.localParticipant.unpublishTrack(track, false);
+      this.localParticipant.publishTrack(track, pub.options);
+    }));
+  };
+
   private handleDisconnect(shouldStopTracks = true) {
     if (this.state === RoomState.Disconnected) {
       return;
@@ -474,7 +514,8 @@ class Room extends EventEmitter {
   private handleParticipantUpdates = (participantInfos: ParticipantInfo[]) => {
     // handle changes to participant state, and send events
     participantInfos.forEach((info) => {
-      if (info.sid === this.localParticipant.sid) {
+      if (info.sid === this.localParticipant.sid
+          || info.identity === this.localParticipant.identity) {
         this.localParticipant.updateInfo(info);
         return;
       }
@@ -775,6 +816,20 @@ class Room extends EventEmitter {
     });
   }
 
+  /**
+   * After resuming, we'll need to notify the server of the current
+   * subscription settings.
+   */
+  private updateSubscriptions() {
+    for (const p of this.participants.values()) {
+      for (const pub of p.videoTracks.values()) {
+        if (pub.isSubscribed && pub instanceof RemoteTrackPublication) {
+          pub.emitTrackUpdate();
+        }
+      }
+    }
+  }
+
   /** @internal */
   emit(event: string | symbol, ...args: any[]): boolean {
     log.debug('room event', event, ...args);

package/src/room/events.ts

@@ -368,9 +368,11 @@ export enum ParticipantEvent {
 export enum EngineEvent {
   Connected = 'connected',
   Disconnected = 'disconnected',
-  Reconnecting = 'reconnecting',
-  Reconnected = 'reconnected',
-  SignalConnected = 'singalConnected',
+  Resuming = 'resuming',
+  Resumed = 'resumed',
+  Restarting = 'restarting',
+  Restarted = 'restarted',
+  SignalResumed = 'signalResumed',
   MediaTrackAdded = 'mediaTrackAdded',
   ActiveSpeakersUpdate = 'activeSpeakersUpdate',
   DataPacketReceived = 'dataPacketReceived',

package/src/room/participant/LocalParticipant.ts

@@ -434,6 +434,7 @@ export default class LocalParticipant extends Participant {
 
   unpublishTrack(
     track: LocalTrack | MediaStreamTrack,
+    stopOnUnpublish?: boolean,
   ): LocalTrackPublication | null {
     // look through all published tracks to find the right ones
     const publication = this.getPublicationForTrack(track);
@@ -455,7 +456,10 @@ export default class LocalParticipant extends Participant {
     track.off(TrackEvent.Unmuted, this.onTrackUnmuted);
     track.off(TrackEvent.Ended, this.onTrackUnpublish);
 
-    if (this.roomOptions?.stopLocalTrackOnUnpublish ?? true) {
+    if (stopOnUnpublish === undefined) {
+      stopOnUnpublish = this.roomOptions?.stopLocalTrackOnUnpublish ?? true;
+    }
+    if (stopOnUnpublish) {
       track.stop();
     }
 
@@ -507,10 +511,6 @@ export default class LocalParticipant extends Participant {
     return publications;
   }
 
-  get publisherMetrics(): any {
-    return null;
-  }
-
   /**
    * Publish a new data payload to the room. Data will be forwarded to each
    * participant in the room if the destination argument is empty

package/src/room/participant/RemoteParticipant.ts

@@ -53,6 +53,9 @@ export default class RemoteParticipant extends Participant {
       },
     );
     publication.on(TrackEvent.UpdateSubscription, (sub: UpdateSubscription) => {
+      sub.participantTracks.forEach((pt) => {
+        pt.participantSid = this.sid;
+      });
       this.signalClient.sendUpdateSubscription(sub);
     });
     publication.on(TrackEvent.Ended, (track: RemoteTrack) => {

package/src/room/track/LocalTrackPublication.ts

@@ -3,12 +3,15 @@ import { TrackEvent } from '../events';
 import LocalAudioTrack from './LocalAudioTrack';
 import LocalTrack from './LocalTrack';
 import LocalVideoTrack from './LocalVideoTrack';
+import { TrackPublishOptions } from './options';
 import { Track } from './Track';
 import { TrackPublication } from './TrackPublication';
 
 export default class LocalTrackPublication extends TrackPublication {
   track?: LocalTrack;
 
+  options?: TrackPublishOptions;
+
   constructor(kind: Track.Kind, ti: TrackInfo, track?: LocalTrack) {
     super(kind, ti.sid, ti.name);
 

package/src/room/track/RemoteTrackPublication.ts

@@ -35,7 +35,12 @@ export default class RemoteTrackPublication extends TrackPublication {
     const sub: UpdateSubscription = {
       trackSids: [this.trackSid],
       subscribe: this.subscribed,
-      participantTracks: [],
+      participantTracks: [{
+        // sending an empty participant id since TrackPublication doesn't keep it
+        // this is filled in by the participant that receives this message
+        participantSid: '',
+        trackSids: [this.trackSid],
+      }],
     };
     this.emit(TrackEvent.UpdateSubscription, sub);
   }
@@ -172,7 +177,8 @@ export default class RemoteTrackPublication extends TrackPublication {
     this.emitTrackUpdate();
   };
 
-  protected emitTrackUpdate() {
+  /* @internal */
+  emitTrackUpdate() {
     const settings: UpdateTrackSettings = UpdateTrackSettings.fromPartial({
       trackSids: [this.trackSid],
       disabled: this.disabled,

package/src/room/track/Track.ts

@@ -76,13 +76,15 @@ export class Track extends EventEmitter {
       element.autoplay = true;
     }
 
-    // already attached
-    if (this.attachedElements.includes(element)) {
-      return element;
+    if (!this.attachedElements.includes(element)) {
+      this.attachedElements.push(element);
     }
 
+    // even if we believe it's already attached to the element, it's possible
+    // the element's srcObject was set to something else out of band.
+    // we'll want to re-attach it in that case
+
     attachToElement(this.mediaStreamTrack, element);
-    this.attachedElements.push(element);
 
     if (element instanceof HTMLAudioElement) {
       // manually play audio to detect audio playback status
@@ -171,7 +173,7 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
     element.srcObject = mediaStream;
   }
 
-  // remove existing tracks of same type from stream
+  // check if track matches existing track
   let existingTracks: MediaStreamTrack[];
   if (track.kind === 'audio') {
     existingTracks = mediaStream.getAudioTracks();
@@ -179,6 +181,10 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
     existingTracks = mediaStream.getVideoTracks();
   }
 
+  if (existingTracks.includes(track)) {
+    return;
+  }
+
   existingTracks.forEach((et) => {
     mediaStream.removeTrack(et);
   });

package/src/version.ts

@@ -1,2 +1,2 @@
-export const version = '0.15.4';
-export const protocolVersion = 5;
+export const version = '0.16.0';
+export const protocolVersion = 6;