livekit-client 0.15.4 → 0.16.3

package/src/proto/livekit_rtc.ts

@@ -110,7 +110,7 @@ export interface SignalRequest {
    */
   updateLayers?: UpdateVideoLayers | undefined;
   /** Update subscriber permissions */
-  subscriptionPermissions?: UpdateSubscriptionPermissions | undefined;
+  subscriptionPermission?: SubscriptionPermission | undefined;
   /** sync client's subscribe state to server during reconnect */
   syncState?: SyncState | undefined;
   /** Simulate conditions, for client validations */
@@ -140,12 +140,17 @@ export interface SignalResponse {
   roomUpdate?: RoomUpdate | undefined;
   /** when connection quality changed */
   connectionQuality?: ConnectionQualityUpdate | undefined;
-  /** when streamed tracks state changed */
+  /**
+   * when streamed tracks state changed, used to notify when any of the streams were paused due to
+   * congestion
+   */
   streamStateUpdate?: StreamStateUpdate | undefined;
-  /** when max subscribe quality changed */
+  /** when max subscribe quality changed, used by dynamic broadcasting to disable unused layers */
   subscribedQualityUpdate?: SubscribedQualityUpdate | undefined;
   /** when subscription permission changed */
   subscriptionPermissionUpdate?: SubscriptionPermissionUpdate | undefined;
+  /** update the token the client was using, to prevent an active client from using an expired token */
+  refreshToken: string | undefined;
 }
 
 export interface AddTrackRequest {
@@ -286,7 +291,7 @@ export interface TrackPermission {
   trackSids: string[];
 }
 
-export interface UpdateSubscriptionPermissions {
+export interface SubscriptionPermission {
   allParticipants: boolean;
   trackPermissions: TrackPermission[];
 }
@@ -366,9 +371,9 @@ export const SignalRequest = {
         writer.uint32(82).fork()
       ).ldelim();
     }
-    if (message.subscriptionPermissions !== undefined) {
-      UpdateSubscriptionPermissions.encode(
-        message.subscriptionPermissions,
+    if (message.subscriptionPermission !== undefined) {
+      SubscriptionPermission.encode(
+        message.subscriptionPermission,
         writer.uint32(90).fork()
       ).ldelim();
     }
@@ -428,8 +433,10 @@ export const SignalRequest = {
           );
           break;
         case 11:
-          message.subscriptionPermissions =
-            UpdateSubscriptionPermissions.decode(reader, reader.uint32());
+          message.subscriptionPermission = SubscriptionPermission.decode(
+            reader,
+            reader.uint32()
+          );
           break;
         case 12:
           message.syncState = SyncState.decode(reader, reader.uint32());
@@ -493,14 +500,14 @@ export const SignalRequest = {
       message.updateLayers = undefined;
     }
     if (
-      object.subscriptionPermissions !== undefined &&
-      object.subscriptionPermissions !== null
+      object.subscriptionPermission !== undefined &&
+      object.subscriptionPermission !== null
     ) {
-      message.subscriptionPermissions = UpdateSubscriptionPermissions.fromJSON(
-        object.subscriptionPermissions
+      message.subscriptionPermission = SubscriptionPermission.fromJSON(
+        object.subscriptionPermission
       );
     } else {
-      message.subscriptionPermissions = undefined;
+      message.subscriptionPermission = undefined;
     }
     if (object.syncState !== undefined && object.syncState !== null) {
       message.syncState = SyncState.fromJSON(object.syncState);
@@ -553,9 +560,9 @@ export const SignalRequest = {
       (obj.updateLayers = message.updateLayers
         ? UpdateVideoLayers.toJSON(message.updateLayers)
         : undefined);
-    message.subscriptionPermissions !== undefined &&
-      (obj.subscriptionPermissions = message.subscriptionPermissions
-        ? UpdateSubscriptionPermissions.toJSON(message.subscriptionPermissions)
+    message.subscriptionPermission !== undefined &&
+      (obj.subscriptionPermission = message.subscriptionPermission
+        ? SubscriptionPermission.toJSON(message.subscriptionPermission)
         : undefined);
     message.syncState !== undefined &&
       (obj.syncState = message.syncState
@@ -620,15 +627,14 @@ export const SignalRequest = {
       message.updateLayers = undefined;
     }
     if (
-      object.subscriptionPermissions !== undefined &&
-      object.subscriptionPermissions !== null
+      object.subscriptionPermission !== undefined &&
+      object.subscriptionPermission !== null
     ) {
-      message.subscriptionPermissions =
-        UpdateSubscriptionPermissions.fromPartial(
-          object.subscriptionPermissions
-        );
+      message.subscriptionPermission = SubscriptionPermission.fromPartial(
+        object.subscriptionPermission
+      );
     } else {
-      message.subscriptionPermissions = undefined;
+      message.subscriptionPermission = undefined;
     }
     if (object.syncState !== undefined && object.syncState !== null) {
       message.syncState = SyncState.fromPartial(object.syncState);
@@ -720,6 +726,9 @@ export const SignalResponse = {
         writer.uint32(122).fork()
       ).ldelim();
     }
+    if (message.refreshToken !== undefined) {
+      writer.uint32(130).string(message.refreshToken);
+    }
     return writer;
   },
 
@@ -788,6 +797,9 @@ export const SignalResponse = {
           message.subscriptionPermissionUpdate =
             SubscriptionPermissionUpdate.decode(reader, reader.uint32());
           break;
+        case 16:
+          message.refreshToken = reader.string();
+          break;
         default:
           reader.skipType(tag & 7);
           break;
@@ -896,6 +908,11 @@ export const SignalResponse = {
     } else {
       message.subscriptionPermissionUpdate = undefined;
     }
+    if (object.refreshToken !== undefined && object.refreshToken !== null) {
+      message.refreshToken = String(object.refreshToken);
+    } else {
+      message.refreshToken = undefined;
+    }
     return message;
   },
 
@@ -957,6 +974,8 @@ export const SignalResponse = {
             message.subscriptionPermissionUpdate
           )
         : undefined);
+    message.refreshToken !== undefined &&
+      (obj.refreshToken = message.refreshToken);
     return obj;
   },
 
@@ -1060,6 +1079,7 @@ export const SignalResponse = {
     } else {
       message.subscriptionPermissionUpdate = undefined;
     }
+    message.refreshToken = object.refreshToken ?? undefined;
     return message;
   },
 };
@@ -2962,11 +2982,11 @@ export const TrackPermission = {
   },
 };
 
-const baseUpdateSubscriptionPermissions: object = { allParticipants: false };
+const baseSubscriptionPermission: object = { allParticipants: false };
 
-export const UpdateSubscriptionPermissions = {
+export const SubscriptionPermission = {
   encode(
-    message: UpdateSubscriptionPermissions,
+    message: SubscriptionPermission,
     writer: _m0.Writer = _m0.Writer.create()
   ): _m0.Writer {
     if (message.allParticipants === true) {
@@ -2981,12 +3001,10 @@ export const UpdateSubscriptionPermissions = {
   decode(
     input: _m0.Reader | Uint8Array,
     length?: number
-  ): UpdateSubscriptionPermissions {
+  ): SubscriptionPermission {
     const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input);
     let end = length === undefined ? reader.len : reader.pos + length;
-    const message = {
-      ...baseUpdateSubscriptionPermissions,
-    } as UpdateSubscriptionPermissions;
+    const message = { ...baseSubscriptionPermission } as SubscriptionPermission;
     message.trackPermissions = [];
     while (reader.pos < end) {
       const tag = reader.uint32();
@@ -3007,10 +3025,8 @@ export const UpdateSubscriptionPermissions = {
     return message;
   },
 
-  fromJSON(object: any): UpdateSubscriptionPermissions {
-    const message = {
-      ...baseUpdateSubscriptionPermissions,
-    } as UpdateSubscriptionPermissions;
+  fromJSON(object: any): SubscriptionPermission {
+    const message = { ...baseSubscriptionPermission } as SubscriptionPermission;
     message.trackPermissions = [];
     if (
       object.allParticipants !== undefined &&
@@ -3031,7 +3047,7 @@ export const UpdateSubscriptionPermissions = {
     return message;
   },
 
-  toJSON(message: UpdateSubscriptionPermissions): unknown {
+  toJSON(message: SubscriptionPermission): unknown {
     const obj: any = {};
     message.allParticipants !== undefined &&
       (obj.allParticipants = message.allParticipants);
@@ -3046,11 +3062,9 @@ export const UpdateSubscriptionPermissions = {
   },
 
   fromPartial(
-    object: DeepPartial<UpdateSubscriptionPermissions>
-  ): UpdateSubscriptionPermissions {
-    const message = {
-      ...baseUpdateSubscriptionPermissions,
-    } as UpdateSubscriptionPermissions;
+    object: DeepPartial<SubscriptionPermission>
+  ): SubscriptionPermission {
+    const message = { ...baseSubscriptionPermission } as SubscriptionPermission;
     message.allParticipants = object.allParticipants ?? false;
     message.trackPermissions = [];
     if (

package/src/room/RTCEngine.ts

@@ -1,24 +1,32 @@
 import { EventEmitter } from 'events';
+import type TypedEventEmitter from 'typed-emitter';
 import { SignalClient, SignalOptions } from '../api/SignalClient';
 import log from '../logger';
-import { DataPacket, DataPacket_Kind, TrackInfo } from '../proto/livekit_models';
+import {
+  DataPacket, DataPacket_Kind, SpeakerInfo, TrackInfo, UserPacket,
+} from '../proto/livekit_models';
 import {
   AddTrackRequest, JoinResponse,
+  LeaveRequest,
   SignalTarget,
   TrackPublishedResponse,
 } from '../proto/livekit_rtc';
 import { ConnectionError, TrackInvalidError, UnexpectedConnectionState } from './errors';
 import { EngineEvent } from './events';
 import PCTransport from './PCTransport';
-import { sleep } from './utils';
+import { isFireFox, sleep } from './utils';
 
 const lossyDataChannel = '_lossy';
 const reliableDataChannel = '_reliable';
-const maxReconnectRetries = 5;
-export const maxICEConnectTimeout = 5 * 1000;
+const maxReconnectRetries = 10;
+const minReconnectWait = 1 * 1000;
+const maxReconnectDuration = 60 * 1000;
+export const maxICEConnectTimeout = 15 * 1000;
 
 /** @internal */
-export default class RTCEngine extends EventEmitter {
+export default class RTCEngine extends (
+  EventEmitter as new () => TypedEventEmitter<EngineEventCallbacks>
+) {
   publisher?: PCTransport;
 
   subscriber?: PCTransport;
@@ -39,7 +47,9 @@ export default class RTCEngine extends EventEmitter {
 
   private subscriberPrimary: boolean = false;
 
-  private iceConnected: boolean = false;
+  private primaryPC?: RTCPeerConnection;
+
+  private pcConnected: boolean = false;
 
   private isClosed: boolean = true;
 
@@ -54,8 +64,14 @@ export default class RTCEngine extends EventEmitter {
 
   private token?: string;
 
+  private signalOpts?: SignalOptions;
+
   private reconnectAttempts: number = 0;
 
+  private reconnectStart: number = 0;
+
+  private fullReconnect: boolean = false;
+
   private connectedServerAddr?: string;
 
   constructor() {
@@ -66,9 +82,9 @@ export default class RTCEngine extends EventEmitter {
   async join(url: string, token: string, opts?: SignalOptions): Promise<JoinResponse> {
     this.url = url;
     this.token = token;
+    this.signalOpts = opts;
 
     const joinResponse = await this.client.join(url, token, opts);
-    this.emit(EngineEvent.SignalConnected);
     this.isClosed = false;
 
     this.subscriberPrimary = joinResponse.subscriberPrimary;
@@ -153,6 +169,8 @@ export default class RTCEngine extends EventEmitter {
     this.publisher = new PCTransport(this.rtcConfig);
     this.subscriber = new PCTransport(this.rtcConfig);
 
+    this.emit(EngineEvent.TransportsCreated, this.publisher, this.subscriber);
+
     this.publisher.pc.onicecandidate = (ev) => {
       if (!ev.candidate) return;
       log.trace('adding ICE candidate for peer', ev.candidate);
@@ -174,21 +192,24 @@ export default class RTCEngine extends EventEmitter {
       // in subscriber primary mode, server side opens sub data channels.
       this.subscriber.pc.ondatachannel = this.handleDataChannel;
     }
-    primaryPC.oniceconnectionstatechange = () => {
-      if (primaryPC.iceConnectionState === 'connected') {
-        log.trace('ICE connected');
-        if (!this.iceConnected) {
-          this.iceConnected = true;
+    this.primaryPC = primaryPC;
+    primaryPC.onconnectionstatechange = async () => {
+      if (primaryPC.connectionState === 'connected') {
+        log.trace('pc connected');
+        try {
+          this.connectedServerAddr = await getConnectedAddress(primaryPC);
+        } catch (e) {
+          log.warn('could not get connected server address', e);
+        }
+        if (!this.pcConnected) {
+          this.pcConnected = true;
           this.emit(EngineEvent.Connected);
         }
-        getConnectedAddress(primaryPC).then((v) => {
-          this.connectedServerAddr = v;
-        });
-      } else if (primaryPC.iceConnectionState === 'failed') {
+      } else if (primaryPC.connectionState === 'failed') {
         // on Safari, PeerConnection will switch to 'disconnected' during renegotiation
-        log.trace('ICE disconnected');
-        if (this.iceConnected) {
-          this.iceConnected = false;
+        log.trace('pc disconnected');
+        if (this.pcConnected) {
+          this.pcConnected = false;
 
           this.handleDisconnect('peerconnection');
         }
@@ -213,6 +234,10 @@ export default class RTCEngine extends EventEmitter {
     this.lossyDC.onmessage = this.handleDataMessage;
     this.reliableDC.onmessage = this.handleDataMessage;
 
+    // handle datachannel errors
+    this.lossyDC.onerror = this.handleDataError;
+    this.reliableDC.onerror = this.handleDataError;
+
     // configure signaling client
     this.client.onAnswer = async (sd) => {
       if (!this.publisher) {
@@ -268,13 +293,22 @@ export default class RTCEngine extends EventEmitter {
       resolve(res.track!);
     };
 
+    this.client.onTokenRefresh = (token: string) => {
+      this.token = token;
+    };
+
     this.client.onClose = () => {
       this.handleDisconnect('signal');
     };
 
-    this.client.onLeave = () => {
-      this.emit(EngineEvent.Disconnected);
-      this.close();
+    this.client.onLeave = (leave?: LeaveRequest) => {
+      if (leave?.canReconnect) {
+        this.fullReconnect = true;
+        this.primaryPC = undefined;
+      } else {
+        this.emit(EngineEvent.Disconnected);
+        this.close();
+      }
     };
   }
 
@@ -312,6 +346,18 @@ export default class RTCEngine extends EventEmitter {
     }
   };
 
+  private handleDataError = (event: Event) => {
+    const channel = event.currentTarget as RTCDataChannel;
+    const channelKind = channel.maxRetransmits === 0 ? 'lossy' : 'reliable';
+
+    if (event instanceof ErrorEvent) {
+      const { error } = event.error;
+      log.error(`DataChannel error on ${channelKind}: ${event.message}`, error);
+    } else {
+      log.error(`Unknown DataChannel Error on ${channelKind}`, event);
+    }
+  };
+
   // websocket reconnect behavior. if websocket is interrupted, and the PeerConnection
   // continues to work, we can reconnect to websocket to continue the session
   // after a number of retries, we'll close and give up permanently
@@ -320,48 +366,110 @@ export default class RTCEngine extends EventEmitter {
       return;
     }
     log.debug(`${connection} disconnected`);
-    if (this.reconnectAttempts >= maxReconnectRetries) {
-      log.info(
-        'could not connect to signal after',
-        maxReconnectRetries,
-        'attempts. giving up',
-      );
-      this.emit(EngineEvent.Disconnected);
-      this.close();
-      return;
+    if (this.reconnectAttempts === 0) {
+      // only reset start time on the first try
+      this.reconnectStart = Date.now();
     }
 
     const delay = (this.reconnectAttempts * this.reconnectAttempts) * 300;
-    setTimeout(() => {
-      this.reconnect()
-        .then(() => {
-          this.reconnectAttempts = 0;
-        })
-        .catch(this.handleDisconnect);
+    setTimeout(async () => {
+      if (this.isClosed) {
+        return;
+      }
+      if (isFireFox()) {
+        // FF does not support DTLS restart.
+        this.fullReconnect = true;
+      }
+
+      try {
+        if (this.fullReconnect) {
+          await this.restartConnection();
+        } else {
+          await this.resumeConnection();
+        }
+        this.reconnectAttempts = 0;
+        this.fullReconnect = false;
+      } catch (e) {
+        this.reconnectAttempts += 1;
+        let recoverable = true;
+        if (e instanceof UnexpectedConnectionState) {
+          log.debug('received unrecoverable error', e.message);
+          // unrecoverable
+          recoverable = false;
+        } else if (!(e instanceof SignalReconnectError)) {
+          // cannot resume
+          this.fullReconnect = true;
+        }
+
+        const duration = Date.now() - this.reconnectStart;
+        if (this.reconnectAttempts >= maxReconnectRetries || duration > maxReconnectDuration) {
+          recoverable = false;
+        }
+
+        if (recoverable) {
+          this.handleDisconnect('reconnect');
+        } else {
+          log.info(
+            `could not recover connection after ${maxReconnectRetries} attempts, ${duration}ms. giving up`,
+          );
+          this.emit(EngineEvent.Disconnected);
+          this.close();
+        }
+      }
     }, delay);
   };
 
-  private async reconnect(): Promise<void> {
-    if (this.isClosed) {
-      return;
-    }
+  private async restartConnection() {
     if (!this.url || !this.token) {
-      throw new ConnectionError('could not reconnect, url or token not saved');
+      // permanent failure, don't attempt reconnection
+      throw new UnexpectedConnectionState('could not reconnect, url or token not saved');
     }
-    log.info('reconnecting to signal connection, attempt', this.reconnectAttempts);
 
+    log.info('reconnecting, attempt', this.reconnectAttempts);
     if (this.reconnectAttempts === 0) {
-      this.emit(EngineEvent.Reconnecting);
+      this.emit(EngineEvent.Restarting);
+    }
+
+    this.primaryPC = undefined;
+    this.publisher?.close();
+    this.publisher = undefined;
+    this.subscriber?.close();
+    this.subscriber = undefined;
+
+    let joinResponse: JoinResponse;
+    try {
+      joinResponse = await this.join(this.url, this.token, this.signalOpts);
+    } catch (e) {
+      throw new SignalReconnectError();
     }
-    this.reconnectAttempts += 1;
 
-    await this.client.reconnect(this.url, this.token);
-    this.emit(EngineEvent.SignalConnected);
+    await this.waitForPCConnected();
 
+    // reconnect success
+    this.emit(EngineEvent.Restarted, joinResponse);
+  }
+
+  private async resumeConnection(): Promise<void> {
+    if (!this.url || !this.token) {
+      // permanent failure, don't attempt reconnection
+      throw new UnexpectedConnectionState('could not reconnect, url or token not saved');
+    }
     // trigger publisher reconnect
     if (!this.publisher || !this.subscriber) {
       throw new UnexpectedConnectionState('publisher and subscriber connections unset');
     }
+    log.info('resuming signal connection, attempt', this.reconnectAttempts);
+    if (this.reconnectAttempts === 0) {
+      this.emit(EngineEvent.Resuming);
+    }
+
+    try {
+      await this.client.reconnect(this.url, this.token);
+    } catch (e) {
+      throw new SignalReconnectError();
+    }
+    this.emit(EngineEvent.SignalResumed);
+
     this.subscriber.restartingIce = true;
 
     // only restart publisher if it's needed
@@ -369,19 +477,35 @@ export default class RTCEngine extends EventEmitter {
       await this.publisher.createAndSendOffer({ iceRestart: true });
     }
 
-    const startTime = (new Date()).getTime();
+    await this.waitForPCConnected();
+
+    // resume success
+    this.emit(EngineEvent.Resumed);
+  }
 
-    while ((new Date()).getTime() - startTime < maxICEConnectTimeout * 2) {
-      if (this.iceConnected) {
-        // reconnect success
-        this.emit(EngineEvent.Reconnected);
+  async waitForPCConnected() {
+    const startTime = (new Date()).getTime();
+    let now = startTime;
+    this.pcConnected = false;
+
+    while (now - startTime < maxICEConnectTimeout) {
+      // if there is no connectionstatechange callback fired
+      // check connectionstate after minReconnectWait
+      if (this.primaryPC === undefined) {
+        // we can abort early, connection is hosed
+        break;
+      } else if (now - startTime > minReconnectWait && this.primaryPC?.connectionState === 'connected') {
+        this.pcConnected = true;
+      }
+      if (this.pcConnected) {
         return;
       }
       await sleep(100);
+      now = (new Date()).getTime();
     }
 
     // have not reconnected, throw
-    throw new ConnectionError('could not establish ICE connection');
+    throw new ConnectionError('could not establish PC connection');
   }
 
   /* @internal */
@@ -482,3 +606,24 @@ async function getConnectedAddress(pc: RTCPeerConnection): Promise<string | unde
   }
   return candidates.get(selectedID);
 }
+
+class SignalReconnectError extends Error {
+}
+
+export type EngineEventCallbacks = {
+  connected: () => void,
+  disconnected: () => void,
+  resuming: () => void,
+  resumed: () => void,
+  restarting: () => void,
+  restarted: (joinResp: JoinResponse) => void,
+  signalResumed: () => void,
+  mediaTrackAdded: (
+    track: MediaStreamTrack,
+    streams: MediaStream,
+    receiver: RTCRtpReceiver
+  ) => void,
+  activeSpeakersUpdate: (speakers: Array<SpeakerInfo>) => void,
+  dataPacketReceived: (userPacket: UserPacket, kind: DataPacket_Kind) => void,
+  transportsCreated: (publisher: PCTransport, subscriber: PCTransport) => void,
+};