livekit-client 0.16.6 → 0.17.0

package/src/proto/livekit_rtc.ts

@@ -6,6 +6,7 @@ import {
   TrackSource,
   Room,
   ParticipantInfo,
+  ClientConfiguration,
   TrackInfo,
   VideoQuality,
   ConnectionQuality,
@@ -192,6 +193,8 @@ export interface JoinResponse {
    * when this is set, the other fields will be largely empty
    */
   alternativeUrl: string;
+  clientConfiguration?: ClientConfiguration;
+  serverRegion: string;
 }
 
 export interface TrackPublishedResponse {
@@ -306,6 +309,12 @@ export interface SyncState {
   answer?: SessionDescription;
   subscription?: UpdateSubscription;
   publishTracks: TrackPublishedResponse[];
+  dataChannels: DataChannelInfo[];
+}
+
+export interface DataChannelInfo {
+  label: string;
+  id: number;
 }
 
 export interface SimulateScenario {
@@ -1405,6 +1414,7 @@ const baseJoinResponse: object = {
   serverVersion: "",
   subscriberPrimary: false,
   alternativeUrl: "",
+  serverRegion: "",
 };
 
 export const JoinResponse = {
@@ -1436,6 +1446,15 @@ export const JoinResponse = {
     if (message.alternativeUrl !== "") {
       writer.uint32(58).string(message.alternativeUrl);
     }
+    if (message.clientConfiguration !== undefined) {
+      ClientConfiguration.encode(
+        message.clientConfiguration,
+        writer.uint32(66).fork()
+      ).ldelim();
+    }
+    if (message.serverRegion !== "") {
+      writer.uint32(74).string(message.serverRegion);
+    }
     return writer;
   },
 
@@ -1471,6 +1490,15 @@ export const JoinResponse = {
         case 7:
           message.alternativeUrl = reader.string();
           break;
+        case 8:
+          message.clientConfiguration = ClientConfiguration.decode(
+            reader,
+            reader.uint32()
+          );
+          break;
+        case 9:
+          message.serverRegion = reader.string();
+          break;
         default:
           reader.skipType(tag & 7);
           break;
@@ -1524,6 +1552,21 @@ export const JoinResponse = {
     } else {
       message.alternativeUrl = "";
     }
+    if (
+      object.clientConfiguration !== undefined &&
+      object.clientConfiguration !== null
+    ) {
+      message.clientConfiguration = ClientConfiguration.fromJSON(
+        object.clientConfiguration
+      );
+    } else {
+      message.clientConfiguration = undefined;
+    }
+    if (object.serverRegion !== undefined && object.serverRegion !== null) {
+      message.serverRegion = String(object.serverRegion);
+    } else {
+      message.serverRegion = "";
+    }
     return message;
   },
 
@@ -1555,6 +1598,12 @@ export const JoinResponse = {
       (obj.subscriberPrimary = message.subscriberPrimary);
     message.alternativeUrl !== undefined &&
       (obj.alternativeUrl = message.alternativeUrl);
+    message.clientConfiguration !== undefined &&
+      (obj.clientConfiguration = message.clientConfiguration
+        ? ClientConfiguration.toJSON(message.clientConfiguration)
+        : undefined);
+    message.serverRegion !== undefined &&
+      (obj.serverRegion = message.serverRegion);
     return obj;
   },
 
@@ -1588,6 +1637,17 @@ export const JoinResponse = {
     }
     message.subscriberPrimary = object.subscriberPrimary ?? false;
     message.alternativeUrl = object.alternativeUrl ?? "";
+    if (
+      object.clientConfiguration !== undefined &&
+      object.clientConfiguration !== null
+    ) {
+      message.clientConfiguration = ClientConfiguration.fromPartial(
+        object.clientConfiguration
+      );
+    } else {
+      message.clientConfiguration = undefined;
+    }
+    message.serverRegion = object.serverRegion ?? "";
     return message;
   },
 };
@@ -3197,6 +3257,9 @@ export const SyncState = {
     for (const v of message.publishTracks) {
       TrackPublishedResponse.encode(v!, writer.uint32(26).fork()).ldelim();
     }
+    for (const v of message.dataChannels) {
+      DataChannelInfo.encode(v!, writer.uint32(34).fork()).ldelim();
+    }
     return writer;
   },
 
@@ -3205,6 +3268,7 @@ export const SyncState = {
     let end = length === undefined ? reader.len : reader.pos + length;
     const message = { ...baseSyncState } as SyncState;
     message.publishTracks = [];
+    message.dataChannels = [];
     while (reader.pos < end) {
       const tag = reader.uint32();
       switch (tag >>> 3) {
@@ -3222,6 +3286,11 @@ export const SyncState = {
             TrackPublishedResponse.decode(reader, reader.uint32())
           );
           break;
+        case 4:
+          message.dataChannels.push(
+            DataChannelInfo.decode(reader, reader.uint32())
+          );
+          break;
         default:
           reader.skipType(tag & 7);
           break;
@@ -3233,6 +3302,7 @@ export const SyncState = {
   fromJSON(object: any): SyncState {
     const message = { ...baseSyncState } as SyncState;
     message.publishTracks = [];
+    message.dataChannels = [];
     if (object.answer !== undefined && object.answer !== null) {
       message.answer = SessionDescription.fromJSON(object.answer);
     } else {
@@ -3248,6 +3318,11 @@ export const SyncState = {
         message.publishTracks.push(TrackPublishedResponse.fromJSON(e));
       }
     }
+    if (object.dataChannels !== undefined && object.dataChannels !== null) {
+      for (const e of object.dataChannels) {
+        message.dataChannels.push(DataChannelInfo.fromJSON(e));
+      }
+    }
     return message;
   },
 
@@ -3268,6 +3343,13 @@ export const SyncState = {
     } else {
       obj.publishTracks = [];
     }
+    if (message.dataChannels) {
+      obj.dataChannels = message.dataChannels.map((e) =>
+        e ? DataChannelInfo.toJSON(e) : undefined
+      );
+    } else {
+      obj.dataChannels = [];
+    }
     return obj;
   },
 
@@ -3291,6 +3373,79 @@ export const SyncState = {
         message.publishTracks.push(TrackPublishedResponse.fromPartial(e));
       }
     }
+    message.dataChannels = [];
+    if (object.dataChannels !== undefined && object.dataChannels !== null) {
+      for (const e of object.dataChannels) {
+        message.dataChannels.push(DataChannelInfo.fromPartial(e));
+      }
+    }
+    return message;
+  },
+};
+
+const baseDataChannelInfo: object = { label: "", id: 0 };
+
+export const DataChannelInfo = {
+  encode(
+    message: DataChannelInfo,
+    writer: _m0.Writer = _m0.Writer.create()
+  ): _m0.Writer {
+    if (message.label !== "") {
+      writer.uint32(10).string(message.label);
+    }
+    if (message.id !== 0) {
+      writer.uint32(16).uint32(message.id);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): DataChannelInfo {
+    const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = { ...baseDataChannelInfo } as DataChannelInfo;
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          message.label = reader.string();
+          break;
+        case 2:
+          message.id = reader.uint32();
+          break;
+        default:
+          reader.skipType(tag & 7);
+          break;
+      }
+    }
+    return message;
+  },
+
+  fromJSON(object: any): DataChannelInfo {
+    const message = { ...baseDataChannelInfo } as DataChannelInfo;
+    if (object.label !== undefined && object.label !== null) {
+      message.label = String(object.label);
+    } else {
+      message.label = "";
+    }
+    if (object.id !== undefined && object.id !== null) {
+      message.id = Number(object.id);
+    } else {
+      message.id = 0;
+    }
+    return message;
+  },
+
+  toJSON(message: DataChannelInfo): unknown {
+    const obj: any = {};
+    message.label !== undefined && (obj.label = message.label);
+    message.id !== undefined && (obj.id = message.id);
+    return obj;
+  },
+
+  fromPartial(object: DeepPartial<DataChannelInfo>): DataChannelInfo {
+    const message = { ...baseDataChannelInfo } as DataChannelInfo;
+    message.label = object.label ?? "";
+    message.id = object.id ?? 0;
     return message;
   },
 };

package/src/room/RTCEngine.ts

@@ -3,6 +3,8 @@ import type TypedEventEmitter from 'typed-emitter';
 import { SignalClient, SignalOptions } from '../api/SignalClient';
 import log from '../logger';
 import {
+  ClientConfigSetting,
+  ClientConfiguration,
   DataPacket, DataPacket_Kind, SpeakerInfo, TrackInfo, UserPacket,
 } from '../proto/livekit_models';
 import {
@@ -70,7 +72,9 @@ export default class RTCEngine extends (
 
   private reconnectStart: number = 0;
 
-  private fullReconnect: boolean = false;
+  private fullReconnectOnNext: boolean = false;
+
+  private clientConfiguration?: ClientConfiguration;
 
   private connectedServerAddr?: string;
 
@@ -96,6 +100,7 @@ export default class RTCEngine extends (
     if (!this.subscriberPrimary) {
       this.negotiate();
     }
+    this.clientConfiguration = joinResponse.clientConfiguration;
 
     return joinResponse;
   }
@@ -303,7 +308,7 @@ export default class RTCEngine extends (
 
     this.client.onLeave = (leave?: LeaveRequest) => {
       if (leave?.canReconnect) {
-        this.fullReconnect = true;
+        this.fullReconnectOnNext = true;
         this.primaryPC = undefined;
       } else {
         this.emit(EngineEvent.Disconnected);
@@ -376,19 +381,19 @@ export default class RTCEngine extends (
       if (this.isClosed) {
         return;
       }
-      if (isFireFox()) {
-        // FF does not support DTLS restart.
-        this.fullReconnect = true;
+      if (isFireFox() // TODO remove once clientConfiguration handles firefox case server side
+        || this.clientConfiguration?.resumeConnection === ClientConfigSetting.DISABLED) {
+        this.fullReconnectOnNext = true;
       }
 
       try {
-        if (this.fullReconnect) {
+        if (this.fullReconnectOnNext) {
           await this.restartConnection();
         } else {
           await this.resumeConnection();
         }
         this.reconnectAttempts = 0;
-        this.fullReconnect = false;
+        this.fullReconnectOnNext = false;
       } catch (e) {
         this.reconnectAttempts += 1;
         let recoverable = true;
@@ -398,7 +403,7 @@ export default class RTCEngine extends (
           recoverable = false;
         } else if (!(e instanceof SignalReconnectError)) {
           // cannot resume
-          this.fullReconnect = true;
+          this.fullReconnectOnNext = true;
         }
 
         const duration = Date.now() - this.reconnectStart;
@@ -444,6 +449,7 @@ export default class RTCEngine extends (
     }
 
     await this.waitForPCConnected();
+    this.client.setReconnected();
 
     // reconnect success
     this.emit(EngineEvent.Restarted, joinResponse);
@@ -478,6 +484,7 @@ export default class RTCEngine extends (
     }
 
     await this.waitForPCConnected();
+    this.client.setReconnected();
 
     // resume success
     this.emit(EngineEvent.Resumed);
@@ -564,7 +571,7 @@ export default class RTCEngine extends (
     this.publisher.negotiate();
   }
 
-  private dataChannelForKind(kind: DataPacket_Kind): RTCDataChannel | undefined {
+  dataChannelForKind(kind: DataPacket_Kind): RTCDataChannel | undefined {
     if (kind === DataPacket_Kind.LOSSY) {
       return this.lossyDC;
     } if (kind === DataPacket_Kind.RELIABLE) {

package/src/room/Room.ts

@@ -28,7 +28,8 @@ import LocalTrackPublication from './track/LocalTrackPublication';
 import RemoteTrackPublication from './track/RemoteTrackPublication';
 import { Track } from './track/Track';
 import { TrackPublication } from './track/TrackPublication';
-import { RemoteTrack } from './track/types';
+import { AdaptiveStreamSettings, RemoteTrack } from './track/types';
+import { getNewAudioContext } from './track/utils';
 import { unpackStreamId } from './utils';
 
 export enum RoomState {
@@ -195,7 +196,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
     try {
       const joinResponse = await this.engine.join(url, token, opts);
-      log.debug('connected to Livekit Server', joinResponse.serverVersion);
+      log.debug(`connected to Livekit Server version: ${joinResponse.serverVersion}, region: ${joinResponse.serverRegion}`);
 
       if (!joinResponse.serverVersion) {
         throw new UnsupportedServer('unknown server version');
@@ -437,12 +438,20 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     if (!trackId || trackId === '') trackId = mediaTrack.id;
 
     const participant = this.getOrCreateParticipant(participantId);
+    let adaptiveStreamSettings: AdaptiveStreamSettings | undefined;
+    if (this.options.adaptiveStream) {
+      if (typeof this.options.adaptiveStream === 'object') {
+        adaptiveStreamSettings = this.options.adaptiveStream;
+      } else {
+        adaptiveStreamSettings = {};
+      }
+    }
     participant.addSubscribedMediaTrack(
       mediaTrack,
       trackId,
       stream,
       receiver,
-      this.options.adaptiveStream,
+      adaptiveStreamSettings,
     );
   }
 
@@ -458,6 +467,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   };
 
   private handleRestarted = async (joinResponse: JoinResponse) => {
+    log.debug('reconnected to server region', joinResponse.serverRegion);
     this.state = RoomState.Connected;
     this.emit(RoomEvent.Reconnected);
     this.emit(RoomEvent.StateChanged, this.state);
@@ -717,10 +727,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
     // by using an AudioContext, it reduces lag on audio elements
     // https://stackoverflow.com/questions/9811429/html5-audio-tag-on-safari-has-a-delay/54119854#54119854
-    // @ts-ignore
-    const AudioContext = window.AudioContext || window.webkitAudioContext;
-    if (AudioContext) {
-      this.audioContext = new AudioContext();
+    const ctx = getNewAudioContext();
+    if (ctx) {
+      this.audioContext = ctx;
     }
   }
 
@@ -826,6 +835,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         participantTracks: [],
       },
       publishTracks: this.localParticipant.publishedTracksInfo(),
+      dataChannels: this.localParticipant.dataChannelsInfo(),
     });
   }
 

package/src/room/events.ts

@@ -185,7 +185,7 @@ export enum RoomEvent {
   ConnectionQualityChanged = 'connectionQualityChanged',
 
   /**
-   * StreamState indicates if a subscribed track has been paused by the SFU
+   * StreamState indicates if a subscribed (remote) track has been paused by the SFU
    * (typically this happens because of subscriber's bandwidth constraints)
    *
    * When bandwidth conditions allow, the track will be resumed automatically.
@@ -400,6 +400,11 @@ export enum TrackEvent {
   AudioPlaybackStarted = 'audioPlaybackStarted',
   /** @internal */
   AudioPlaybackFailed = 'audioPlaybackFailed',
+  /**
+   * @internal
+   * Only fires on LocalAudioTrack instances
+  */
+  AudioSilenceDetected = 'audioSilenceDetected',
   /** @internal */
   VisibilityChanged = 'visibilityChanged',
   /** @internal */

package/src/room/participant/LocalParticipant.ts

@@ -3,7 +3,9 @@ import { RoomOptions } from '../../options';
 import {
   DataPacket, DataPacket_Kind,
 } from '../../proto/livekit_models';
-import { AddTrackRequest, SubscribedQualityUpdate, TrackPublishedResponse } from '../../proto/livekit_rtc';
+import {
+  AddTrackRequest, DataChannelInfo, SubscribedQualityUpdate, TrackPublishedResponse,
+} from '../../proto/livekit_rtc';
 import {
   TrackInvalidError,
   UnexpectedConnectionState,
@@ -17,7 +19,8 @@ import LocalVideoTrack, { videoLayersFromEncodings } from '../track/LocalVideoTr
 import {
   CreateLocalTracksOptions,
   ScreenShareCaptureOptions,
-  TrackPublishOptions, VideoCodec, VideoPresets,
+  ScreenSharePresets,
+  TrackPublishOptions, VideoCodec,
 } from '../track/options';
 import { Track } from '../track/Track';
 import { constraintsForOptions, mergeDefaultOptions } from '../track/utils';
@@ -272,7 +275,7 @@ export default class LocalParticipant extends Participant {
       options = {};
     }
     if (options.resolution === undefined) {
-      options.resolution = VideoPresets.fhd.resolution;
+      options.resolution = ScreenSharePresets.h720fps15.resolution;
     }
 
     let videoConstraints: MediaTrackConstraints | boolean = true;
@@ -525,7 +528,7 @@ export default class LocalParticipant extends Participant {
     destination?: RemoteParticipant[] | string[]) {
     const dest: string[] = [];
     if (destination !== undefined) {
-      destination.forEach((val : any) => {
+      destination.forEach((val: any) => {
         if (val instanceof RemoteParticipant) {
           dest.push(val.sid);
         } else {
@@ -681,4 +684,20 @@ export default class LocalParticipant extends Participant {
     });
     return infos;
   }
+
+  /** @internal */
+  dataChannelsInfo(): DataChannelInfo[] {
+    const infos: DataChannelInfo[] = [];
+    const getInfo = (dc: RTCDataChannel | undefined) => {
+      if (dc?.id !== undefined && dc.id !== null) {
+        infos.push({
+          label: dc.label,
+          id: dc.id,
+        });
+      }
+    };
+    getInfo(this.engine.dataChannelForKind(DataPacket_Kind.LOSSY));
+    getInfo(this.engine.dataChannelForKind(DataPacket_Kind.RELIABLE));
+    return infos;
+  }
 }

package/src/room/participant/RemoteParticipant.ts

@@ -10,7 +10,7 @@ import RemoteAudioTrack from '../track/RemoteAudioTrack';
 import RemoteTrackPublication from '../track/RemoteTrackPublication';
 import RemoteVideoTrack from '../track/RemoteVideoTrack';
 import { Track } from '../track/Track';
-import { RemoteTrack } from '../track/types';
+import { AdaptiveStreamSettings, RemoteTrack } from '../track/types';
 import Participant, { ParticipantEventCallbacks } from './Participant';
 
 export default class RemoteParticipant extends Participant {
@@ -82,7 +82,7 @@ export default class RemoteParticipant extends Participant {
     sid: Track.SID,
     mediaStream: MediaStream,
     receiver?: RTCRtpReceiver,
-    adaptiveStream?: boolean,
+    adaptiveStreamSettings?: AdaptiveStreamSettings,
     triesLeft?: number,
   ) {
     // find the track publication
@@ -114,7 +114,7 @@ export default class RemoteParticipant extends Participant {
       if (triesLeft === undefined) triesLeft = 20;
       setTimeout(() => {
         this.addSubscribedMediaTrack(mediaTrack, sid, mediaStream,
-          receiver, adaptiveStream, triesLeft! - 1);
+          receiver, adaptiveStreamSettings, triesLeft! - 1);
       }, 150);
       return;
     }
@@ -122,7 +122,7 @@ export default class RemoteParticipant extends Participant {
     const isVideo = mediaTrack.kind === 'video';
     let track: RemoteTrack;
     if (isVideo) {
-      track = new RemoteVideoTrack(mediaTrack, sid, receiver, adaptiveStream);
+      track = new RemoteVideoTrack(mediaTrack, sid, receiver, adaptiveStreamSettings);
     } else {
       track = new RemoteAudioTrack(mediaTrack, sid, receiver);
     }

package/src/room/participant/publishUtils.test.ts

@@ -1,11 +1,15 @@
-import { VideoPresets, VideoPresets43 } from '../track/options';
 import {
+  ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43,
+} from '../track/options';
+import {
+  computeDefaultScreenShareSimulcastPresets,
   computeVideoEncodings,
   determineAppropriateEncoding,
   presets169,
   presets43,
   presetsForResolution,
   presetsScreenShare,
+  sortPresets,
 } from './publishUtils';
 
 describe('presetsForResolution', () => {
@@ -63,7 +67,7 @@ describe('computeVideoEncodings', () => {
 
     // ensure they are what we expect
     expect(encodings![0].rid).toBe('q');
-    expect(encodings![0].maxBitrate).toBe(VideoPresets.qvga.encoding.maxBitrate);
+    expect(encodings![0].maxBitrate).toBe(VideoPresets.h180.encoding.maxBitrate);
     expect(encodings![0].scaleResolutionDownBy).toBe(3);
     expect(encodings![1].rid).toBe('h');
     expect(encodings![1].scaleResolutionDownBy).toBe(1.5);
@@ -77,7 +81,7 @@ describe('computeVideoEncodings', () => {
     expect(encodings).toHaveLength(3);
     expect(encodings![0].scaleResolutionDownBy).toBe(3);
     expect(encodings![1].scaleResolutionDownBy).toBe(1.5);
-    expect(encodings![2].maxBitrate).toBe(VideoPresets.qhd.encoding.maxBitrate);
+    expect(encodings![2].maxBitrate).toBe(VideoPresets.h540.encoding.maxBitrate);
   });
 
   it('returns two encodings for lower-res simulcast', () => {
@@ -88,9 +92,9 @@ describe('computeVideoEncodings', () => {
 
     // ensure they are what we expect
     expect(encodings![0].rid).toBe('q');
-    expect(encodings![0].maxBitrate).toBe(VideoPresets.qvga.encoding.maxBitrate);
+    expect(encodings![0].maxBitrate).toBe(VideoPresets.h180.encoding.maxBitrate);
     expect(encodings![1].rid).toBe('h');
-    expect(encodings![1].maxBitrate).toBe(VideoPresets.vga.encoding.maxBitrate);
+    expect(encodings![1].maxBitrate).toBe(VideoPresets.h360.encoding.maxBitrate);
   });
 
   it('respects provided min resolution', () => {
@@ -99,7 +103,43 @@ describe('computeVideoEncodings', () => {
     });
     expect(encodings).toHaveLength(1);
     expect(encodings![0].rid).toBe('q');
-    expect(encodings![0].maxBitrate).toBe(VideoPresets43.qvga.encoding.maxBitrate);
+    expect(encodings![0].maxBitrate).toBe(VideoPresets43.h120.encoding.maxBitrate);
     expect(encodings![0].scaleResolutionDownBy).toBe(1);
   });
 });
+
+describe('customSimulcastLayers', () => {
+  it('sorts presets from lowest to highest', () => {
+    const sortedPresets = sortPresets(
+      [VideoPresets.h1440, VideoPresets.h360, VideoPresets.h1080, VideoPresets.h90],
+    ) as Array<VideoPreset>;
+    expect(sortPresets).not.toBeUndefined();
+    expect(sortedPresets[0]).toBe(VideoPresets.h90);
+    expect(sortedPresets[1]).toBe(VideoPresets.h360);
+    expect(sortedPresets[2]).toBe(VideoPresets.h1080);
+    expect(sortedPresets[3]).toBe(VideoPresets.h1440);
+  });
+  it('sorts presets from lowest to highest, even when dimensions are the same', () => {
+    const sortedPresets = sortPresets([
+      new VideoPreset(1920, 1080, 3_000_000, 20),
+      new VideoPreset(1920, 1080, 2_000_000, 15),
+      new VideoPreset(1920, 1080, 3_000_000, 15),
+    ]) as Array<VideoPreset>;
+    expect(sortPresets).not.toBeUndefined();
+    expect(sortedPresets[0].encoding.maxBitrate).toBe(2_000_000);
+    expect(sortedPresets[1].encoding.maxFramerate).toBe(15);
+    expect(sortedPresets[2].encoding.maxFramerate).toBe(20);
+  });
+});
+
+describe('screenShareSimulcastDefaults', () => {
+  it('computes appropriate bitrate from original preset', () => {
+    const defaultSimulcastLayers = computeDefaultScreenShareSimulcastPresets(
+      ScreenSharePresets.h720fps15,
+    );
+    expect(defaultSimulcastLayers[0].width).toBe(640);
+    expect(defaultSimulcastLayers[0].height).toBe(360);
+    expect(defaultSimulcastLayers[0].encoding.maxFramerate).toBe(3);
+    expect(defaultSimulcastLayers[0].encoding.maxBitrate).toBe(150_000);
+  });
+});