livekit-client 1.11.3 → 1.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (136) hide show
  1. package/README.md +1 -3
  2. package/dist/livekit-client.e2ee.worker.js +2 -0
  3. package/dist/livekit-client.e2ee.worker.js.map +1 -0
  4. package/dist/livekit-client.e2ee.worker.mjs +1525 -0
  5. package/dist/livekit-client.e2ee.worker.mjs.map +1 -0
  6. package/dist/livekit-client.esm.mjs +1462 -660
  7. package/dist/livekit-client.esm.mjs.map +1 -1
  8. package/dist/livekit-client.umd.js +1 -1
  9. package/dist/livekit-client.umd.js.map +1 -1
  10. package/dist/src/api/SignalClient.d.ts +4 -1
  11. package/dist/src/api/SignalClient.d.ts.map +1 -1
  12. package/dist/src/connectionHelper/checks/turn.d.ts.map +1 -1
  13. package/dist/src/connectionHelper/checks/websocket.d.ts.map +1 -1
  14. package/dist/src/e2ee/E2eeManager.d.ts +45 -0
  15. package/dist/src/e2ee/E2eeManager.d.ts.map +1 -0
  16. package/dist/src/e2ee/KeyProvider.d.ts +42 -0
  17. package/dist/src/e2ee/KeyProvider.d.ts.map +1 -0
  18. package/dist/src/e2ee/constants.d.ts +14 -0
  19. package/dist/src/e2ee/constants.d.ts.map +1 -0
  20. package/dist/src/e2ee/errors.d.ts +11 -0
  21. package/dist/src/e2ee/errors.d.ts.map +1 -0
  22. package/dist/src/e2ee/index.d.ts +4 -0
  23. package/dist/src/e2ee/index.d.ts.map +1 -0
  24. package/dist/src/e2ee/types.d.ts +129 -0
  25. package/dist/src/e2ee/types.d.ts.map +1 -0
  26. package/dist/src/e2ee/utils.d.ts +24 -0
  27. package/dist/src/e2ee/utils.d.ts.map +1 -0
  28. package/dist/src/e2ee/worker/FrameCryptor.d.ts +175 -0
  29. package/dist/src/e2ee/worker/FrameCryptor.d.ts.map +1 -0
  30. package/dist/src/e2ee/worker/ParticipantKeyHandler.d.ts +46 -0
  31. package/dist/src/e2ee/worker/ParticipantKeyHandler.d.ts.map +1 -0
  32. package/dist/src/e2ee/worker/e2ee.worker.d.ts +2 -0
  33. package/dist/src/e2ee/worker/e2ee.worker.d.ts.map +1 -0
  34. package/dist/src/index.d.ts +2 -0
  35. package/dist/src/index.d.ts.map +1 -1
  36. package/dist/src/logger.d.ts +4 -1
  37. package/dist/src/logger.d.ts.map +1 -1
  38. package/dist/src/options.d.ts +5 -0
  39. package/dist/src/options.d.ts.map +1 -1
  40. package/dist/src/proto/livekit_models.d.ts +2 -2
  41. package/dist/src/proto/livekit_models.d.ts.map +1 -1
  42. package/dist/src/room/PCTransport.d.ts +3 -1
  43. package/dist/src/room/PCTransport.d.ts.map +1 -1
  44. package/dist/src/room/RTCEngine.d.ts +17 -3
  45. package/dist/src/room/RTCEngine.d.ts.map +1 -1
  46. package/dist/src/room/Room.d.ts +10 -1
  47. package/dist/src/room/Room.d.ts.map +1 -1
  48. package/dist/src/room/events.d.ts +14 -2
  49. package/dist/src/room/events.d.ts.map +1 -1
  50. package/dist/src/room/participant/LocalParticipant.d.ts +9 -2
  51. package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
  52. package/dist/src/room/participant/Participant.d.ts +1 -0
  53. package/dist/src/room/participant/Participant.d.ts.map +1 -1
  54. package/dist/src/room/track/LocalAudioTrack.d.ts.map +1 -1
  55. package/dist/src/room/track/LocalTrack.d.ts +3 -2
  56. package/dist/src/room/track/LocalTrack.d.ts.map +1 -1
  57. package/dist/src/room/track/LocalVideoTrack.d.ts.map +1 -1
  58. package/dist/src/room/track/RemoteVideoTrack.d.ts.map +1 -1
  59. package/dist/src/room/track/TrackPublication.d.ts +3 -0
  60. package/dist/src/room/track/TrackPublication.d.ts.map +1 -1
  61. package/dist/src/room/track/facingMode.d.ts +41 -0
  62. package/dist/src/room/track/facingMode.d.ts.map +1 -0
  63. package/dist/src/room/track/options.d.ts +2 -2
  64. package/dist/src/room/track/options.d.ts.map +1 -1
  65. package/dist/src/room/track/utils.d.ts +5 -35
  66. package/dist/src/room/track/utils.d.ts.map +1 -1
  67. package/dist/src/room/utils.d.ts +2 -0
  68. package/dist/src/room/utils.d.ts.map +1 -1
  69. package/dist/src/test/MockMediaStreamTrack.d.ts.map +1 -1
  70. package/dist/ts4.2/src/api/SignalClient.d.ts +4 -1
  71. package/dist/ts4.2/src/e2ee/E2eeManager.d.ts +45 -0
  72. package/dist/ts4.2/src/e2ee/KeyProvider.d.ts +42 -0
  73. package/dist/ts4.2/src/e2ee/constants.d.ts +14 -0
  74. package/dist/ts4.2/src/e2ee/errors.d.ts +11 -0
  75. package/dist/ts4.2/src/e2ee/index.d.ts +4 -0
  76. package/dist/ts4.2/src/e2ee/types.d.ts +129 -0
  77. package/dist/ts4.2/src/e2ee/utils.d.ts +24 -0
  78. package/dist/ts4.2/src/e2ee/worker/FrameCryptor.d.ts +175 -0
  79. package/dist/ts4.2/src/e2ee/worker/ParticipantKeyHandler.d.ts +46 -0
  80. package/dist/ts4.2/src/e2ee/worker/e2ee.worker.d.ts +2 -0
  81. package/dist/ts4.2/src/index.d.ts +2 -0
  82. package/dist/ts4.2/src/logger.d.ts +4 -1
  83. package/dist/ts4.2/src/options.d.ts +5 -0
  84. package/dist/ts4.2/src/proto/livekit_models.d.ts +2 -2
  85. package/dist/ts4.2/src/room/PCTransport.d.ts +3 -1
  86. package/dist/ts4.2/src/room/RTCEngine.d.ts +17 -3
  87. package/dist/ts4.2/src/room/Room.d.ts +10 -1
  88. package/dist/ts4.2/src/room/events.d.ts +14 -2
  89. package/dist/ts4.2/src/room/participant/LocalParticipant.d.ts +9 -2
  90. package/dist/ts4.2/src/room/participant/Participant.d.ts +1 -0
  91. package/dist/ts4.2/src/room/track/LocalTrack.d.ts +3 -2
  92. package/dist/ts4.2/src/room/track/TrackPublication.d.ts +3 -0
  93. package/dist/ts4.2/src/room/track/facingMode.d.ts +41 -0
  94. package/dist/ts4.2/src/room/track/options.d.ts +6 -6
  95. package/dist/ts4.2/src/room/track/utils.d.ts +5 -35
  96. package/dist/ts4.2/src/room/utils.d.ts +2 -0
  97. package/package.json +17 -7
  98. package/src/api/SignalClient.ts +28 -9
  99. package/src/connectionHelper/checks/turn.ts +1 -0
  100. package/src/connectionHelper/checks/websocket.ts +1 -0
  101. package/src/e2ee/E2eeManager.ts +374 -0
  102. package/src/e2ee/KeyProvider.ts +77 -0
  103. package/src/e2ee/constants.ts +40 -0
  104. package/src/e2ee/errors.ts +16 -0
  105. package/src/e2ee/index.ts +3 -0
  106. package/src/e2ee/types.ts +160 -0
  107. package/src/e2ee/utils.ts +127 -0
  108. package/src/e2ee/worker/FrameCryptor.test.ts +21 -0
  109. package/src/e2ee/worker/FrameCryptor.ts +614 -0
  110. package/src/e2ee/worker/ParticipantKeyHandler.ts +129 -0
  111. package/src/e2ee/worker/e2ee.worker.ts +217 -0
  112. package/src/e2ee/worker/tsconfig.json +6 -0
  113. package/src/index.ts +2 -0
  114. package/src/logger.ts +10 -2
  115. package/src/options.ts +6 -0
  116. package/src/proto/livekit_models.ts +12 -12
  117. package/src/room/PCTransport.ts +39 -9
  118. package/src/room/RTCEngine.ts +127 -34
  119. package/src/room/Room.ts +83 -30
  120. package/src/room/defaults.ts +1 -1
  121. package/src/room/events.ts +14 -0
  122. package/src/room/participant/LocalParticipant.ts +82 -10
  123. package/src/room/participant/Participant.ts +4 -0
  124. package/src/room/track/LocalAudioTrack.ts +11 -4
  125. package/src/room/track/LocalTrack.ts +50 -43
  126. package/src/room/track/LocalVideoTrack.ts +5 -3
  127. package/src/room/track/RemoteVideoTrack.ts +2 -2
  128. package/src/room/track/TrackPublication.ts +9 -1
  129. package/src/room/track/facingMode.test.ts +30 -0
  130. package/src/room/track/facingMode.ts +103 -0
  131. package/src/room/track/options.ts +3 -2
  132. package/src/room/track/utils.test.ts +1 -30
  133. package/src/room/track/utils.ts +16 -91
  134. package/src/room/utils.ts +5 -0
  135. package/src/room/worker.d.ts +4 -0
  136. package/src/test/MockMediaStreamTrack.ts +1 -0
@@ -1,6 +1,12 @@
1
1
  import log from '../../logger';
2
2
  import type { InternalRoomOptions } from '../../options';
3
- import { DataPacket, DataPacket_Kind, ParticipantInfo } from '../../proto/livekit_models';
3
+ import {
4
+ DataPacket,
5
+ DataPacket_Kind,
6
+ Encryption_Type,
7
+ ParticipantInfo,
8
+ ParticipantPermission,
9
+ } from '../../proto/livekit_models';
4
10
  import {
5
11
  AddTrackRequest,
6
12
  DataChannelInfo,
@@ -50,6 +56,9 @@ export default class LocalParticipant extends Participant {
50
56
  /** @internal */
51
57
  engine: RTCEngine;
52
58
 
59
+ /** @internal */
60
+ activeDeviceMap: Map<MediaDeviceKind, string>;
61
+
53
62
  private pendingPublishing = new Set<Track.Source>();
54
63
 
55
64
  private pendingPublishPromises = new Map<LocalTrack, Promise<LocalTrackPublication>>();
@@ -65,6 +74,8 @@ export default class LocalParticipant extends Participant {
65
74
  // keep a pointer to room options
66
75
  private roomOptions: InternalRoomOptions;
67
76
 
77
+ private encryptionType: Encryption_Type = Encryption_Type.NONE;
78
+
68
79
  private reconnectFuture?: Future<void>;
69
80
 
70
81
  /** @internal */
@@ -76,6 +87,7 @@ export default class LocalParticipant extends Participant {
76
87
  this.engine = engine;
77
88
  this.roomOptions = options;
78
89
  this.setupEngine(engine);
90
+ this.activeDeviceMap = new Map();
79
91
  }
80
92
 
81
93
  get lastCameraError(): Error | undefined {
@@ -210,6 +222,22 @@ export default class LocalParticipant extends Participant {
210
222
  return this.setTrackEnabled(Track.Source.ScreenShare, enabled, options, publishOptions);
211
223
  }
212
224
 
225
+ /** @internal */
226
+ setPermissions(permissions: ParticipantPermission): boolean {
227
+ const prevPermissions = this.permissions;
228
+ const changed = super.setPermissions(permissions);
229
+ if (changed && prevPermissions) {
230
+ this.emit(ParticipantEvent.ParticipantPermissionsChanged, prevPermissions);
231
+ }
232
+ return changed;
233
+ }
234
+
235
+ /** @internal */
236
+ async setE2EEEnabled(enabled: boolean) {
237
+ this.encryptionType = enabled ? Encryption_Type.GCM : Encryption_Type.NONE;
238
+ await this.republishAllTracks(undefined, false);
239
+ }
240
+
213
241
  /**
214
242
  * Enable or disable publishing for a track by source. This serves as a simple
215
243
  * way to manage the common tracks (camera, mic, or screen share).
@@ -464,14 +492,38 @@ export default class LocalParticipant extends Participant {
464
492
  if (track instanceof LocalTrack && this.pendingPublishPromises.has(track)) {
465
493
  await this.pendingPublishPromises.get(track);
466
494
  }
495
+ let defaultConstraints: MediaTrackConstraints | undefined;
496
+ if (track instanceof MediaStreamTrack) {
497
+ defaultConstraints = track.getConstraints();
498
+ } else {
499
+ // we want to access constraints directly as `track.mediaStreamTrack`
500
+ // might be pointing to a non-device track (e.g. processed track) already
501
+ defaultConstraints = track.constraints;
502
+ let deviceKind: MediaDeviceKind | undefined = undefined;
503
+ switch (track.source) {
504
+ case Track.Source.Microphone:
505
+ deviceKind = 'audioinput';
506
+ break;
507
+ case Track.Source.Camera:
508
+ deviceKind = 'videoinput';
509
+ default:
510
+ break;
511
+ }
512
+ if (deviceKind && this.activeDeviceMap.has(deviceKind)) {
513
+ defaultConstraints = {
514
+ ...defaultConstraints,
515
+ deviceId: this.activeDeviceMap.get(deviceKind),
516
+ };
517
+ }
518
+ }
467
519
  // convert raw media track into audio or video track
468
520
  if (track instanceof MediaStreamTrack) {
469
521
  switch (track.kind) {
470
522
  case 'audio':
471
- track = new LocalAudioTrack(track, undefined, true);
523
+ track = new LocalAudioTrack(track, defaultConstraints, true);
472
524
  break;
473
525
  case 'video':
474
- track = new LocalVideoTrack(track, undefined, true);
526
+ track = new LocalVideoTrack(track, defaultConstraints, true);
475
527
  break;
476
528
  default:
477
529
  throw new TrackInvalidError(`unsupported MediaStreamTrack kind ${track.kind}`);
@@ -524,6 +576,12 @@ export default class LocalParticipant extends Participant {
524
576
  ...options,
525
577
  };
526
578
 
579
+ // disable simulcast if e2ee is set on safari
580
+ if (isSafari() && this.roomOptions.e2ee) {
581
+ log.info(`End-to-end encryption is set up, simulcast publishing will be disabled on Safari`);
582
+ opts.simulcast = false;
583
+ }
584
+
527
585
  if (opts.source) {
528
586
  track.source = opts.source;
529
587
  }
@@ -596,8 +654,9 @@ export default class LocalParticipant extends Participant {
596
654
  muted: track.isMuted,
597
655
  source: Track.sourceToProto(track.source),
598
656
  disableDtx: !(opts.dtx ?? true),
657
+ encryption: this.encryptionType,
599
658
  stereo: isStereo,
600
- disableRed: !(opts.red ?? true),
659
+ // disableRed: !(opts.red ?? true),
601
660
  });
602
661
 
603
662
  // compute encodings and layers for video
@@ -732,11 +791,11 @@ export default class LocalParticipant extends Participant {
732
791
 
733
792
  if (encodings) {
734
793
  if (isFireFox() && track.kind === Track.Kind.Audio) {
735
- /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
794
+ /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
736
795
  livekit-server uses maxaveragebitrate=510000in the answer sdp to permit client to
737
- publish high quality audio track. But firefox always uses this value as the actual
796
+ publish high quality audio track. But firefox always uses this value as the actual
738
797
  bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
739
- So the client need to modify maxaverragebitrates in answer sdp to user provided value to
798
+ So the client need to modify maxaverragebitrates in answer sdp to user provided value to
740
799
  fix the issue.
741
800
  */
742
801
  let trackTransceiver: RTCRtpTransceiver | undefined = undefined;
@@ -762,7 +821,7 @@ export default class LocalParticipant extends Participant {
762
821
  }
763
822
  }
764
823
 
765
- this.engine.negotiate();
824
+ await this.engine.negotiate();
766
825
 
767
826
  if (track instanceof LocalVideoTrack) {
768
827
  track.startMonitor(this.engine.client);
@@ -848,7 +907,7 @@ export default class LocalParticipant extends Participant {
848
907
  }
849
908
  await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
850
909
 
851
- this.engine.negotiate();
910
+ await this.engine.negotiate();
852
911
  log.debug(`published ${videoCodec} for track ${track.sid}`, { encodings, trackInfo: ti });
853
912
  }
854
913
 
@@ -952,7 +1011,7 @@ export default class LocalParticipant extends Participant {
952
1011
  ) as LocalTrackPublication[];
953
1012
  }
954
1013
 
955
- async republishAllTracks(options?: TrackPublishOptions) {
1014
+ async republishAllTracks(options?: TrackPublishOptions, restartTracks: boolean = true) {
956
1015
  const localPubs: LocalTrackPublication[] = [];
957
1016
  this.tracks.forEach((pub) => {
958
1017
  if (pub.track) {
@@ -967,6 +1026,19 @@ export default class LocalParticipant extends Participant {
967
1026
  localPubs.map(async (pub) => {
968
1027
  const track = pub.track!;
969
1028
  await this.unpublishTrack(track, false);
1029
+ if (
1030
+ restartTracks &&
1031
+ !track.isMuted &&
1032
+ (track instanceof LocalAudioTrack || track instanceof LocalVideoTrack) &&
1033
+ !track.isUserProvided
1034
+ ) {
1035
+ // generally we need to restart the track before publishing, often a full reconnect
1036
+ // is necessary because computer had gone to sleep.
1037
+ log.debug('restarting existing track', {
1038
+ track: pub.trackSid,
1039
+ });
1040
+ await track.restartTrack();
1041
+ }
970
1042
  await this.publishTrack(track, pub.options);
971
1043
  }),
972
1044
  );
@@ -68,6 +68,10 @@ export default class Participant extends EventEmitter<ParticipantEventCallbacks>
68
68
 
69
69
  private _connectionQuality: ConnectionQuality = ConnectionQuality.Unknown;
70
70
 
71
+ get isEncrypted() {
72
+ return this.tracks.size > 0 && Array.from(this.tracks.values()).every((tr) => tr.isEncrypted);
73
+ }
74
+
71
75
  /** @internal */
72
76
  constructor(sid: string, identity: string, name?: string, metadata?: string) {
73
77
  super();
@@ -30,14 +30,16 @@ export default class LocalAudioTrack extends LocalTrack {
30
30
  }
31
31
 
32
32
  async setDeviceId(deviceId: ConstrainDOMString): Promise<boolean> {
33
- if (this.constraints.deviceId === deviceId) {
33
+ if (this._constraints.deviceId === deviceId) {
34
34
  return true;
35
35
  }
36
- this.constraints.deviceId = deviceId;
36
+ this._constraints.deviceId = deviceId;
37
37
  if (!this.isMuted) {
38
38
  await this.restartTrack();
39
39
  }
40
- return unwrapConstraint(deviceId) === this.mediaStreamTrack.getSettings().deviceId;
40
+ return (
41
+ this.isMuted || unwrapConstraint(deviceId) === this.mediaStreamTrack.getSettings().deviceId
42
+ );
41
43
  }
42
44
 
43
45
  async mute(): Promise<LocalAudioTrack> {
@@ -59,9 +61,14 @@ export default class LocalAudioTrack extends LocalTrack {
59
61
  async unmute(): Promise<LocalAudioTrack> {
60
62
  const unlock = await this.muteLock.lock();
61
63
  try {
64
+ const deviceHasChanged =
65
+ this._constraints.deviceId &&
66
+ this._mediaStreamTrack.getSettings().deviceId !==
67
+ unwrapConstraint(this._constraints.deviceId);
68
+
62
69
  if (
63
70
  this.source === Track.Source.Microphone &&
64
- (this.stopOnMute || this._mediaStreamTrack.readyState === 'ended') &&
71
+ (this.stopOnMute || this._mediaStreamTrack.readyState === 'ended' || deviceHasChanged) &&
65
72
  !this.isUserProvided
66
73
  ) {
67
74
  log.debug('reacquiring mic track');
@@ -17,7 +17,11 @@ export default abstract class LocalTrack extends Track {
17
17
  /** @internal */
18
18
  codec?: VideoCodec;
19
19
 
20
- protected constraints: MediaTrackConstraints;
20
+ get constraints() {
21
+ return this._constraints;
22
+ }
23
+
24
+ protected _constraints: MediaTrackConstraints;
21
25
 
22
26
  protected reacquireTrack: boolean;
23
27
 
@@ -31,7 +35,7 @@ export default abstract class LocalTrack extends Track {
31
35
 
32
36
  protected processor?: TrackProcessor<typeof this.kind>;
33
37
 
34
- protected isSettingUpProcessor: boolean = false;
38
+ protected processorLock: Mutex;
35
39
 
36
40
  /**
37
41
  *
@@ -51,11 +55,13 @@ export default abstract class LocalTrack extends Track {
51
55
  this.providedByUser = userProvidedTrack;
52
56
  this.muteLock = new Mutex();
53
57
  this.pauseUpstreamLock = new Mutex();
58
+ this.processorLock = new Mutex();
59
+ this.setMediaStreamTrack(mediaTrack, true);
60
+
54
61
  // added to satisfy TS compiler, constraints are synced with MediaStreamTrack
55
- this.constraints = mediaTrack.getConstraints();
56
- this.setMediaStreamTrack(mediaTrack);
62
+ this._constraints = mediaTrack.getConstraints();
57
63
  if (constraints) {
58
- this.constraints = constraints;
64
+ this._constraints = constraints;
59
65
  }
60
66
  }
61
67
 
@@ -92,8 +98,8 @@ export default abstract class LocalTrack extends Track {
92
98
  return this.processor?.processedTrack ?? this._mediaStreamTrack;
93
99
  }
94
100
 
95
- private async setMediaStreamTrack(newTrack: MediaStreamTrack) {
96
- if (newTrack === this._mediaStreamTrack) {
101
+ private async setMediaStreamTrack(newTrack: MediaStreamTrack, force?: boolean) {
102
+ if (newTrack === this._mediaStreamTrack && !force) {
97
103
  return;
98
104
  }
99
105
  if (this._mediaStreamTrack) {
@@ -104,7 +110,7 @@ export default abstract class LocalTrack extends Track {
104
110
  this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
105
111
  this._mediaStreamTrack.removeEventListener('mute', this.pauseUpstream);
106
112
  this._mediaStreamTrack.removeEventListener('unmute', this.resumeUpstream);
107
- if (!this.providedByUser) {
113
+ if (!this.providedByUser && this._mediaStreamTrack !== newTrack) {
108
114
  this._mediaStreamTrack.stop();
109
115
  }
110
116
  }
@@ -119,7 +125,7 @@ export default abstract class LocalTrack extends Track {
119
125
  // touch MediaStreamTrack.enabled
120
126
  newTrack.addEventListener('mute', this.pauseUpstream);
121
127
  newTrack.addEventListener('unmute', this.resumeUpstream);
122
- this.constraints = newTrack.getConstraints();
128
+ this._constraints = newTrack.getConstraints();
123
129
  }
124
130
  if (this.sender) {
125
131
  await this.sender.replaceTrack(newTrack);
@@ -195,7 +201,7 @@ export default abstract class LocalTrack extends Track {
195
201
 
196
202
  protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
197
203
  if (!constraints) {
198
- constraints = this.constraints;
204
+ constraints = this._constraints;
199
205
  }
200
206
  log.debug('restarting track with constraints', constraints);
201
207
 
@@ -228,7 +234,7 @@ export default abstract class LocalTrack extends Track {
228
234
  log.debug('re-acquired MediaStreamTrack');
229
235
 
230
236
  await this.setMediaStreamTrack(newTrack);
231
- this.constraints = constraints;
237
+ this._constraints = constraints;
232
238
  if (this.processor) {
233
239
  const processor = this.processor;
234
240
  await this.setProcessor(processor);
@@ -357,42 +363,43 @@ export default abstract class LocalTrack extends Track {
357
363
  processor: TrackProcessor<typeof this.kind>,
358
364
  showProcessedStreamLocally = true,
359
365
  ) {
360
- if (this.isSettingUpProcessor) {
361
- log.warn('already trying to set up a processor');
362
- return;
363
- }
364
- log.debug('setting up processor');
365
- this.isSettingUpProcessor = true;
366
- if (this.processor) {
367
- await this.stopProcessor();
368
- }
369
- if (this.kind === 'unknown') {
370
- throw TypeError('cannot set processor on track of unknown kind');
371
- }
372
- this.processorElement = this.processorElement ?? document.createElement(this.kind);
373
- this.processorElement.muted = true;
374
-
375
- attachToElement(this._mediaStreamTrack, this.processorElement);
376
- this.processorElement.play().catch((e) => log.error(e));
377
-
378
- const processorOptions = {
379
- kind: this.kind,
380
- track: this._mediaStreamTrack,
381
- element: this.processorElement,
382
- };
366
+ const unlock = await this.processorLock.lock();
367
+ try {
368
+ log.debug('setting up processor');
369
+ if (this.processor) {
370
+ await this.stopProcessor();
371
+ }
372
+ if (this.kind === 'unknown') {
373
+ throw TypeError('cannot set processor on track of unknown kind');
374
+ }
375
+ this.processorElement = this.processorElement ?? document.createElement(this.kind);
376
+ this.processorElement.muted = true;
377
+
378
+ attachToElement(this._mediaStreamTrack, this.processorElement);
379
+ this.processorElement
380
+ .play()
381
+ .catch((error) => log.error('failed to play processor element', { error }));
382
+
383
+ const processorOptions = {
384
+ kind: this.kind,
385
+ track: this._mediaStreamTrack,
386
+ element: this.processorElement,
387
+ };
383
388
 
384
- await processor.init(processorOptions);
385
- this.processor = processor;
386
- if (this.processor.processedTrack) {
387
- for (const el of this.attachedElements) {
388
- if (el !== this.processorElement && showProcessedStreamLocally) {
389
- detachTrack(this._mediaStreamTrack, el);
390
- attachToElement(this.processor.processedTrack, el);
389
+ await processor.init(processorOptions);
390
+ this.processor = processor;
391
+ if (this.processor.processedTrack) {
392
+ for (const el of this.attachedElements) {
393
+ if (el !== this.processorElement && showProcessedStreamLocally) {
394
+ detachTrack(this._mediaStreamTrack, el);
395
+ attachToElement(this.processor.processedTrack, el);
396
+ }
391
397
  }
398
+ await this.sender?.replaceTrack(this.processor.processedTrack);
392
399
  }
393
- await this.sender?.replaceTrack(this.processor.processedTrack);
400
+ } finally {
401
+ unlock();
394
402
  }
395
- this.isSettingUpProcessor = false;
396
403
  }
397
404
 
398
405
  getProcessor() {
@@ -184,18 +184,20 @@ export default class LocalVideoTrack extends LocalTrack {
184
184
 
185
185
  async setDeviceId(deviceId: ConstrainDOMString): Promise<boolean> {
186
186
  if (
187
- this.constraints.deviceId === deviceId &&
187
+ this._constraints.deviceId === deviceId &&
188
188
  this._mediaStreamTrack.getSettings().deviceId === unwrapConstraint(deviceId)
189
189
  ) {
190
190
  return true;
191
191
  }
192
- this.constraints.deviceId = deviceId;
192
+ this._constraints.deviceId = deviceId;
193
193
  // when video is muted, underlying media stream track is stopped and
194
194
  // will be restarted later
195
195
  if (!this.isMuted) {
196
196
  await this.restartTrack();
197
197
  }
198
- return unwrapConstraint(deviceId) === this._mediaStreamTrack.getSettings().deviceId;
198
+ return (
199
+ this.isMuted || unwrapConstraint(deviceId) === this._mediaStreamTrack.getSettings().deviceId
200
+ );
199
201
  }
200
202
 
201
203
  async restartTrack(options?: VideoCaptureOptions) {
@@ -123,6 +123,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
123
123
  }
124
124
  this.elementInfos = this.elementInfos.filter((info) => info !== elementInfo);
125
125
  this.updateVisibility();
126
+ this.debouncedHandleResize();
126
127
  }
127
128
 
128
129
  detach(): HTMLMediaElement[];
@@ -195,9 +196,8 @@ export default class RemoteVideoTrack extends RemoteTrack {
195
196
  private stopObservingElement(element: HTMLMediaElement) {
196
197
  const stopElementInfos = this.elementInfos.filter((info) => info.element === element);
197
198
  for (const info of stopElementInfos) {
198
- info.stopObserving();
199
+ this.stopObservingElementInfo(info);
199
200
  }
200
- this.elementInfos = this.elementInfos.filter((info) => info.element !== element);
201
201
  }
202
202
 
203
203
  protected async handleAppVisibilityChanged() {
@@ -1,5 +1,6 @@
1
1
  import EventEmitter from 'eventemitter3';
2
2
  import log from '../../logger';
3
+ import { Encryption_Type } from '../../proto/livekit_models';
3
4
  import type { SubscriptionError, TrackInfo } from '../../proto/livekit_models';
4
5
  import type { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
5
6
  import { TrackEvent } from '../events';
@@ -35,6 +36,8 @@ export class TrackPublication extends EventEmitter<PublicationEventCallbacks> {
35
36
 
36
37
  protected metadataMuted: boolean = false;
37
38
 
39
+ protected encryption: Encryption_Type = Encryption_Type.NONE;
40
+
38
41
  constructor(kind: Track.Kind, id: string, name: string) {
39
42
  super();
40
43
  this.kind = kind;
@@ -71,6 +74,10 @@ export class TrackPublication extends EventEmitter<PublicationEventCallbacks> {
71
74
  return this.track !== undefined;
72
75
  }
73
76
 
77
+ get isEncrypted(): boolean {
78
+ return this.encryption !== Encryption_Type.NONE;
79
+ }
80
+
74
81
  /**
75
82
  * an [AudioTrack] if this publication holds an audio track
76
83
  */
@@ -110,8 +117,9 @@ export class TrackPublication extends EventEmitter<PublicationEventCallbacks> {
110
117
  };
111
118
  this.simulcasted = info.simulcast;
112
119
  }
120
+ this.encryption = info.encryption;
113
121
  this.trackInfo = info;
114
- log.trace('update publication info', { info });
122
+ log.debug('update publication info', { info });
115
123
  }
116
124
  }
117
125
 
@@ -0,0 +1,30 @@
1
+ import { facingModeFromDeviceLabel } from './facingMode';
2
+
3
+ describe('Test facingMode detection', () => {
4
+ test('OBS virtual camera should be detected.', () => {
5
+ const result = facingModeFromDeviceLabel('OBS Virtual Camera');
6
+ expect(result?.facingMode).toEqual('environment');
7
+ expect(result?.confidence).toEqual('medium');
8
+ });
9
+
10
+ test.each([
11
+ ['Peter’s iPhone Camera', { facingMode: 'environment', confidence: 'medium' }],
12
+ ['iPhone de Théo Camera', { facingMode: 'environment', confidence: 'medium' }],
13
+ ])(
14
+ 'Device labels that contain "iphone" should return facingMode "environment".',
15
+ (label, expected) => {
16
+ const result = facingModeFromDeviceLabel(label);
17
+ expect(result?.facingMode).toEqual(expected.facingMode);
18
+ expect(result?.confidence).toEqual(expected.confidence);
19
+ },
20
+ );
21
+
22
+ test.each([
23
+ ['Peter’s iPad Camera', { facingMode: 'environment', confidence: 'medium' }],
24
+ ['iPad de Théo Camera', { facingMode: 'environment', confidence: 'medium' }],
25
+ ])('Device label that contain "ipad" should detect.', (label, expected) => {
26
+ const result = facingModeFromDeviceLabel(label);
27
+ expect(result?.facingMode).toEqual(expected.facingMode);
28
+ expect(result?.confidence).toEqual(expected.confidence);
29
+ });
30
+ });
@@ -0,0 +1,103 @@
1
+ import log from 'loglevel';
2
+ import LocalTrack from './LocalTrack';
3
+ import type { VideoCaptureOptions } from './options';
4
+
5
+ type FacingMode = NonNullable<VideoCaptureOptions['facingMode']>;
6
+ type FacingModeFromLocalTrackOptions = {
7
+ /**
8
+ * If no facing mode can be determined, this value will be used.
9
+ * @defaultValue 'user'
10
+ */
11
+ defaultFacingMode?: FacingMode;
12
+ };
13
+ type FacingModeFromLocalTrackReturnValue = {
14
+ /**
15
+ * The (probable) facingMode of the track.
16
+ * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackConstraints/facingMode | MDN docs on facingMode}
17
+ */
18
+ facingMode: FacingMode;
19
+ /**
20
+ * The confidence that the returned facingMode is correct.
21
+ */
22
+ confidence: 'high' | 'medium' | 'low';
23
+ };
24
+
25
+ /**
26
+ * Try to analyze the local track to determine the facing mode of a track.
27
+ *
28
+ * @remarks
29
+ * There is no property supported by all browsers to detect whether a video track originated from a user- or environment-facing camera device.
30
+ * For this reason, we use the `facingMode` property when available, but will fall back on a string-based analysis of the device label to determine the facing mode.
31
+ * If both methods fail, the default facing mode will be used.
32
+ *
33
+ * @see {@link https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackConstraints/facingMode | MDN docs on facingMode}
34
+ * @experimental
35
+ */
36
+ export function facingModeFromLocalTrack(
37
+ localTrack: LocalTrack | MediaStreamTrack,
38
+ options: FacingModeFromLocalTrackOptions = {},
39
+ ): FacingModeFromLocalTrackReturnValue {
40
+ const track = localTrack instanceof LocalTrack ? localTrack.mediaStreamTrack : localTrack;
41
+ const trackSettings = track.getSettings();
42
+ let result: FacingModeFromLocalTrackReturnValue = {
43
+ facingMode: options.defaultFacingMode ?? 'user',
44
+ confidence: 'low',
45
+ };
46
+
47
+ // 1. Try to get facingMode from track settings.
48
+ if ('facingMode' in trackSettings) {
49
+ const rawFacingMode = trackSettings.facingMode;
50
+ log.debug('rawFacingMode', { rawFacingMode });
51
+ if (rawFacingMode && typeof rawFacingMode === 'string' && isFacingModeValue(rawFacingMode)) {
52
+ result = { facingMode: rawFacingMode, confidence: 'high' };
53
+ }
54
+ }
55
+
56
+ // 2. If we don't have a high confidence we try to get the facing mode from the device label.
57
+ if (['low', 'medium'].includes(result.confidence)) {
58
+ log.debug(`Try to get facing mode from device label: (${track.label})`);
59
+ const labelAnalysisResult = facingModeFromDeviceLabel(track.label);
60
+ if (labelAnalysisResult !== undefined) {
61
+ result = labelAnalysisResult;
62
+ }
63
+ }
64
+
65
+ return result;
66
+ }
67
+
68
+ const knownDeviceLabels = new Map<string, FacingModeFromLocalTrackReturnValue>([
69
+ ['obs virtual camera', { facingMode: 'environment', confidence: 'medium' }],
70
+ ]);
71
+ const knownDeviceLabelSections = new Map<string, FacingModeFromLocalTrackReturnValue>([
72
+ ['iphone', { facingMode: 'environment', confidence: 'medium' }],
73
+ ['ipad', { facingMode: 'environment', confidence: 'medium' }],
74
+ ]);
75
+ /**
76
+ * Attempt to analyze the device label to determine the facing mode.
77
+ *
78
+ * @experimental
79
+ */
80
+ export function facingModeFromDeviceLabel(
81
+ deviceLabel: string,
82
+ ): FacingModeFromLocalTrackReturnValue | undefined {
83
+ const label = deviceLabel.trim().toLowerCase();
84
+ // Empty string is a valid device label but we can't infer anything from it.
85
+ if (label === '') {
86
+ return undefined;
87
+ }
88
+
89
+ // Can we match against widely known device labels.
90
+ if (knownDeviceLabels.has(label)) {
91
+ return knownDeviceLabels.get(label);
92
+ }
93
+
94
+ // Can we match against sections of the device label.
95
+ return Array.from(knownDeviceLabelSections.entries()).find(([section]) =>
96
+ label.includes(section),
97
+ )?.[1];
98
+ }
99
+
100
+ function isFacingModeValue(item: string): item is FacingMode {
101
+ const allowedValues: FacingMode[] = ['user', 'environment', 'left', 'right'];
102
+ return item === undefined || allowedValues.includes(item as FacingMode);
103
+ }
@@ -272,10 +272,11 @@ export interface AudioPreset {
272
272
  priority?: RTCPriorityType;
273
273
  }
274
274
 
275
- const codecs = ['vp8', 'h264', 'vp9', 'av1'] as const;
276
275
  const backupCodecs = ['vp8', 'h264'] as const;
277
276
 
278
- export type VideoCodec = (typeof codecs)[number];
277
+ export const videoCodecs = ['vp8', 'h264', 'vp9', 'av1'] as const;
278
+
279
+ export type VideoCodec = (typeof videoCodecs)[number];
279
280
 
280
281
  export type BackupVideoCodec = (typeof backupCodecs)[number];
281
282
 
@@ -1,5 +1,5 @@
1
1
  import { AudioCaptureOptions, VideoCaptureOptions, VideoPresets } from './options';
2
- import { constraintsForOptions, facingModeFromDeviceLabel, mergeDefaultOptions } from './utils';
2
+ import { constraintsForOptions, mergeDefaultOptions } from './utils';
3
3
 
4
4
  describe('mergeDefaultOptions', () => {
5
5
  const audioDefaults: AudioCaptureOptions = {
@@ -108,32 +108,3 @@ describe('constraintsForOptions', () => {
108
108
  expect(videoOpts.aspectRatio).toEqual(VideoPresets.h720.resolution.aspectRatio);
109
109
  });
110
110
  });
111
-
112
- describe('Test facingMode detection', () => {
113
- test('OBS virtual camera should be detected.', () => {
114
- const result = facingModeFromDeviceLabel('OBS Virtual Camera');
115
- expect(result?.facingMode).toEqual('environment');
116
- expect(result?.confidence).toEqual('medium');
117
- });
118
-
119
- test.each([
120
- ['Peter’s iPhone Camera', { facingMode: 'environment', confidence: 'medium' }],
121
- ['iPhone de Théo Camera', { facingMode: 'environment', confidence: 'medium' }],
122
- ])(
123
- 'Device labels that contain "iphone" should return facingMode "environment".',
124
- (label, expected) => {
125
- const result = facingModeFromDeviceLabel(label);
126
- expect(result?.facingMode).toEqual(expected.facingMode);
127
- expect(result?.confidence).toEqual(expected.confidence);
128
- },
129
- );
130
-
131
- test.each([
132
- ['Peter’s iPad Camera', { facingMode: 'environment', confidence: 'medium' }],
133
- ['iPad de Théo Camera', { facingMode: 'environment', confidence: 'medium' }],
134
- ])('Device label that contain "ipad" should detect.', (label, expected) => {
135
- const result = facingModeFromDeviceLabel(label);
136
- expect(result?.facingMode).toEqual(expected.facingMode);
137
- expect(result?.confidence).toEqual(expected.confidence);
138
- });
139
- });