livekit-client 1.12.2 → 1.13.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (114) hide show
  1. package/dist/livekit-client.e2ee.worker.js +1 -1
  2. package/dist/livekit-client.e2ee.worker.js.map +1 -1
  3. package/dist/livekit-client.e2ee.worker.mjs +83 -9
  4. package/dist/livekit-client.e2ee.worker.mjs.map +1 -1
  5. package/dist/livekit-client.esm.mjs +2239 -1975
  6. package/dist/livekit-client.esm.mjs.map +1 -1
  7. package/dist/livekit-client.umd.js +1 -1
  8. package/dist/livekit-client.umd.js.map +1 -1
  9. package/dist/src/api/SignalClient.d.ts +2 -5
  10. package/dist/src/api/SignalClient.d.ts.map +1 -1
  11. package/dist/src/connectionHelper/checks/turn.d.ts.map +1 -1
  12. package/dist/src/connectionHelper/checks/webrtc.d.ts.map +1 -1
  13. package/dist/src/connectionHelper/checks/websocket.d.ts.map +1 -1
  14. package/dist/src/e2ee/E2eeManager.d.ts +5 -0
  15. package/dist/src/e2ee/E2eeManager.d.ts.map +1 -1
  16. package/dist/src/e2ee/KeyProvider.d.ts +4 -2
  17. package/dist/src/e2ee/KeyProvider.d.ts.map +1 -1
  18. package/dist/src/e2ee/constants.d.ts +2 -0
  19. package/dist/src/e2ee/constants.d.ts.map +1 -1
  20. package/dist/src/e2ee/types.d.ts +7 -1
  21. package/dist/src/e2ee/types.d.ts.map +1 -1
  22. package/dist/src/e2ee/utils.d.ts +1 -0
  23. package/dist/src/e2ee/utils.d.ts.map +1 -1
  24. package/dist/src/e2ee/worker/FrameCryptor.d.ts +4 -2
  25. package/dist/src/e2ee/worker/FrameCryptor.d.ts.map +1 -1
  26. package/dist/src/e2ee/worker/ParticipantKeyHandler.d.ts.map +1 -1
  27. package/dist/src/e2ee/worker/SifGuard.d.ts +11 -0
  28. package/dist/src/e2ee/worker/SifGuard.d.ts.map +1 -0
  29. package/dist/src/options.d.ts +5 -0
  30. package/dist/src/options.d.ts.map +1 -1
  31. package/dist/src/proto/livekit_models_pb.d.ts.map +1 -1
  32. package/dist/src/proto/livekit_rtc_pb.d.ts.map +1 -1
  33. package/dist/src/room/DeviceManager.d.ts +1 -0
  34. package/dist/src/room/DeviceManager.d.ts.map +1 -1
  35. package/dist/src/room/RTCEngine.d.ts.map +1 -1
  36. package/dist/src/room/Room.d.ts +1 -1
  37. package/dist/src/room/Room.d.ts.map +1 -1
  38. package/dist/src/room/defaults.d.ts.map +1 -1
  39. package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
  40. package/dist/src/room/participant/Participant.d.ts +5 -0
  41. package/dist/src/room/participant/Participant.d.ts.map +1 -1
  42. package/dist/src/room/participant/RemoteParticipant.d.ts +0 -5
  43. package/dist/src/room/participant/RemoteParticipant.d.ts.map +1 -1
  44. package/dist/src/room/participant/publishUtils.d.ts +1 -1
  45. package/dist/src/room/participant/publishUtils.d.ts.map +1 -1
  46. package/dist/src/room/timers.d.ts +2 -2
  47. package/dist/src/room/timers.d.ts.map +1 -1
  48. package/dist/src/room/track/LocalAudioTrack.d.ts +9 -1
  49. package/dist/src/room/track/LocalAudioTrack.d.ts.map +1 -1
  50. package/dist/src/room/track/LocalTrack.d.ts +3 -3
  51. package/dist/src/room/track/LocalTrack.d.ts.map +1 -1
  52. package/dist/src/room/track/LocalVideoTrack.d.ts +6 -0
  53. package/dist/src/room/track/LocalVideoTrack.d.ts.map +1 -1
  54. package/dist/src/room/track/RemoteAudioTrack.d.ts.map +1 -1
  55. package/dist/src/room/track/options.d.ts +1 -0
  56. package/dist/src/room/track/options.d.ts.map +1 -1
  57. package/dist/src/room/track/processor/types.d.ts +13 -2
  58. package/dist/src/room/track/processor/types.d.ts.map +1 -1
  59. package/dist/src/room/types.d.ts +1 -1
  60. package/dist/src/room/types.d.ts.map +1 -1
  61. package/dist/ts4.2/src/api/SignalClient.d.ts +2 -5
  62. package/dist/ts4.2/src/e2ee/E2eeManager.d.ts +5 -0
  63. package/dist/ts4.2/src/e2ee/KeyProvider.d.ts +4 -2
  64. package/dist/ts4.2/src/e2ee/constants.d.ts +2 -0
  65. package/dist/ts4.2/src/e2ee/types.d.ts +7 -1
  66. package/dist/ts4.2/src/e2ee/utils.d.ts +1 -0
  67. package/dist/ts4.2/src/e2ee/worker/FrameCryptor.d.ts +4 -2
  68. package/dist/ts4.2/src/e2ee/worker/SifGuard.d.ts +11 -0
  69. package/dist/ts4.2/src/options.d.ts +5 -0
  70. package/dist/ts4.2/src/room/DeviceManager.d.ts +1 -0
  71. package/dist/ts4.2/src/room/Room.d.ts +1 -1
  72. package/dist/ts4.2/src/room/participant/Participant.d.ts +5 -0
  73. package/dist/ts4.2/src/room/participant/RemoteParticipant.d.ts +0 -5
  74. package/dist/ts4.2/src/room/participant/publishUtils.d.ts +1 -1
  75. package/dist/ts4.2/src/room/timers.d.ts +2 -2
  76. package/dist/ts4.2/src/room/track/LocalAudioTrack.d.ts +9 -1
  77. package/dist/ts4.2/src/room/track/LocalTrack.d.ts +3 -3
  78. package/dist/ts4.2/src/room/track/LocalVideoTrack.d.ts +6 -0
  79. package/dist/ts4.2/src/room/track/options.d.ts +1 -0
  80. package/dist/ts4.2/src/room/track/processor/types.d.ts +13 -2
  81. package/dist/ts4.2/src/room/types.d.ts +1 -1
  82. package/package.json +15 -16
  83. package/src/api/SignalClient.ts +13 -9
  84. package/src/connectionHelper/checks/turn.ts +1 -0
  85. package/src/connectionHelper/checks/webrtc.ts +9 -7
  86. package/src/connectionHelper/checks/websocket.ts +1 -0
  87. package/src/e2ee/E2eeManager.ts +27 -2
  88. package/src/e2ee/KeyProvider.ts +9 -4
  89. package/src/e2ee/constants.ts +3 -0
  90. package/src/e2ee/types.ts +9 -1
  91. package/src/e2ee/utils.ts +9 -0
  92. package/src/e2ee/worker/FrameCryptor.ts +46 -17
  93. package/src/e2ee/worker/ParticipantKeyHandler.ts +1 -0
  94. package/src/e2ee/worker/SifGuard.ts +47 -0
  95. package/src/e2ee/worker/e2ee.worker.ts +14 -0
  96. package/src/options.ts +6 -0
  97. package/src/proto/livekit_models_pb.ts +14 -0
  98. package/src/proto/livekit_rtc_pb.ts +14 -0
  99. package/src/room/DeviceManager.ts +7 -2
  100. package/src/room/RTCEngine.ts +7 -5
  101. package/src/room/Room.ts +27 -7
  102. package/src/room/defaults.ts +1 -0
  103. package/src/room/participant/LocalParticipant.ts +14 -2
  104. package/src/room/participant/Participant.ts +16 -0
  105. package/src/room/participant/RemoteParticipant.ts +0 -12
  106. package/src/room/participant/publishUtils.ts +4 -2
  107. package/src/room/track/LocalAudioTrack.ts +45 -0
  108. package/src/room/track/LocalTrack.ts +4 -4
  109. package/src/room/track/LocalVideoTrack.ts +49 -8
  110. package/src/room/track/RemoteAudioTrack.ts +9 -1
  111. package/src/room/track/RemoteTrackPublication.ts +2 -2
  112. package/src/room/track/options.ts +17 -16
  113. package/src/room/track/processor/types.ts +17 -2
  114. package/src/room/types.ts +5 -1
@@ -24,6 +24,8 @@ let useSharedKey: boolean = false;
24
24
 
25
25
  let sharedKey: CryptoKey | undefined;
26
26
 
27
+ let sifTrailer: Uint8Array | undefined;
28
+
27
29
  let keyProviderOptions: KeyProviderOptions = KEY_PROVIDER_DEFAULTS;
28
30
 
29
31
  workerLogger.setDefaultLevel('info');
@@ -94,6 +96,10 @@ onmessage = (ev) => {
94
96
  break;
95
97
  case 'ratchetRequest':
96
98
  handleRatchetRequest(data);
99
+ break;
100
+ case 'setSifTrailer':
101
+ handleSifTrailer(data.trailer);
102
+ break;
97
103
  default:
98
104
  break;
99
105
  }
@@ -116,6 +122,7 @@ function getTrackCryptor(participantId: string, trackId: string) {
116
122
  participantId,
117
123
  keys: getParticipantKeyHandler(participantId),
118
124
  keyProviderOptions,
125
+ sifTrailer,
119
126
  });
120
127
 
121
128
  setupCryptorErrorEvents(cryptor);
@@ -205,6 +212,13 @@ function emitRatchetedKeys(material: CryptoKey, keyIndex?: number) {
205
212
  postMessage(msg);
206
213
  }
207
214
 
215
+ function handleSifTrailer(trailer: Uint8Array) {
216
+ sifTrailer = trailer;
217
+ participantCryptors.forEach((c) => {
218
+ c.setSifTrailer(trailer);
219
+ });
220
+ }
221
+
208
222
  // Operations using RTCRtpScriptTransform.
209
223
  // @ts-ignore
210
224
  if (self.RTCTransformEvent) {
package/src/options.ts CHANGED
@@ -31,6 +31,9 @@ export interface InternalRoomOptions {
31
31
  * enable Dynacast, off by default. With Dynacast dynamically pauses
32
32
  * video layers that are not being consumed by any subscribers, significantly
33
33
  * reducing publishing CPU and bandwidth usage.
34
+ *
35
+ * Dynacast will be enabled if SVC codecs (VP9/AV1) are used. Multi-codec simulcast
36
+ * requires dynacast
34
37
  */
35
38
  dynacast: boolean;
36
39
 
@@ -119,6 +122,9 @@ export interface InternalRoomConnectOptions {
119
122
 
120
123
  /** specifies how often an initial join connection is allowed to retry (only applicable if server is not reachable) */
121
124
  maxRetries: number;
125
+
126
+ /** amount of time for Websocket connection to be established, defaults to 15s */
127
+ websocketTimeout: number;
122
128
  }
123
129
 
124
130
  /**
@@ -1,3 +1,17 @@
1
+ // Copyright 2023 LiveKit, Inc.
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+
1
15
  // @generated by protoc-gen-es v1.3.0 with parameter "target=ts"
2
16
  // @generated from file livekit_models.proto (package livekit, syntax proto3)
3
17
  /* eslint-disable */
@@ -1,3 +1,17 @@
1
+ // Copyright 2023 LiveKit, Inc.
2
+ //
3
+ // Licensed under the Apache License, Version 2.0 (the "License");
4
+ // you may not use this file except in compliance with the License.
5
+ // You may obtain a copy of the License at
6
+ //
7
+ // http://www.apache.org/licenses/LICENSE-2.0
8
+ //
9
+ // Unless required by applicable law or agreed to in writing, software
10
+ // distributed under the License is distributed on an "AS IS" BASIS,
11
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ // See the License for the specific language governing permissions and
13
+ // limitations under the License.
14
+
1
15
  // @generated by protoc-gen-es v1.3.0 with parameter "target=ts"
2
16
  // @generated from file livekit_rtc.proto (package livekit, syntax proto3)
3
17
  /* eslint-disable */
@@ -37,9 +37,8 @@ export default class DeviceManager {
37
37
 
38
38
  if (
39
39
  requestPermissions &&
40
- kind &&
41
40
  // for safari we need to skip this check, as otherwise it will re-acquire user media and fail on iOS https://bugs.webkit.org/show_bug.cgi?id=179363
42
- (!DeviceManager.userMediaPromiseMap.get(kind) || !isSafari())
41
+ !(isSafari() && this.hasDeviceInUse(kind))
43
42
  ) {
44
43
  const isDummyDeviceOrEmpty =
45
44
  devices.length === 0 ||
@@ -85,4 +84,10 @@ export default class DeviceManager {
85
84
 
86
85
  return device?.deviceId;
87
86
  }
87
+
88
+ private hasDeviceInUse(kind?: MediaDeviceKind): boolean {
89
+ return kind
90
+ ? DeviceManager.userMediaPromiseMap.has(kind)
91
+ : DeviceManager.userMediaPromiseMap.size > 0;
92
+ }
88
93
  }
@@ -191,8 +191,11 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
191
191
  this.url = url;
192
192
  this.token = token;
193
193
  this.signalOpts = opts;
194
+ this.maxJoinAttempts = opts.maxRetries;
194
195
  try {
195
196
  this.joinAttempts += 1;
197
+
198
+ this.setupSignalClientCallbacks();
196
199
  const joinResponse = await this.client.join(url, token, opts, abortSignal);
197
200
  this._isClosed = false;
198
201
  this.latestJoinResponse = joinResponse;
@@ -206,7 +209,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
206
209
  if (!this.subscriberPrimary) {
207
210
  this.negotiate();
208
211
  }
209
- this.setupSignalClientCallbacks();
210
212
 
211
213
  this.clientConfiguration = joinResponse.clientConfiguration;
212
214
 
@@ -497,11 +499,11 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
497
499
 
498
500
  this.client.onLocalTrackPublished = (res: TrackPublishedResponse) => {
499
501
  log.debug('received trackPublishedResponse', res);
500
- const { resolve } = this.pendingTrackResolvers[res.cid];
501
- if (!resolve) {
502
+ if (!this.pendingTrackResolvers[res.cid]) {
502
503
  log.error(`missing track resolver for ${res.cid}`);
503
504
  return;
504
505
  }
506
+ const { resolve } = this.pendingTrackResolvers[res.cid];
505
507
  delete this.pendingTrackResolvers[res.cid];
506
508
  resolve(res.track!);
507
509
  };
@@ -649,11 +651,11 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
649
651
  const channel = event.currentTarget as RTCDataChannel;
650
652
  const channelKind = channel.maxRetransmits === 0 ? 'lossy' : 'reliable';
651
653
 
652
- if (event instanceof ErrorEvent) {
654
+ if (event instanceof ErrorEvent && event.error) {
653
655
  const { error } = event.error;
654
656
  log.error(`DataChannel error on ${channelKind}: ${event.message}`, error);
655
657
  } else {
656
- log.error(`Unknown DataChannel Error on ${channelKind}`, event);
658
+ log.error(`Unknown DataChannel error on ${channelKind}`, event);
657
659
  }
658
660
  };
659
661
 
package/src/room/Room.ts CHANGED
@@ -424,13 +424,15 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
424
424
  this.abortController.abort();
425
425
  }
426
426
 
427
- this.abortController = new AbortController();
427
+ // explicit creation as local var needed to satisfy TS compiler when passing it to `attemptConnection` further down
428
+ const abortController = new AbortController();
429
+ this.abortController = abortController;
428
430
 
429
431
  // at this point the intention to connect has been signalled so we can allow cancelling of the connection via disconnect() again
430
432
  unlockDisconnect?.();
431
433
 
432
434
  try {
433
- await this.attemptConnection(regionUrl ?? url, token, opts, this.abortController);
435
+ await this.attemptConnection(regionUrl ?? url, token, opts, abortController);
434
436
  this.abortController = undefined;
435
437
  resolve();
436
438
  } catch (e) {
@@ -500,6 +502,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
500
502
  typeof roomOptions.adaptiveStream === 'object' ? true : roomOptions.adaptiveStream,
501
503
  maxRetries: connectOptions.maxRetries,
502
504
  e2eeEnabled: !!this.e2eeManager,
505
+ websocketTimeout: connectOptions.websocketTimeout,
503
506
  },
504
507
  abortController.signal,
505
508
  );
@@ -540,6 +543,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
540
543
  if (joinResponse.room) {
541
544
  this.handleRoomUpdate(joinResponse.room);
542
545
  }
546
+
547
+ if (this.options.e2ee && this.e2eeManager) {
548
+ this.e2eeManager.setSifTrailer(joinResponse.sifTrailer);
549
+ }
543
550
  };
544
551
 
545
552
  private attemptConnection = async (
@@ -692,7 +699,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
692
699
  /**
693
700
  * @internal for testing
694
701
  */
695
- async simulateScenario(scenario: SimulationScenario) {
702
+ async simulateScenario(scenario: SimulationScenario, arg?: any) {
696
703
  let postAction = () => {};
697
704
  let req: SimulateScenario | undefined;
698
705
  switch (scenario) {
@@ -762,6 +769,17 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
762
769
  }
763
770
  };
764
771
  break;
772
+ case 'subscriber-bandwidth':
773
+ if (arg === undefined || typeof arg !== 'number') {
774
+ throw new Error('subscriber-bandwidth requires a number as argument');
775
+ }
776
+ req = new SimulateScenario({
777
+ scenario: {
778
+ case: 'subscriberBandwidth',
779
+ value: BigInt(arg),
780
+ },
781
+ });
782
+ break;
765
783
  default:
766
784
  }
767
785
  if (req) {
@@ -782,7 +800,6 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
782
800
  * - `getUserMedia`
783
801
  */
784
802
  async startAudio() {
785
- await this.acquireAudioContext();
786
803
  const elements: Array<HTMLMediaElement> = [];
787
804
  const browser = getBrowser();
788
805
  if (browser && browser.os === 'iOS') {
@@ -831,12 +848,13 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
831
848
  });
832
849
 
833
850
  try {
834
- await Promise.all(
835
- elements.map((e) => {
851
+ await Promise.all([
852
+ this.acquireAudioContext(),
853
+ ...elements.map((e) => {
836
854
  e.muted = false;
837
855
  return e.play();
838
856
  }),
839
- );
857
+ ]);
840
858
  this.handleAudioPlaybackStarted();
841
859
  } catch (err) {
842
860
  this.handleAudioPlaybackFailed(err);
@@ -1405,6 +1423,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
1405
1423
  this.participants.forEach((participant) => participant.setAudioContext(this.audioContext));
1406
1424
  }
1407
1425
 
1426
+ this.localParticipant.setAudioContext(this.audioContext);
1427
+
1408
1428
  const newContextIsRunning = this.audioContext?.state === 'running';
1409
1429
  if (newContextIsRunning !== this.canPlaybackAudio) {
1410
1430
  this.audioEnabled = newContextIsRunning;
@@ -46,4 +46,5 @@ export const roomConnectOptionDefaults: InternalRoomConnectOptions = {
46
46
  autoSubscribe: true,
47
47
  maxRetries: 1,
48
48
  peerConnectionTimeout: 15_000,
49
+ websocketTimeout: 15_000,
49
50
  } as const;
@@ -457,7 +457,12 @@ export default class LocalParticipant extends Participant {
457
457
  screenVideo.source = Track.Source.ScreenShare;
458
458
  const localTracks: Array<LocalTrack> = [screenVideo];
459
459
  if (stream.getAudioTracks().length > 0) {
460
- const screenAudio = new LocalAudioTrack(stream.getAudioTracks()[0], undefined, false);
460
+ const screenAudio = new LocalAudioTrack(
461
+ stream.getAudioTracks()[0],
462
+ undefined,
463
+ false,
464
+ this.audioContext,
465
+ );
461
466
  screenAudio.source = Track.Source.ScreenShareAudio;
462
467
  localTracks.push(screenAudio);
463
468
  }
@@ -505,7 +510,7 @@ export default class LocalParticipant extends Participant {
505
510
  if (track instanceof MediaStreamTrack) {
506
511
  switch (track.kind) {
507
512
  case 'audio':
508
- track = new LocalAudioTrack(track, defaultConstraints, true);
513
+ track = new LocalAudioTrack(track, defaultConstraints, true, this.audioContext);
509
514
  break;
510
515
  case 'video':
511
516
  track = new LocalVideoTrack(track, defaultConstraints, true);
@@ -515,6 +520,10 @@ export default class LocalParticipant extends Participant {
515
520
  }
516
521
  }
517
522
 
523
+ if (track instanceof LocalAudioTrack) {
524
+ track.setAudioContext(this.audioContext);
525
+ }
526
+
518
527
  // is it already published? if so skip
519
528
  let existingPublication: LocalTrackPublication | undefined;
520
529
  this.tracks.forEach((publication) => {
@@ -673,6 +682,9 @@ export default class LocalParticipant extends Participant {
673
682
 
674
683
  // set up backup
675
684
  if (opts.videoCodec && opts.backupCodec && opts.videoCodec !== opts.backupCodec.codec) {
685
+ if (!this.roomOptions.dynacast) {
686
+ this.roomOptions.dynacast = true;
687
+ }
676
688
  const simOpts = { ...opts };
677
689
  simOpts.simulcast = true;
678
690
  simEncodings = computeTrackBackupEncodings(track, opts.backupCodec.codec, simOpts);
@@ -9,7 +9,9 @@ import {
9
9
  SubscriptionError,
10
10
  } from '../../proto/livekit_models_pb';
11
11
  import { ParticipantEvent, TrackEvent } from '../events';
12
+ import LocalAudioTrack from '../track/LocalAudioTrack';
12
13
  import type LocalTrackPublication from '../track/LocalTrackPublication';
14
+ import RemoteAudioTrack from '../track/RemoteAudioTrack';
13
15
  import type RemoteTrack from '../track/RemoteTrack';
14
16
  import type RemoteTrackPublication from '../track/RemoteTrackPublication';
15
17
  import { Track } from '../track/Track';
@@ -69,6 +71,8 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
69
71
 
70
72
  private _connectionQuality: ConnectionQuality = ConnectionQuality.Unknown;
71
73
 
74
+ protected audioContext?: AudioContext;
75
+
72
76
  get isEncrypted() {
73
77
  return this.tracks.size > 0 && Array.from(this.tracks.values()).every((tr) => tr.isEncrypted);
74
78
  }
@@ -238,6 +242,18 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
238
242
  }
239
243
  }
240
244
 
245
+ /**
246
+ * @internal
247
+ */
248
+ setAudioContext(ctx: AudioContext | undefined) {
249
+ this.audioContext = ctx;
250
+ this.audioTracks.forEach(
251
+ (track) =>
252
+ (track.track instanceof RemoteAudioTrack || track.track instanceof LocalAudioTrack) &&
253
+ track.track.setAudioContext(ctx),
254
+ );
255
+ }
256
+
241
257
  protected addTrackPublication(publication: TrackPublication) {
242
258
  // forward publication driven events
243
259
  publication.on(TrackEvent.Muted, () => {
@@ -25,8 +25,6 @@ export default class RemoteParticipant extends Participant {
25
25
 
26
26
  private volumeMap: Map<Track.Source, number>;
27
27
 
28
- private audioContext?: AudioContext;
29
-
30
28
  private audioOutput?: AudioOutputOptions;
31
29
 
32
30
  /** @internal */
@@ -324,16 +322,6 @@ export default class RemoteParticipant extends Participant {
324
322
  }
325
323
  }
326
324
 
327
- /**
328
- * @internal
329
- */
330
- setAudioContext(ctx: AudioContext | undefined) {
331
- this.audioContext = ctx;
332
- this.audioTracks.forEach(
333
- (track) => track.track instanceof RemoteAudioTrack && track.track.setAudioContext(ctx),
334
- );
335
- }
336
-
337
325
  /**
338
326
  * @internal
339
327
  */
@@ -3,13 +3,13 @@ import { TrackInvalidError } from '../errors';
3
3
  import LocalAudioTrack from '../track/LocalAudioTrack';
4
4
  import LocalVideoTrack from '../track/LocalVideoTrack';
5
5
  import { Track } from '../track/Track';
6
- import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options';
7
6
  import type {
8
7
  BackupVideoCodec,
9
8
  TrackPublishOptions,
10
9
  VideoCodec,
11
10
  VideoEncoding,
12
11
  } from '../track/options';
12
+ import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options';
13
13
  import { getReactNativeOs, isFireFox, isReactNative, isSVCCodec } from '../utils';
14
14
 
15
15
  /** @internal */
@@ -240,7 +240,9 @@ export function determineAppropriateEncoding(
240
240
  }
241
241
  // presets are based on the assumption of vp8 as a codec
242
242
  // for other codecs we adjust the maxBitrate if no specific videoEncoding has been provided
243
- // TODO make the bitrate multipliers configurable per codec
243
+ // users should override these with ones that are optimized for their use case
244
+ // NOTE: SVC codec bitrates are inclusive of all scalability layers. while
245
+ // bitrate for non-SVC codecs does not include other simulcast layers.
244
246
  if (codec) {
245
247
  switch (codec) {
246
248
  case 'av1':
@@ -6,12 +6,15 @@ import { isWeb, unwrapConstraint } from '../utils';
6
6
  import LocalTrack from './LocalTrack';
7
7
  import { Track } from './Track';
8
8
  import type { AudioCaptureOptions } from './options';
9
+ import type { TrackProcessor } from './processor/types';
9
10
  import { constraintsForOptions, detectSilence } from './utils';
10
11
 
11
12
  export default class LocalAudioTrack extends LocalTrack {
12
13
  /** @internal */
13
14
  stopOnMute: boolean = false;
14
15
 
16
+ private audioContext?: AudioContext;
17
+
15
18
  private prevStats?: AudioSenderStats;
16
19
 
17
20
  /**
@@ -24,8 +27,10 @@ export default class LocalAudioTrack extends LocalTrack {
24
27
  mediaTrack: MediaStreamTrack,
25
28
  constraints?: MediaTrackConstraints,
26
29
  userProvidedTrack = true,
30
+ audioContext?: AudioContext,
27
31
  ) {
28
32
  super(mediaTrack, Track.Kind.Audio, constraints, userProvidedTrack);
33
+ this.audioContext = audioContext;
29
34
  this.checkForSilence();
30
35
  }
31
36
 
@@ -133,6 +138,46 @@ export default class LocalAudioTrack extends LocalTrack {
133
138
  this.prevStats = stats;
134
139
  };
135
140
 
141
+ async setProcessor(processor: TrackProcessor<typeof this.kind>) {
142
+ const unlock = await this.processorLock.lock();
143
+ try {
144
+ if (!this.audioContext) {
145
+ throw Error(
146
+ 'Audio context needs to be set on LocalAudioTrack in order to enable processors',
147
+ );
148
+ }
149
+ if (this.processor) {
150
+ await this.stopProcessor();
151
+ }
152
+ if (this.kind === 'unknown') {
153
+ throw TypeError('cannot set processor on track of unknown kind');
154
+ }
155
+
156
+ const processorOptions = {
157
+ kind: this.kind,
158
+ track: this._mediaStreamTrack,
159
+ audioContext: this.audioContext,
160
+ };
161
+ log.debug(`setting up audio processor ${processor.name}`);
162
+
163
+ await processor.init(processorOptions);
164
+ this.processor = processor;
165
+ if (this.processor.processedTrack) {
166
+ await this.sender?.replaceTrack(this.processor.processedTrack);
167
+ }
168
+ } finally {
169
+ unlock();
170
+ }
171
+ }
172
+
173
+ /**
174
+ * @internal
175
+ * @experimental
176
+ */
177
+ setAudioContext(audioContext: AudioContext | undefined) {
178
+ this.audioContext = audioContext;
179
+ }
180
+
136
181
  async getSenderStats(): Promise<AudioSenderStats | undefined> {
137
182
  if (!this.sender?.getStats) {
138
183
  return undefined;
@@ -320,7 +320,7 @@ export default abstract class LocalTrack extends Track {
320
320
  * the server.
321
321
  * this API is unsupported on Safari < 12 due to a bug
322
322
  **/
323
- pauseUpstream = async () => {
323
+ async pauseUpstream() {
324
324
  const unlock = await this.pauseUpstreamLock.lock();
325
325
  try {
326
326
  if (this._isUpstreamPaused === true) {
@@ -342,9 +342,9 @@ export default abstract class LocalTrack extends Track {
342
342
  } finally {
343
343
  unlock();
344
344
  }
345
- };
345
+ }
346
346
 
347
- resumeUpstream = async () => {
347
+ async resumeUpstream() {
348
348
  const unlock = await this.pauseUpstreamLock.lock();
349
349
  try {
350
350
  if (this._isUpstreamPaused === false) {
@@ -362,7 +362,7 @@ export default abstract class LocalTrack extends Track {
362
362
  } finally {
363
363
  unlock();
364
364
  }
365
- };
365
+ }
366
366
 
367
367
  /**
368
368
  * Sets a processor on this track.
@@ -2,12 +2,14 @@ import type { SignalClient } from '../../api/SignalClient';
2
2
  import log from '../../logger';
3
3
  import { VideoLayer, VideoQuality } from '../../proto/livekit_models_pb';
4
4
  import { SubscribedCodec, SubscribedQuality } from '../../proto/livekit_rtc_pb';
5
- import { computeBitrate, monitorFrequency } from '../stats';
5
+ import { ScalabilityMode } from '../participant/publishUtils';
6
6
  import type { VideoSenderStats } from '../stats';
7
+ import { computeBitrate, monitorFrequency } from '../stats';
7
8
  import { Mutex, isFireFox, isMobile, isWeb, unwrapConstraint } from '../utils';
8
9
  import LocalTrack from './LocalTrack';
9
10
  import { Track } from './Track';
10
11
  import type { VideoCaptureOptions, VideoCodec } from './options';
12
+ import type { TrackProcessor } from './processor/types';
11
13
  import { constraintsForOptions } from './utils';
12
14
 
13
15
  export class SimulcastTrackInfo {
@@ -97,6 +99,20 @@ export default class LocalVideoTrack extends LocalTrack {
97
99
  super.stop();
98
100
  }
99
101
 
102
+ async pauseUpstream() {
103
+ await super.pauseUpstream();
104
+ for await (const sc of this.simulcastCodecs.values()) {
105
+ await sc.sender?.replaceTrack(null);
106
+ }
107
+ }
108
+
109
+ async resumeUpstream() {
110
+ await super.resumeUpstream();
111
+ for await (const sc of this.simulcastCodecs.values()) {
112
+ await sc.sender?.replaceTrack(sc.mediaStreamTrack);
113
+ }
114
+ }
115
+
100
116
  async mute(): Promise<LocalVideoTrack> {
101
117
  const unlock = await this.muteLock.lock();
102
118
  try {
@@ -126,6 +142,13 @@ export default class LocalVideoTrack extends LocalTrack {
126
142
  }
127
143
  }
128
144
 
145
+ protected setTrackMuted(muted: boolean) {
146
+ super.setTrackMuted(muted);
147
+ for (const sc of this.simulcastCodecs.values()) {
148
+ sc.mediaStreamTrack.enabled = !muted;
149
+ }
150
+ }
151
+
129
152
  async getSenderStats(): Promise<VideoSenderStats[]> {
130
153
  if (!this.sender?.getStats) {
131
154
  return [];
@@ -210,6 +233,23 @@ export default class LocalVideoTrack extends LocalTrack {
210
233
  }
211
234
  }
212
235
  await this.restart(constraints);
236
+
237
+ for await (const sc of this.simulcastCodecs.values()) {
238
+ if (sc.sender) {
239
+ sc.mediaStreamTrack = this.mediaStreamTrack.clone();
240
+ await sc.sender.replaceTrack(sc.mediaStreamTrack);
241
+ }
242
+ }
243
+ }
244
+
245
+ async setProcessor(processor: TrackProcessor<Track.Kind>, showProcessedStreamLocally = true) {
246
+ await super.setProcessor(processor, showProcessedStreamLocally);
247
+
248
+ if (this.processor?.processedTrack) {
249
+ for await (const sc of this.simulcastCodecs.values()) {
250
+ await sc.sender?.replaceTrack(this.processor.processedTrack);
251
+ }
252
+ }
213
253
  }
214
254
 
215
255
  addSimulcastTrack(codec: VideoCodec, encodings?: RTCRtpEncodingParameters[]): SimulcastTrackInfo {
@@ -359,7 +399,7 @@ async function setPublishingLayersForSender(
359
399
  let hasChanged = false;
360
400
 
361
401
  /* disable closable spatial layer as it has video blur / frozen issue with current server / client
362
- 1. chrome 113: when switching to up layer with scalability Mode change, it will generate a
402
+ 1. chrome 113: when switching to up layer with scalability Mode change, it will generate a
363
403
  low resolution frame and recover very quickly, but noticable
364
404
  2. livekit sfu: additional pli request cause video frozen for a few frames, also noticable */
365
405
  const closableSpatial = false;
@@ -483,15 +523,16 @@ export function videoLayersFromEncodings(
483
523
  if (svc) {
484
524
  // svc layers
485
525
  /* @ts-ignore */
486
- const sm = new ScalabilityMode(encodings[0].scalabilityMode);
526
+ const encodingSM = encodings[0].scalabilityMode as string;
527
+ const sm = new ScalabilityMode(encodingSM);
487
528
  const layers = [];
488
529
  for (let i = 0; i < sm.spatial; i += 1) {
489
530
  layers.push(
490
531
  new VideoLayer({
491
532
  quality: VideoQuality.HIGH - i,
492
- width: width / 2 ** i,
493
- height: height / 2 ** i,
494
- bitrate: encodings[0].maxBitrate ? encodings[0].maxBitrate / 3 ** i : 0,
533
+ width: Math.ceil(width / 2 ** i),
534
+ height: Math.ceil(height / 2 ** i),
535
+ bitrate: encodings[0].maxBitrate ? Math.ceil(encodings[0].maxBitrate / 3 ** i) : 0,
495
536
  ssrc: 0,
496
537
  }),
497
538
  );
@@ -504,8 +545,8 @@ export function videoLayersFromEncodings(
504
545
  let quality = videoQualityForRid(encoding.rid ?? '');
505
546
  return new VideoLayer({
506
547
  quality,
507
- width: width / scale,
508
- height: height / scale,
548
+ width: Math.ceil(width / scale),
549
+ height: Math.ceil(height / scale),
509
550
  bitrate: encoding.maxBitrate ?? 0,
510
551
  ssrc: 0,
511
552
  });
@@ -2,7 +2,7 @@ import log from '../../logger';
2
2
  import { TrackEvent } from '../events';
3
3
  import { computeBitrate } from '../stats';
4
4
  import type { AudioReceiverStats } from '../stats';
5
- import { supportsSetSinkId } from '../utils';
5
+ import { isReactNative, supportsSetSinkId } from '../utils';
6
6
  import RemoteTrack from './RemoteTrack';
7
7
  import { Track } from './Track';
8
8
  import type { AudioOutputOptions } from './options';
@@ -48,6 +48,10 @@ export default class RemoteAudioTrack extends RemoteTrack {
48
48
  el.volume = volume;
49
49
  }
50
50
  }
51
+ if (isReactNative()) {
52
+ // @ts-ignore
53
+ this._mediaStreamTrack._setVolume(volume);
54
+ }
51
55
  this.elementVolume = volume;
52
56
  }
53
57
 
@@ -58,6 +62,10 @@ export default class RemoteAudioTrack extends RemoteTrack {
58
62
  if (this.elementVolume) {
59
63
  return this.elementVolume;
60
64
  }
65
+ if (isReactNative()) {
66
+ // RN volume value defaults to 1.0 if hasn't been changed.
67
+ return 1.0;
68
+ }
61
69
  let highestVolume = 0;
62
70
  this.attachedElements.forEach((element) => {
63
71
  if (element.volume > highestVolume) {
@@ -297,8 +297,8 @@ export default class RemoteTrackPublication extends TrackPublication {
297
297
  fps: this.fps,
298
298
  });
299
299
  if (this.videoDimensions) {
300
- settings.width = this.videoDimensions.width;
301
- settings.height = this.videoDimensions.height;
300
+ settings.width = Math.ceil(this.videoDimensions.width);
301
+ settings.height = Math.ceil(this.videoDimensions.height);
302
302
  } else if (this.currentVideoQuality !== undefined) {
303
303
  settings.quality = this.currentVideoQuality;
304
304
  } else {