livekit-client 1.9.6 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/livekit-client.esm.mjs +1318 -885
  2. package/dist/livekit-client.esm.mjs.map +1 -1
  3. package/dist/livekit-client.umd.js +1 -1
  4. package/dist/livekit-client.umd.js.map +1 -1
  5. package/dist/src/api/SignalClient.d.ts +2 -1
  6. package/dist/src/api/SignalClient.d.ts.map +1 -1
  7. package/dist/src/index.d.ts +1 -0
  8. package/dist/src/index.d.ts.map +1 -1
  9. package/dist/src/proto/livekit_models.d.ts +108 -10
  10. package/dist/src/proto/livekit_models.d.ts.map +1 -1
  11. package/dist/src/proto/livekit_rtc.d.ts +513 -194
  12. package/dist/src/proto/livekit_rtc.d.ts.map +1 -1
  13. package/dist/src/room/Room.d.ts +3 -2
  14. package/dist/src/room/Room.d.ts.map +1 -1
  15. package/dist/src/room/events.d.ts +5 -1
  16. package/dist/src/room/events.d.ts.map +1 -1
  17. package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
  18. package/dist/src/room/participant/Participant.d.ts +2 -2
  19. package/dist/src/room/participant/Participant.d.ts.map +1 -1
  20. package/dist/src/room/participant/RemoteParticipant.d.ts.map +1 -1
  21. package/dist/src/room/participant/publishUtils.d.ts +8 -0
  22. package/dist/src/room/participant/publishUtils.d.ts.map +1 -1
  23. package/dist/src/room/track/LocalTrack.d.ts +32 -0
  24. package/dist/src/room/track/LocalTrack.d.ts.map +1 -1
  25. package/dist/src/room/track/LocalVideoTrack.d.ts.map +1 -1
  26. package/dist/src/room/track/RemoteTrackPublication.d.ts +4 -1
  27. package/dist/src/room/track/RemoteTrackPublication.d.ts.map +1 -1
  28. package/dist/src/room/track/TrackPublication.d.ts +2 -1
  29. package/dist/src/room/track/TrackPublication.d.ts.map +1 -1
  30. package/dist/src/room/track/options.d.ts +1 -1
  31. package/dist/src/room/track/options.d.ts.map +1 -1
  32. package/dist/src/room/track/processor/types.d.ts +19 -0
  33. package/dist/src/room/track/processor/types.d.ts.map +1 -0
  34. package/dist/src/utils/browserParser.d.ts.map +1 -1
  35. package/dist/ts4.2/src/api/SignalClient.d.ts +2 -1
  36. package/dist/ts4.2/src/index.d.ts +1 -0
  37. package/dist/ts4.2/src/proto/livekit_models.d.ts +126 -12
  38. package/dist/ts4.2/src/proto/livekit_rtc.d.ts +617 -254
  39. package/dist/ts4.2/src/room/Room.d.ts +3 -2
  40. package/dist/ts4.2/src/room/events.d.ts +5 -1
  41. package/dist/ts4.2/src/room/participant/Participant.d.ts +2 -2
  42. package/dist/ts4.2/src/room/participant/publishUtils.d.ts +8 -0
  43. package/dist/ts4.2/src/room/track/LocalTrack.d.ts +32 -0
  44. package/dist/ts4.2/src/room/track/RemoteTrackPublication.d.ts +4 -1
  45. package/dist/ts4.2/src/room/track/TrackPublication.d.ts +2 -1
  46. package/dist/ts4.2/src/room/track/options.d.ts +1 -1
  47. package/dist/ts4.2/src/room/track/processor/types.d.ts +19 -0
  48. package/package.json +14 -13
  49. package/src/api/SignalClient.ts +8 -1
  50. package/src/index.ts +1 -0
  51. package/src/proto/google/protobuf/timestamp.ts +3 -3
  52. package/src/proto/livekit_models.ts +254 -161
  53. package/src/proto/livekit_rtc.ts +334 -180
  54. package/src/room/Room.ts +26 -1
  55. package/src/room/events.ts +4 -0
  56. package/src/room/participant/LocalParticipant.ts +23 -3
  57. package/src/room/participant/Participant.ts +2 -1
  58. package/src/room/participant/RemoteParticipant.ts +4 -1
  59. package/src/room/participant/publishUtils.ts +68 -12
  60. package/src/room/track/LocalTrack.ts +120 -16
  61. package/src/room/track/LocalVideoTrack.ts +96 -33
  62. package/src/room/track/RemoteTrackPublication.ts +8 -1
  63. package/src/room/track/Track.ts +3 -3
  64. package/src/room/track/TrackPublication.ts +2 -1
  65. package/src/room/track/options.ts +1 -1
  66. package/src/room/track/processor/types.ts +20 -0
  67. package/src/utils/browserParser.ts +1 -4
package/src/room/Room.ts CHANGED
@@ -18,6 +18,7 @@ import {
18
18
  Room as RoomModel,
19
19
  ServerInfo,
20
20
  SpeakerInfo,
21
+ SubscriptionError,
21
22
  TrackInfo,
22
23
  TrackSource,
23
24
  TrackType,
@@ -29,6 +30,7 @@ import {
29
30
  SimulateScenario,
30
31
  StreamStateUpdate,
31
32
  SubscriptionPermissionUpdate,
33
+ SubscriptionResponse,
32
34
  } from '../proto/livekit_rtc';
33
35
  import DeviceManager from './DeviceManager';
34
36
  import RTCEngine from './RTCEngine';
@@ -207,6 +209,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
207
209
  this.engine.client.onStreamStateUpdate = this.handleStreamStateUpdate;
208
210
  this.engine.client.onSubscriptionPermissionUpdate = this.handleSubscriptionPermissionUpdate;
209
211
  this.engine.client.onConnectionQuality = this.handleConnectionQualityUpdate;
212
+ this.engine.client.onSubscriptionError = this.handleSubscriptionError;
210
213
 
211
214
  this.engine
212
215
  .on(
@@ -1114,6 +1117,21 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
1114
1117
  pub.setAllowed(update.allowed);
1115
1118
  };
1116
1119
 
1120
+ private handleSubscriptionError = (update: SubscriptionResponse) => {
1121
+ const participant = Array.from(this.participants.values()).find((p) =>
1122
+ p.tracks.has(update.trackSid),
1123
+ );
1124
+ if (!participant) {
1125
+ return;
1126
+ }
1127
+ const pub = participant.getTrackPublication(update.trackSid);
1128
+ if (!pub) {
1129
+ return;
1130
+ }
1131
+
1132
+ pub.setSubscriptionError(update.err);
1133
+ };
1134
+
1117
1135
  private handleDataPacket = (userPacket: UserPacket, kind: DataPacket_Kind) => {
1118
1136
  // find the participant
1119
1137
  const participant = this.participants.get(userPacket.participantSid);
@@ -1280,6 +1298,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
1280
1298
  .on(ParticipantEvent.TrackSubscriptionStatusChanged, (pub, status) => {
1281
1299
  this.emitWhenConnected(RoomEvent.TrackSubscriptionStatusChanged, pub, status, participant);
1282
1300
  })
1301
+ .on(ParticipantEvent.TrackSubscriptionFailed, (trackSid, error) => {
1302
+ this.emit(RoomEvent.TrackSubscriptionFailed, trackSid, participant, error);
1303
+ })
1283
1304
  .on(ParticipantEvent.TrackSubscriptionPermissionChanged, (pub, status) => {
1284
1305
  this.emitWhenConnected(
1285
1306
  RoomEvent.TrackSubscriptionPermissionChanged,
@@ -1609,7 +1630,11 @@ export type RoomEventCallbacks = {
1609
1630
  publication: RemoteTrackPublication,
1610
1631
  participant: RemoteParticipant,
1611
1632
  ) => void;
1612
- trackSubscriptionFailed: (trackSid: string, participant: RemoteParticipant) => void;
1633
+ trackSubscriptionFailed: (
1634
+ trackSid: string,
1635
+ participant: RemoteParticipant,
1636
+ reason?: SubscriptionError,
1637
+ ) => void;
1613
1638
  trackUnpublished: (publication: RemoteTrackPublication, participant: RemoteParticipant) => void;
1614
1639
  trackUnsubscribed: (
1615
1640
  track: RemoteTrack,
@@ -508,4 +508,8 @@ export enum TrackEvent {
508
508
  * Fires on RemoteTrackPublication
509
509
  */
510
510
  SubscriptionStatusChanged = 'subscriptionStatusChanged',
511
+ /**
512
+ * Fires on RemoteTrackPublication
513
+ */
514
+ SubscriptionFailed = 'subscriptionFailed',
511
515
  }
@@ -625,8 +625,8 @@ export default class LocalParticipant extends Participant {
625
625
  // for svc codecs, disable simulcast and use vp8 for backup codec
626
626
  if (track instanceof LocalVideoTrack) {
627
627
  if (isSVCCodec(opts.videoCodec)) {
628
- // set scalabilityMode to 'L3T3' by default
629
- opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
628
+ // set scalabilityMode to 'L3T3_KEY' by default
629
+ opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY';
630
630
  }
631
631
 
632
632
  // set up backup
@@ -647,6 +647,16 @@ export default class LocalParticipant extends Participant {
647
647
  enableSimulcastLayers: true,
648
648
  },
649
649
  ];
650
+ } else if (opts.videoCodec) {
651
+ // pass codec info to sfu so it can prefer codec for the client which don't support
652
+ // setCodecPreferences
653
+ req.simulcastCodecs = [
654
+ {
655
+ codec: opts.videoCodec,
656
+ cid: track.mediaStreamTrack.id,
657
+ enableSimulcastLayers: opts.simulcast ?? false,
658
+ },
659
+ ];
650
660
  }
651
661
  }
652
662
 
@@ -656,7 +666,7 @@ export default class LocalParticipant extends Participant {
656
666
  dims.height,
657
667
  opts,
658
668
  );
659
- req.layers = videoLayersFromEncodings(req.width, req.height, simEncodings ?? encodings);
669
+ req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
660
670
  } else if (track.kind === Track.Kind.Audio) {
661
671
  encodings = [
662
672
  {
@@ -850,6 +860,16 @@ export default class LocalParticipant extends Participant {
850
860
  trackSender
851
861
  ) {
852
862
  try {
863
+ for (const transceiver of this.engine.publisher.pc.getTransceivers()) {
864
+ // if sender is not currently sending (after replaceTrack(null))
865
+ // removeTrack would have no effect.
866
+ // to ensure we end up successfully removing the track, manually set
867
+ // the transceiver to inactive
868
+ if (transceiver.sender === trackSender) {
869
+ transceiver.direction = 'inactive';
870
+ negotiationNeeded = true;
871
+ }
872
+ }
853
873
  if (this.engine.removeTrack(trackSender)) {
854
874
  negotiationNeeded = true;
855
875
  }
@@ -6,6 +6,7 @@ import {
6
6
  ParticipantInfo,
7
7
  ParticipantPermission,
8
8
  ConnectionQuality as ProtoQuality,
9
+ SubscriptionError,
9
10
  } from '../../proto/livekit_models';
10
11
  import { ParticipantEvent, TrackEvent } from '../events';
11
12
  import type LocalTrackPublication from '../track/LocalTrackPublication';
@@ -265,7 +266,7 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
265
266
  export type ParticipantEventCallbacks = {
266
267
  trackPublished: (publication: RemoteTrackPublication) => void;
267
268
  trackSubscribed: (track: RemoteTrack, publication: RemoteTrackPublication) => void;
268
- trackSubscriptionFailed: (trackSid: string) => void;
269
+ trackSubscriptionFailed: (trackSid: string, reason?: SubscriptionError) => void;
269
270
  trackUnpublished: (publication: RemoteTrackPublication) => void;
270
271
  trackUnsubscribed: (track: RemoteTrack, publication: RemoteTrackPublication) => void;
271
272
  trackMuted: (publication: TrackPublication) => void;
@@ -1,6 +1,6 @@
1
1
  import type { SignalClient } from '../../api/SignalClient';
2
2
  import log from '../../logger';
3
- import type { ParticipantInfo } from '../../proto/livekit_models';
3
+ import type { ParticipantInfo, SubscriptionError } from '../../proto/livekit_models';
4
4
  import type { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
5
5
  import { ParticipantEvent, TrackEvent } from '../events';
6
6
  import RemoteAudioTrack from '../track/RemoteAudioTrack';
@@ -81,6 +81,9 @@ export default class RemoteParticipant extends Participant {
81
81
  publication.on(TrackEvent.Unsubscribed, (previousTrack: RemoteTrack) => {
82
82
  this.emit(ParticipantEvent.TrackUnsubscribed, previousTrack, publication);
83
83
  });
84
+ publication.on(TrackEvent.SubscriptionFailed, (error: SubscriptionError) => {
85
+ this.emit(ParticipantEvent.TrackSubscriptionFailed, publication.trackSid, error);
86
+ });
84
87
  }
85
88
 
86
89
  getTrack(source: Track.Source): RemoteTrackPublication | undefined {
@@ -10,7 +10,7 @@ import type {
10
10
  VideoCodec,
11
11
  VideoEncoding,
12
12
  } from '../track/options';
13
- import { isSVCCodec } from '../utils';
13
+ import { getReactNativeOs, isReactNative, isSVCCodec } from '../utils';
14
14
 
15
15
  /** @internal */
16
16
  export function mediaTrackToLocalTrack(
@@ -128,17 +128,15 @@ export function computeVideoEncodings(
128
128
  // svc use first encoding as the original, so we sort encoding from high to low
129
129
  switch (scalabilityMode) {
130
130
  case 'L3T3':
131
- for (let i = 0; i < 3; i += 1) {
132
- encodings.push({
133
- rid: videoRids[2 - i],
134
- scaleResolutionDownBy: 2 ** i,
135
- maxBitrate: videoEncoding.maxBitrate / 3 ** i,
136
- /* @ts-ignore */
137
- maxFramerate: original.encoding.maxFramerate,
138
- /* @ts-ignore */
139
- scalabilityMode: 'L3T3',
140
- });
141
- }
131
+ case 'L3T3_KEY':
132
+ encodings.push({
133
+ rid: videoRids[2],
134
+ maxBitrate: videoEncoding.maxBitrate,
135
+ /* @ts-ignore */
136
+ maxFramerate: original.encoding.maxFramerate,
137
+ /* @ts-ignore */
138
+ scalabilityMode: scalabilityMode,
139
+ });
142
140
  log.debug('encodings', encodings);
143
141
  return encodings;
144
142
 
@@ -321,6 +319,33 @@ function encodingsFromPresets(
321
319
  }
322
320
  encodings.push(encoding);
323
321
  });
322
+
323
+ // RN ios simulcast requires all same framerates.
324
+ if (isReactNative() && getReactNativeOs() === 'ios') {
325
+ let topFramerate: number | undefined = undefined;
326
+ encodings.forEach((encoding) => {
327
+ if (!topFramerate) {
328
+ topFramerate = encoding.maxFramerate;
329
+ } else if (encoding.maxFramerate && encoding.maxFramerate > topFramerate) {
330
+ topFramerate = encoding.maxFramerate;
331
+ }
332
+ });
333
+
334
+ let notifyOnce = true;
335
+ encodings.forEach((encoding) => {
336
+ if (encoding.maxFramerate != topFramerate) {
337
+ if (notifyOnce) {
338
+ notifyOnce = false;
339
+ log.info(
340
+ `Simulcast on iOS React-Native requires all encodings to share the same framerate.`,
341
+ );
342
+ }
343
+ log.info(`Setting framerate of encoding \"${encoding.rid ?? ''}\" to ${topFramerate}`);
344
+ encoding.maxFramerate = topFramerate;
345
+ }
346
+ });
347
+ }
348
+
324
349
  return encodings;
325
350
  }
326
351
 
@@ -341,3 +366,34 @@ export function sortPresets(presets: Array<VideoPreset> | undefined) {
341
366
  return 0;
342
367
  });
343
368
  }
369
+
370
+ /** @internal */
371
+ export class ScalabilityMode {
372
+ spatial: number;
373
+
374
+ temporal: number;
375
+
376
+ suffix: undefined | 'h' | '_KEY' | '_KEY_SHIFT';
377
+
378
+ constructor(scalabilityMode: string) {
379
+ const results = scalabilityMode.match(/^L(\d)T(\d)(h|_KEY|_KEY_SHIFT){0,1}$/);
380
+ if (!results) {
381
+ throw new Error('invalid scalability mode');
382
+ }
383
+
384
+ this.spatial = parseInt(results[1]);
385
+ this.temporal = parseInt(results[2]);
386
+ if (results.length > 3) {
387
+ switch (results[3]) {
388
+ case 'h':
389
+ case '_KEY':
390
+ case '_KEY_SHIFT':
391
+ this.suffix = results[3];
392
+ }
393
+ }
394
+ }
395
+
396
+ toString(): string {
397
+ return `L${this.spatial}T${this.temporal}${this.suffix ?? ''}`;
398
+ }
399
+ }
@@ -1,16 +1,12 @@
1
1
  import log from '../../logger';
2
+ import { getBrowser } from '../../utils/browserParser';
2
3
  import DeviceManager from '../DeviceManager';
3
- import { TrackInvalidError } from '../errors';
4
+ import { DeviceUnsupportedError, TrackInvalidError } from '../errors';
4
5
  import { TrackEvent } from '../events';
5
- import {
6
- Mutex,
7
- getEmptyAudioStreamTrack,
8
- getEmptyVideoStreamTrack,
9
- isMobile,
10
- sleep,
11
- } from '../utils';
6
+ import { Mutex, compareVersions, isMobile, sleep } from '../utils';
12
7
  import { Track, attachToElement, detachTrack } from './Track';
13
8
  import type { VideoCodec } from './options';
9
+ import type { TrackProcessor } from './processor/types';
14
10
 
15
11
  const defaultDimensionsTimeout = 1000;
16
12
 
@@ -31,6 +27,12 @@ export default abstract class LocalTrack extends Track {
31
27
 
32
28
  protected pauseUpstreamLock: Mutex;
33
29
 
30
+ protected processorElement?: HTMLMediaElement;
31
+
32
+ protected processor?: TrackProcessor<typeof this.kind>;
33
+
34
+ protected isSettingUpProcessor: boolean = false;
35
+
34
36
  /**
35
37
  *
36
38
  * @param mediaTrack
@@ -82,6 +84,10 @@ export default abstract class LocalTrack extends Track {
82
84
  return this.providedByUser;
83
85
  }
84
86
 
87
+ get mediaStreamTrack() {
88
+ return this.processor?.processedTrack ?? this._mediaStreamTrack;
89
+ }
90
+
85
91
  async waitForDimensions(timeout = defaultDimensionsTimeout): Promise<Track.Dimensions> {
86
92
  if (this.kind === Track.Kind.Audio) {
87
93
  throw new Error('cannot get dimensions for audio tracks');
@@ -158,6 +164,9 @@ export default abstract class LocalTrack extends Track {
158
164
 
159
165
  this.mediaStream = new MediaStream([track]);
160
166
  this.providedByUser = userProvidedTrack;
167
+ if (this.processor) {
168
+ await this.stopProcessor();
169
+ }
161
170
  return this;
162
171
  }
163
172
 
@@ -180,7 +189,7 @@ export default abstract class LocalTrack extends Track {
180
189
 
181
190
  // detach
182
191
  this.attachedElements.forEach((el) => {
183
- detachTrack(this._mediaStreamTrack, el);
192
+ detachTrack(this.mediaStreamTrack, el);
184
193
  });
185
194
  this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
186
195
  // on Safari, the old audio track must be stopped before attempting to acquire
@@ -203,12 +212,16 @@ export default abstract class LocalTrack extends Track {
203
212
 
204
213
  await this.resumeUpstream();
205
214
 
206
- this.attachedElements.forEach((el) => {
207
- attachToElement(newTrack, el);
208
- });
209
-
210
215
  this.mediaStream = mediaStream;
211
216
  this.constraints = constraints;
217
+ if (this.processor) {
218
+ const processor = this.processor;
219
+ await this.setProcessor(processor);
220
+ } else {
221
+ this.attachedElements.forEach((el) => {
222
+ attachToElement(this._mediaStreamTrack, el);
223
+ });
224
+ }
212
225
  this.emit(TrackEvent.Restarted, this);
213
226
  return this;
214
227
  }
@@ -253,6 +266,18 @@ export default abstract class LocalTrack extends Track {
253
266
  this.emit(TrackEvent.Ended, this);
254
267
  };
255
268
 
269
+ stop() {
270
+ super.stop();
271
+ this.processor?.destroy();
272
+ this.processor = undefined;
273
+ }
274
+
275
+ /**
276
+ * pauses publishing to the server without disabling the local MediaStreamTrack
277
+ * this is used to display a user's own video locally while pausing publishing to
278
+ * the server.
279
+ * this API is unsupported on Safari < 12 due to a bug
280
+ **/
256
281
  async pauseUpstream() {
257
282
  const unlock = await this.pauseUpstreamLock.lock();
258
283
  try {
@@ -266,9 +291,12 @@ export default abstract class LocalTrack extends Track {
266
291
 
267
292
  this._isUpstreamPaused = true;
268
293
  this.emit(TrackEvent.UpstreamPaused, this);
269
- const emptyTrack =
270
- this.kind === Track.Kind.Audio ? getEmptyAudioStreamTrack() : getEmptyVideoStreamTrack();
271
- await this.sender.replaceTrack(emptyTrack);
294
+ const browser = getBrowser();
295
+ if (browser?.name === 'Safari' && compareVersions(browser.version, '12.0') < 0) {
296
+ // https://bugs.webkit.org/show_bug.cgi?id=184911
297
+ throw new DeviceUnsupportedError('pauseUpstream is not supported on Safari < 12.');
298
+ }
299
+ await this.sender.replaceTrack(null);
272
300
  } finally {
273
301
  unlock();
274
302
  }
@@ -293,5 +321,81 @@ export default abstract class LocalTrack extends Track {
293
321
  }
294
322
  }
295
323
 
324
+ /**
325
+ * Sets a processor on this track.
326
+ * See https://github.com/livekit/track-processors-js for example usage
327
+ *
328
+ * @experimental
329
+ *
330
+ * @param processor
331
+ * @param showProcessedStreamLocally
332
+ * @returns
333
+ */
334
+ async setProcessor(
335
+ processor: TrackProcessor<typeof this.kind>,
336
+ showProcessedStreamLocally = true,
337
+ ) {
338
+ if (this.isSettingUpProcessor) {
339
+ log.warn('already trying to set up a processor');
340
+ return;
341
+ }
342
+ log.debug('setting up processor');
343
+ this.isSettingUpProcessor = true;
344
+ if (this.processor) {
345
+ await this.stopProcessor();
346
+ }
347
+ if (this.kind === 'unknown') {
348
+ throw TypeError('cannot set processor on track of unknown kind');
349
+ }
350
+ this.processorElement = this.processorElement ?? document.createElement(this.kind);
351
+ this.processorElement.muted = true;
352
+
353
+ attachToElement(this._mediaStreamTrack, this.processorElement);
354
+ this.processorElement.play().catch((e) => log.error(e));
355
+
356
+ const processorOptions = {
357
+ kind: this.kind,
358
+ track: this._mediaStreamTrack,
359
+ element: this.processorElement,
360
+ };
361
+
362
+ await processor.init(processorOptions);
363
+ this.processor = processor;
364
+ if (this.processor.processedTrack) {
365
+ for (const el of this.attachedElements) {
366
+ if (el !== this.processorElement && showProcessedStreamLocally) {
367
+ detachTrack(this._mediaStreamTrack, el);
368
+ attachToElement(this.processor.processedTrack, el);
369
+ }
370
+ }
371
+ await this.sender?.replaceTrack(this.processor.processedTrack);
372
+ }
373
+ this.isSettingUpProcessor = false;
374
+ }
375
+
376
+ getProcessor() {
377
+ return this.processor;
378
+ }
379
+
380
+ /**
381
+ * Stops the track processor
382
+ * See https://github.com/livekit/track-processors-js for example usage
383
+ *
384
+ * @experimental
385
+ * @returns
386
+ */
387
+ async stopProcessor() {
388
+ if (!this.processor) return;
389
+
390
+ log.debug('stopping processor');
391
+ this.processor.processedTrack?.stop();
392
+ await this.processor.destroy();
393
+ this.processor = undefined;
394
+ this.processorElement?.remove();
395
+ this.processorElement = undefined;
396
+
397
+ await this.restart();
398
+ }
399
+
296
400
  protected abstract monitorSender(): void;
297
401
  }
@@ -2,6 +2,7 @@ import type { SignalClient } from '../../api/SignalClient';
2
2
  import log from '../../logger';
3
3
  import { VideoLayer, VideoQuality } from '../../proto/livekit_models';
4
4
  import type { SubscribedCodec, SubscribedQuality } from '../../proto/livekit_rtc';
5
+ import { ScalabilityMode } from '../participant/publishUtils';
5
6
  import { computeBitrate, monitorFrequency } from '../stats';
6
7
  import type { VideoSenderStats } from '../stats';
7
8
  import { Mutex, isFireFox, isMobile, isWeb } from '../utils';
@@ -349,45 +350,88 @@ async function setPublishingLayersForSender(
349
350
  }
350
351
 
351
352
  let hasChanged = false;
352
- encodings.forEach((encoding, idx) => {
353
- let rid = encoding.rid ?? '';
354
- if (rid === '') {
355
- rid = 'q';
356
- }
357
- const quality = videoQualityForRid(rid);
358
- const subscribedQuality = qualities.find((q) => q.quality === quality);
359
- if (!subscribedQuality) {
360
- return;
361
- }
362
- if (encoding.active !== subscribedQuality.enabled) {
353
+
354
+ /* @ts-ignore */
355
+ if (encodings.length === 1 && encodings[0].scalabilityMode) {
356
+ // svc dynacast encodings
357
+ const encoding = encodings[0];
358
+ /* @ts-ignore */
359
+ // const mode = new ScalabilityMode(encoding.scalabilityMode);
360
+ let maxQuality = VideoQuality.OFF;
361
+ qualities.forEach((q) => {
362
+ if (q.enabled && (maxQuality === VideoQuality.OFF || q.quality > maxQuality)) {
363
+ maxQuality = q.quality;
364
+ }
365
+ });
366
+
367
+ if (maxQuality === VideoQuality.OFF) {
368
+ if (encoding.active) {
369
+ encoding.active = false;
370
+ hasChanged = true;
371
+ }
372
+ } else if (!encoding.active /* || mode.spatial !== maxQuality + 1*/) {
363
373
  hasChanged = true;
364
- encoding.active = subscribedQuality.enabled;
365
- log.debug(
366
- `setting layer ${subscribedQuality.quality} to ${
367
- encoding.active ? 'enabled' : 'disabled'
368
- }`,
369
- );
370
-
371
- // FireFox does not support setting encoding.active to false, so we
372
- // have a workaround of lowering its bitrate and resolution to the min.
373
- if (isFireFox()) {
374
- if (subscribedQuality.enabled) {
375
- encoding.scaleResolutionDownBy = senderEncodings[idx].scaleResolutionDownBy;
376
- encoding.maxBitrate = senderEncodings[idx].maxBitrate;
377
- /* @ts-ignore */
378
- encoding.maxFrameRate = senderEncodings[idx].maxFrameRate;
379
- } else {
380
- encoding.scaleResolutionDownBy = 4;
381
- encoding.maxBitrate = 10;
382
- /* @ts-ignore */
383
- encoding.maxFrameRate = 2;
384
- }
374
+ encoding.active = true;
375
+ /* disable closable spatial layer as it has video blur/frozen issue with current server/client
376
+ 1. chrome 113: when switching to up layer with scalability Mode change, it will generate a
377
+ low resolution frame and recover very quickly, but noticable
378
+ 2. livekit sfu: additional pli request cause video frozen for a few frames, also noticable
379
+ @ts-ignore
380
+ const originalMode = new ScalabilityMode(senderEncodings[0].scalabilityMode)
381
+ mode.spatial = maxQuality + 1;
382
+ mode.suffix = originalMode.suffix;
383
+ if (mode.spatial === 1) {
384
+ // no suffix for L1Tx
385
+ mode.suffix = undefined;
385
386
  }
387
+ @ts-ignore
388
+ encoding.scalabilityMode = mode.toString();
389
+ encoding.scaleResolutionDownBy = 2 ** (2 - maxQuality);
390
+ */
386
391
  }
387
- });
392
+ } else {
393
+ // simulcast dynacast encodings
394
+ encodings.forEach((encoding, idx) => {
395
+ let rid = encoding.rid ?? '';
396
+ if (rid === '') {
397
+ rid = 'q';
398
+ }
399
+ const quality = videoQualityForRid(rid);
400
+ const subscribedQuality = qualities.find((q) => q.quality === quality);
401
+ if (!subscribedQuality) {
402
+ return;
403
+ }
404
+ if (encoding.active !== subscribedQuality.enabled) {
405
+ hasChanged = true;
406
+ encoding.active = subscribedQuality.enabled;
407
+ log.debug(
408
+ `setting layer ${subscribedQuality.quality} to ${
409
+ encoding.active ? 'enabled' : 'disabled'
410
+ }`,
411
+ );
412
+
413
+ // FireFox does not support setting encoding.active to false, so we
414
+ // have a workaround of lowering its bitrate and resolution to the min.
415
+ if (isFireFox()) {
416
+ if (subscribedQuality.enabled) {
417
+ encoding.scaleResolutionDownBy = senderEncodings[idx].scaleResolutionDownBy;
418
+ encoding.maxBitrate = senderEncodings[idx].maxBitrate;
419
+ /* @ts-ignore */
420
+ encoding.maxFrameRate = senderEncodings[idx].maxFrameRate;
421
+ } else {
422
+ encoding.scaleResolutionDownBy = 4;
423
+ encoding.maxBitrate = 10;
424
+ /* @ts-ignore */
425
+ encoding.maxFrameRate = 2;
426
+ }
427
+ }
428
+ }
429
+ });
430
+ }
388
431
 
389
432
  if (hasChanged) {
390
433
  params.encodings = encodings;
434
+ log.debug(`setting encodings`, params.encodings);
391
435
  await sender.setParameters(params);
392
436
  }
393
437
  } finally {
@@ -425,6 +469,25 @@ export function videoLayersFromEncodings(
425
469
  },
426
470
  ];
427
471
  }
472
+
473
+ /* @ts-ignore */
474
+ if (encodings.length === 1 && encodings[0].scalabilityMode) {
475
+ // svc layers
476
+ /* @ts-ignore */
477
+ const sm = new ScalabilityMode(encodings[0].scalabilityMode);
478
+ const layers = [];
479
+ for (let i = 0; i < sm.spatial; i += 1) {
480
+ layers.push({
481
+ quality: VideoQuality.HIGH - i,
482
+ width: width / 2 ** i,
483
+ height: height / 2 ** i,
484
+ bitrate: encodings[0].maxBitrate ? encodings[0].maxBitrate / 3 ** i : 0,
485
+ ssrc: 0,
486
+ });
487
+ }
488
+ return layers;
489
+ }
490
+
428
491
  return encodings.map((encoding) => {
429
492
  const scale = encoding.scaleResolutionDownBy ?? 1;
430
493
  let quality = videoQualityForRid(encoding.rid ?? '');
@@ -1,5 +1,5 @@
1
1
  import log from '../../logger';
2
- import { TrackInfo, VideoQuality } from '../../proto/livekit_models';
2
+ import { SubscriptionError, TrackInfo, VideoQuality } from '../../proto/livekit_models';
3
3
  import { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
4
4
  import { TrackEvent } from '../events';
5
5
  import type RemoteTrack from './RemoteTrack';
@@ -24,6 +24,8 @@ export default class RemoteTrackPublication extends TrackPublication {
24
24
 
25
25
  protected fps?: number;
26
26
 
27
+ protected subscriptionError?: SubscriptionError;
28
+
27
29
  constructor(kind: Track.Kind, ti: TrackInfo, autoSubscribe: boolean | undefined) {
28
30
  super(kind, ti.sid, ti.name);
29
31
  this.subscribed = autoSubscribe;
@@ -205,6 +207,11 @@ export default class RemoteTrackPublication extends TrackPublication {
205
207
  this.emitSubscriptionUpdateIfChanged(prevStatus);
206
208
  }
207
209
 
210
+ /** @internal */
211
+ setSubscriptionError(error: SubscriptionError) {
212
+ this.emit(TrackEvent.SubscriptionFailed, error);
213
+ }
214
+
208
215
  /** @internal */
209
216
  updateInfo(info: TrackInfo) {
210
217
  super.updateInfo(info);
@@ -118,7 +118,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
118
118
  // even if we believe it's already attached to the element, it's possible
119
119
  // the element's srcObject was set to something else out of band.
120
120
  // we'll want to re-attach it in that case
121
- attachToElement(this._mediaStreamTrack, element);
121
+ attachToElement(this.mediaStreamTrack, element);
122
122
 
123
123
  // handle auto playback failures
124
124
  const allMediaStreamTracks = (element.srcObject as MediaStream).getTracks();
@@ -167,7 +167,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
167
167
  try {
168
168
  // detach from a single element
169
169
  if (element) {
170
- detachTrack(this._mediaStreamTrack, element);
170
+ detachTrack(this.mediaStreamTrack, element);
171
171
  const idx = this.attachedElements.indexOf(element);
172
172
  if (idx >= 0) {
173
173
  this.attachedElements.splice(idx, 1);
@@ -179,7 +179,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
179
179
 
180
180
  const detached: HTMLMediaElement[] = [];
181
181
  this.attachedElements.forEach((elm) => {
182
- detachTrack(this._mediaStreamTrack, elm);
182
+ detachTrack(this.mediaStreamTrack, elm);
183
183
  detached.push(elm);
184
184
  this.recycleElement(elm);
185
185
  this.emit(TrackEvent.ElementDetached, elm);