livekit-client 1.9.6 → 1.10.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. package/dist/livekit-client.esm.mjs +1318 -885
  2. package/dist/livekit-client.esm.mjs.map +1 -1
  3. package/dist/livekit-client.umd.js +1 -1
  4. package/dist/livekit-client.umd.js.map +1 -1
  5. package/dist/src/api/SignalClient.d.ts +2 -1
  6. package/dist/src/api/SignalClient.d.ts.map +1 -1
  7. package/dist/src/index.d.ts +1 -0
  8. package/dist/src/index.d.ts.map +1 -1
  9. package/dist/src/proto/livekit_models.d.ts +108 -10
  10. package/dist/src/proto/livekit_models.d.ts.map +1 -1
  11. package/dist/src/proto/livekit_rtc.d.ts +513 -194
  12. package/dist/src/proto/livekit_rtc.d.ts.map +1 -1
  13. package/dist/src/room/Room.d.ts +3 -2
  14. package/dist/src/room/Room.d.ts.map +1 -1
  15. package/dist/src/room/events.d.ts +5 -1
  16. package/dist/src/room/events.d.ts.map +1 -1
  17. package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
  18. package/dist/src/room/participant/Participant.d.ts +2 -2
  19. package/dist/src/room/participant/Participant.d.ts.map +1 -1
  20. package/dist/src/room/participant/RemoteParticipant.d.ts.map +1 -1
  21. package/dist/src/room/participant/publishUtils.d.ts +8 -0
  22. package/dist/src/room/participant/publishUtils.d.ts.map +1 -1
  23. package/dist/src/room/track/LocalTrack.d.ts +32 -0
  24. package/dist/src/room/track/LocalTrack.d.ts.map +1 -1
  25. package/dist/src/room/track/LocalVideoTrack.d.ts.map +1 -1
  26. package/dist/src/room/track/RemoteTrackPublication.d.ts +4 -1
  27. package/dist/src/room/track/RemoteTrackPublication.d.ts.map +1 -1
  28. package/dist/src/room/track/TrackPublication.d.ts +2 -1
  29. package/dist/src/room/track/TrackPublication.d.ts.map +1 -1
  30. package/dist/src/room/track/options.d.ts +1 -1
  31. package/dist/src/room/track/options.d.ts.map +1 -1
  32. package/dist/src/room/track/processor/types.d.ts +19 -0
  33. package/dist/src/room/track/processor/types.d.ts.map +1 -0
  34. package/dist/src/utils/browserParser.d.ts.map +1 -1
  35. package/dist/ts4.2/src/api/SignalClient.d.ts +2 -1
  36. package/dist/ts4.2/src/index.d.ts +1 -0
  37. package/dist/ts4.2/src/proto/livekit_models.d.ts +126 -12
  38. package/dist/ts4.2/src/proto/livekit_rtc.d.ts +617 -254
  39. package/dist/ts4.2/src/room/Room.d.ts +3 -2
  40. package/dist/ts4.2/src/room/events.d.ts +5 -1
  41. package/dist/ts4.2/src/room/participant/Participant.d.ts +2 -2
  42. package/dist/ts4.2/src/room/participant/publishUtils.d.ts +8 -0
  43. package/dist/ts4.2/src/room/track/LocalTrack.d.ts +32 -0
  44. package/dist/ts4.2/src/room/track/RemoteTrackPublication.d.ts +4 -1
  45. package/dist/ts4.2/src/room/track/TrackPublication.d.ts +2 -1
  46. package/dist/ts4.2/src/room/track/options.d.ts +1 -1
  47. package/dist/ts4.2/src/room/track/processor/types.d.ts +19 -0
  48. package/package.json +14 -13
  49. package/src/api/SignalClient.ts +8 -1
  50. package/src/index.ts +1 -0
  51. package/src/proto/google/protobuf/timestamp.ts +3 -3
  52. package/src/proto/livekit_models.ts +254 -161
  53. package/src/proto/livekit_rtc.ts +334 -180
  54. package/src/room/Room.ts +26 -1
  55. package/src/room/events.ts +4 -0
  56. package/src/room/participant/LocalParticipant.ts +23 -3
  57. package/src/room/participant/Participant.ts +2 -1
  58. package/src/room/participant/RemoteParticipant.ts +4 -1
  59. package/src/room/participant/publishUtils.ts +68 -12
  60. package/src/room/track/LocalTrack.ts +120 -16
  61. package/src/room/track/LocalVideoTrack.ts +96 -33
  62. package/src/room/track/RemoteTrackPublication.ts +8 -1
  63. package/src/room/track/Track.ts +3 -3
  64. package/src/room/track/TrackPublication.ts +2 -1
  65. package/src/room/track/options.ts +1 -1
  66. package/src/room/track/processor/types.ts +20 -0
  67. package/src/utils/browserParser.ts +1 -4
package/src/room/Room.ts CHANGED
@@ -18,6 +18,7 @@ import {
18
18
  Room as RoomModel,
19
19
  ServerInfo,
20
20
  SpeakerInfo,
21
+ SubscriptionError,
21
22
  TrackInfo,
22
23
  TrackSource,
23
24
  TrackType,
@@ -29,6 +30,7 @@ import {
29
30
  SimulateScenario,
30
31
  StreamStateUpdate,
31
32
  SubscriptionPermissionUpdate,
33
+ SubscriptionResponse,
32
34
  } from '../proto/livekit_rtc';
33
35
  import DeviceManager from './DeviceManager';
34
36
  import RTCEngine from './RTCEngine';
@@ -207,6 +209,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
207
209
  this.engine.client.onStreamStateUpdate = this.handleStreamStateUpdate;
208
210
  this.engine.client.onSubscriptionPermissionUpdate = this.handleSubscriptionPermissionUpdate;
209
211
  this.engine.client.onConnectionQuality = this.handleConnectionQualityUpdate;
212
+ this.engine.client.onSubscriptionError = this.handleSubscriptionError;
210
213
 
211
214
  this.engine
212
215
  .on(
@@ -1114,6 +1117,21 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
1114
1117
  pub.setAllowed(update.allowed);
1115
1118
  };
1116
1119
 
1120
+ private handleSubscriptionError = (update: SubscriptionResponse) => {
1121
+ const participant = Array.from(this.participants.values()).find((p) =>
1122
+ p.tracks.has(update.trackSid),
1123
+ );
1124
+ if (!participant) {
1125
+ return;
1126
+ }
1127
+ const pub = participant.getTrackPublication(update.trackSid);
1128
+ if (!pub) {
1129
+ return;
1130
+ }
1131
+
1132
+ pub.setSubscriptionError(update.err);
1133
+ };
1134
+
1117
1135
  private handleDataPacket = (userPacket: UserPacket, kind: DataPacket_Kind) => {
1118
1136
  // find the participant
1119
1137
  const participant = this.participants.get(userPacket.participantSid);
@@ -1280,6 +1298,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
1280
1298
  .on(ParticipantEvent.TrackSubscriptionStatusChanged, (pub, status) => {
1281
1299
  this.emitWhenConnected(RoomEvent.TrackSubscriptionStatusChanged, pub, status, participant);
1282
1300
  })
1301
+ .on(ParticipantEvent.TrackSubscriptionFailed, (trackSid, error) => {
1302
+ this.emit(RoomEvent.TrackSubscriptionFailed, trackSid, participant, error);
1303
+ })
1283
1304
  .on(ParticipantEvent.TrackSubscriptionPermissionChanged, (pub, status) => {
1284
1305
  this.emitWhenConnected(
1285
1306
  RoomEvent.TrackSubscriptionPermissionChanged,
@@ -1609,7 +1630,11 @@ export type RoomEventCallbacks = {
1609
1630
  publication: RemoteTrackPublication,
1610
1631
  participant: RemoteParticipant,
1611
1632
  ) => void;
1612
- trackSubscriptionFailed: (trackSid: string, participant: RemoteParticipant) => void;
1633
+ trackSubscriptionFailed: (
1634
+ trackSid: string,
1635
+ participant: RemoteParticipant,
1636
+ reason?: SubscriptionError,
1637
+ ) => void;
1613
1638
  trackUnpublished: (publication: RemoteTrackPublication, participant: RemoteParticipant) => void;
1614
1639
  trackUnsubscribed: (
1615
1640
  track: RemoteTrack,
@@ -508,4 +508,8 @@ export enum TrackEvent {
508
508
  * Fires on RemoteTrackPublication
509
509
  */
510
510
  SubscriptionStatusChanged = 'subscriptionStatusChanged',
511
+ /**
512
+ * Fires on RemoteTrackPublication
513
+ */
514
+ SubscriptionFailed = 'subscriptionFailed',
511
515
  }
@@ -625,8 +625,8 @@ export default class LocalParticipant extends Participant {
625
625
  // for svc codecs, disable simulcast and use vp8 for backup codec
626
626
  if (track instanceof LocalVideoTrack) {
627
627
  if (isSVCCodec(opts.videoCodec)) {
628
- // set scalabilityMode to 'L3T3' by default
629
- opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
628
+ // set scalabilityMode to 'L3T3_KEY' by default
629
+ opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY';
630
630
  }
631
631
 
632
632
  // set up backup
@@ -647,6 +647,16 @@ export default class LocalParticipant extends Participant {
647
647
  enableSimulcastLayers: true,
648
648
  },
649
649
  ];
650
+ } else if (opts.videoCodec) {
651
+ // pass codec info to sfu so it can prefer codec for the client which don't support
652
+ // setCodecPreferences
653
+ req.simulcastCodecs = [
654
+ {
655
+ codec: opts.videoCodec,
656
+ cid: track.mediaStreamTrack.id,
657
+ enableSimulcastLayers: opts.simulcast ?? false,
658
+ },
659
+ ];
650
660
  }
651
661
  }
652
662
 
@@ -656,7 +666,7 @@ export default class LocalParticipant extends Participant {
656
666
  dims.height,
657
667
  opts,
658
668
  );
659
- req.layers = videoLayersFromEncodings(req.width, req.height, simEncodings ?? encodings);
669
+ req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
660
670
  } else if (track.kind === Track.Kind.Audio) {
661
671
  encodings = [
662
672
  {
@@ -850,6 +860,16 @@ export default class LocalParticipant extends Participant {
850
860
  trackSender
851
861
  ) {
852
862
  try {
863
+ for (const transceiver of this.engine.publisher.pc.getTransceivers()) {
864
+ // if sender is not currently sending (after replaceTrack(null))
865
+ // removeTrack would have no effect.
866
+ // to ensure we end up successfully removing the track, manually set
867
+ // the transceiver to inactive
868
+ if (transceiver.sender === trackSender) {
869
+ transceiver.direction = 'inactive';
870
+ negotiationNeeded = true;
871
+ }
872
+ }
853
873
  if (this.engine.removeTrack(trackSender)) {
854
874
  negotiationNeeded = true;
855
875
  }
@@ -6,6 +6,7 @@ import {
6
6
  ParticipantInfo,
7
7
  ParticipantPermission,
8
8
  ConnectionQuality as ProtoQuality,
9
+ SubscriptionError,
9
10
  } from '../../proto/livekit_models';
10
11
  import { ParticipantEvent, TrackEvent } from '../events';
11
12
  import type LocalTrackPublication from '../track/LocalTrackPublication';
@@ -265,7 +266,7 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
265
266
  export type ParticipantEventCallbacks = {
266
267
  trackPublished: (publication: RemoteTrackPublication) => void;
267
268
  trackSubscribed: (track: RemoteTrack, publication: RemoteTrackPublication) => void;
268
- trackSubscriptionFailed: (trackSid: string) => void;
269
+ trackSubscriptionFailed: (trackSid: string, reason?: SubscriptionError) => void;
269
270
  trackUnpublished: (publication: RemoteTrackPublication) => void;
270
271
  trackUnsubscribed: (track: RemoteTrack, publication: RemoteTrackPublication) => void;
271
272
  trackMuted: (publication: TrackPublication) => void;
@@ -1,6 +1,6 @@
1
1
  import type { SignalClient } from '../../api/SignalClient';
2
2
  import log from '../../logger';
3
- import type { ParticipantInfo } from '../../proto/livekit_models';
3
+ import type { ParticipantInfo, SubscriptionError } from '../../proto/livekit_models';
4
4
  import type { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
5
5
  import { ParticipantEvent, TrackEvent } from '../events';
6
6
  import RemoteAudioTrack from '../track/RemoteAudioTrack';
@@ -81,6 +81,9 @@ export default class RemoteParticipant extends Participant {
81
81
  publication.on(TrackEvent.Unsubscribed, (previousTrack: RemoteTrack) => {
82
82
  this.emit(ParticipantEvent.TrackUnsubscribed, previousTrack, publication);
83
83
  });
84
+ publication.on(TrackEvent.SubscriptionFailed, (error: SubscriptionError) => {
85
+ this.emit(ParticipantEvent.TrackSubscriptionFailed, publication.trackSid, error);
86
+ });
84
87
  }
85
88
 
86
89
  getTrack(source: Track.Source): RemoteTrackPublication | undefined {
@@ -10,7 +10,7 @@ import type {
10
10
  VideoCodec,
11
11
  VideoEncoding,
12
12
  } from '../track/options';
13
- import { isSVCCodec } from '../utils';
13
+ import { getReactNativeOs, isReactNative, isSVCCodec } from '../utils';
14
14
 
15
15
  /** @internal */
16
16
  export function mediaTrackToLocalTrack(
@@ -128,17 +128,15 @@ export function computeVideoEncodings(
128
128
  // svc use first encoding as the original, so we sort encoding from high to low
129
129
  switch (scalabilityMode) {
130
130
  case 'L3T3':
131
- for (let i = 0; i < 3; i += 1) {
132
- encodings.push({
133
- rid: videoRids[2 - i],
134
- scaleResolutionDownBy: 2 ** i,
135
- maxBitrate: videoEncoding.maxBitrate / 3 ** i,
136
- /* @ts-ignore */
137
- maxFramerate: original.encoding.maxFramerate,
138
- /* @ts-ignore */
139
- scalabilityMode: 'L3T3',
140
- });
141
- }
131
+ case 'L3T3_KEY':
132
+ encodings.push({
133
+ rid: videoRids[2],
134
+ maxBitrate: videoEncoding.maxBitrate,
135
+ /* @ts-ignore */
136
+ maxFramerate: original.encoding.maxFramerate,
137
+ /* @ts-ignore */
138
+ scalabilityMode: scalabilityMode,
139
+ });
142
140
  log.debug('encodings', encodings);
143
141
  return encodings;
144
142
 
@@ -321,6 +319,33 @@ function encodingsFromPresets(
321
319
  }
322
320
  encodings.push(encoding);
323
321
  });
322
+
323
+ // RN ios simulcast requires all same framerates.
324
+ if (isReactNative() && getReactNativeOs() === 'ios') {
325
+ let topFramerate: number | undefined = undefined;
326
+ encodings.forEach((encoding) => {
327
+ if (!topFramerate) {
328
+ topFramerate = encoding.maxFramerate;
329
+ } else if (encoding.maxFramerate && encoding.maxFramerate > topFramerate) {
330
+ topFramerate = encoding.maxFramerate;
331
+ }
332
+ });
333
+
334
+ let notifyOnce = true;
335
+ encodings.forEach((encoding) => {
336
+ if (encoding.maxFramerate != topFramerate) {
337
+ if (notifyOnce) {
338
+ notifyOnce = false;
339
+ log.info(
340
+ `Simulcast on iOS React-Native requires all encodings to share the same framerate.`,
341
+ );
342
+ }
343
+ log.info(`Setting framerate of encoding \"${encoding.rid ?? ''}\" to ${topFramerate}`);
344
+ encoding.maxFramerate = topFramerate;
345
+ }
346
+ });
347
+ }
348
+
324
349
  return encodings;
325
350
  }
326
351
 
@@ -341,3 +366,34 @@ export function sortPresets(presets: Array<VideoPreset> | undefined) {
341
366
  return 0;
342
367
  });
343
368
  }
369
+
370
+ /** @internal */
371
+ export class ScalabilityMode {
372
+ spatial: number;
373
+
374
+ temporal: number;
375
+
376
+ suffix: undefined | 'h' | '_KEY' | '_KEY_SHIFT';
377
+
378
+ constructor(scalabilityMode: string) {
379
+ const results = scalabilityMode.match(/^L(\d)T(\d)(h|_KEY|_KEY_SHIFT){0,1}$/);
380
+ if (!results) {
381
+ throw new Error('invalid scalability mode');
382
+ }
383
+
384
+ this.spatial = parseInt(results[1]);
385
+ this.temporal = parseInt(results[2]);
386
+ if (results.length > 3) {
387
+ switch (results[3]) {
388
+ case 'h':
389
+ case '_KEY':
390
+ case '_KEY_SHIFT':
391
+ this.suffix = results[3];
392
+ }
393
+ }
394
+ }
395
+
396
+ toString(): string {
397
+ return `L${this.spatial}T${this.temporal}${this.suffix ?? ''}`;
398
+ }
399
+ }
@@ -1,16 +1,12 @@
1
1
  import log from '../../logger';
2
+ import { getBrowser } from '../../utils/browserParser';
2
3
  import DeviceManager from '../DeviceManager';
3
- import { TrackInvalidError } from '../errors';
4
+ import { DeviceUnsupportedError, TrackInvalidError } from '../errors';
4
5
  import { TrackEvent } from '../events';
5
- import {
6
- Mutex,
7
- getEmptyAudioStreamTrack,
8
- getEmptyVideoStreamTrack,
9
- isMobile,
10
- sleep,
11
- } from '../utils';
6
+ import { Mutex, compareVersions, isMobile, sleep } from '../utils';
12
7
  import { Track, attachToElement, detachTrack } from './Track';
13
8
  import type { VideoCodec } from './options';
9
+ import type { TrackProcessor } from './processor/types';
14
10
 
15
11
  const defaultDimensionsTimeout = 1000;
16
12
 
@@ -31,6 +27,12 @@ export default abstract class LocalTrack extends Track {
31
27
 
32
28
  protected pauseUpstreamLock: Mutex;
33
29
 
30
+ protected processorElement?: HTMLMediaElement;
31
+
32
+ protected processor?: TrackProcessor<typeof this.kind>;
33
+
34
+ protected isSettingUpProcessor: boolean = false;
35
+
34
36
  /**
35
37
  *
36
38
  * @param mediaTrack
@@ -82,6 +84,10 @@ export default abstract class LocalTrack extends Track {
82
84
  return this.providedByUser;
83
85
  }
84
86
 
87
+ get mediaStreamTrack() {
88
+ return this.processor?.processedTrack ?? this._mediaStreamTrack;
89
+ }
90
+
85
91
  async waitForDimensions(timeout = defaultDimensionsTimeout): Promise<Track.Dimensions> {
86
92
  if (this.kind === Track.Kind.Audio) {
87
93
  throw new Error('cannot get dimensions for audio tracks');
@@ -158,6 +164,9 @@ export default abstract class LocalTrack extends Track {
158
164
 
159
165
  this.mediaStream = new MediaStream([track]);
160
166
  this.providedByUser = userProvidedTrack;
167
+ if (this.processor) {
168
+ await this.stopProcessor();
169
+ }
161
170
  return this;
162
171
  }
163
172
 
@@ -180,7 +189,7 @@ export default abstract class LocalTrack extends Track {
180
189
 
181
190
  // detach
182
191
  this.attachedElements.forEach((el) => {
183
- detachTrack(this._mediaStreamTrack, el);
192
+ detachTrack(this.mediaStreamTrack, el);
184
193
  });
185
194
  this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
186
195
  // on Safari, the old audio track must be stopped before attempting to acquire
@@ -203,12 +212,16 @@ export default abstract class LocalTrack extends Track {
203
212
 
204
213
  await this.resumeUpstream();
205
214
 
206
- this.attachedElements.forEach((el) => {
207
- attachToElement(newTrack, el);
208
- });
209
-
210
215
  this.mediaStream = mediaStream;
211
216
  this.constraints = constraints;
217
+ if (this.processor) {
218
+ const processor = this.processor;
219
+ await this.setProcessor(processor);
220
+ } else {
221
+ this.attachedElements.forEach((el) => {
222
+ attachToElement(this._mediaStreamTrack, el);
223
+ });
224
+ }
212
225
  this.emit(TrackEvent.Restarted, this);
213
226
  return this;
214
227
  }
@@ -253,6 +266,18 @@ export default abstract class LocalTrack extends Track {
253
266
  this.emit(TrackEvent.Ended, this);
254
267
  };
255
268
 
269
+ stop() {
270
+ super.stop();
271
+ this.processor?.destroy();
272
+ this.processor = undefined;
273
+ }
274
+
275
+ /**
276
+ * pauses publishing to the server without disabling the local MediaStreamTrack
277
+ * this is used to display a user's own video locally while pausing publishing to
278
+ * the server.
279
+ * this API is unsupported on Safari < 12 due to a bug
280
+ **/
256
281
  async pauseUpstream() {
257
282
  const unlock = await this.pauseUpstreamLock.lock();
258
283
  try {
@@ -266,9 +291,12 @@ export default abstract class LocalTrack extends Track {
266
291
 
267
292
  this._isUpstreamPaused = true;
268
293
  this.emit(TrackEvent.UpstreamPaused, this);
269
- const emptyTrack =
270
- this.kind === Track.Kind.Audio ? getEmptyAudioStreamTrack() : getEmptyVideoStreamTrack();
271
- await this.sender.replaceTrack(emptyTrack);
294
+ const browser = getBrowser();
295
+ if (browser?.name === 'Safari' && compareVersions(browser.version, '12.0') < 0) {
296
+ // https://bugs.webkit.org/show_bug.cgi?id=184911
297
+ throw new DeviceUnsupportedError('pauseUpstream is not supported on Safari < 12.');
298
+ }
299
+ await this.sender.replaceTrack(null);
272
300
  } finally {
273
301
  unlock();
274
302
  }
@@ -293,5 +321,81 @@ export default abstract class LocalTrack extends Track {
293
321
  }
294
322
  }
295
323
 
324
+ /**
325
+ * Sets a processor on this track.
326
+ * See https://github.com/livekit/track-processors-js for example usage
327
+ *
328
+ * @experimental
329
+ *
330
+ * @param processor
331
+ * @param showProcessedStreamLocally
332
+ * @returns
333
+ */
334
+ async setProcessor(
335
+ processor: TrackProcessor<typeof this.kind>,
336
+ showProcessedStreamLocally = true,
337
+ ) {
338
+ if (this.isSettingUpProcessor) {
339
+ log.warn('already trying to set up a processor');
340
+ return;
341
+ }
342
+ log.debug('setting up processor');
343
+ this.isSettingUpProcessor = true;
344
+ if (this.processor) {
345
+ await this.stopProcessor();
346
+ }
347
+ if (this.kind === 'unknown') {
348
+ throw TypeError('cannot set processor on track of unknown kind');
349
+ }
350
+ this.processorElement = this.processorElement ?? document.createElement(this.kind);
351
+ this.processorElement.muted = true;
352
+
353
+ attachToElement(this._mediaStreamTrack, this.processorElement);
354
+ this.processorElement.play().catch((e) => log.error(e));
355
+
356
+ const processorOptions = {
357
+ kind: this.kind,
358
+ track: this._mediaStreamTrack,
359
+ element: this.processorElement,
360
+ };
361
+
362
+ await processor.init(processorOptions);
363
+ this.processor = processor;
364
+ if (this.processor.processedTrack) {
365
+ for (const el of this.attachedElements) {
366
+ if (el !== this.processorElement && showProcessedStreamLocally) {
367
+ detachTrack(this._mediaStreamTrack, el);
368
+ attachToElement(this.processor.processedTrack, el);
369
+ }
370
+ }
371
+ await this.sender?.replaceTrack(this.processor.processedTrack);
372
+ }
373
+ this.isSettingUpProcessor = false;
374
+ }
375
+
376
+ getProcessor() {
377
+ return this.processor;
378
+ }
379
+
380
+ /**
381
+ * Stops the track processor
382
+ * See https://github.com/livekit/track-processors-js for example usage
383
+ *
384
+ * @experimental
385
+ * @returns
386
+ */
387
+ async stopProcessor() {
388
+ if (!this.processor) return;
389
+
390
+ log.debug('stopping processor');
391
+ this.processor.processedTrack?.stop();
392
+ await this.processor.destroy();
393
+ this.processor = undefined;
394
+ this.processorElement?.remove();
395
+ this.processorElement = undefined;
396
+
397
+ await this.restart();
398
+ }
399
+
296
400
  protected abstract monitorSender(): void;
297
401
  }
@@ -2,6 +2,7 @@ import type { SignalClient } from '../../api/SignalClient';
2
2
  import log from '../../logger';
3
3
  import { VideoLayer, VideoQuality } from '../../proto/livekit_models';
4
4
  import type { SubscribedCodec, SubscribedQuality } from '../../proto/livekit_rtc';
5
+ import { ScalabilityMode } from '../participant/publishUtils';
5
6
  import { computeBitrate, monitorFrequency } from '../stats';
6
7
  import type { VideoSenderStats } from '../stats';
7
8
  import { Mutex, isFireFox, isMobile, isWeb } from '../utils';
@@ -349,45 +350,88 @@ async function setPublishingLayersForSender(
349
350
  }
350
351
 
351
352
  let hasChanged = false;
352
- encodings.forEach((encoding, idx) => {
353
- let rid = encoding.rid ?? '';
354
- if (rid === '') {
355
- rid = 'q';
356
- }
357
- const quality = videoQualityForRid(rid);
358
- const subscribedQuality = qualities.find((q) => q.quality === quality);
359
- if (!subscribedQuality) {
360
- return;
361
- }
362
- if (encoding.active !== subscribedQuality.enabled) {
353
+
354
+ /* @ts-ignore */
355
+ if (encodings.length === 1 && encodings[0].scalabilityMode) {
356
+ // svc dynacast encodings
357
+ const encoding = encodings[0];
358
+ /* @ts-ignore */
359
+ // const mode = new ScalabilityMode(encoding.scalabilityMode);
360
+ let maxQuality = VideoQuality.OFF;
361
+ qualities.forEach((q) => {
362
+ if (q.enabled && (maxQuality === VideoQuality.OFF || q.quality > maxQuality)) {
363
+ maxQuality = q.quality;
364
+ }
365
+ });
366
+
367
+ if (maxQuality === VideoQuality.OFF) {
368
+ if (encoding.active) {
369
+ encoding.active = false;
370
+ hasChanged = true;
371
+ }
372
+ } else if (!encoding.active /* || mode.spatial !== maxQuality + 1*/) {
363
373
  hasChanged = true;
364
- encoding.active = subscribedQuality.enabled;
365
- log.debug(
366
- `setting layer ${subscribedQuality.quality} to ${
367
- encoding.active ? 'enabled' : 'disabled'
368
- }`,
369
- );
370
-
371
- // FireFox does not support setting encoding.active to false, so we
372
- // have a workaround of lowering its bitrate and resolution to the min.
373
- if (isFireFox()) {
374
- if (subscribedQuality.enabled) {
375
- encoding.scaleResolutionDownBy = senderEncodings[idx].scaleResolutionDownBy;
376
- encoding.maxBitrate = senderEncodings[idx].maxBitrate;
377
- /* @ts-ignore */
378
- encoding.maxFrameRate = senderEncodings[idx].maxFrameRate;
379
- } else {
380
- encoding.scaleResolutionDownBy = 4;
381
- encoding.maxBitrate = 10;
382
- /* @ts-ignore */
383
- encoding.maxFrameRate = 2;
384
- }
374
+ encoding.active = true;
375
+ /* disable closable spatial layer as it has video blur/frozen issue with current server/client
376
+ 1. chrome 113: when switching to up layer with scalability Mode change, it will generate a
377
+ low resolution frame and recover very quickly, but noticable
378
+ 2. livekit sfu: additional pli request cause video frozen for a few frames, also noticable
379
+ @ts-ignore
380
+ const originalMode = new ScalabilityMode(senderEncodings[0].scalabilityMode)
381
+ mode.spatial = maxQuality + 1;
382
+ mode.suffix = originalMode.suffix;
383
+ if (mode.spatial === 1) {
384
+ // no suffix for L1Tx
385
+ mode.suffix = undefined;
385
386
  }
387
+ @ts-ignore
388
+ encoding.scalabilityMode = mode.toString();
389
+ encoding.scaleResolutionDownBy = 2 ** (2 - maxQuality);
390
+ */
386
391
  }
387
- });
392
+ } else {
393
+ // simulcast dynacast encodings
394
+ encodings.forEach((encoding, idx) => {
395
+ let rid = encoding.rid ?? '';
396
+ if (rid === '') {
397
+ rid = 'q';
398
+ }
399
+ const quality = videoQualityForRid(rid);
400
+ const subscribedQuality = qualities.find((q) => q.quality === quality);
401
+ if (!subscribedQuality) {
402
+ return;
403
+ }
404
+ if (encoding.active !== subscribedQuality.enabled) {
405
+ hasChanged = true;
406
+ encoding.active = subscribedQuality.enabled;
407
+ log.debug(
408
+ `setting layer ${subscribedQuality.quality} to ${
409
+ encoding.active ? 'enabled' : 'disabled'
410
+ }`,
411
+ );
412
+
413
+ // FireFox does not support setting encoding.active to false, so we
414
+ // have a workaround of lowering its bitrate and resolution to the min.
415
+ if (isFireFox()) {
416
+ if (subscribedQuality.enabled) {
417
+ encoding.scaleResolutionDownBy = senderEncodings[idx].scaleResolutionDownBy;
418
+ encoding.maxBitrate = senderEncodings[idx].maxBitrate;
419
+ /* @ts-ignore */
420
+ encoding.maxFrameRate = senderEncodings[idx].maxFrameRate;
421
+ } else {
422
+ encoding.scaleResolutionDownBy = 4;
423
+ encoding.maxBitrate = 10;
424
+ /* @ts-ignore */
425
+ encoding.maxFrameRate = 2;
426
+ }
427
+ }
428
+ }
429
+ });
430
+ }
388
431
 
389
432
  if (hasChanged) {
390
433
  params.encodings = encodings;
434
+ log.debug(`setting encodings`, params.encodings);
391
435
  await sender.setParameters(params);
392
436
  }
393
437
  } finally {
@@ -425,6 +469,25 @@ export function videoLayersFromEncodings(
425
469
  },
426
470
  ];
427
471
  }
472
+
473
+ /* @ts-ignore */
474
+ if (encodings.length === 1 && encodings[0].scalabilityMode) {
475
+ // svc layers
476
+ /* @ts-ignore */
477
+ const sm = new ScalabilityMode(encodings[0].scalabilityMode);
478
+ const layers = [];
479
+ for (let i = 0; i < sm.spatial; i += 1) {
480
+ layers.push({
481
+ quality: VideoQuality.HIGH - i,
482
+ width: width / 2 ** i,
483
+ height: height / 2 ** i,
484
+ bitrate: encodings[0].maxBitrate ? encodings[0].maxBitrate / 3 ** i : 0,
485
+ ssrc: 0,
486
+ });
487
+ }
488
+ return layers;
489
+ }
490
+
428
491
  return encodings.map((encoding) => {
429
492
  const scale = encoding.scaleResolutionDownBy ?? 1;
430
493
  let quality = videoQualityForRid(encoding.rid ?? '');
@@ -1,5 +1,5 @@
1
1
  import log from '../../logger';
2
- import { TrackInfo, VideoQuality } from '../../proto/livekit_models';
2
+ import { SubscriptionError, TrackInfo, VideoQuality } from '../../proto/livekit_models';
3
3
  import { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
4
4
  import { TrackEvent } from '../events';
5
5
  import type RemoteTrack from './RemoteTrack';
@@ -24,6 +24,8 @@ export default class RemoteTrackPublication extends TrackPublication {
24
24
 
25
25
  protected fps?: number;
26
26
 
27
+ protected subscriptionError?: SubscriptionError;
28
+
27
29
  constructor(kind: Track.Kind, ti: TrackInfo, autoSubscribe: boolean | undefined) {
28
30
  super(kind, ti.sid, ti.name);
29
31
  this.subscribed = autoSubscribe;
@@ -205,6 +207,11 @@ export default class RemoteTrackPublication extends TrackPublication {
205
207
  this.emitSubscriptionUpdateIfChanged(prevStatus);
206
208
  }
207
209
 
210
+ /** @internal */
211
+ setSubscriptionError(error: SubscriptionError) {
212
+ this.emit(TrackEvent.SubscriptionFailed, error);
213
+ }
214
+
208
215
  /** @internal */
209
216
  updateInfo(info: TrackInfo) {
210
217
  super.updateInfo(info);
@@ -118,7 +118,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
118
118
  // even if we believe it's already attached to the element, it's possible
119
119
  // the element's srcObject was set to something else out of band.
120
120
  // we'll want to re-attach it in that case
121
- attachToElement(this._mediaStreamTrack, element);
121
+ attachToElement(this.mediaStreamTrack, element);
122
122
 
123
123
  // handle auto playback failures
124
124
  const allMediaStreamTracks = (element.srcObject as MediaStream).getTracks();
@@ -167,7 +167,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
167
167
  try {
168
168
  // detach from a single element
169
169
  if (element) {
170
- detachTrack(this._mediaStreamTrack, element);
170
+ detachTrack(this.mediaStreamTrack, element);
171
171
  const idx = this.attachedElements.indexOf(element);
172
172
  if (idx >= 0) {
173
173
  this.attachedElements.splice(idx, 1);
@@ -179,7 +179,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
179
179
 
180
180
  const detached: HTMLMediaElement[] = [];
181
181
  this.attachedElements.forEach((elm) => {
182
- detachTrack(this._mediaStreamTrack, elm);
182
+ detachTrack(this.mediaStreamTrack, elm);
183
183
  detached.push(elm);
184
184
  this.recycleElement(elm);
185
185
  this.emit(TrackEvent.ElementDetached, elm);