livekit-client 0.16.5 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/dist/api/RequestQueue.js +6 -6
  2. package/dist/api/RequestQueue.js.map +1 -1
  3. package/dist/api/SignalClient.d.ts +3 -0
  4. package/dist/api/SignalClient.js +24 -3
  5. package/dist/api/SignalClient.js.map +1 -1
  6. package/dist/connect.js +1 -1
  7. package/dist/connect.js.map +1 -1
  8. package/dist/index.d.ts +1 -1
  9. package/dist/index.js +1 -1
  10. package/dist/index.js.map +1 -1
  11. package/dist/options.d.ts +7 -2
  12. package/dist/proto/livekit_models.d.ts +33 -0
  13. package/dist/proto/livekit_models.js +213 -3
  14. package/dist/proto/livekit_models.js.map +1 -1
  15. package/dist/proto/livekit_rtc.d.ts +15 -1
  16. package/dist/proto/livekit_rtc.js +128 -2
  17. package/dist/proto/livekit_rtc.js.map +1 -1
  18. package/dist/room/RTCEngine.d.ts +21 -6
  19. package/dist/room/RTCEngine.js +13 -8
  20. package/dist/room/RTCEngine.js.map +1 -1
  21. package/dist/room/Room.d.ts +43 -6
  22. package/dist/room/Room.js +81 -59
  23. package/dist/room/Room.js.map +1 -1
  24. package/dist/room/events.d.ts +14 -2
  25. package/dist/room/events.js +16 -4
  26. package/dist/room/events.js.map +1 -1
  27. package/dist/room/participant/LocalParticipant.d.ts +3 -1
  28. package/dist/room/participant/LocalParticipant.js +17 -1
  29. package/dist/room/participant/LocalParticipant.js.map +1 -1
  30. package/dist/room/participant/Participant.d.ts +30 -4
  31. package/dist/room/participant/Participant.js +2 -2
  32. package/dist/room/participant/Participant.js.map +1 -1
  33. package/dist/room/participant/RemoteParticipant.d.ts +5 -5
  34. package/dist/room/participant/RemoteParticipant.js +3 -3
  35. package/dist/room/participant/RemoteParticipant.js.map +1 -1
  36. package/dist/room/participant/publishUtils.d.ts +6 -0
  37. package/dist/room/participant/publishUtils.js +65 -24
  38. package/dist/room/participant/publishUtils.js.map +1 -1
  39. package/dist/room/participant/publishUtils.test.js +35 -5
  40. package/dist/room/participant/publishUtils.test.js.map +1 -1
  41. package/dist/room/track/LocalAudioTrack.d.ts +2 -0
  42. package/dist/room/track/LocalAudioTrack.js +23 -0
  43. package/dist/room/track/LocalAudioTrack.js.map +1 -1
  44. package/dist/room/track/LocalTrack.d.ts +4 -0
  45. package/dist/room/track/LocalTrack.js +34 -0
  46. package/dist/room/track/LocalTrack.js.map +1 -1
  47. package/dist/room/track/LocalVideoTrack.d.ts +1 -0
  48. package/dist/room/track/LocalVideoTrack.js +13 -0
  49. package/dist/room/track/LocalVideoTrack.js.map +1 -1
  50. package/dist/room/track/RemoteTrack.d.ts +1 -0
  51. package/dist/room/track/RemoteTrack.js +1 -0
  52. package/dist/room/track/RemoteTrack.js.map +1 -1
  53. package/dist/room/track/RemoteVideoTrack.d.ts +4 -2
  54. package/dist/room/track/RemoteVideoTrack.js +23 -8
  55. package/dist/room/track/RemoteVideoTrack.js.map +1 -1
  56. package/dist/room/track/Track.d.ts +20 -4
  57. package/dist/room/track/Track.js +20 -1
  58. package/dist/room/track/Track.js.map +1 -1
  59. package/dist/room/track/defaults.js +2 -2
  60. package/dist/room/track/defaults.js.map +1 -1
  61. package/dist/room/track/options.d.ts +65 -15
  62. package/dist/room/track/options.js +38 -0
  63. package/dist/room/track/options.js.map +1 -1
  64. package/dist/room/track/types.d.ts +15 -4
  65. package/dist/room/track/utils.d.ts +10 -0
  66. package/dist/room/track/utils.js +46 -1
  67. package/dist/room/track/utils.js.map +1 -1
  68. package/dist/room/utils.d.ts +1 -0
  69. package/dist/room/utils.js +5 -1
  70. package/dist/room/utils.js.map +1 -1
  71. package/dist/version.d.ts +1 -1
  72. package/dist/version.js +1 -1
  73. package/package.json +2 -1
  74. package/src/api/RequestQueue.ts +7 -7
  75. package/src/api/SignalClient.ts +31 -4
  76. package/src/connect.ts +1 -1
  77. package/src/index.ts +1 -1
  78. package/src/options.ts +12 -3
  79. package/src/proto/livekit_models.ts +249 -0
  80. package/src/proto/livekit_rtc.ts +155 -0
  81. package/src/room/RTCEngine.ts +43 -11
  82. package/src/room/Room.ts +152 -66
  83. package/src/room/events.ts +16 -2
  84. package/src/room/participant/LocalParticipant.ts +23 -4
  85. package/src/room/participant/Participant.ts +39 -4
  86. package/src/room/participant/RemoteParticipant.ts +10 -8
  87. package/src/room/participant/publishUtils.test.ts +46 -6
  88. package/src/room/participant/publishUtils.ts +72 -27
  89. package/src/room/track/LocalAudioTrack.ts +19 -1
  90. package/src/room/track/LocalTrack.ts +36 -0
  91. package/src/room/track/LocalVideoTrack.ts +9 -1
  92. package/src/room/track/RemoteTrack.ts +2 -0
  93. package/src/room/track/RemoteVideoTrack.ts +22 -9
  94. package/src/room/track/Track.ts +29 -3
  95. package/src/room/track/defaults.ts +2 -2
  96. package/src/room/track/options.ts +55 -3
  97. package/src/room/track/types.ts +16 -4
  98. package/src/room/track/utils.ts +39 -0
  99. package/src/room/utils.ts +4 -0
  100. package/src/version.ts +1 -1
@@ -10,9 +10,8 @@ import RemoteAudioTrack from '../track/RemoteAudioTrack';
10
10
  import RemoteTrackPublication from '../track/RemoteTrackPublication';
11
11
  import RemoteVideoTrack from '../track/RemoteVideoTrack';
12
12
  import { Track } from '../track/Track';
13
- import { TrackPublication } from '../track/TrackPublication';
14
- import { RemoteTrack } from '../track/types';
15
- import Participant from './Participant';
13
+ import { AdaptiveStreamSettings, RemoteTrack } from '../track/types';
14
+ import Participant, { ParticipantEventCallbacks } from './Participant';
16
15
 
17
16
  export default class RemoteParticipant extends Participant {
18
17
  audioTracks: Map<string, RemoteTrackPublication>;
@@ -42,7 +41,7 @@ export default class RemoteParticipant extends Participant {
42
41
  this.videoTracks = new Map();
43
42
  }
44
43
 
45
- protected addTrackPublication(publication: TrackPublication) {
44
+ protected addTrackPublication(publication: RemoteTrackPublication) {
46
45
  super.addTrackPublication(publication);
47
46
 
48
47
  // register action events
@@ -83,7 +82,7 @@ export default class RemoteParticipant extends Participant {
83
82
  sid: Track.SID,
84
83
  mediaStream: MediaStream,
85
84
  receiver?: RTCRtpReceiver,
86
- adaptiveStream?: boolean,
85
+ adaptiveStreamSettings?: AdaptiveStreamSettings,
87
86
  triesLeft?: number,
88
87
  ) {
89
88
  // find the track publication
@@ -115,7 +114,7 @@ export default class RemoteParticipant extends Participant {
115
114
  if (triesLeft === undefined) triesLeft = 20;
116
115
  setTimeout(() => {
117
116
  this.addSubscribedMediaTrack(mediaTrack, sid, mediaStream,
118
- receiver, adaptiveStream, triesLeft! - 1);
117
+ receiver, adaptiveStreamSettings, triesLeft! - 1);
119
118
  }, 150);
120
119
  return;
121
120
  }
@@ -123,7 +122,7 @@ export default class RemoteParticipant extends Participant {
123
122
  const isVideo = mediaTrack.kind === 'video';
124
123
  let track: RemoteTrack;
125
124
  if (isVideo) {
126
- track = new RemoteVideoTrack(mediaTrack, sid, receiver, adaptiveStream);
125
+ track = new RemoteVideoTrack(mediaTrack, sid, receiver, adaptiveStreamSettings);
127
126
  } else {
128
127
  track = new RemoteAudioTrack(mediaTrack, sid, receiver);
129
128
  }
@@ -234,7 +233,10 @@ export default class RemoteParticipant extends Participant {
234
233
  }
235
234
 
236
235
  /** @internal */
237
- emit(event: string | symbol, ...args: any[]): boolean {
236
+ emit<E extends keyof ParticipantEventCallbacks>(
237
+ event: E,
238
+ ...args: Parameters<ParticipantEventCallbacks[E]>
239
+ ): boolean {
238
240
  log.trace('participant event', this.sid, event, ...args);
239
241
  return super.emit(event, ...args);
240
242
  }
@@ -1,11 +1,15 @@
1
- import { VideoPresets, VideoPresets43 } from '../track/options';
2
1
  import {
2
+ ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43,
3
+ } from '../track/options';
4
+ import {
5
+ computeDefaultScreenShareSimulcastPresets,
3
6
  computeVideoEncodings,
4
7
  determineAppropriateEncoding,
5
8
  presets169,
6
9
  presets43,
7
10
  presetsForResolution,
8
11
  presetsScreenShare,
12
+ sortPresets,
9
13
  } from './publishUtils';
10
14
 
11
15
  describe('presetsForResolution', () => {
@@ -63,7 +67,7 @@ describe('computeVideoEncodings', () => {
63
67
 
64
68
  // ensure they are what we expect
65
69
  expect(encodings![0].rid).toBe('q');
66
- expect(encodings![0].maxBitrate).toBe(VideoPresets.qvga.encoding.maxBitrate);
70
+ expect(encodings![0].maxBitrate).toBe(VideoPresets.h180.encoding.maxBitrate);
67
71
  expect(encodings![0].scaleResolutionDownBy).toBe(3);
68
72
  expect(encodings![1].rid).toBe('h');
69
73
  expect(encodings![1].scaleResolutionDownBy).toBe(1.5);
@@ -77,7 +81,7 @@ describe('computeVideoEncodings', () => {
77
81
  expect(encodings).toHaveLength(3);
78
82
  expect(encodings![0].scaleResolutionDownBy).toBe(3);
79
83
  expect(encodings![1].scaleResolutionDownBy).toBe(1.5);
80
- expect(encodings![2].maxBitrate).toBe(VideoPresets.qhd.encoding.maxBitrate);
84
+ expect(encodings![2].maxBitrate).toBe(VideoPresets.h540.encoding.maxBitrate);
81
85
  });
82
86
 
83
87
  it('returns two encodings for lower-res simulcast', () => {
@@ -88,9 +92,9 @@ describe('computeVideoEncodings', () => {
88
92
 
89
93
  // ensure they are what we expect
90
94
  expect(encodings![0].rid).toBe('q');
91
- expect(encodings![0].maxBitrate).toBe(VideoPresets.qvga.encoding.maxBitrate);
95
+ expect(encodings![0].maxBitrate).toBe(VideoPresets.h180.encoding.maxBitrate);
92
96
  expect(encodings![1].rid).toBe('h');
93
- expect(encodings![1].maxBitrate).toBe(VideoPresets.vga.encoding.maxBitrate);
97
+ expect(encodings![1].maxBitrate).toBe(VideoPresets.h360.encoding.maxBitrate);
94
98
  });
95
99
 
96
100
  it('respects provided min resolution', () => {
@@ -99,7 +103,43 @@ describe('computeVideoEncodings', () => {
99
103
  });
100
104
  expect(encodings).toHaveLength(1);
101
105
  expect(encodings![0].rid).toBe('q');
102
- expect(encodings![0].maxBitrate).toBe(VideoPresets43.qvga.encoding.maxBitrate);
106
+ expect(encodings![0].maxBitrate).toBe(VideoPresets43.h120.encoding.maxBitrate);
103
107
  expect(encodings![0].scaleResolutionDownBy).toBe(1);
104
108
  });
105
109
  });
110
+
111
+ describe('customSimulcastLayers', () => {
112
+ it('sorts presets from lowest to highest', () => {
113
+ const sortedPresets = sortPresets(
114
+ [VideoPresets.h1440, VideoPresets.h360, VideoPresets.h1080, VideoPresets.h90],
115
+ ) as Array<VideoPreset>;
116
+ expect(sortPresets).not.toBeUndefined();
117
+ expect(sortedPresets[0]).toBe(VideoPresets.h90);
118
+ expect(sortedPresets[1]).toBe(VideoPresets.h360);
119
+ expect(sortedPresets[2]).toBe(VideoPresets.h1080);
120
+ expect(sortedPresets[3]).toBe(VideoPresets.h1440);
121
+ });
122
+ it('sorts presets from lowest to highest, even when dimensions are the same', () => {
123
+ const sortedPresets = sortPresets([
124
+ new VideoPreset(1920, 1080, 3_000_000, 20),
125
+ new VideoPreset(1920, 1080, 2_000_000, 15),
126
+ new VideoPreset(1920, 1080, 3_000_000, 15),
127
+ ]) as Array<VideoPreset>;
128
+ expect(sortPresets).not.toBeUndefined();
129
+ expect(sortedPresets[0].encoding.maxBitrate).toBe(2_000_000);
130
+ expect(sortedPresets[1].encoding.maxFramerate).toBe(15);
131
+ expect(sortedPresets[2].encoding.maxFramerate).toBe(20);
132
+ });
133
+ });
134
+
135
+ describe('screenShareSimulcastDefaults', () => {
136
+ it('computes appropriate bitrate from original preset', () => {
137
+ const defaultSimulcastLayers = computeDefaultScreenShareSimulcastPresets(
138
+ ScreenSharePresets.h720fps15,
139
+ );
140
+ expect(defaultSimulcastLayers[0].width).toBe(640);
141
+ expect(defaultSimulcastLayers[0].height).toBe(360);
142
+ expect(defaultSimulcastLayers[0].encoding.maxFramerate).toBe(3);
143
+ expect(defaultSimulcastLayers[0].encoding.maxBitrate).toBe(150_000);
144
+ });
145
+ });
@@ -26,32 +26,38 @@ export function mediaTrackToLocalTrack(
26
26
  }
27
27
 
28
28
  /* @internal */
29
- export const presets169 = [
30
- VideoPresets.qvga,
31
- VideoPresets.vga,
32
- VideoPresets.qhd,
33
- VideoPresets.hd,
34
- VideoPresets.fhd,
35
- ];
29
+ export const presets169 = Object.values(VideoPresets);
30
+
31
+ /* @internal */
32
+ export const presets43 = Object.values(VideoPresets43);
33
+
34
+ /* @internal */
35
+ export const presetsScreenShare = Object.values(ScreenSharePresets);
36
36
 
37
37
  /* @internal */
38
- export const presets43 = [
39
- VideoPresets43.qvga,
40
- VideoPresets43.vga,
41
- VideoPresets43.qhd,
42
- VideoPresets43.hd,
43
- VideoPresets43.fhd,
38
+ export const defaultSimulcastPresets169 = [
39
+ VideoPresets.h180,
40
+ VideoPresets.h360,
44
41
  ];
45
42
 
46
43
  /* @internal */
47
- export const presetsScreenShare = [
48
- ScreenSharePresets.vga,
49
- ScreenSharePresets.hd_8,
50
- ScreenSharePresets.hd_15,
51
- ScreenSharePresets.fhd_15,
52
- ScreenSharePresets.fhd_30,
44
+ export const defaultSimulcastPresets43 = [
45
+ VideoPresets43.h180,
46
+ VideoPresets43.h360,
53
47
  ];
54
48
 
49
+ /* @internal */
50
+ export const computeDefaultScreenShareSimulcastPresets = (fromPreset: VideoPreset) => {
51
+ const layers = [{ scaleResolutionDownBy: 2, fps: 3 }];
52
+ return layers.map((t) => new VideoPreset(
53
+ Math.floor(fromPreset.width / t.scaleResolutionDownBy),
54
+ Math.floor(fromPreset.height / t.scaleResolutionDownBy),
55
+ Math.max(150_000, Math.floor(fromPreset.encoding.maxBitrate
56
+ / (t.scaleResolutionDownBy ** 2 * ((fromPreset.encoding.maxFramerate ?? 30) / t.fps)))),
57
+ t.fps,
58
+ ));
59
+ };
60
+
55
61
  const videoRids = ['q', 'h', 'f'];
56
62
 
57
63
  /* @internal */
@@ -65,7 +71,7 @@ export function computeVideoEncodings(
65
71
  if (isScreenShare) {
66
72
  videoEncoding = options?.screenShareEncoding;
67
73
  }
68
- const useSimulcast = !isScreenShare && options?.simulcast;
74
+ const useSimulcast = options?.simulcast;
69
75
 
70
76
  if ((!videoEncoding && !useSimulcast) || !width || !height) {
71
77
  // when we aren't simulcasting, will need to return a single encoding without
@@ -82,16 +88,22 @@ export function computeVideoEncodings(
82
88
  if (!useSimulcast) {
83
89
  return [videoEncoding];
84
90
  }
85
-
86
- const presets = presetsForResolution(isScreenShare, width, height);
91
+ const original = new VideoPreset(
92
+ width, height, videoEncoding.maxBitrate, videoEncoding.maxFramerate,
93
+ );
94
+ let presets: Array<VideoPreset> = [];
95
+ if (isScreenShare) {
96
+ presets = sortPresets(options?.screenShareSimulcastLayers)
97
+ ?? defaultSimulcastLayers(isScreenShare, original);
98
+ } else {
99
+ presets = sortPresets(options?.videoSimulcastLayers)
100
+ ?? defaultSimulcastLayers(isScreenShare, original);
101
+ }
87
102
  let midPreset: VideoPreset | undefined;
88
103
  const lowPreset = presets[0];
89
104
  if (presets.length > 1) {
90
- [,midPreset] = presets;
105
+ [, midPreset] = presets;
91
106
  }
92
- const original = new VideoPreset(
93
- width, height, videoEncoding.maxBitrate, videoEncoding.maxFramerate,
94
- );
95
107
 
96
108
  // NOTE:
97
109
  // 1. Ordering of these encodings is important. Chrome seems
@@ -108,7 +120,7 @@ export function computeVideoEncodings(
108
120
  lowPreset, midPreset, original,
109
121
  ]);
110
122
  }
111
- if (size >= 500) {
123
+ if (size >= 480) {
112
124
  return encodingsFromPresets(width, height, [
113
125
  lowPreset, original,
114
126
  ]);
@@ -155,6 +167,21 @@ export function presetsForResolution(
155
167
  return presets43;
156
168
  }
157
169
 
170
+ /* @internal */
171
+ export function defaultSimulcastLayers(
172
+ isScreenShare: boolean, original: VideoPreset,
173
+ ): VideoPreset[] {
174
+ if (isScreenShare) {
175
+ return computeDefaultScreenShareSimulcastPresets(original);
176
+ }
177
+ const { width, height } = original;
178
+ const aspect = width > height ? width / height : height / width;
179
+ if (Math.abs(aspect - 16.0 / 9) < Math.abs(aspect - 4.0 / 3)) {
180
+ return defaultSimulcastPresets169;
181
+ }
182
+ return defaultSimulcastPresets43;
183
+ }
184
+
158
185
  // presets should be ordered by low, medium, high
159
186
  function encodingsFromPresets(
160
187
  width: number,
@@ -178,3 +205,21 @@ function encodingsFromPresets(
178
205
  });
179
206
  return encodings;
180
207
  }
208
+
209
+ /** @internal */
210
+ export function sortPresets(presets: Array<VideoPreset> | undefined) {
211
+ if (!presets) return;
212
+ return presets.sort((a, b) => {
213
+ const { encoding: aEnc } = a;
214
+ const { encoding: bEnc } = b;
215
+
216
+ if (aEnc.maxBitrate > bEnc.maxBitrate) {
217
+ return 1;
218
+ }
219
+ if (aEnc.maxBitrate < bEnc.maxBitrate) return -1;
220
+ if (aEnc.maxBitrate === bEnc.maxBitrate && aEnc.maxFramerate && bEnc.maxFramerate) {
221
+ return aEnc.maxFramerate > bEnc.maxFramerate ? 1 : -1;
222
+ }
223
+ return 0;
224
+ });
225
+ }
@@ -1,9 +1,10 @@
1
1
  import log from '../../logger';
2
+ import { TrackEvent } from '../events';
2
3
  import { AudioSenderStats, computeBitrate, monitorFrequency } from '../stats';
3
4
  import LocalTrack from './LocalTrack';
4
5
  import { AudioCaptureOptions } from './options';
5
6
  import { Track } from './Track';
6
- import { constraintsForOptions } from './utils';
7
+ import { constraintsForOptions, detectSilence } from './utils';
7
8
 
8
9
  export default class LocalAudioTrack extends LocalTrack {
9
10
  sender?: RTCRtpSender;
@@ -18,6 +19,7 @@ export default class LocalAudioTrack extends LocalTrack {
18
19
  constraints?: MediaTrackConstraints,
19
20
  ) {
20
21
  super(mediaTrack, Track.Kind.Audio, constraints);
22
+ this.checkForSilence();
21
23
  }
22
24
 
23
25
  async setDeviceId(deviceId: string) {
@@ -61,6 +63,12 @@ export default class LocalAudioTrack extends LocalTrack {
61
63
  await this.restart(constraints);
62
64
  }
63
65
 
66
+ protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
67
+ const track = await super.restart(constraints);
68
+ this.checkForSilence();
69
+ return track;
70
+ }
71
+
64
72
  /* @internal */
65
73
  startMonitor() {
66
74
  setTimeout(() => {
@@ -116,4 +124,14 @@ export default class LocalAudioTrack extends LocalTrack {
116
124
 
117
125
  return audioStats;
118
126
  }
127
+
128
+ async checkForSilence() {
129
+ const trackIsSilent = await detectSilence(this);
130
+ if (trackIsSilent) {
131
+ if (!this.isMuted) {
132
+ log.warn('silence detected on local audio track');
133
+ }
134
+ this.emit(TrackEvent.AudioSilenceDetected);
135
+ }
136
+ }
119
137
  }
@@ -2,6 +2,7 @@ import log from '../../logger';
2
2
  import DeviceManager from '../DeviceManager';
3
3
  import { TrackInvalidError } from '../errors';
4
4
  import { TrackEvent } from '../events';
5
+ import { isMobile } from '../utils';
5
6
  import { attachToElement, detachTrack, Track } from './Track';
6
7
 
7
8
  export default class LocalTrack extends Track {
@@ -10,12 +11,18 @@ export default class LocalTrack extends Track {
10
11
 
11
12
  protected constraints: MediaTrackConstraints;
12
13
 
14
+ protected wasMuted: boolean;
15
+
16
+ protected reacquireTrack: boolean;
17
+
13
18
  protected constructor(
14
19
  mediaTrack: MediaStreamTrack, kind: Track.Kind, constraints?: MediaTrackConstraints,
15
20
  ) {
16
21
  super(mediaTrack, kind);
17
22
  this.mediaStreamTrack.addEventListener('ended', this.handleEnded);
18
23
  this.constraints = constraints ?? mediaTrack.getConstraints();
24
+ this.reacquireTrack = false;
25
+ this.wasMuted = false;
19
26
  }
20
27
 
21
28
  get id(): string {
@@ -118,7 +125,36 @@ export default class LocalTrack extends Track {
118
125
  this.emit(muted ? TrackEvent.Muted : TrackEvent.Unmuted, this);
119
126
  }
120
127
 
128
+ protected get needsReAcquisition(): boolean {
129
+ return this.mediaStreamTrack.readyState !== 'live'
130
+ || this.mediaStreamTrack.muted
131
+ || !this.mediaStreamTrack.enabled
132
+ || this.reacquireTrack;
133
+ }
134
+
135
+ protected async handleAppVisibilityChanged() {
136
+ await super.handleAppVisibilityChanged();
137
+ if (!isMobile()) return;
138
+ log.debug('visibility changed, is in Background: ', this.isInBackground);
139
+
140
+ if (!this.isInBackground && this.needsReAcquisition) {
141
+ log.debug('track needs to be reaquired, restarting', this.source);
142
+ await this.restart();
143
+ this.reacquireTrack = false;
144
+ // Restore muted state if had to be restarted
145
+ this.setTrackMuted(this.wasMuted);
146
+ }
147
+
148
+ // store muted state each time app goes to background
149
+ if (this.isInBackground) {
150
+ this.wasMuted = this.isMuted;
151
+ }
152
+ }
153
+
121
154
  private handleEnded = () => {
155
+ if (this.isInBackground) {
156
+ this.reacquireTrack = true;
157
+ }
122
158
  this.emit(TrackEvent.Ended, this);
123
159
  };
124
160
  }
@@ -3,7 +3,7 @@ import log from '../../logger';
3
3
  import { VideoLayer, VideoQuality } from '../../proto/livekit_models';
4
4
  import { SubscribedQuality } from '../../proto/livekit_rtc';
5
5
  import { computeBitrate, monitorFrequency, VideoSenderStats } from '../stats';
6
- import { isFireFox } from '../utils';
6
+ import { isFireFox, isMobile } from '../utils';
7
7
  import LocalTrack from './LocalTrack';
8
8
  import { VideoCaptureOptions } from './options';
9
9
  import { Track } from './Track';
@@ -238,6 +238,14 @@ export default class LocalVideoTrack extends LocalTrack {
238
238
  this.monitorSender();
239
239
  }, monitorFrequency);
240
240
  };
241
+
242
+ protected async handleAppVisibilityChanged() {
243
+ await super.handleAppVisibilityChanged();
244
+ if (!isMobile()) return;
245
+ if (this.isInBackground && this.source === Track.Source.Camera) {
246
+ this.mediaStreamTrack.enabled = false;
247
+ }
248
+ }
241
249
  }
242
250
 
243
251
  export function videoQualityForRid(rid: string): VideoQuality {
@@ -6,6 +6,8 @@ export default abstract class RemoteTrack extends Track {
6
6
  /** @internal */
7
7
  receiver?: RTCRtpReceiver;
8
8
 
9
+ streamState: Track.StreamState = Track.StreamState.Active;
10
+
9
11
  constructor(
10
12
  mediaTrack: MediaStreamTrack,
11
13
  sid: string,
@@ -1,9 +1,12 @@
1
1
  import { debounce } from 'ts-debounce';
2
2
  import { TrackEvent } from '../events';
3
3
  import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
4
- import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
4
+ import {
5
+ getIntersectionObserver, getResizeObserver, isMobile, ObservableMediaElement,
6
+ } from '../utils';
5
7
  import RemoteTrack from './RemoteTrack';
6
8
  import { attachToElement, detachTrack, Track } from './Track';
9
+ import { AdaptiveStreamSettings } from './types';
7
10
 
8
11
  const REACTION_DELAY = 100;
9
12
 
@@ -15,7 +18,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
15
18
 
16
19
  private elementInfos: ElementInfo[] = [];
17
20
 
18
- private adaptiveStream?: boolean;
21
+ private adaptiveStreamSettings?: AdaptiveStreamSettings;
19
22
 
20
23
  private lastVisible?: boolean;
21
24
 
@@ -25,14 +28,14 @@ export default class RemoteVideoTrack extends RemoteTrack {
25
28
  mediaTrack: MediaStreamTrack,
26
29
  sid: string,
27
30
  receiver?: RTCRtpReceiver,
28
- adaptiveStream?: boolean,
31
+ adaptiveStreamSettings?: AdaptiveStreamSettings,
29
32
  ) {
30
33
  super(mediaTrack, sid, Track.Kind.Video, receiver);
31
- this.adaptiveStream = adaptiveStream;
34
+ this.adaptiveStreamSettings = adaptiveStreamSettings;
32
35
  }
33
36
 
34
37
  get isAdaptiveStream(): boolean {
35
- return this.adaptiveStream ?? false;
38
+ return this.adaptiveStreamSettings !== undefined;
36
39
  }
37
40
 
38
41
  /** @internal */
@@ -60,7 +63,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
60
63
 
61
64
  // It's possible attach is called multiple times on an element. When that's
62
65
  // the case, we'd want to avoid adding duplicate elementInfos
63
- if (this.adaptiveStream
66
+ if (this.adaptiveStreamSettings
64
67
  && this.elementInfos.find((info) => info.element === element) === undefined
65
68
  ) {
66
69
  this.elementInfos.push({
@@ -164,6 +167,14 @@ export default class RemoteVideoTrack extends RemoteTrack {
164
167
  this.updateVisibility();
165
168
  };
166
169
 
170
+ protected async handleAppVisibilityChanged() {
171
+ await super.handleAppVisibilityChanged();
172
+ if (!this.isAdaptiveStream) return;
173
+ // on desktop don't pause when tab is backgrounded
174
+ if (!isMobile()) return;
175
+ this.updateVisibility();
176
+ }
177
+
167
178
  private readonly debouncedHandleResize = debounce(() => {
168
179
  this.updateDimensions();
169
180
  }, REACTION_DELAY);
@@ -173,7 +184,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
173
184
  (prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
174
185
  0,
175
186
  );
176
- const isVisible = this.elementInfos.some((info) => info.visible);
187
+ const isVisible = this.elementInfos.some((info) => info.visible) && !this.isInBackground;
177
188
 
178
189
  if (this.lastVisible === isVisible) {
179
190
  return;
@@ -195,8 +206,10 @@ export default class RemoteVideoTrack extends RemoteTrack {
195
206
  let maxWidth = 0;
196
207
  let maxHeight = 0;
197
208
  for (const info of this.elementInfos) {
198
- const currentElementWidth = info.element.clientWidth * (window.devicePixelRatio ?? 1);
199
- const currentElementHeight = info.element.clientHeight * (window.devicePixelRatio ?? 1);
209
+ const pixelDensity = this.adaptiveStreamSettings?.pixelDensity ?? 1;
210
+ const pixelDensityValue = pixelDensity === 'screen' ? window.devicePixelRatio : pixelDensity;
211
+ const currentElementWidth = info.element.clientWidth * pixelDensityValue;
212
+ const currentElementHeight = info.element.clientHeight * pixelDensityValue;
200
213
  if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {
201
214
  maxWidth = currentElementWidth;
202
215
  maxHeight = currentElementHeight;
@@ -1,4 +1,5 @@
1
1
  import { EventEmitter } from 'events';
2
+ import type TypedEventEmitter from 'typed-emitter';
2
3
  import { TrackSource, TrackType } from '../../proto/livekit_models';
3
4
  import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc';
4
5
  import { TrackEvent } from '../events';
@@ -8,7 +9,7 @@ import { isFireFox, isSafari } from '../utils';
8
9
  // Safari tracks which audio elements have been "blessed" by the user.
9
10
  const recycledElements: Array<HTMLAudioElement> = [];
10
11
 
11
- export class Track extends EventEmitter {
12
+ export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
12
13
  kind: Track.Kind;
13
14
 
14
15
  mediaStreamTrack: MediaStreamTrack;
@@ -17,10 +18,10 @@ export class Track extends EventEmitter {
17
18
 
18
19
  isMuted: boolean = false;
19
20
 
20
- streamState: Track.StreamState = Track.StreamState.Active;
21
-
22
21
  source: Track.Source;
23
22
 
23
+ protected isInBackground: boolean;
24
+
24
25
  /**
25
26
  * sid is set after track is published to server, or if it's a remote track
26
27
  */
@@ -33,6 +34,8 @@ export class Track extends EventEmitter {
33
34
  this.kind = kind;
34
35
  this.mediaStreamTrack = mediaTrack;
35
36
  this.source = Track.Source.Unknown;
37
+ this.isInBackground = document.visibilityState === 'hidden';
38
+ document.addEventListener('visibilitychange', this.appVisibilityChangedListener);
36
39
  }
37
40
 
38
41
  /** current receive bits per second */
@@ -130,6 +133,7 @@ export class Track extends EventEmitter {
130
133
 
131
134
  stop() {
132
135
  this.mediaStreamTrack.stop();
136
+ document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
133
137
  }
134
138
 
135
139
  protected enable() {
@@ -155,6 +159,14 @@ export class Track extends EventEmitter {
155
159
  }
156
160
  }
157
161
  }
162
+
163
+ appVisibilityChangedListener = () => {
164
+ this.handleAppVisibilityChanged();
165
+ };
166
+
167
+ protected async handleAppVisibilityChanged() {
168
+ this.isInBackground = document.visibilityState === 'hidden';
169
+ }
158
170
  }
159
171
 
160
172
  /** @internal */
@@ -307,3 +319,17 @@ export namespace Track {
307
319
  }
308
320
  }
309
321
  }
322
+
323
+ export type TrackEventCallbacks = {
324
+ message: () => void,
325
+ muted: (track?: any) => void,
326
+ unmuted: (track?: any) => void,
327
+ ended: (track?: any) => void,
328
+ updateSettings: () => void,
329
+ updateSubscription: () => void,
330
+ audioPlaybackStarted: () => void,
331
+ audioPlaybackFailed: (error: Error) => void,
332
+ audioSilenceDetected: () => void,
333
+ visibilityChanged: (visible: boolean, track?: any) => void,
334
+ videoDimensionsChanged: (dimensions: Track.Dimensions, track?: any) => void,
335
+ };
@@ -7,7 +7,7 @@ export const publishDefaults: TrackPublishDefaults = {
7
7
  audioBitrate: AudioPresets.speech.maxBitrate,
8
8
  dtx: true,
9
9
  simulcast: true,
10
- screenShareEncoding: ScreenSharePresets.hd_15.encoding,
10
+ screenShareEncoding: ScreenSharePresets.h1080fps15.encoding,
11
11
  stopMicTrackOnMute: false,
12
12
  };
13
13
 
@@ -19,5 +19,5 @@ export const audioDefaults: AudioCaptureOptions = {
19
19
  };
20
20
 
21
21
  export const videoDefaults: VideoCaptureOptions = {
22
- resolution: VideoPresets.qhd.resolution,
22
+ resolution: VideoPresets.h540.resolution,
23
23
  };