livekit-client 0.16.5 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/RequestQueue.js +6 -6
- package/dist/api/RequestQueue.js.map +1 -1
- package/dist/api/SignalClient.d.ts +3 -0
- package/dist/api/SignalClient.js +24 -3
- package/dist/api/SignalClient.js.map +1 -1
- package/dist/connect.js +1 -1
- package/dist/connect.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/options.d.ts +7 -2
- package/dist/proto/livekit_models.d.ts +33 -0
- package/dist/proto/livekit_models.js +213 -3
- package/dist/proto/livekit_models.js.map +1 -1
- package/dist/proto/livekit_rtc.d.ts +15 -1
- package/dist/proto/livekit_rtc.js +128 -2
- package/dist/proto/livekit_rtc.js.map +1 -1
- package/dist/room/RTCEngine.d.ts +21 -6
- package/dist/room/RTCEngine.js +13 -8
- package/dist/room/RTCEngine.js.map +1 -1
- package/dist/room/Room.d.ts +43 -6
- package/dist/room/Room.js +81 -59
- package/dist/room/Room.js.map +1 -1
- package/dist/room/events.d.ts +14 -2
- package/dist/room/events.js +16 -4
- package/dist/room/events.js.map +1 -1
- package/dist/room/participant/LocalParticipant.d.ts +3 -1
- package/dist/room/participant/LocalParticipant.js +17 -1
- package/dist/room/participant/LocalParticipant.js.map +1 -1
- package/dist/room/participant/Participant.d.ts +30 -4
- package/dist/room/participant/Participant.js +2 -2
- package/dist/room/participant/Participant.js.map +1 -1
- package/dist/room/participant/RemoteParticipant.d.ts +5 -5
- package/dist/room/participant/RemoteParticipant.js +3 -3
- package/dist/room/participant/RemoteParticipant.js.map +1 -1
- package/dist/room/participant/publishUtils.d.ts +6 -0
- package/dist/room/participant/publishUtils.js +65 -24
- package/dist/room/participant/publishUtils.js.map +1 -1
- package/dist/room/participant/publishUtils.test.js +35 -5
- package/dist/room/participant/publishUtils.test.js.map +1 -1
- package/dist/room/track/LocalAudioTrack.d.ts +2 -0
- package/dist/room/track/LocalAudioTrack.js +23 -0
- package/dist/room/track/LocalAudioTrack.js.map +1 -1
- package/dist/room/track/LocalTrack.d.ts +4 -0
- package/dist/room/track/LocalTrack.js +34 -0
- package/dist/room/track/LocalTrack.js.map +1 -1
- package/dist/room/track/LocalVideoTrack.d.ts +1 -0
- package/dist/room/track/LocalVideoTrack.js +13 -0
- package/dist/room/track/LocalVideoTrack.js.map +1 -1
- package/dist/room/track/RemoteTrack.d.ts +1 -0
- package/dist/room/track/RemoteTrack.js +1 -0
- package/dist/room/track/RemoteTrack.js.map +1 -1
- package/dist/room/track/RemoteVideoTrack.d.ts +4 -2
- package/dist/room/track/RemoteVideoTrack.js +23 -8
- package/dist/room/track/RemoteVideoTrack.js.map +1 -1
- package/dist/room/track/Track.d.ts +20 -4
- package/dist/room/track/Track.js +20 -1
- package/dist/room/track/Track.js.map +1 -1
- package/dist/room/track/defaults.js +2 -2
- package/dist/room/track/defaults.js.map +1 -1
- package/dist/room/track/options.d.ts +65 -15
- package/dist/room/track/options.js +38 -0
- package/dist/room/track/options.js.map +1 -1
- package/dist/room/track/types.d.ts +15 -4
- package/dist/room/track/utils.d.ts +10 -0
- package/dist/room/track/utils.js +46 -1
- package/dist/room/track/utils.js.map +1 -1
- package/dist/room/utils.d.ts +1 -0
- package/dist/room/utils.js +5 -1
- package/dist/room/utils.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +2 -1
- package/src/api/RequestQueue.ts +7 -7
- package/src/api/SignalClient.ts +31 -4
- package/src/connect.ts +1 -1
- package/src/index.ts +1 -1
- package/src/options.ts +12 -3
- package/src/proto/livekit_models.ts +249 -0
- package/src/proto/livekit_rtc.ts +155 -0
- package/src/room/RTCEngine.ts +43 -11
- package/src/room/Room.ts +152 -66
- package/src/room/events.ts +16 -2
- package/src/room/participant/LocalParticipant.ts +23 -4
- package/src/room/participant/Participant.ts +39 -4
- package/src/room/participant/RemoteParticipant.ts +10 -8
- package/src/room/participant/publishUtils.test.ts +46 -6
- package/src/room/participant/publishUtils.ts +72 -27
- package/src/room/track/LocalAudioTrack.ts +19 -1
- package/src/room/track/LocalTrack.ts +36 -0
- package/src/room/track/LocalVideoTrack.ts +9 -1
- package/src/room/track/RemoteTrack.ts +2 -0
- package/src/room/track/RemoteVideoTrack.ts +22 -9
- package/src/room/track/Track.ts +29 -3
- package/src/room/track/defaults.ts +2 -2
- package/src/room/track/options.ts +55 -3
- package/src/room/track/types.ts +16 -4
- package/src/room/track/utils.ts +39 -0
- package/src/room/utils.ts +4 -0
- package/src/version.ts +1 -1
@@ -10,9 +10,8 @@ import RemoteAudioTrack from '../track/RemoteAudioTrack';
|
|
10
10
|
import RemoteTrackPublication from '../track/RemoteTrackPublication';
|
11
11
|
import RemoteVideoTrack from '../track/RemoteVideoTrack';
|
12
12
|
import { Track } from '../track/Track';
|
13
|
-
import {
|
14
|
-
import {
|
15
|
-
import Participant from './Participant';
|
13
|
+
import { AdaptiveStreamSettings, RemoteTrack } from '../track/types';
|
14
|
+
import Participant, { ParticipantEventCallbacks } from './Participant';
|
16
15
|
|
17
16
|
export default class RemoteParticipant extends Participant {
|
18
17
|
audioTracks: Map<string, RemoteTrackPublication>;
|
@@ -42,7 +41,7 @@ export default class RemoteParticipant extends Participant {
|
|
42
41
|
this.videoTracks = new Map();
|
43
42
|
}
|
44
43
|
|
45
|
-
protected addTrackPublication(publication:
|
44
|
+
protected addTrackPublication(publication: RemoteTrackPublication) {
|
46
45
|
super.addTrackPublication(publication);
|
47
46
|
|
48
47
|
// register action events
|
@@ -83,7 +82,7 @@ export default class RemoteParticipant extends Participant {
|
|
83
82
|
sid: Track.SID,
|
84
83
|
mediaStream: MediaStream,
|
85
84
|
receiver?: RTCRtpReceiver,
|
86
|
-
|
85
|
+
adaptiveStreamSettings?: AdaptiveStreamSettings,
|
87
86
|
triesLeft?: number,
|
88
87
|
) {
|
89
88
|
// find the track publication
|
@@ -115,7 +114,7 @@ export default class RemoteParticipant extends Participant {
|
|
115
114
|
if (triesLeft === undefined) triesLeft = 20;
|
116
115
|
setTimeout(() => {
|
117
116
|
this.addSubscribedMediaTrack(mediaTrack, sid, mediaStream,
|
118
|
-
receiver,
|
117
|
+
receiver, adaptiveStreamSettings, triesLeft! - 1);
|
119
118
|
}, 150);
|
120
119
|
return;
|
121
120
|
}
|
@@ -123,7 +122,7 @@ export default class RemoteParticipant extends Participant {
|
|
123
122
|
const isVideo = mediaTrack.kind === 'video';
|
124
123
|
let track: RemoteTrack;
|
125
124
|
if (isVideo) {
|
126
|
-
track = new RemoteVideoTrack(mediaTrack, sid, receiver,
|
125
|
+
track = new RemoteVideoTrack(mediaTrack, sid, receiver, adaptiveStreamSettings);
|
127
126
|
} else {
|
128
127
|
track = new RemoteAudioTrack(mediaTrack, sid, receiver);
|
129
128
|
}
|
@@ -234,7 +233,10 @@ export default class RemoteParticipant extends Participant {
|
|
234
233
|
}
|
235
234
|
|
236
235
|
/** @internal */
|
237
|
-
emit
|
236
|
+
emit<E extends keyof ParticipantEventCallbacks>(
|
237
|
+
event: E,
|
238
|
+
...args: Parameters<ParticipantEventCallbacks[E]>
|
239
|
+
): boolean {
|
238
240
|
log.trace('participant event', this.sid, event, ...args);
|
239
241
|
return super.emit(event, ...args);
|
240
242
|
}
|
@@ -1,11 +1,15 @@
|
|
1
|
-
import { VideoPresets, VideoPresets43 } from '../track/options';
|
2
1
|
import {
|
2
|
+
ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43,
|
3
|
+
} from '../track/options';
|
4
|
+
import {
|
5
|
+
computeDefaultScreenShareSimulcastPresets,
|
3
6
|
computeVideoEncodings,
|
4
7
|
determineAppropriateEncoding,
|
5
8
|
presets169,
|
6
9
|
presets43,
|
7
10
|
presetsForResolution,
|
8
11
|
presetsScreenShare,
|
12
|
+
sortPresets,
|
9
13
|
} from './publishUtils';
|
10
14
|
|
11
15
|
describe('presetsForResolution', () => {
|
@@ -63,7 +67,7 @@ describe('computeVideoEncodings', () => {
|
|
63
67
|
|
64
68
|
// ensure they are what we expect
|
65
69
|
expect(encodings![0].rid).toBe('q');
|
66
|
-
expect(encodings![0].maxBitrate).toBe(VideoPresets.
|
70
|
+
expect(encodings![0].maxBitrate).toBe(VideoPresets.h180.encoding.maxBitrate);
|
67
71
|
expect(encodings![0].scaleResolutionDownBy).toBe(3);
|
68
72
|
expect(encodings![1].rid).toBe('h');
|
69
73
|
expect(encodings![1].scaleResolutionDownBy).toBe(1.5);
|
@@ -77,7 +81,7 @@ describe('computeVideoEncodings', () => {
|
|
77
81
|
expect(encodings).toHaveLength(3);
|
78
82
|
expect(encodings![0].scaleResolutionDownBy).toBe(3);
|
79
83
|
expect(encodings![1].scaleResolutionDownBy).toBe(1.5);
|
80
|
-
expect(encodings![2].maxBitrate).toBe(VideoPresets.
|
84
|
+
expect(encodings![2].maxBitrate).toBe(VideoPresets.h540.encoding.maxBitrate);
|
81
85
|
});
|
82
86
|
|
83
87
|
it('returns two encodings for lower-res simulcast', () => {
|
@@ -88,9 +92,9 @@ describe('computeVideoEncodings', () => {
|
|
88
92
|
|
89
93
|
// ensure they are what we expect
|
90
94
|
expect(encodings![0].rid).toBe('q');
|
91
|
-
expect(encodings![0].maxBitrate).toBe(VideoPresets.
|
95
|
+
expect(encodings![0].maxBitrate).toBe(VideoPresets.h180.encoding.maxBitrate);
|
92
96
|
expect(encodings![1].rid).toBe('h');
|
93
|
-
expect(encodings![1].maxBitrate).toBe(VideoPresets.
|
97
|
+
expect(encodings![1].maxBitrate).toBe(VideoPresets.h360.encoding.maxBitrate);
|
94
98
|
});
|
95
99
|
|
96
100
|
it('respects provided min resolution', () => {
|
@@ -99,7 +103,43 @@ describe('computeVideoEncodings', () => {
|
|
99
103
|
});
|
100
104
|
expect(encodings).toHaveLength(1);
|
101
105
|
expect(encodings![0].rid).toBe('q');
|
102
|
-
expect(encodings![0].maxBitrate).toBe(VideoPresets43.
|
106
|
+
expect(encodings![0].maxBitrate).toBe(VideoPresets43.h120.encoding.maxBitrate);
|
103
107
|
expect(encodings![0].scaleResolutionDownBy).toBe(1);
|
104
108
|
});
|
105
109
|
});
|
110
|
+
|
111
|
+
describe('customSimulcastLayers', () => {
|
112
|
+
it('sorts presets from lowest to highest', () => {
|
113
|
+
const sortedPresets = sortPresets(
|
114
|
+
[VideoPresets.h1440, VideoPresets.h360, VideoPresets.h1080, VideoPresets.h90],
|
115
|
+
) as Array<VideoPreset>;
|
116
|
+
expect(sortPresets).not.toBeUndefined();
|
117
|
+
expect(sortedPresets[0]).toBe(VideoPresets.h90);
|
118
|
+
expect(sortedPresets[1]).toBe(VideoPresets.h360);
|
119
|
+
expect(sortedPresets[2]).toBe(VideoPresets.h1080);
|
120
|
+
expect(sortedPresets[3]).toBe(VideoPresets.h1440);
|
121
|
+
});
|
122
|
+
it('sorts presets from lowest to highest, even when dimensions are the same', () => {
|
123
|
+
const sortedPresets = sortPresets([
|
124
|
+
new VideoPreset(1920, 1080, 3_000_000, 20),
|
125
|
+
new VideoPreset(1920, 1080, 2_000_000, 15),
|
126
|
+
new VideoPreset(1920, 1080, 3_000_000, 15),
|
127
|
+
]) as Array<VideoPreset>;
|
128
|
+
expect(sortPresets).not.toBeUndefined();
|
129
|
+
expect(sortedPresets[0].encoding.maxBitrate).toBe(2_000_000);
|
130
|
+
expect(sortedPresets[1].encoding.maxFramerate).toBe(15);
|
131
|
+
expect(sortedPresets[2].encoding.maxFramerate).toBe(20);
|
132
|
+
});
|
133
|
+
});
|
134
|
+
|
135
|
+
describe('screenShareSimulcastDefaults', () => {
|
136
|
+
it('computes appropriate bitrate from original preset', () => {
|
137
|
+
const defaultSimulcastLayers = computeDefaultScreenShareSimulcastPresets(
|
138
|
+
ScreenSharePresets.h720fps15,
|
139
|
+
);
|
140
|
+
expect(defaultSimulcastLayers[0].width).toBe(640);
|
141
|
+
expect(defaultSimulcastLayers[0].height).toBe(360);
|
142
|
+
expect(defaultSimulcastLayers[0].encoding.maxFramerate).toBe(3);
|
143
|
+
expect(defaultSimulcastLayers[0].encoding.maxBitrate).toBe(150_000);
|
144
|
+
});
|
145
|
+
});
|
@@ -26,32 +26,38 @@ export function mediaTrackToLocalTrack(
|
|
26
26
|
}
|
27
27
|
|
28
28
|
/* @internal */
|
29
|
-
export const presets169 =
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
29
|
+
export const presets169 = Object.values(VideoPresets);
|
30
|
+
|
31
|
+
/* @internal */
|
32
|
+
export const presets43 = Object.values(VideoPresets43);
|
33
|
+
|
34
|
+
/* @internal */
|
35
|
+
export const presetsScreenShare = Object.values(ScreenSharePresets);
|
36
36
|
|
37
37
|
/* @internal */
|
38
|
-
export const
|
39
|
-
|
40
|
-
|
41
|
-
VideoPresets43.qhd,
|
42
|
-
VideoPresets43.hd,
|
43
|
-
VideoPresets43.fhd,
|
38
|
+
export const defaultSimulcastPresets169 = [
|
39
|
+
VideoPresets.h180,
|
40
|
+
VideoPresets.h360,
|
44
41
|
];
|
45
42
|
|
46
43
|
/* @internal */
|
47
|
-
export const
|
48
|
-
|
49
|
-
|
50
|
-
ScreenSharePresets.hd_15,
|
51
|
-
ScreenSharePresets.fhd_15,
|
52
|
-
ScreenSharePresets.fhd_30,
|
44
|
+
export const defaultSimulcastPresets43 = [
|
45
|
+
VideoPresets43.h180,
|
46
|
+
VideoPresets43.h360,
|
53
47
|
];
|
54
48
|
|
49
|
+
/* @internal */
|
50
|
+
export const computeDefaultScreenShareSimulcastPresets = (fromPreset: VideoPreset) => {
|
51
|
+
const layers = [{ scaleResolutionDownBy: 2, fps: 3 }];
|
52
|
+
return layers.map((t) => new VideoPreset(
|
53
|
+
Math.floor(fromPreset.width / t.scaleResolutionDownBy),
|
54
|
+
Math.floor(fromPreset.height / t.scaleResolutionDownBy),
|
55
|
+
Math.max(150_000, Math.floor(fromPreset.encoding.maxBitrate
|
56
|
+
/ (t.scaleResolutionDownBy ** 2 * ((fromPreset.encoding.maxFramerate ?? 30) / t.fps)))),
|
57
|
+
t.fps,
|
58
|
+
));
|
59
|
+
};
|
60
|
+
|
55
61
|
const videoRids = ['q', 'h', 'f'];
|
56
62
|
|
57
63
|
/* @internal */
|
@@ -65,7 +71,7 @@ export function computeVideoEncodings(
|
|
65
71
|
if (isScreenShare) {
|
66
72
|
videoEncoding = options?.screenShareEncoding;
|
67
73
|
}
|
68
|
-
const useSimulcast =
|
74
|
+
const useSimulcast = options?.simulcast;
|
69
75
|
|
70
76
|
if ((!videoEncoding && !useSimulcast) || !width || !height) {
|
71
77
|
// when we aren't simulcasting, will need to return a single encoding without
|
@@ -82,16 +88,22 @@ export function computeVideoEncodings(
|
|
82
88
|
if (!useSimulcast) {
|
83
89
|
return [videoEncoding];
|
84
90
|
}
|
85
|
-
|
86
|
-
|
91
|
+
const original = new VideoPreset(
|
92
|
+
width, height, videoEncoding.maxBitrate, videoEncoding.maxFramerate,
|
93
|
+
);
|
94
|
+
let presets: Array<VideoPreset> = [];
|
95
|
+
if (isScreenShare) {
|
96
|
+
presets = sortPresets(options?.screenShareSimulcastLayers)
|
97
|
+
?? defaultSimulcastLayers(isScreenShare, original);
|
98
|
+
} else {
|
99
|
+
presets = sortPresets(options?.videoSimulcastLayers)
|
100
|
+
?? defaultSimulcastLayers(isScreenShare, original);
|
101
|
+
}
|
87
102
|
let midPreset: VideoPreset | undefined;
|
88
103
|
const lowPreset = presets[0];
|
89
104
|
if (presets.length > 1) {
|
90
|
-
[,midPreset] = presets;
|
105
|
+
[, midPreset] = presets;
|
91
106
|
}
|
92
|
-
const original = new VideoPreset(
|
93
|
-
width, height, videoEncoding.maxBitrate, videoEncoding.maxFramerate,
|
94
|
-
);
|
95
107
|
|
96
108
|
// NOTE:
|
97
109
|
// 1. Ordering of these encodings is important. Chrome seems
|
@@ -108,7 +120,7 @@ export function computeVideoEncodings(
|
|
108
120
|
lowPreset, midPreset, original,
|
109
121
|
]);
|
110
122
|
}
|
111
|
-
if (size >=
|
123
|
+
if (size >= 480) {
|
112
124
|
return encodingsFromPresets(width, height, [
|
113
125
|
lowPreset, original,
|
114
126
|
]);
|
@@ -155,6 +167,21 @@ export function presetsForResolution(
|
|
155
167
|
return presets43;
|
156
168
|
}
|
157
169
|
|
170
|
+
/* @internal */
|
171
|
+
export function defaultSimulcastLayers(
|
172
|
+
isScreenShare: boolean, original: VideoPreset,
|
173
|
+
): VideoPreset[] {
|
174
|
+
if (isScreenShare) {
|
175
|
+
return computeDefaultScreenShareSimulcastPresets(original);
|
176
|
+
}
|
177
|
+
const { width, height } = original;
|
178
|
+
const aspect = width > height ? width / height : height / width;
|
179
|
+
if (Math.abs(aspect - 16.0 / 9) < Math.abs(aspect - 4.0 / 3)) {
|
180
|
+
return defaultSimulcastPresets169;
|
181
|
+
}
|
182
|
+
return defaultSimulcastPresets43;
|
183
|
+
}
|
184
|
+
|
158
185
|
// presets should be ordered by low, medium, high
|
159
186
|
function encodingsFromPresets(
|
160
187
|
width: number,
|
@@ -178,3 +205,21 @@ function encodingsFromPresets(
|
|
178
205
|
});
|
179
206
|
return encodings;
|
180
207
|
}
|
208
|
+
|
209
|
+
/** @internal */
|
210
|
+
export function sortPresets(presets: Array<VideoPreset> | undefined) {
|
211
|
+
if (!presets) return;
|
212
|
+
return presets.sort((a, b) => {
|
213
|
+
const { encoding: aEnc } = a;
|
214
|
+
const { encoding: bEnc } = b;
|
215
|
+
|
216
|
+
if (aEnc.maxBitrate > bEnc.maxBitrate) {
|
217
|
+
return 1;
|
218
|
+
}
|
219
|
+
if (aEnc.maxBitrate < bEnc.maxBitrate) return -1;
|
220
|
+
if (aEnc.maxBitrate === bEnc.maxBitrate && aEnc.maxFramerate && bEnc.maxFramerate) {
|
221
|
+
return aEnc.maxFramerate > bEnc.maxFramerate ? 1 : -1;
|
222
|
+
}
|
223
|
+
return 0;
|
224
|
+
});
|
225
|
+
}
|
@@ -1,9 +1,10 @@
|
|
1
1
|
import log from '../../logger';
|
2
|
+
import { TrackEvent } from '../events';
|
2
3
|
import { AudioSenderStats, computeBitrate, monitorFrequency } from '../stats';
|
3
4
|
import LocalTrack from './LocalTrack';
|
4
5
|
import { AudioCaptureOptions } from './options';
|
5
6
|
import { Track } from './Track';
|
6
|
-
import { constraintsForOptions } from './utils';
|
7
|
+
import { constraintsForOptions, detectSilence } from './utils';
|
7
8
|
|
8
9
|
export default class LocalAudioTrack extends LocalTrack {
|
9
10
|
sender?: RTCRtpSender;
|
@@ -18,6 +19,7 @@ export default class LocalAudioTrack extends LocalTrack {
|
|
18
19
|
constraints?: MediaTrackConstraints,
|
19
20
|
) {
|
20
21
|
super(mediaTrack, Track.Kind.Audio, constraints);
|
22
|
+
this.checkForSilence();
|
21
23
|
}
|
22
24
|
|
23
25
|
async setDeviceId(deviceId: string) {
|
@@ -61,6 +63,12 @@ export default class LocalAudioTrack extends LocalTrack {
|
|
61
63
|
await this.restart(constraints);
|
62
64
|
}
|
63
65
|
|
66
|
+
protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
|
67
|
+
const track = await super.restart(constraints);
|
68
|
+
this.checkForSilence();
|
69
|
+
return track;
|
70
|
+
}
|
71
|
+
|
64
72
|
/* @internal */
|
65
73
|
startMonitor() {
|
66
74
|
setTimeout(() => {
|
@@ -116,4 +124,14 @@ export default class LocalAudioTrack extends LocalTrack {
|
|
116
124
|
|
117
125
|
return audioStats;
|
118
126
|
}
|
127
|
+
|
128
|
+
async checkForSilence() {
|
129
|
+
const trackIsSilent = await detectSilence(this);
|
130
|
+
if (trackIsSilent) {
|
131
|
+
if (!this.isMuted) {
|
132
|
+
log.warn('silence detected on local audio track');
|
133
|
+
}
|
134
|
+
this.emit(TrackEvent.AudioSilenceDetected);
|
135
|
+
}
|
136
|
+
}
|
119
137
|
}
|
@@ -2,6 +2,7 @@ import log from '../../logger';
|
|
2
2
|
import DeviceManager from '../DeviceManager';
|
3
3
|
import { TrackInvalidError } from '../errors';
|
4
4
|
import { TrackEvent } from '../events';
|
5
|
+
import { isMobile } from '../utils';
|
5
6
|
import { attachToElement, detachTrack, Track } from './Track';
|
6
7
|
|
7
8
|
export default class LocalTrack extends Track {
|
@@ -10,12 +11,18 @@ export default class LocalTrack extends Track {
|
|
10
11
|
|
11
12
|
protected constraints: MediaTrackConstraints;
|
12
13
|
|
14
|
+
protected wasMuted: boolean;
|
15
|
+
|
16
|
+
protected reacquireTrack: boolean;
|
17
|
+
|
13
18
|
protected constructor(
|
14
19
|
mediaTrack: MediaStreamTrack, kind: Track.Kind, constraints?: MediaTrackConstraints,
|
15
20
|
) {
|
16
21
|
super(mediaTrack, kind);
|
17
22
|
this.mediaStreamTrack.addEventListener('ended', this.handleEnded);
|
18
23
|
this.constraints = constraints ?? mediaTrack.getConstraints();
|
24
|
+
this.reacquireTrack = false;
|
25
|
+
this.wasMuted = false;
|
19
26
|
}
|
20
27
|
|
21
28
|
get id(): string {
|
@@ -118,7 +125,36 @@ export default class LocalTrack extends Track {
|
|
118
125
|
this.emit(muted ? TrackEvent.Muted : TrackEvent.Unmuted, this);
|
119
126
|
}
|
120
127
|
|
128
|
+
protected get needsReAcquisition(): boolean {
|
129
|
+
return this.mediaStreamTrack.readyState !== 'live'
|
130
|
+
|| this.mediaStreamTrack.muted
|
131
|
+
|| !this.mediaStreamTrack.enabled
|
132
|
+
|| this.reacquireTrack;
|
133
|
+
}
|
134
|
+
|
135
|
+
protected async handleAppVisibilityChanged() {
|
136
|
+
await super.handleAppVisibilityChanged();
|
137
|
+
if (!isMobile()) return;
|
138
|
+
log.debug('visibility changed, is in Background: ', this.isInBackground);
|
139
|
+
|
140
|
+
if (!this.isInBackground && this.needsReAcquisition) {
|
141
|
+
log.debug('track needs to be reaquired, restarting', this.source);
|
142
|
+
await this.restart();
|
143
|
+
this.reacquireTrack = false;
|
144
|
+
// Restore muted state if had to be restarted
|
145
|
+
this.setTrackMuted(this.wasMuted);
|
146
|
+
}
|
147
|
+
|
148
|
+
// store muted state each time app goes to background
|
149
|
+
if (this.isInBackground) {
|
150
|
+
this.wasMuted = this.isMuted;
|
151
|
+
}
|
152
|
+
}
|
153
|
+
|
121
154
|
private handleEnded = () => {
|
155
|
+
if (this.isInBackground) {
|
156
|
+
this.reacquireTrack = true;
|
157
|
+
}
|
122
158
|
this.emit(TrackEvent.Ended, this);
|
123
159
|
};
|
124
160
|
}
|
@@ -3,7 +3,7 @@ import log from '../../logger';
|
|
3
3
|
import { VideoLayer, VideoQuality } from '../../proto/livekit_models';
|
4
4
|
import { SubscribedQuality } from '../../proto/livekit_rtc';
|
5
5
|
import { computeBitrate, monitorFrequency, VideoSenderStats } from '../stats';
|
6
|
-
import { isFireFox } from '../utils';
|
6
|
+
import { isFireFox, isMobile } from '../utils';
|
7
7
|
import LocalTrack from './LocalTrack';
|
8
8
|
import { VideoCaptureOptions } from './options';
|
9
9
|
import { Track } from './Track';
|
@@ -238,6 +238,14 @@ export default class LocalVideoTrack extends LocalTrack {
|
|
238
238
|
this.monitorSender();
|
239
239
|
}, monitorFrequency);
|
240
240
|
};
|
241
|
+
|
242
|
+
protected async handleAppVisibilityChanged() {
|
243
|
+
await super.handleAppVisibilityChanged();
|
244
|
+
if (!isMobile()) return;
|
245
|
+
if (this.isInBackground && this.source === Track.Source.Camera) {
|
246
|
+
this.mediaStreamTrack.enabled = false;
|
247
|
+
}
|
248
|
+
}
|
241
249
|
}
|
242
250
|
|
243
251
|
export function videoQualityForRid(rid: string): VideoQuality {
|
@@ -1,9 +1,12 @@
|
|
1
1
|
import { debounce } from 'ts-debounce';
|
2
2
|
import { TrackEvent } from '../events';
|
3
3
|
import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
|
4
|
-
import {
|
4
|
+
import {
|
5
|
+
getIntersectionObserver, getResizeObserver, isMobile, ObservableMediaElement,
|
6
|
+
} from '../utils';
|
5
7
|
import RemoteTrack from './RemoteTrack';
|
6
8
|
import { attachToElement, detachTrack, Track } from './Track';
|
9
|
+
import { AdaptiveStreamSettings } from './types';
|
7
10
|
|
8
11
|
const REACTION_DELAY = 100;
|
9
12
|
|
@@ -15,7 +18,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
|
|
15
18
|
|
16
19
|
private elementInfos: ElementInfo[] = [];
|
17
20
|
|
18
|
-
private
|
21
|
+
private adaptiveStreamSettings?: AdaptiveStreamSettings;
|
19
22
|
|
20
23
|
private lastVisible?: boolean;
|
21
24
|
|
@@ -25,14 +28,14 @@ export default class RemoteVideoTrack extends RemoteTrack {
|
|
25
28
|
mediaTrack: MediaStreamTrack,
|
26
29
|
sid: string,
|
27
30
|
receiver?: RTCRtpReceiver,
|
28
|
-
|
31
|
+
adaptiveStreamSettings?: AdaptiveStreamSettings,
|
29
32
|
) {
|
30
33
|
super(mediaTrack, sid, Track.Kind.Video, receiver);
|
31
|
-
this.
|
34
|
+
this.adaptiveStreamSettings = adaptiveStreamSettings;
|
32
35
|
}
|
33
36
|
|
34
37
|
get isAdaptiveStream(): boolean {
|
35
|
-
return this.
|
38
|
+
return this.adaptiveStreamSettings !== undefined;
|
36
39
|
}
|
37
40
|
|
38
41
|
/** @internal */
|
@@ -60,7 +63,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
|
|
60
63
|
|
61
64
|
// It's possible attach is called multiple times on an element. When that's
|
62
65
|
// the case, we'd want to avoid adding duplicate elementInfos
|
63
|
-
if (this.
|
66
|
+
if (this.adaptiveStreamSettings
|
64
67
|
&& this.elementInfos.find((info) => info.element === element) === undefined
|
65
68
|
) {
|
66
69
|
this.elementInfos.push({
|
@@ -164,6 +167,14 @@ export default class RemoteVideoTrack extends RemoteTrack {
|
|
164
167
|
this.updateVisibility();
|
165
168
|
};
|
166
169
|
|
170
|
+
protected async handleAppVisibilityChanged() {
|
171
|
+
await super.handleAppVisibilityChanged();
|
172
|
+
if (!this.isAdaptiveStream) return;
|
173
|
+
// on desktop don't pause when tab is backgrounded
|
174
|
+
if (!isMobile()) return;
|
175
|
+
this.updateVisibility();
|
176
|
+
}
|
177
|
+
|
167
178
|
private readonly debouncedHandleResize = debounce(() => {
|
168
179
|
this.updateDimensions();
|
169
180
|
}, REACTION_DELAY);
|
@@ -173,7 +184,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
|
|
173
184
|
(prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
|
174
185
|
0,
|
175
186
|
);
|
176
|
-
const isVisible = this.elementInfos.some((info) => info.visible);
|
187
|
+
const isVisible = this.elementInfos.some((info) => info.visible) && !this.isInBackground;
|
177
188
|
|
178
189
|
if (this.lastVisible === isVisible) {
|
179
190
|
return;
|
@@ -195,8 +206,10 @@ export default class RemoteVideoTrack extends RemoteTrack {
|
|
195
206
|
let maxWidth = 0;
|
196
207
|
let maxHeight = 0;
|
197
208
|
for (const info of this.elementInfos) {
|
198
|
-
const
|
199
|
-
const
|
209
|
+
const pixelDensity = this.adaptiveStreamSettings?.pixelDensity ?? 1;
|
210
|
+
const pixelDensityValue = pixelDensity === 'screen' ? window.devicePixelRatio : pixelDensity;
|
211
|
+
const currentElementWidth = info.element.clientWidth * pixelDensityValue;
|
212
|
+
const currentElementHeight = info.element.clientHeight * pixelDensityValue;
|
200
213
|
if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {
|
201
214
|
maxWidth = currentElementWidth;
|
202
215
|
maxHeight = currentElementHeight;
|
package/src/room/track/Track.ts
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
import { EventEmitter } from 'events';
|
2
|
+
import type TypedEventEmitter from 'typed-emitter';
|
2
3
|
import { TrackSource, TrackType } from '../../proto/livekit_models';
|
3
4
|
import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc';
|
4
5
|
import { TrackEvent } from '../events';
|
@@ -8,7 +9,7 @@ import { isFireFox, isSafari } from '../utils';
|
|
8
9
|
// Safari tracks which audio elements have been "blessed" by the user.
|
9
10
|
const recycledElements: Array<HTMLAudioElement> = [];
|
10
11
|
|
11
|
-
export class Track extends EventEmitter {
|
12
|
+
export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
|
12
13
|
kind: Track.Kind;
|
13
14
|
|
14
15
|
mediaStreamTrack: MediaStreamTrack;
|
@@ -17,10 +18,10 @@ export class Track extends EventEmitter {
|
|
17
18
|
|
18
19
|
isMuted: boolean = false;
|
19
20
|
|
20
|
-
streamState: Track.StreamState = Track.StreamState.Active;
|
21
|
-
|
22
21
|
source: Track.Source;
|
23
22
|
|
23
|
+
protected isInBackground: boolean;
|
24
|
+
|
24
25
|
/**
|
25
26
|
* sid is set after track is published to server, or if it's a remote track
|
26
27
|
*/
|
@@ -33,6 +34,8 @@ export class Track extends EventEmitter {
|
|
33
34
|
this.kind = kind;
|
34
35
|
this.mediaStreamTrack = mediaTrack;
|
35
36
|
this.source = Track.Source.Unknown;
|
37
|
+
this.isInBackground = document.visibilityState === 'hidden';
|
38
|
+
document.addEventListener('visibilitychange', this.appVisibilityChangedListener);
|
36
39
|
}
|
37
40
|
|
38
41
|
/** current receive bits per second */
|
@@ -130,6 +133,7 @@ export class Track extends EventEmitter {
|
|
130
133
|
|
131
134
|
stop() {
|
132
135
|
this.mediaStreamTrack.stop();
|
136
|
+
document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
|
133
137
|
}
|
134
138
|
|
135
139
|
protected enable() {
|
@@ -155,6 +159,14 @@ export class Track extends EventEmitter {
|
|
155
159
|
}
|
156
160
|
}
|
157
161
|
}
|
162
|
+
|
163
|
+
appVisibilityChangedListener = () => {
|
164
|
+
this.handleAppVisibilityChanged();
|
165
|
+
};
|
166
|
+
|
167
|
+
protected async handleAppVisibilityChanged() {
|
168
|
+
this.isInBackground = document.visibilityState === 'hidden';
|
169
|
+
}
|
158
170
|
}
|
159
171
|
|
160
172
|
/** @internal */
|
@@ -307,3 +319,17 @@ export namespace Track {
|
|
307
319
|
}
|
308
320
|
}
|
309
321
|
}
|
322
|
+
|
323
|
+
export type TrackEventCallbacks = {
|
324
|
+
message: () => void,
|
325
|
+
muted: (track?: any) => void,
|
326
|
+
unmuted: (track?: any) => void,
|
327
|
+
ended: (track?: any) => void,
|
328
|
+
updateSettings: () => void,
|
329
|
+
updateSubscription: () => void,
|
330
|
+
audioPlaybackStarted: () => void,
|
331
|
+
audioPlaybackFailed: (error: Error) => void,
|
332
|
+
audioSilenceDetected: () => void,
|
333
|
+
visibilityChanged: (visible: boolean, track?: any) => void,
|
334
|
+
videoDimensionsChanged: (dimensions: Track.Dimensions, track?: any) => void,
|
335
|
+
};
|
@@ -7,7 +7,7 @@ export const publishDefaults: TrackPublishDefaults = {
|
|
7
7
|
audioBitrate: AudioPresets.speech.maxBitrate,
|
8
8
|
dtx: true,
|
9
9
|
simulcast: true,
|
10
|
-
screenShareEncoding: ScreenSharePresets.
|
10
|
+
screenShareEncoding: ScreenSharePresets.h1080fps15.encoding,
|
11
11
|
stopMicTrackOnMute: false,
|
12
12
|
};
|
13
13
|
|
@@ -19,5 +19,5 @@ export const audioDefaults: AudioCaptureOptions = {
|
|
19
19
|
};
|
20
20
|
|
21
21
|
export const videoDefaults: VideoCaptureOptions = {
|
22
|
-
resolution: VideoPresets.
|
22
|
+
resolution: VideoPresets.h540.resolution,
|
23
23
|
};
|