@stream-io/video-client 1.14.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/index.browser.es.js +1532 -1784
- package/dist/index.browser.es.js.map +1 -1
- package/dist/index.cjs.js +1512 -1783
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.es.js +1532 -1784
- package/dist/index.es.js.map +1 -1
- package/dist/src/Call.d.ts +43 -28
- package/dist/src/StreamSfuClient.d.ts +4 -5
- package/dist/src/devices/CameraManager.d.ts +5 -8
- package/dist/src/devices/InputMediaDeviceManager.d.ts +5 -5
- package/dist/src/devices/MicrophoneManager.d.ts +7 -2
- package/dist/src/devices/ScreenShareManager.d.ts +1 -2
- package/dist/src/gen/video/sfu/event/events.d.ts +38 -19
- package/dist/src/gen/video/sfu/models/models.d.ts +76 -9
- package/dist/src/helpers/array.d.ts +7 -0
- package/dist/src/permissions/PermissionsContext.d.ts +6 -0
- package/dist/src/rtc/BasePeerConnection.d.ts +90 -0
- package/dist/src/rtc/Dispatcher.d.ts +0 -1
- package/dist/src/rtc/IceTrickleBuffer.d.ts +3 -2
- package/dist/src/rtc/Publisher.d.ts +32 -86
- package/dist/src/rtc/Subscriber.d.ts +4 -56
- package/dist/src/rtc/TransceiverCache.d.ts +55 -0
- package/dist/src/rtc/codecs.d.ts +1 -15
- package/dist/src/rtc/helpers/sdp.d.ts +8 -0
- package/dist/src/rtc/helpers/tracks.d.ts +1 -0
- package/dist/src/rtc/index.d.ts +3 -0
- package/dist/src/rtc/videoLayers.d.ts +11 -25
- package/dist/src/stats/{stateStoreStatsReporter.d.ts → CallStateStatsReporter.d.ts} +5 -1
- package/dist/src/stats/SfuStatsReporter.d.ts +4 -2
- package/dist/src/stats/index.d.ts +1 -1
- package/dist/src/stats/types.d.ts +8 -0
- package/dist/src/types.d.ts +12 -22
- package/package.json +1 -1
- package/src/Call.ts +254 -268
- package/src/StreamSfuClient.ts +9 -14
- package/src/StreamVideoClient.ts +1 -1
- package/src/__tests__/Call.publishing.test.ts +306 -0
- package/src/devices/CameraManager.ts +33 -16
- package/src/devices/InputMediaDeviceManager.ts +36 -27
- package/src/devices/MicrophoneManager.ts +29 -8
- package/src/devices/ScreenShareManager.ts +6 -8
- package/src/devices/__tests__/CameraManager.test.ts +111 -14
- package/src/devices/__tests__/InputMediaDeviceManager.test.ts +4 -4
- package/src/devices/__tests__/MicrophoneManager.test.ts +59 -21
- package/src/devices/__tests__/ScreenShareManager.test.ts +5 -5
- package/src/devices/__tests__/mocks.ts +1 -0
- package/src/events/__tests__/internal.test.ts +132 -0
- package/src/events/__tests__/mutes.test.ts +0 -3
- package/src/events/__tests__/speaker.test.ts +92 -0
- package/src/events/participant.ts +3 -4
- package/src/gen/video/sfu/event/events.ts +91 -30
- package/src/gen/video/sfu/models/models.ts +105 -13
- package/src/helpers/array.ts +14 -0
- package/src/permissions/PermissionsContext.ts +22 -0
- package/src/permissions/__tests__/PermissionsContext.test.ts +40 -0
- package/src/rpc/__tests__/createClient.test.ts +38 -0
- package/src/rpc/createClient.ts +11 -5
- package/src/rtc/BasePeerConnection.ts +240 -0
- package/src/rtc/Dispatcher.ts +0 -9
- package/src/rtc/IceTrickleBuffer.ts +24 -4
- package/src/rtc/Publisher.ts +210 -528
- package/src/rtc/Subscriber.ts +26 -200
- package/src/rtc/TransceiverCache.ts +120 -0
- package/src/rtc/__tests__/Publisher.test.ts +407 -210
- package/src/rtc/__tests__/Subscriber.test.ts +88 -36
- package/src/rtc/__tests__/mocks/webrtc.mocks.ts +22 -2
- package/src/rtc/__tests__/videoLayers.test.ts +161 -54
- package/src/rtc/codecs.ts +1 -131
- package/src/rtc/helpers/__tests__/rtcConfiguration.test.ts +34 -0
- package/src/rtc/helpers/__tests__/sdp.test.ts +59 -0
- package/src/rtc/helpers/sdp.ts +30 -0
- package/src/rtc/helpers/tracks.ts +3 -0
- package/src/rtc/index.ts +4 -0
- package/src/rtc/videoLayers.ts +68 -76
- package/src/stats/{stateStoreStatsReporter.ts → CallStateStatsReporter.ts} +58 -27
- package/src/stats/SfuStatsReporter.ts +31 -3
- package/src/stats/index.ts +1 -1
- package/src/stats/types.ts +12 -0
- package/src/types.ts +12 -22
- package/dist/src/helpers/sdp-munging.d.ts +0 -24
- package/dist/src/rtc/bitrateLookup.d.ts +0 -2
- package/dist/src/rtc/helpers/iceCandidate.d.ts +0 -2
- package/src/helpers/__tests__/hq-audio-sdp.ts +0 -332
- package/src/helpers/__tests__/sdp-munging.test.ts +0 -283
- package/src/helpers/sdp-munging.ts +0 -265
- package/src/rtc/__tests__/bitrateLookup.test.ts +0 -12
- package/src/rtc/__tests__/codecs.test.ts +0 -145
- package/src/rtc/bitrateLookup.ts +0 -61
- package/src/rtc/helpers/iceCandidate.ts +0 -16
- /package/dist/src/{compatibility.d.ts → helpers/compatibility.d.ts} +0 -0
- /package/src/{compatibility.ts → helpers/compatibility.ts} +0 -0
|
@@ -5,7 +5,7 @@ import { DispatchableMessage, Dispatcher } from '../Dispatcher';
|
|
|
5
5
|
import { StreamSfuClient } from '../../StreamSfuClient';
|
|
6
6
|
import { Subscriber } from '../Subscriber';
|
|
7
7
|
import { CallState } from '../../store';
|
|
8
|
-
import { SfuEvent } from '../../gen/video/sfu/event/events';
|
|
8
|
+
import { SfuEvent, SubscriberOffer } from '../../gen/video/sfu/event/events';
|
|
9
9
|
import { PeerType, TrackType } from '../../gen/video/sfu/models/models';
|
|
10
10
|
import { IceTrickleBuffer } from '../IceTrickleBuffer';
|
|
11
11
|
import { StreamClient } from '../../coordinator/connection/client';
|
|
@@ -55,44 +55,10 @@ describe('Subscriber', () => {
|
|
|
55
55
|
afterEach(() => {
|
|
56
56
|
vi.clearAllMocks();
|
|
57
57
|
vi.resetModules();
|
|
58
|
-
|
|
58
|
+
subscriber.dispose();
|
|
59
59
|
});
|
|
60
60
|
|
|
61
61
|
describe('Subscriber ICE restart', () => {
|
|
62
|
-
it('should perform ICE restart when iceRestart event is received', () => {
|
|
63
|
-
sfuClient.iceRestart = vi.fn();
|
|
64
|
-
dispatcher.dispatch(
|
|
65
|
-
SfuEvent.create({
|
|
66
|
-
eventPayload: {
|
|
67
|
-
oneofKind: 'iceRestart',
|
|
68
|
-
iceRestart: {
|
|
69
|
-
peerType: PeerType.SUBSCRIBER,
|
|
70
|
-
},
|
|
71
|
-
},
|
|
72
|
-
}) as DispatchableMessage<'iceRestart'>,
|
|
73
|
-
);
|
|
74
|
-
|
|
75
|
-
expect(sfuClient.iceRestart).toHaveBeenCalledWith({
|
|
76
|
-
peerType: PeerType.SUBSCRIBER,
|
|
77
|
-
});
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
it('should not perform ICE restart when iceRestart event is received for a different peer type', () => {
|
|
81
|
-
sfuClient.iceRestart = vi.fn();
|
|
82
|
-
dispatcher.dispatch(
|
|
83
|
-
SfuEvent.create({
|
|
84
|
-
eventPayload: {
|
|
85
|
-
oneofKind: 'iceRestart',
|
|
86
|
-
iceRestart: {
|
|
87
|
-
peerType: PeerType.PUBLISHER_UNSPECIFIED,
|
|
88
|
-
},
|
|
89
|
-
},
|
|
90
|
-
}) as DispatchableMessage<'iceRestart'>,
|
|
91
|
-
);
|
|
92
|
-
|
|
93
|
-
expect(sfuClient.iceRestart).not.toHaveBeenCalled();
|
|
94
|
-
});
|
|
95
|
-
|
|
96
62
|
it(`should drop consequent ICE restart requests`, async () => {
|
|
97
63
|
sfuClient.iceRestart = vi.fn();
|
|
98
64
|
// @ts-ignore
|
|
@@ -111,6 +77,17 @@ describe('Subscriber', () => {
|
|
|
111
77
|
expect(sfuClient.iceRestart).not.toHaveBeenCalled();
|
|
112
78
|
});
|
|
113
79
|
|
|
80
|
+
it('should ask the SFU for ICE restart', async () => {
|
|
81
|
+
sfuClient.iceRestart = vi.fn();
|
|
82
|
+
// @ts-ignore
|
|
83
|
+
subscriber['pc'].connectionState = 'connected';
|
|
84
|
+
|
|
85
|
+
await subscriber.restartIce();
|
|
86
|
+
expect(sfuClient.iceRestart).toHaveBeenCalledWith({
|
|
87
|
+
peerType: PeerType.SUBSCRIBER,
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
|
|
114
91
|
it(`should perform ICE restart when connection state changes to 'failed'`, () => {
|
|
115
92
|
vi.spyOn(subscriber, 'restartIce').mockResolvedValue();
|
|
116
93
|
// @ts-ignore
|
|
@@ -175,5 +152,80 @@ describe('Subscriber', () => {
|
|
|
175
152
|
videoStream: mediaStream,
|
|
176
153
|
});
|
|
177
154
|
});
|
|
155
|
+
|
|
156
|
+
it('should replace participant stream when a new one arrives', () => {
|
|
157
|
+
const mediaStream = new MediaStream();
|
|
158
|
+
const mediaStreamTrack = new MediaStreamTrack();
|
|
159
|
+
// @ts-ignore - mock
|
|
160
|
+
mediaStream.id = '123:TRACK_TYPE_VIDEO';
|
|
161
|
+
|
|
162
|
+
const updateParticipantSpy = vi.spyOn(state, 'updateParticipant');
|
|
163
|
+
|
|
164
|
+
const baseStream = new MediaStream();
|
|
165
|
+
const baseTrack = new MediaStreamTrack();
|
|
166
|
+
vi.spyOn(baseStream, 'getTracks').mockReturnValue([baseTrack]);
|
|
167
|
+
// @ts-expect-error - incomplete mock
|
|
168
|
+
state.updateOrAddParticipant('session-id', {
|
|
169
|
+
sessionId: 'session-id',
|
|
170
|
+
trackLookupPrefix: '123',
|
|
171
|
+
videoStream: baseStream,
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
const onTrack = subscriber['handleOnTrack'];
|
|
175
|
+
// @ts-expect-error - incomplete mock
|
|
176
|
+
onTrack({ streams: [mediaStream], track: mediaStreamTrack });
|
|
177
|
+
|
|
178
|
+
expect(updateParticipantSpy).toHaveBeenCalledWith('session-id', {
|
|
179
|
+
videoStream: mediaStream,
|
|
180
|
+
});
|
|
181
|
+
expect(baseStream.getTracks).toHaveBeenCalled();
|
|
182
|
+
expect(baseTrack.stop).toHaveBeenCalled();
|
|
183
|
+
expect(baseStream.removeTrack).toHaveBeenCalledWith(baseTrack);
|
|
184
|
+
});
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
describe('Negotiation', () => {
|
|
188
|
+
it('negotiates with the SFU', async () => {
|
|
189
|
+
sfuClient.sendAnswer = vi.fn();
|
|
190
|
+
subscriber['pc'].createAnswer = vi
|
|
191
|
+
.fn()
|
|
192
|
+
.mockResolvedValue({ sdp: 'answer-sdp' });
|
|
193
|
+
|
|
194
|
+
const offer = SubscriberOffer.create({ sdp: 'offer-sdp' });
|
|
195
|
+
// @ts-expect-error - private method
|
|
196
|
+
await subscriber.negotiate(offer);
|
|
197
|
+
expect(subscriber['pc'].setRemoteDescription).toHaveBeenCalledWith({
|
|
198
|
+
type: 'offer',
|
|
199
|
+
sdp: 'offer-sdp',
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
expect(subscriber['pc'].createAnswer).toHaveBeenCalled();
|
|
203
|
+
expect(sfuClient.sendAnswer).toHaveBeenCalledWith({
|
|
204
|
+
peerType: PeerType.SUBSCRIBER,
|
|
205
|
+
sdp: 'answer-sdp',
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
describe('Event handling', () => {
|
|
211
|
+
it('handles SubscriberOffer', async () => {
|
|
212
|
+
// @ts-expect-error - private method
|
|
213
|
+
subscriber.negotiate = vi.fn();
|
|
214
|
+
const subscriberOffer = SubscriberOffer.create({
|
|
215
|
+
sdp: 'offer-sdp',
|
|
216
|
+
iceRestart: false,
|
|
217
|
+
});
|
|
218
|
+
dispatcher.dispatch(
|
|
219
|
+
SfuEvent.create({
|
|
220
|
+
eventPayload: {
|
|
221
|
+
oneofKind: 'subscriberOffer',
|
|
222
|
+
subscriberOffer,
|
|
223
|
+
},
|
|
224
|
+
}) as DispatchableMessage<'subscriberOffer'>,
|
|
225
|
+
);
|
|
226
|
+
|
|
227
|
+
// @ts-expect-error - private method
|
|
228
|
+
expect(subscriber.negotiate).toHaveBeenCalledWith(subscriberOffer);
|
|
229
|
+
});
|
|
178
230
|
});
|
|
179
231
|
});
|
|
@@ -3,6 +3,7 @@ import { vi } from 'vitest';
|
|
|
3
3
|
const RTCPeerConnectionMock = vi.fn((): Partial<RTCPeerConnection> => {
|
|
4
4
|
return {
|
|
5
5
|
addEventListener: vi.fn(),
|
|
6
|
+
addIceCandidate: vi.fn(),
|
|
6
7
|
removeEventListener: vi.fn(),
|
|
7
8
|
getTransceivers: vi.fn(),
|
|
8
9
|
addTransceiver: vi.fn(),
|
|
@@ -14,15 +15,21 @@ const RTCPeerConnectionMock = vi.fn((): Partial<RTCPeerConnection> => {
|
|
|
14
15
|
setRemoteDescription: vi.fn().mockResolvedValue({}),
|
|
15
16
|
close: vi.fn(),
|
|
16
17
|
connectionState: 'connected',
|
|
18
|
+
signalingState: 'stable',
|
|
17
19
|
getReceivers: vi.fn(),
|
|
20
|
+
getSenders: vi.fn(),
|
|
21
|
+
removeTrack: vi.fn(),
|
|
18
22
|
};
|
|
19
23
|
});
|
|
20
24
|
vi.stubGlobal('RTCPeerConnection', RTCPeerConnectionMock);
|
|
21
25
|
|
|
22
26
|
const MediaStreamMock = vi.fn((): Partial<MediaStream> => {
|
|
23
27
|
return {
|
|
24
|
-
getTracks: vi.fn(),
|
|
28
|
+
getTracks: vi.fn().mockReturnValue([]),
|
|
25
29
|
addTrack: vi.fn(),
|
|
30
|
+
removeTrack: vi.fn(),
|
|
31
|
+
getAudioTracks: vi.fn().mockReturnValue([]),
|
|
32
|
+
getVideoTracks: vi.fn().mockReturnValue([]),
|
|
26
33
|
};
|
|
27
34
|
});
|
|
28
35
|
vi.stubGlobal('MediaStream', MediaStreamMock);
|
|
@@ -31,10 +38,13 @@ const MediaStreamTrackMock = vi.fn((): Partial<MediaStreamTrack> => {
|
|
|
31
38
|
return {
|
|
32
39
|
addEventListener: vi.fn(),
|
|
33
40
|
removeEventListener: vi.fn(),
|
|
34
|
-
getSettings: vi.fn(),
|
|
41
|
+
getSettings: vi.fn().mockReturnValue({ width: 1280, height: 720 }),
|
|
35
42
|
stop: vi.fn(),
|
|
43
|
+
clone: vi.fn(),
|
|
36
44
|
readyState: 'live',
|
|
37
45
|
kind: 'video',
|
|
46
|
+
enabled: true,
|
|
47
|
+
id: crypto.randomUUID(),
|
|
38
48
|
};
|
|
39
49
|
});
|
|
40
50
|
vi.stubGlobal('MediaStreamTrack', MediaStreamTrackMock);
|
|
@@ -49,6 +59,7 @@ const RTCRtpTransceiverMock = vi.fn((): Partial<RTCRtpTransceiver> => {
|
|
|
49
59
|
setParameters: vi.fn(),
|
|
50
60
|
},
|
|
51
61
|
setCodecPreferences: vi.fn(),
|
|
62
|
+
mid: '',
|
|
52
63
|
};
|
|
53
64
|
});
|
|
54
65
|
vi.stubGlobal('RTCRtpTransceiver', RTCRtpTransceiverMock);
|
|
@@ -69,3 +80,12 @@ const RTCRtpReceiverMock = vi.fn((): Partial<typeof RTCRtpReceiver> => {
|
|
|
69
80
|
};
|
|
70
81
|
});
|
|
71
82
|
vi.stubGlobal('RTCRtpReceiver', RTCRtpReceiverMock);
|
|
83
|
+
|
|
84
|
+
const RTCRtpSenderMock = vi.fn((): Partial<typeof RTCRtpSender> => {
|
|
85
|
+
return {
|
|
86
|
+
getCapabilities: vi.fn(),
|
|
87
|
+
// @ts-ignore
|
|
88
|
+
track: vi.fn(),
|
|
89
|
+
};
|
|
90
|
+
});
|
|
91
|
+
vi.stubGlobal('RTCRtpSender', RTCRtpSenderMock);
|
|
@@ -1,48 +1,20 @@
|
|
|
1
1
|
import './mocks/webrtc.mocks';
|
|
2
2
|
import { describe, expect, it, vi } from 'vitest';
|
|
3
3
|
import {
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
PublishOption,
|
|
5
|
+
TrackType,
|
|
6
|
+
VideoQuality,
|
|
7
|
+
} from '../../gen/video/sfu/models/models';
|
|
8
|
+
import {
|
|
9
|
+
computeVideoLayers,
|
|
6
10
|
getComputedMaxBitrate,
|
|
7
11
|
OptimalVideoLayer,
|
|
8
12
|
ridToVideoQuality,
|
|
9
13
|
toSvcEncodings,
|
|
14
|
+
toVideoLayers,
|
|
10
15
|
} from '../videoLayers';
|
|
11
|
-
import { VideoQuality } from '../../gen/video/sfu/models/models';
|
|
12
16
|
|
|
13
17
|
describe('videoLayers', () => {
|
|
14
|
-
it('should find optimal screen sharing layers', () => {
|
|
15
|
-
const track = new MediaStreamTrack();
|
|
16
|
-
vi.spyOn(track, 'getSettings').mockReturnValue({
|
|
17
|
-
width: 1920,
|
|
18
|
-
height: 1080,
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
const layers = findOptimalScreenSharingLayers(track);
|
|
22
|
-
expect(layers).toEqual([
|
|
23
|
-
{
|
|
24
|
-
active: true,
|
|
25
|
-
rid: 'q',
|
|
26
|
-
width: 1920,
|
|
27
|
-
height: 1080,
|
|
28
|
-
maxBitrate: 3000000,
|
|
29
|
-
scaleResolutionDownBy: 1,
|
|
30
|
-
maxFramerate: 30,
|
|
31
|
-
},
|
|
32
|
-
]);
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
it('should use default max bitrate if none is provided in preferences', () => {
|
|
36
|
-
const track = new MediaStreamTrack();
|
|
37
|
-
vi.spyOn(track, 'getSettings').mockReturnValue({
|
|
38
|
-
width: 1920,
|
|
39
|
-
height: 1080,
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
const layers = findOptimalScreenSharingLayers(track, undefined, 192000);
|
|
43
|
-
expect(layers).toMatchObject([{ maxBitrate: 192000 }]);
|
|
44
|
-
});
|
|
45
|
-
|
|
46
18
|
it('should find optimal video layers', () => {
|
|
47
19
|
const track = new MediaStreamTrack();
|
|
48
20
|
const width = 1920;
|
|
@@ -50,11 +22,14 @@ describe('videoLayers', () => {
|
|
|
50
22
|
const targetBitrate = 3000000;
|
|
51
23
|
vi.spyOn(track, 'getSettings').mockReturnValue({ width, height });
|
|
52
24
|
|
|
53
|
-
const
|
|
54
|
-
width,
|
|
55
|
-
height,
|
|
25
|
+
const publishOption: PublishOption = {
|
|
56
26
|
bitrate: targetBitrate,
|
|
57
|
-
|
|
27
|
+
// @ts-expect-error - incomplete data
|
|
28
|
+
codec: { name: 'vp8' },
|
|
29
|
+
videoDimension: { width, height },
|
|
30
|
+
fps: 30,
|
|
31
|
+
};
|
|
32
|
+
const layers = computeVideoLayers(track, publishOption);
|
|
58
33
|
expect(layers).toEqual([
|
|
59
34
|
{
|
|
60
35
|
active: true,
|
|
@@ -86,13 +61,31 @@ describe('videoLayers', () => {
|
|
|
86
61
|
]);
|
|
87
62
|
});
|
|
88
63
|
|
|
64
|
+
it('should return undefined for audio track', () => {
|
|
65
|
+
const track = new MediaStreamTrack();
|
|
66
|
+
expect(
|
|
67
|
+
// @ts-expect-error - incomplete data
|
|
68
|
+
computeVideoLayers(track, { trackType: TrackType.AUDIO }),
|
|
69
|
+
).toBeUndefined();
|
|
70
|
+
expect(
|
|
71
|
+
// @ts-expect-error - incomplete data
|
|
72
|
+
computeVideoLayers(track, { trackType: TrackType.SCREEN_SHARE_AUDIO }),
|
|
73
|
+
).toBeUndefined();
|
|
74
|
+
});
|
|
75
|
+
|
|
89
76
|
it('should use predefined bitrate values when track dimensions cant be determined', () => {
|
|
90
77
|
const width = 0;
|
|
91
78
|
const height = 0;
|
|
92
79
|
const bitrate = 3000000;
|
|
93
80
|
const track = new MediaStreamTrack();
|
|
94
81
|
vi.spyOn(track, 'getSettings').mockReturnValue({ width, height });
|
|
95
|
-
const layers =
|
|
82
|
+
const layers = computeVideoLayers(track, {
|
|
83
|
+
bitrate,
|
|
84
|
+
// @ts-expect-error - incomplete data
|
|
85
|
+
codec: { name: 'vp8' },
|
|
86
|
+
fps: 30,
|
|
87
|
+
videoDimension: { width, height },
|
|
88
|
+
});
|
|
96
89
|
expect(layers).toEqual([
|
|
97
90
|
{
|
|
98
91
|
active: true,
|
|
@@ -111,7 +104,13 @@ describe('videoLayers', () => {
|
|
|
111
104
|
const width = 320;
|
|
112
105
|
const height = 240;
|
|
113
106
|
vi.spyOn(track, 'getSettings').mockReturnValue({ width, height });
|
|
114
|
-
const layers =
|
|
107
|
+
const layers = computeVideoLayers(track, {
|
|
108
|
+
bitrate: 0,
|
|
109
|
+
// @ts-expect-error - incomplete data
|
|
110
|
+
codec: { name: 'vp8' },
|
|
111
|
+
fps: 30,
|
|
112
|
+
videoDimension: { width, height },
|
|
113
|
+
});
|
|
115
114
|
expect(layers.length).toBe(1);
|
|
116
115
|
expect(layers[0].rid).toBe('q');
|
|
117
116
|
});
|
|
@@ -121,7 +120,13 @@ describe('videoLayers', () => {
|
|
|
121
120
|
const width = 640;
|
|
122
121
|
const height = 480;
|
|
123
122
|
vi.spyOn(track, 'getSettings').mockReturnValue({ width, height });
|
|
124
|
-
const layers =
|
|
123
|
+
const layers = computeVideoLayers(track, {
|
|
124
|
+
bitrate: 0,
|
|
125
|
+
// @ts-expect-error - incomplete data
|
|
126
|
+
codec: { name: 'vp8' },
|
|
127
|
+
fps: 30,
|
|
128
|
+
videoDimension: { width, height },
|
|
129
|
+
});
|
|
125
130
|
expect(layers.length).toBe(2);
|
|
126
131
|
expect(layers[0].rid).toBe('q');
|
|
127
132
|
expect(layers[1].rid).toBe('h');
|
|
@@ -132,7 +137,13 @@ describe('videoLayers', () => {
|
|
|
132
137
|
const width = 1280;
|
|
133
138
|
const height = 720;
|
|
134
139
|
vi.spyOn(track, 'getSettings').mockReturnValue({ width, height });
|
|
135
|
-
const layers =
|
|
140
|
+
const layers = computeVideoLayers(track, {
|
|
141
|
+
bitrate: 0,
|
|
142
|
+
// @ts-expect-error - incomplete data
|
|
143
|
+
codec: { name: 'vp8' },
|
|
144
|
+
fps: 30,
|
|
145
|
+
videoDimension: { width, height },
|
|
146
|
+
});
|
|
136
147
|
expect(layers.length).toBe(3);
|
|
137
148
|
expect(layers[0].rid).toBe('q');
|
|
138
149
|
expect(layers[1].rid).toBe('h');
|
|
@@ -145,12 +156,15 @@ describe('videoLayers', () => {
|
|
|
145
156
|
width: 1280,
|
|
146
157
|
height: 720,
|
|
147
158
|
});
|
|
148
|
-
const layers =
|
|
149
|
-
|
|
150
|
-
|
|
159
|
+
const layers = computeVideoLayers(track, {
|
|
160
|
+
maxTemporalLayers: 3,
|
|
161
|
+
maxSpatialLayers: 3,
|
|
162
|
+
// @ts-expect-error - incomplete data
|
|
163
|
+
codec: { name: 'vp9' },
|
|
164
|
+
videoDimension: { width: 1280, height: 720 },
|
|
151
165
|
});
|
|
152
166
|
expect(layers.length).toBe(3);
|
|
153
|
-
expect(layers[0].scalabilityMode).toBe('
|
|
167
|
+
expect(layers[0].scalabilityMode).toBe('L3T3_KEY');
|
|
154
168
|
expect(layers[0].rid).toBe('q');
|
|
155
169
|
expect(layers[1].rid).toBe('h');
|
|
156
170
|
expect(layers[2].rid).toBe('f');
|
|
@@ -163,7 +177,39 @@ describe('videoLayers', () => {
|
|
|
163
177
|
expect(ridToVideoQuality('')).toBe(VideoQuality.HIGH);
|
|
164
178
|
});
|
|
165
179
|
|
|
166
|
-
it('should map
|
|
180
|
+
it('should map optimal video layers to SFU VideoLayers', () => {
|
|
181
|
+
const layers: Array<Partial<OptimalVideoLayer>> = [
|
|
182
|
+
{ rid: 'f', width: 1920, height: 1080, maxBitrate: 3000000 },
|
|
183
|
+
{ rid: 'h', width: 960, height: 540, maxBitrate: 750000 },
|
|
184
|
+
{ rid: 'q', width: 480, height: 270, maxBitrate: 187500 },
|
|
185
|
+
];
|
|
186
|
+
|
|
187
|
+
const videoLayers = toVideoLayers(layers as OptimalVideoLayer[]);
|
|
188
|
+
expect(videoLayers.length).toBe(3);
|
|
189
|
+
expect(videoLayers[0]).toEqual({
|
|
190
|
+
rid: 'f',
|
|
191
|
+
bitrate: 3000000,
|
|
192
|
+
fps: 0,
|
|
193
|
+
quality: VideoQuality.HIGH,
|
|
194
|
+
videoDimension: { width: 1920, height: 1080 },
|
|
195
|
+
});
|
|
196
|
+
expect(videoLayers[1]).toEqual({
|
|
197
|
+
rid: 'h',
|
|
198
|
+
bitrate: 750000,
|
|
199
|
+
fps: 0,
|
|
200
|
+
quality: VideoQuality.MID,
|
|
201
|
+
videoDimension: { width: 960, height: 540 },
|
|
202
|
+
});
|
|
203
|
+
expect(videoLayers[2]).toEqual({
|
|
204
|
+
rid: 'q',
|
|
205
|
+
bitrate: 187500,
|
|
206
|
+
fps: 0,
|
|
207
|
+
quality: VideoQuality.LOW_UNSPECIFIED,
|
|
208
|
+
videoDimension: { width: 480, height: 270 },
|
|
209
|
+
});
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
it('should map OptimalVideoLayer to SVC encodings (three layers)', () => {
|
|
167
213
|
const layers: Array<Partial<OptimalVideoLayer>> = [
|
|
168
214
|
{ rid: 'f', width: 1920, height: 1080, maxBitrate: 3000000 },
|
|
169
215
|
{ rid: 'h', width: 960, height: 540, maxBitrate: 750000 },
|
|
@@ -180,10 +226,46 @@ describe('videoLayers', () => {
|
|
|
180
226
|
});
|
|
181
227
|
});
|
|
182
228
|
|
|
229
|
+
it('should map OptimalVideoLayer to SVC encodings (two layers)', () => {
|
|
230
|
+
const layers: Array<Partial<OptimalVideoLayer>> = [
|
|
231
|
+
{ rid: 'h', width: 960, height: 540, maxBitrate: 750000 },
|
|
232
|
+
{ rid: 'q', width: 480, height: 270, maxBitrate: 187500 },
|
|
233
|
+
];
|
|
234
|
+
|
|
235
|
+
const svcLayers = toSvcEncodings(layers as OptimalVideoLayer[]);
|
|
236
|
+
expect(svcLayers.length).toBe(1);
|
|
237
|
+
expect(svcLayers[0]).toEqual({
|
|
238
|
+
rid: 'q',
|
|
239
|
+
width: 960,
|
|
240
|
+
height: 540,
|
|
241
|
+
maxBitrate: 750000,
|
|
242
|
+
});
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
it('should map OptimalVideoLayer to SVC encodings (two layers)', () => {
|
|
246
|
+
const layers: Array<Partial<OptimalVideoLayer>> = [
|
|
247
|
+
{ rid: 'q', width: 480, height: 270, maxBitrate: 187500 },
|
|
248
|
+
];
|
|
249
|
+
|
|
250
|
+
const svcLayers = toSvcEncodings(layers as OptimalVideoLayer[]);
|
|
251
|
+
expect(svcLayers.length).toBe(1);
|
|
252
|
+
expect(svcLayers[0]).toEqual({
|
|
253
|
+
rid: 'q',
|
|
254
|
+
width: 480,
|
|
255
|
+
height: 270,
|
|
256
|
+
maxBitrate: 187500,
|
|
257
|
+
});
|
|
258
|
+
});
|
|
259
|
+
|
|
183
260
|
describe('getComputedMaxBitrate', () => {
|
|
184
261
|
it('should scale target bitrate down if resolution is smaller than target resolution', () => {
|
|
185
262
|
const targetResolution = { width: 1920, height: 1080, bitrate: 3000000 };
|
|
186
|
-
const scaledBitrate = getComputedMaxBitrate(
|
|
263
|
+
const scaledBitrate = getComputedMaxBitrate(
|
|
264
|
+
targetResolution,
|
|
265
|
+
1280,
|
|
266
|
+
720,
|
|
267
|
+
3000000,
|
|
268
|
+
);
|
|
187
269
|
expect(scaledBitrate).toBe(1333333);
|
|
188
270
|
});
|
|
189
271
|
|
|
@@ -193,7 +275,12 @@ describe('videoLayers', () => {
|
|
|
193
275
|
const targetBitrates = ['f', 'h', 'q'].map((rid) => {
|
|
194
276
|
const width = targetResolution.width / downscaleFactor;
|
|
195
277
|
const height = targetResolution.height / downscaleFactor;
|
|
196
|
-
const bitrate = getComputedMaxBitrate(
|
|
278
|
+
const bitrate = getComputedMaxBitrate(
|
|
279
|
+
targetResolution,
|
|
280
|
+
width,
|
|
281
|
+
height,
|
|
282
|
+
3000000,
|
|
283
|
+
);
|
|
197
284
|
downscaleFactor *= 2;
|
|
198
285
|
return {
|
|
199
286
|
rid,
|
|
@@ -211,25 +298,45 @@ describe('videoLayers', () => {
|
|
|
211
298
|
|
|
212
299
|
it('should not scale target bitrate if resolution is larger than target resolution', () => {
|
|
213
300
|
const targetResolution = { width: 1280, height: 720, bitrate: 1000000 };
|
|
214
|
-
const scaledBitrate = getComputedMaxBitrate(
|
|
301
|
+
const scaledBitrate = getComputedMaxBitrate(
|
|
302
|
+
targetResolution,
|
|
303
|
+
2560,
|
|
304
|
+
1440,
|
|
305
|
+
1000000,
|
|
306
|
+
);
|
|
215
307
|
expect(scaledBitrate).toBe(1000000);
|
|
216
308
|
});
|
|
217
309
|
|
|
218
310
|
it('should not scale target bitrate if resolution is equal to target resolution', () => {
|
|
219
311
|
const targetResolution = { width: 1280, height: 720, bitrate: 1000000 };
|
|
220
|
-
const scaledBitrate = getComputedMaxBitrate(
|
|
312
|
+
const scaledBitrate = getComputedMaxBitrate(
|
|
313
|
+
targetResolution,
|
|
314
|
+
1280,
|
|
315
|
+
720,
|
|
316
|
+
1000000,
|
|
317
|
+
);
|
|
221
318
|
expect(scaledBitrate).toBe(1000000);
|
|
222
319
|
});
|
|
223
320
|
|
|
224
321
|
it('should handle 0 width and height', () => {
|
|
225
322
|
const targetResolution = { width: 1280, height: 720, bitrate: 1000000 };
|
|
226
|
-
const scaledBitrate = getComputedMaxBitrate(
|
|
323
|
+
const scaledBitrate = getComputedMaxBitrate(
|
|
324
|
+
targetResolution,
|
|
325
|
+
0,
|
|
326
|
+
0,
|
|
327
|
+
1000000,
|
|
328
|
+
);
|
|
227
329
|
expect(scaledBitrate).toBe(0);
|
|
228
330
|
});
|
|
229
331
|
|
|
230
332
|
it('should handle 4k target resolution', () => {
|
|
231
333
|
const targetResolution = { width: 3840, height: 2160, bitrate: 15000000 };
|
|
232
|
-
const scaledBitrate = getComputedMaxBitrate(
|
|
334
|
+
const scaledBitrate = getComputedMaxBitrate(
|
|
335
|
+
targetResolution,
|
|
336
|
+
1280,
|
|
337
|
+
720,
|
|
338
|
+
15000000,
|
|
339
|
+
);
|
|
233
340
|
expect(scaledBitrate).toBe(1666667);
|
|
234
341
|
});
|
|
235
342
|
});
|
package/src/rtc/codecs.ts
CHANGED
|
@@ -1,80 +1,7 @@
|
|
|
1
|
-
import { getOSInfo } from '../client-details';
|
|
2
|
-
import { isReactNative } from '../helpers/platforms';
|
|
3
|
-
import { isFirefox, isSafari } from '../helpers/browsers';
|
|
4
|
-
import type { PreferredCodec } from '../types';
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Returns back a list of sorted codecs, with the preferred codec first.
|
|
8
|
-
*
|
|
9
|
-
* @param kind the kind of codec to get.
|
|
10
|
-
* @param preferredCodec the codec to prioritize (vp8, h264, vp9, av1...).
|
|
11
|
-
* @param codecToRemove the codec to exclude from the list.
|
|
12
|
-
* @param codecPreferencesSource the source of the codec preferences.
|
|
13
|
-
*/
|
|
14
|
-
export const getPreferredCodecs = (
|
|
15
|
-
kind: 'audio' | 'video',
|
|
16
|
-
preferredCodec: string,
|
|
17
|
-
codecToRemove: string | undefined,
|
|
18
|
-
codecPreferencesSource: 'sender' | 'receiver',
|
|
19
|
-
): RTCRtpCodec[] | undefined => {
|
|
20
|
-
const source =
|
|
21
|
-
codecPreferencesSource === 'receiver' ? RTCRtpReceiver : RTCRtpSender;
|
|
22
|
-
if (!('getCapabilities' in source)) return;
|
|
23
|
-
|
|
24
|
-
const capabilities = source.getCapabilities(kind);
|
|
25
|
-
if (!capabilities) return;
|
|
26
|
-
|
|
27
|
-
const preferred: RTCRtpCodecCapability[] = [];
|
|
28
|
-
const partiallyPreferred: RTCRtpCodecCapability[] = [];
|
|
29
|
-
const unpreferred: RTCRtpCodecCapability[] = [];
|
|
30
|
-
|
|
31
|
-
const preferredCodecMimeType = `${kind}/${preferredCodec.toLowerCase()}`;
|
|
32
|
-
const codecToRemoveMimeType =
|
|
33
|
-
codecToRemove && `${kind}/${codecToRemove.toLowerCase()}`;
|
|
34
|
-
|
|
35
|
-
for (const codec of capabilities.codecs) {
|
|
36
|
-
const codecMimeType = codec.mimeType.toLowerCase();
|
|
37
|
-
|
|
38
|
-
const shouldRemoveCodec = codecMimeType === codecToRemoveMimeType;
|
|
39
|
-
if (shouldRemoveCodec) continue; // skip this codec
|
|
40
|
-
|
|
41
|
-
const isPreferredCodec = codecMimeType === preferredCodecMimeType;
|
|
42
|
-
if (!isPreferredCodec) {
|
|
43
|
-
unpreferred.push(codec);
|
|
44
|
-
continue;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
// h264 is a special case, we want to prioritize the baseline codec with
|
|
48
|
-
// profile-level-id is 42e01f and packetization-mode=0 for maximum
|
|
49
|
-
// cross-browser compatibility.
|
|
50
|
-
// this branch covers the other cases, such as vp8.
|
|
51
|
-
if (codecMimeType !== 'video/h264') {
|
|
52
|
-
preferred.push(codec);
|
|
53
|
-
continue;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
const sdpFmtpLine = codec.sdpFmtpLine;
|
|
57
|
-
if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42')) {
|
|
58
|
-
// this is not the baseline h264 codec, prioritize it lower
|
|
59
|
-
partiallyPreferred.push(codec);
|
|
60
|
-
continue;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
if (sdpFmtpLine.includes('packetization-mode=1')) {
|
|
64
|
-
preferred.unshift(codec);
|
|
65
|
-
} else {
|
|
66
|
-
preferred.push(codec);
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
// return a sorted list of codecs, with the preferred codecs first
|
|
71
|
-
return [...preferred, ...partiallyPreferred, ...unpreferred];
|
|
72
|
-
};
|
|
73
|
-
|
|
74
1
|
/**
|
|
75
2
|
* Returns a generic SDP for the given direction.
|
|
76
3
|
* We use this SDP to send it as part of our JoinRequest so that the SFU
|
|
77
|
-
* can use it to determine client's codec capabilities.
|
|
4
|
+
* can use it to determine the client's codec capabilities.
|
|
78
5
|
*
|
|
79
6
|
* @param direction the direction of the transceiver.
|
|
80
7
|
*/
|
|
@@ -93,63 +20,6 @@ export const getGenericSdp = async (direction: RTCRtpTransceiverDirection) => {
|
|
|
93
20
|
return sdp;
|
|
94
21
|
};
|
|
95
22
|
|
|
96
|
-
/**
|
|
97
|
-
* Returns the optimal video codec for the device.
|
|
98
|
-
*/
|
|
99
|
-
export const getOptimalVideoCodec = (
|
|
100
|
-
preferredCodec: PreferredCodec | undefined,
|
|
101
|
-
): PreferredCodec => {
|
|
102
|
-
if (isReactNative()) {
|
|
103
|
-
const os = getOSInfo()?.name.toLowerCase();
|
|
104
|
-
if (os === 'android') return preferredOr(preferredCodec, 'vp8');
|
|
105
|
-
if (os === 'ios' || os === 'ipados') {
|
|
106
|
-
return supportsH264Baseline() ? 'h264' : 'vp8';
|
|
107
|
-
}
|
|
108
|
-
return preferredOr(preferredCodec, 'h264');
|
|
109
|
-
}
|
|
110
|
-
if (isSafari()) return 'h264';
|
|
111
|
-
if (isFirefox()) return 'vp8';
|
|
112
|
-
return preferredOr(preferredCodec, 'vp8');
|
|
113
|
-
};
|
|
114
|
-
|
|
115
|
-
/**
|
|
116
|
-
* Determines if the platform supports the preferred codec.
|
|
117
|
-
* If not, it returns the fallback codec.
|
|
118
|
-
*/
|
|
119
|
-
const preferredOr = (
|
|
120
|
-
codec: PreferredCodec | undefined,
|
|
121
|
-
fallback: PreferredCodec,
|
|
122
|
-
): PreferredCodec => {
|
|
123
|
-
if (!codec) return fallback;
|
|
124
|
-
if (!('getCapabilities' in RTCRtpSender)) return fallback;
|
|
125
|
-
const capabilities = RTCRtpSender.getCapabilities('video');
|
|
126
|
-
if (!capabilities) return fallback;
|
|
127
|
-
|
|
128
|
-
// Safari and Firefox do not have a good support encoding to SVC codecs,
|
|
129
|
-
// so we disable it for them.
|
|
130
|
-
if (isSvcCodec(codec) && (isSafari() || isFirefox())) return fallback;
|
|
131
|
-
|
|
132
|
-
const { codecs } = capabilities;
|
|
133
|
-
const codecMimeType = `video/${codec}`.toLowerCase();
|
|
134
|
-
return codecs.some((c) => c.mimeType.toLowerCase() === codecMimeType)
|
|
135
|
-
? codec
|
|
136
|
-
: fallback;
|
|
137
|
-
};
|
|
138
|
-
|
|
139
|
-
/**
|
|
140
|
-
* Returns whether the platform supports the H264 baseline codec.
|
|
141
|
-
*/
|
|
142
|
-
const supportsH264Baseline = (): boolean => {
|
|
143
|
-
if (!('getCapabilities' in RTCRtpSender)) return false;
|
|
144
|
-
const capabilities = RTCRtpSender.getCapabilities('video');
|
|
145
|
-
if (!capabilities) return false;
|
|
146
|
-
return capabilities.codecs.some(
|
|
147
|
-
(c) =>
|
|
148
|
-
c.mimeType.toLowerCase() === 'video/h264' &&
|
|
149
|
-
c.sdpFmtpLine?.includes('profile-level-id=42e01f'),
|
|
150
|
-
);
|
|
151
|
-
};
|
|
152
|
-
|
|
153
23
|
/**
|
|
154
24
|
* Returns whether the codec is an SVC codec.
|
|
155
25
|
*
|