livekit-client 2.22.1 → 2.22.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/livekit-client.e2ee.worker.js +1 -1
- package/dist/livekit-client.e2ee.worker.js.map +1 -1
- package/dist/livekit-client.e2ee.worker.mjs +663 -476
- package/dist/livekit-client.e2ee.worker.mjs.map +1 -1
- package/dist/livekit-client.esm.mjs +480 -157
- package/dist/livekit-client.esm.mjs.map +1 -1
- package/dist/livekit-client.fm.worker.js +1 -1
- package/dist/livekit-client.fm.worker.js.map +1 -1
- package/dist/livekit-client.fm.worker.mjs +8 -1
- package/dist/livekit-client.fm.worker.mjs.map +1 -1
- package/dist/livekit-client.umd.js +1 -1
- package/dist/livekit-client.umd.js.map +1 -1
- package/dist/src/api/SignalClient.d.ts.map +1 -1
- package/dist/src/api/WebSocketStream.d.ts.map +1 -1
- package/dist/src/api/utils.d.ts +1 -0
- package/dist/src/api/utils.d.ts.map +1 -1
- package/dist/src/e2ee/E2eeManager.d.ts +33 -0
- package/dist/src/e2ee/E2eeManager.d.ts.map +1 -1
- package/dist/src/e2ee/constants.d.ts +5 -0
- package/dist/src/e2ee/constants.d.ts.map +1 -1
- package/dist/src/e2ee/types.d.ts +24 -3
- package/dist/src/e2ee/types.d.ts.map +1 -1
- package/dist/src/e2ee/worker/DataCryptor.d.ts.map +1 -1
- package/dist/src/e2ee/worker/ErrorRateLimiter.d.ts +21 -0
- package/dist/src/e2ee/worker/ErrorRateLimiter.d.ts.map +1 -0
- package/dist/src/e2ee/worker/FrameCryptor.d.ts +43 -19
- package/dist/src/e2ee/worker/FrameCryptor.d.ts.map +1 -1
- package/dist/src/logger.d.ts +4 -0
- package/dist/src/logger.d.ts.map +1 -1
- package/dist/src/room/PCTransportManager.d.ts +12 -0
- package/dist/src/room/PCTransportManager.d.ts.map +1 -1
- package/dist/src/room/RTCEngine.d.ts +1 -0
- package/dist/src/room/RTCEngine.d.ts.map +1 -1
- package/dist/src/room/data-stream/incoming/StreamReader.d.ts +17 -17
- package/dist/src/room/data-stream/incoming/StreamReader.d.ts.map +1 -1
- package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
- package/dist/src/room/participant/publishUtils.d.ts +16 -0
- package/dist/src/room/participant/publishUtils.d.ts.map +1 -1
- package/dist/src/room/track/LocalVideoTrack.d.ts +7 -0
- package/dist/src/room/track/LocalVideoTrack.d.ts.map +1 -1
- package/dist/src/room/track/options.d.ts +1 -1
- package/dist/src/room/utils.d.ts +34 -0
- package/dist/src/room/utils.d.ts.map +1 -1
- package/dist/ts4.2/api/utils.d.ts +1 -0
- package/dist/ts4.2/e2ee/E2eeManager.d.ts +33 -0
- package/dist/ts4.2/e2ee/constants.d.ts +5 -0
- package/dist/ts4.2/e2ee/types.d.ts +24 -3
- package/dist/ts4.2/e2ee/worker/ErrorRateLimiter.d.ts +21 -0
- package/dist/ts4.2/e2ee/worker/FrameCryptor.d.ts +43 -19
- package/dist/ts4.2/logger.d.ts +4 -0
- package/dist/ts4.2/room/PCTransportManager.d.ts +12 -0
- package/dist/ts4.2/room/RTCEngine.d.ts +1 -0
- package/dist/ts4.2/room/data-stream/incoming/StreamReader.d.ts +17 -17
- package/dist/ts4.2/room/participant/publishUtils.d.ts +16 -0
- package/dist/ts4.2/room/track/LocalVideoTrack.d.ts +7 -0
- package/dist/ts4.2/room/track/options.d.ts +1 -1
- package/dist/ts4.2/room/utils.d.ts +34 -0
- package/package.json +1 -1
- package/src/api/SignalClient.ts +2 -1
- package/src/api/WebSocketStream.ts +3 -8
- package/src/api/utils.ts +10 -0
- package/src/e2ee/E2eeManager.test.ts +196 -0
- package/src/e2ee/E2eeManager.ts +150 -31
- package/src/e2ee/constants.ts +6 -0
- package/src/e2ee/subscriberBlackScreen.test.ts +544 -0
- package/src/e2ee/types.ts +28 -3
- package/src/e2ee/worker/DataCryptor.ts +2 -1
- package/src/e2ee/worker/ErrorRateLimiter.test.ts +53 -0
- package/src/e2ee/worker/ErrorRateLimiter.ts +52 -0
- package/src/e2ee/worker/FrameCryptor.race.test.ts +9 -26
- package/src/e2ee/worker/FrameCryptor.test.ts +0 -1
- package/src/e2ee/worker/FrameCryptor.ts +202 -119
- package/src/e2ee/worker/e2ee.worker.ts +72 -17
- package/src/logger.ts +22 -0
- package/src/room/PCTransportManager.test.ts +35 -0
- package/src/room/PCTransportManager.ts +12 -4
- package/src/room/RTCEngine.ts +28 -7
- package/src/room/Room.ts +1 -1
- package/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +171 -0
- package/src/room/data-stream/incoming/IncomingDataStreamManager.ts +17 -18
- package/src/room/data-stream/incoming/StreamReader.ts +20 -50
- package/src/room/participant/LocalParticipant.ts +30 -14
- package/src/room/participant/publishUtils.test.ts +133 -0
- package/src/room/participant/publishUtils.ts +54 -19
- package/src/room/track/LocalVideoTrack.ts +15 -5
- package/src/room/track/options.ts +1 -1
- package/src/room/utils.test.ts +87 -0
- package/src/room/utils.ts +59 -0
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { DataStream_Chunk } from '@livekit/protocol';
|
|
2
2
|
import { DataStreamError, DataStreamErrorReason } from '../../errors';
|
|
3
3
|
import type { BaseStreamInfo, ByteStreamInfo, TextStreamInfo } from '../../types';
|
|
4
|
-
import { bigIntToNumber } from '../../utils';
|
|
5
4
|
|
|
6
5
|
export type BaseStreamReaderReadAllOpts = {
|
|
7
6
|
/** An AbortSignal can be used to terminate reads early. */
|
|
@@ -47,14 +46,11 @@ abstract class BaseStreamReader<T extends BaseStreamInfo> {
|
|
|
47
46
|
this.bytesReceived = 0;
|
|
48
47
|
}
|
|
49
48
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export class ByteStreamReader extends BaseStreamReader<ByteStreamInfo> {
|
|
49
|
+
/**
|
|
50
|
+
* Counts a chunk's bytes against `totalByteSize` and reports progress. Chunk ordering and
|
|
51
|
+
* de-duplication happen upstream in the manager's `ensureOrderedChunks`, so every chunk reaching
|
|
52
|
+
* here is new and in order.
|
|
53
|
+
*/
|
|
58
54
|
protected handleChunkReceived(chunk: DataStream_Chunk) {
|
|
59
55
|
this.bytesReceived += chunk.content.byteLength;
|
|
60
56
|
this.validateBytesReceived();
|
|
@@ -65,8 +61,15 @@ export class ByteStreamReader extends BaseStreamReader<ByteStreamInfo> {
|
|
|
65
61
|
this.onProgress?.(currentProgress);
|
|
66
62
|
}
|
|
67
63
|
|
|
64
|
+
/**
|
|
65
|
+
* @param progress - progress of the stream between 0 and 1. Undefined for streams of unknown size
|
|
66
|
+
*/
|
|
68
67
|
onProgress?: (progress: number | undefined) => void;
|
|
69
68
|
|
|
69
|
+
abstract readAll(opts?: BaseStreamReaderReadAllOpts): Promise<string | Array<Uint8Array>>;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export class ByteStreamReader extends BaseStreamReader<ByteStreamInfo> {
|
|
70
73
|
signal?: AbortSignal;
|
|
71
74
|
|
|
72
75
|
[Symbol.asyncIterator]() {
|
|
@@ -151,53 +154,20 @@ export class ByteStreamReader extends BaseStreamReader<ByteStreamInfo> {
|
|
|
151
154
|
}
|
|
152
155
|
|
|
153
156
|
/**
|
|
154
|
-
* A class to read chunks from a ReadableStream and
|
|
157
|
+
* A class to read chunks from a ReadableStream and decode them as UTF-8 text.
|
|
158
|
+
*
|
|
159
|
+
* NOTE: chunk-level `version` (resending a chunk at an already-received `chunkIndex` to supersede
|
|
160
|
+
* it) is not supported. The reader used to rebuild the whole string from a per-index chunk map and
|
|
161
|
+
* yield it as `TextStreamChunk.collected`, which made superseding work; 5d4a6346 (#1410, text auto
|
|
162
|
+
* chunking) changed the iterator to yield each chunk's text as it arrives, and a streaming reader
|
|
163
|
+
* cannot retract text it has already handed to the consumer. No sender emits a versioned chunk.
|
|
155
164
|
*/
|
|
156
165
|
export class TextStreamReader extends BaseStreamReader<TextStreamInfo> {
|
|
157
|
-
private receivedChunks: Map<number /* chunk index */, DataStream_Chunk>;
|
|
158
|
-
|
|
159
166
|
signal?: AbortSignal;
|
|
160
167
|
|
|
161
|
-
/**
|
|
162
|
-
* A TextStreamReader instance can be used as an AsyncIterator that returns the entire string
|
|
163
|
-
* that has been received up to the current point in time.
|
|
164
|
-
*/
|
|
165
|
-
constructor(
|
|
166
|
-
info: TextStreamInfo,
|
|
167
|
-
stream: ReadableStream<DataStream_Chunk>,
|
|
168
|
-
totalChunkCount?: number,
|
|
169
|
-
) {
|
|
170
|
-
super(info, stream, totalChunkCount);
|
|
171
|
-
this.receivedChunks = new Map();
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
protected handleChunkReceived(chunk: DataStream_Chunk) {
|
|
175
|
-
const index = bigIntToNumber(chunk.chunkIndex);
|
|
176
|
-
const previousChunkAtIndex = this.receivedChunks.get(index);
|
|
177
|
-
if (previousChunkAtIndex && previousChunkAtIndex.version > chunk.version) {
|
|
178
|
-
// we have a newer version already, dropping the old one
|
|
179
|
-
return;
|
|
180
|
-
}
|
|
181
|
-
this.receivedChunks.set(index, chunk);
|
|
182
|
-
|
|
183
|
-
this.bytesReceived += chunk.content.byteLength;
|
|
184
|
-
this.validateBytesReceived();
|
|
185
|
-
|
|
186
|
-
const currentProgress = this.totalByteSize
|
|
187
|
-
? this.bytesReceived / this.totalByteSize
|
|
188
|
-
: undefined;
|
|
189
|
-
this.onProgress?.(currentProgress);
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
/**
|
|
193
|
-
* @param progress - progress of the stream between 0 and 1. Undefined for streams of unknown size
|
|
194
|
-
*/
|
|
195
|
-
onProgress?: (progress: number | undefined) => void;
|
|
196
|
-
|
|
197
168
|
/**
|
|
198
169
|
* Async iterator implementation to allow usage of `for await...of` syntax.
|
|
199
|
-
* Yields
|
|
200
|
-
*
|
|
170
|
+
* Yields each chunk's decoded text as it arrives - a delta, not the string accumulated so far.
|
|
201
171
|
*/
|
|
202
172
|
[Symbol.asyncIterator]() {
|
|
203
173
|
const reader = this.reader.getReader();
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
TrackInfo,
|
|
20
20
|
TrackUnpublishedResponse,
|
|
21
21
|
UserPacket,
|
|
22
|
+
VideoLayer_Mode,
|
|
22
23
|
protoInt64,
|
|
23
24
|
} from '@livekit/protocol';
|
|
24
25
|
import { SignalConnectionState } from '../../api/SignalClient';
|
|
@@ -101,6 +102,8 @@ import {
|
|
|
101
102
|
isLocalTrack,
|
|
102
103
|
isLocalVideoTrack,
|
|
103
104
|
isSVCCodec,
|
|
105
|
+
isSVCSimulcast,
|
|
106
|
+
isSVCSimulcastSupportedByServer,
|
|
104
107
|
isSafari17Based,
|
|
105
108
|
isVideoCodec,
|
|
106
109
|
isVideoTrack,
|
|
@@ -108,12 +111,14 @@ import {
|
|
|
108
111
|
sleep,
|
|
109
112
|
supportsAV1,
|
|
110
113
|
supportsVP9,
|
|
114
|
+
usesLegacySVCEncodings,
|
|
111
115
|
} from '../utils';
|
|
112
116
|
import Participant from './Participant';
|
|
113
117
|
import type { ParticipantTrackPermission } from './ParticipantTrackPermission';
|
|
114
118
|
import { trackPermissionToProto } from './ParticipantTrackPermission';
|
|
115
119
|
import type RemoteParticipant from './RemoteParticipant';
|
|
116
120
|
import {
|
|
121
|
+
computeStartTargetBitrate,
|
|
117
122
|
computeTrackBackupEncodings,
|
|
118
123
|
computeVideoEncodings,
|
|
119
124
|
getDefaultDegradationPreference,
|
|
@@ -1135,7 +1140,19 @@ export default class LocalParticipant extends Participant {
|
|
|
1135
1140
|
req.height = dims.height;
|
|
1136
1141
|
// for svc codecs, disable simulcast and use vp8 for backup codec
|
|
1137
1142
|
if (isLocalVideoTrack(track)) {
|
|
1138
|
-
if (
|
|
1143
|
+
if (
|
|
1144
|
+
isSVCSimulcast(videoCodec, opts) &&
|
|
1145
|
+
(usesLegacySVCEncodings() || !isSVCSimulcastSupportedByServer(this.engine?.serverVersion))
|
|
1146
|
+
) {
|
|
1147
|
+
opts.simulcast = false;
|
|
1148
|
+
this.log.info(
|
|
1149
|
+
'SVC simulcast is not supported, disabling simulcast.',
|
|
1150
|
+
getLogContextFromTrack(track),
|
|
1151
|
+
);
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
const svcSimulcast = isSVCSimulcast(videoCodec, opts);
|
|
1155
|
+
if (isSVCCodec(videoCodec) && !svcSimulcast) {
|
|
1139
1156
|
if (track.source === Track.Source.ScreenShare) {
|
|
1140
1157
|
// vp9 svc with screenshare cannot encode multiple spatial layers
|
|
1141
1158
|
// doing so reduces publish resolution to minimal resolution
|
|
@@ -1157,12 +1174,14 @@ export default class LocalParticipant extends Participant {
|
|
|
1157
1174
|
opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY';
|
|
1158
1175
|
}
|
|
1159
1176
|
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1177
|
+
const primaryCodec = new SimulcastCodec({
|
|
1178
|
+
codec: videoCodec,
|
|
1179
|
+
cid: track.mediaStreamTrack.id,
|
|
1180
|
+
});
|
|
1181
|
+
if (svcSimulcast) {
|
|
1182
|
+
primaryCodec.videoLayerMode = VideoLayer_Mode.ONE_SPATIAL_LAYER_PER_STREAM;
|
|
1183
|
+
}
|
|
1184
|
+
req.simulcastCodecs = [primaryCodec];
|
|
1166
1185
|
|
|
1167
1186
|
// set up backup
|
|
1168
1187
|
if (opts.backupCodec === true) {
|
|
@@ -1197,7 +1216,7 @@ export default class LocalParticipant extends Participant {
|
|
|
1197
1216
|
req.width,
|
|
1198
1217
|
req.height,
|
|
1199
1218
|
encodings,
|
|
1200
|
-
isSVCCodec(opts.videoCodec),
|
|
1219
|
+
isSVCCodec(opts.videoCodec) && !isSVCSimulcast(opts.videoCodec, opts),
|
|
1201
1220
|
);
|
|
1202
1221
|
} else if (track.kind === Track.Kind.Audio) {
|
|
1203
1222
|
encodings = [
|
|
@@ -1253,12 +1272,9 @@ export default class LocalParticipant extends Participant {
|
|
|
1253
1272
|
});
|
|
1254
1273
|
}
|
|
1255
1274
|
} else if (track.codec && isVideoCodec(track.codec)) {
|
|
1256
|
-
// Apply start bitrate for all video codecs to prevent initial blurriness
|
|
1257
|
-
//
|
|
1258
|
-
|
|
1259
|
-
const targetBitrate = isSVCCodec(track.codec)
|
|
1260
|
-
? (encodings[0]?.maxBitrate ?? 0)
|
|
1261
|
-
: encodings.reduce((sum, enc) => sum + (enc.maxBitrate ?? 0), 0);
|
|
1275
|
+
// Apply start bitrate for all video codecs to prevent initial blurriness,
|
|
1276
|
+
// see computeStartTargetBitrate
|
|
1277
|
+
const targetBitrate = computeStartTargetBitrate(track.codec, opts, encodings);
|
|
1262
1278
|
if (targetBitrate > 0) {
|
|
1263
1279
|
this.engine.pcManager.publisher.setTrackCodecBitrate({
|
|
1264
1280
|
cid: req.cid,
|
|
@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest';
|
|
|
2
2
|
import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options';
|
|
3
3
|
import {
|
|
4
4
|
computeDefaultScreenShareSimulcastPresets,
|
|
5
|
+
computeStartTargetBitrate,
|
|
5
6
|
computeVideoEncodings,
|
|
6
7
|
determineAppropriateEncoding,
|
|
7
8
|
presets43,
|
|
@@ -116,6 +117,78 @@ describe('computeVideoEncodings', () => {
|
|
|
116
117
|
expect(encodings![0].scaleResolutionDownBy).toBe(1);
|
|
117
118
|
});
|
|
118
119
|
|
|
120
|
+
// svc carries the scalabilityMode on the first encoding only (whether it emits a
|
|
121
|
+
// single encoding or the legacy multi-encoding shape), simulcast carries it on all
|
|
122
|
+
const countScalabilityModes = (encodings?: RTCRtpEncodingParameters[]) =>
|
|
123
|
+
/* @ts-ignore */
|
|
124
|
+
encodings!.filter((encoding) => encoding.scalabilityMode !== undefined).length;
|
|
125
|
+
|
|
126
|
+
it('keeps svc for an svc codec without simulcast', () => {
|
|
127
|
+
const encodings = computeVideoEncodings(false, 960, 540, {
|
|
128
|
+
simulcast: false,
|
|
129
|
+
videoCodec: 'vp9',
|
|
130
|
+
scalabilityMode: 'L3T3_KEY',
|
|
131
|
+
});
|
|
132
|
+
/* @ts-ignore */
|
|
133
|
+
expect(encodings![0].scalabilityMode).toBe('L3T3_KEY');
|
|
134
|
+
expect(countScalabilityModes(encodings)).toBe(1);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
it('keeps svc for an svc codec with a multi spatial layer mode even if simulcast is set', () => {
|
|
138
|
+
const encodings = computeVideoEncodings(false, 960, 540, {
|
|
139
|
+
simulcast: true,
|
|
140
|
+
videoCodec: 'vp9',
|
|
141
|
+
scalabilityMode: 'L3T3_KEY',
|
|
142
|
+
});
|
|
143
|
+
/* @ts-ignore */
|
|
144
|
+
expect(encodings![0].scalabilityMode).toBe('L3T3_KEY');
|
|
145
|
+
expect(countScalabilityModes(encodings)).toBe(1);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
it('returns a simulcast ladder for an svc codec with simulcast and an L1Tx mode', () => {
|
|
149
|
+
for (const videoCodec of ['vp9', 'av1'] as const) {
|
|
150
|
+
const encodings = computeVideoEncodings(false, 960, 540, {
|
|
151
|
+
simulcast: true,
|
|
152
|
+
videoCodec,
|
|
153
|
+
scalabilityMode: 'L1T2',
|
|
154
|
+
});
|
|
155
|
+
expect(encodings).toHaveLength(3);
|
|
156
|
+
expect(encodings!.map((e) => e.rid)).toEqual(['q', 'h', 'f']);
|
|
157
|
+
// every encoding needs both scalabilityMode and scaleResolutionDownBy for chrome
|
|
158
|
+
// M113+ to treat them as real simulcast rather than legacy svc
|
|
159
|
+
encodings!.forEach((encoding) => {
|
|
160
|
+
/* @ts-ignore */
|
|
161
|
+
expect(encoding.scalabilityMode).toBe('L1T2');
|
|
162
|
+
expect(encoding.scaleResolutionDownBy).toBeGreaterThanOrEqual(1);
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
it('sets the scalability mode on a single encoding svc simulcast ladder', () => {
|
|
168
|
+
const encodings = computeVideoEncodings(false, 100, 120, {
|
|
169
|
+
simulcast: true,
|
|
170
|
+
videoCodec: 'vp9',
|
|
171
|
+
scalabilityMode: 'L1T3',
|
|
172
|
+
});
|
|
173
|
+
expect(encodings).toHaveLength(1);
|
|
174
|
+
expect(encodings![0].rid).toBe('q');
|
|
175
|
+
/* @ts-ignore */
|
|
176
|
+
expect(encodings![0].scalabilityMode).toBe('L1T3');
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it('does not set a scalability mode for non-svc simulcast', () => {
|
|
180
|
+
const encodings = computeVideoEncodings(false, 960, 540, {
|
|
181
|
+
simulcast: true,
|
|
182
|
+
videoCodec: 'vp8',
|
|
183
|
+
scalabilityMode: 'L1T2',
|
|
184
|
+
});
|
|
185
|
+
expect(encodings).toHaveLength(3);
|
|
186
|
+
encodings!.forEach((encoding) => {
|
|
187
|
+
/* @ts-ignore */
|
|
188
|
+
expect(encoding.scalabilityMode).toBeUndefined();
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
|
|
119
192
|
// it('respects default backup codec encoding', () => {
|
|
120
193
|
// const vp8Encodings = computeTrackBackupEncodings(false, 100, 120, { simulcast: true });
|
|
121
194
|
// const h264Encodings = computeVideoEncodings(false, 100, 120, {
|
|
@@ -193,3 +266,63 @@ describe('screenShareSimulcastDefaults', () => {
|
|
|
193
266
|
expect(defaultSimulcastLayers[0].encoding.maxBitrate).toBe(375000);
|
|
194
267
|
});
|
|
195
268
|
});
|
|
269
|
+
|
|
270
|
+
describe('computeStartTargetBitrate', () => {
|
|
271
|
+
// ordered q..f, as encodingsFromPresets builds them
|
|
272
|
+
const simulcastLadder: RTCRtpEncodingParameters[] = [
|
|
273
|
+
{ rid: 'q', maxBitrate: 160_000 },
|
|
274
|
+
{ rid: 'h', maxBitrate: 450_000 },
|
|
275
|
+
{ rid: 'f', maxBitrate: 680_000 },
|
|
276
|
+
];
|
|
277
|
+
|
|
278
|
+
it('sums the ladder for plain simulcast', () => {
|
|
279
|
+
expect(computeStartTargetBitrate('vp8', { simulcast: true }, simulcastLadder)).toBe(1_290_000);
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
it('sums the ladder for vp9/av1 published as rid simulcast', () => {
|
|
283
|
+
// encodings[0] is the *smallest* layer here, so taking it would under-hint BWE
|
|
284
|
+
for (const codec of ['vp9', 'av1'] as const) {
|
|
285
|
+
expect(
|
|
286
|
+
computeStartTargetBitrate(
|
|
287
|
+
codec,
|
|
288
|
+
{ simulcast: true, videoCodec: codec, scalabilityMode: 'L1T2' },
|
|
289
|
+
simulcastLadder,
|
|
290
|
+
),
|
|
291
|
+
).toBe(1_290_000);
|
|
292
|
+
}
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
it('uses the single encoding for vp9/av1 svc', () => {
|
|
296
|
+
for (const codec of ['vp9', 'av1'] as const) {
|
|
297
|
+
expect(
|
|
298
|
+
computeStartTargetBitrate(
|
|
299
|
+
codec,
|
|
300
|
+
{ simulcast: false, videoCodec: codec, scalabilityMode: 'L3T3_KEY' },
|
|
301
|
+
[{ maxBitrate: 680_000 }],
|
|
302
|
+
),
|
|
303
|
+
).toBe(680_000);
|
|
304
|
+
}
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
it('uses the first encoding for the legacy svc shape, which is ordered f..q', () => {
|
|
308
|
+
// legacy SVC pushes videoRids[2 - i], so encodings[0] carries the full bitrate
|
|
309
|
+
const legacySvc: RTCRtpEncodingParameters[] = [
|
|
310
|
+
{ rid: 'f', maxBitrate: 680_000 },
|
|
311
|
+
{ rid: 'h', maxBitrate: 226_667 },
|
|
312
|
+
{ rid: 'q', maxBitrate: 75_556 },
|
|
313
|
+
];
|
|
314
|
+
expect(
|
|
315
|
+
computeStartTargetBitrate(
|
|
316
|
+
'vp9',
|
|
317
|
+
{ simulcast: false, videoCodec: 'vp9', scalabilityMode: 'L3T3_KEY' },
|
|
318
|
+
legacySvc,
|
|
319
|
+
),
|
|
320
|
+
).toBe(680_000);
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
it('handles missing bitrates and empty encodings', () => {
|
|
324
|
+
expect(computeStartTargetBitrate('vp8', { simulcast: true }, [])).toBe(0);
|
|
325
|
+
expect(computeStartTargetBitrate('vp9', undefined, [])).toBe(0);
|
|
326
|
+
expect(computeStartTargetBitrate('vp8', { simulcast: true }, [{ rid: 'q' }])).toBe(0);
|
|
327
|
+
});
|
|
328
|
+
});
|
|
@@ -13,12 +13,12 @@ import type {
|
|
|
13
13
|
import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options';
|
|
14
14
|
import type { LoggerOptions } from '../types';
|
|
15
15
|
import {
|
|
16
|
-
compareVersions,
|
|
17
16
|
getReactNativeOs,
|
|
18
17
|
isReactNative,
|
|
19
18
|
isSVCCodec,
|
|
20
|
-
|
|
19
|
+
isSVCSimulcast,
|
|
21
20
|
isSafariSvcApi,
|
|
21
|
+
usesLegacySVCEncodings,
|
|
22
22
|
} from '../utils';
|
|
23
23
|
|
|
24
24
|
/** @internal */
|
|
@@ -111,6 +111,8 @@ export function computeVideoEncodings(
|
|
|
111
111
|
const useSimulcast = options?.simulcast;
|
|
112
112
|
const scalabilityMode = options?.scalabilityMode;
|
|
113
113
|
const videoCodec = options?.videoCodec;
|
|
114
|
+
// VP9/AV1 published as rid based simulcast rather than SVC, see isSVCSimulcast
|
|
115
|
+
const useSVCSimulcast = isSVCSimulcast(videoCodec, options);
|
|
114
116
|
|
|
115
117
|
if ((!videoEncoding && !useSimulcast && !scalabilityMode) || !width || !height) {
|
|
116
118
|
// when we aren't simulcasting or svc, will need to return a single encoding without
|
|
@@ -134,7 +136,7 @@ export function computeVideoEncodings(
|
|
|
134
136
|
videoEncoding.priority,
|
|
135
137
|
);
|
|
136
138
|
|
|
137
|
-
if (scalabilityMode && isSVCCodec(videoCodec)) {
|
|
139
|
+
if (scalabilityMode && isSVCCodec(videoCodec) && !useSVCSimulcast) {
|
|
138
140
|
const sm = new ScalabilityMode(scalabilityMode);
|
|
139
141
|
|
|
140
142
|
const encodings: RTCRtpEncodingParameters[] = [];
|
|
@@ -142,20 +144,10 @@ export function computeVideoEncodings(
|
|
|
142
144
|
if (sm.spatial > 3) {
|
|
143
145
|
throw new Error(`unsupported scalabilityMode: ${scalabilityMode}`);
|
|
144
146
|
}
|
|
145
|
-
//
|
|
146
|
-
//
|
|
147
|
-
// This is a bit confusing but is due to how libwebrtc interpreted the encodings field
|
|
148
|
-
// before M113.
|
|
149
|
-
// Announced here: https://groups.google.com/g/discuss-webrtc/c/-QQ3pxrl-fw?pli=1
|
|
147
|
+
// Browsers that read multiple encodings on an SVC codec as SVC rather than as
|
|
148
|
+
// simulcast, see usesLegacySVCEncodings
|
|
150
149
|
const browser = getBrowser();
|
|
151
|
-
if (
|
|
152
|
-
isSafariBased() ||
|
|
153
|
-
// Even tho RN runs M114, it does not produce SVC layers when a single encoding
|
|
154
|
-
// is provided. So we'll use the legacy SVC specification for now.
|
|
155
|
-
// TODO: when we upstream libwebrtc, this will need additional verification
|
|
156
|
-
isReactNative() ||
|
|
157
|
-
(browser?.name === 'Chrome' && compareVersions(browser?.version, '113') < 0)
|
|
158
|
-
) {
|
|
150
|
+
if (usesLegacySVCEncodings()) {
|
|
159
151
|
const bitratesRatio = sm.suffix == 'h' ? 2 : 3;
|
|
160
152
|
// safari 18.4 uses a different svc API that requires scaleResolutionDownBy to be set.
|
|
161
153
|
const requireScale = isSafariSvcApi(browser);
|
|
@@ -193,6 +185,19 @@ export function computeVideoEncodings(
|
|
|
193
185
|
return [videoEncoding];
|
|
194
186
|
}
|
|
195
187
|
|
|
188
|
+
// Chrome M113+ only treats multiple encodings on an SVC capable codec as real
|
|
189
|
+
// simulcast when every encoding carries an explicit scalabilityMode next to its
|
|
190
|
+
// scaleResolutionDownBy. Without it the encodings are interpreted as legacy SVC.
|
|
191
|
+
const applySVCSimulcastMode = (encodings: RTCRtpEncodingParameters[]) => {
|
|
192
|
+
if (useSVCSimulcast) {
|
|
193
|
+
encodings.forEach((encoding) => {
|
|
194
|
+
/* @ts-ignore */
|
|
195
|
+
encoding.scalabilityMode = scalabilityMode;
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
return encodings;
|
|
199
|
+
};
|
|
200
|
+
|
|
196
201
|
let presets: Array<VideoPreset>;
|
|
197
202
|
if (isScreenShare) {
|
|
198
203
|
presets =
|
|
@@ -220,13 +225,43 @@ export function computeVideoEncodings(
|
|
|
220
225
|
// based on other conditions.
|
|
221
226
|
const size = Math.max(width, height);
|
|
222
227
|
if (size >= 960 && midPreset) {
|
|
223
|
-
return
|
|
228
|
+
return applySVCSimulcastMode(
|
|
229
|
+
encodingsFromPresets(width, height, [lowPreset, midPreset, original], sourceFramerate),
|
|
230
|
+
);
|
|
224
231
|
}
|
|
225
232
|
if (size >= 480) {
|
|
226
|
-
return
|
|
233
|
+
return applySVCSimulcastMode(
|
|
234
|
+
encodingsFromPresets(width, height, [lowPreset, original], sourceFramerate),
|
|
235
|
+
);
|
|
227
236
|
}
|
|
228
237
|
}
|
|
229
|
-
return encodingsFromPresets(width, height, [original]);
|
|
238
|
+
return applySVCSimulcastMode(encodingsFromPresets(width, height, [original]));
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Bitrate to hint to the bandwidth estimator through `x-google-start-bitrate`, so that
|
|
243
|
+
* a publish does not spend its first seconds ramping up from a very low rate.
|
|
244
|
+
*
|
|
245
|
+
* It has to be the total the encoder will put on the wire, which means picking the
|
|
246
|
+
* encoding that carries the inclusive bitrate:
|
|
247
|
+
* - SVC publishes a single stream with the layers built in. `encodings[0]` holds the
|
|
248
|
+
* full bitrate — the legacy SVC shape orders its encodings `f`..`q`, so that holds
|
|
249
|
+
* for both SVC shapes.
|
|
250
|
+
* - Simulcast publishes independent streams ordered `q`..`f`, so the total is the sum.
|
|
251
|
+
* This includes VP9/AV1 published as rid based simulcast, where `encodings[0]` is
|
|
252
|
+
* the *smallest* layer even though the codec is SVC capable.
|
|
253
|
+
*
|
|
254
|
+
* @internal
|
|
255
|
+
*/
|
|
256
|
+
export function computeStartTargetBitrate(
|
|
257
|
+
codec: string,
|
|
258
|
+
options: TrackPublishOptions | undefined,
|
|
259
|
+
encodings: RTCRtpEncodingParameters[],
|
|
260
|
+
): number {
|
|
261
|
+
if (isSVCCodec(codec) && !isSVCSimulcast(codec, options)) {
|
|
262
|
+
return encodings[0]?.maxBitrate ?? 0;
|
|
263
|
+
}
|
|
264
|
+
return encodings.reduce((sum, enc) => sum + (enc.maxBitrate ?? 0), 0);
|
|
230
265
|
}
|
|
231
266
|
|
|
232
267
|
export function computeTrackBackupEncodings(
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
import type { VideoSenderStats } from '../stats';
|
|
17
17
|
import { computeBitrate, monitorFrequency } from '../stats';
|
|
18
18
|
import type { LoggerOptions } from '../types';
|
|
19
|
-
import { isFireFox, isMobile, isSVCCodec, isWeb } from '../utils';
|
|
19
|
+
import { isFireFox, isMobile, isSVCCodec, isSVCSimulcast, isWeb } from '../utils';
|
|
20
20
|
import LocalTrack from './LocalTrack';
|
|
21
21
|
import { Track, VideoQuality } from './Track';
|
|
22
22
|
import type { TrackPublishOptions, VideoCaptureOptions, VideoCodec } from './options';
|
|
@@ -243,6 +243,16 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
|
|
|
243
243
|
return items;
|
|
244
244
|
}
|
|
245
245
|
|
|
246
|
+
/**
|
|
247
|
+
* Whether `codec` is being published as SVC (a single stream carrying all spatial
|
|
248
|
+
* layers) as opposed to rid based simulcast. VP9/AV1 are SVC unless the publisher
|
|
249
|
+
* opted into simulcast, in which case each rid is an independent stream and the
|
|
250
|
+
* layers can be enabled/disabled individually.
|
|
251
|
+
*/
|
|
252
|
+
private isSvcPublish(codec?: string): boolean {
|
|
253
|
+
return isSVCCodec(codec) && !isSVCSimulcast(codec, this.publishOptions);
|
|
254
|
+
}
|
|
255
|
+
|
|
246
256
|
setPublishingQuality(maxQuality: VideoQuality) {
|
|
247
257
|
const qualities: SubscribedQuality[] = [];
|
|
248
258
|
for (let q = VideoQuality.LOW; q <= VideoQuality.HIGH; q += 1) {
|
|
@@ -254,7 +264,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
|
|
|
254
264
|
);
|
|
255
265
|
}
|
|
256
266
|
this.log.debug(`setting publishing quality. max quality ${maxQuality}`, this.logContext);
|
|
257
|
-
this.setPublishingLayers(
|
|
267
|
+
this.setPublishingLayers(this.isSvcPublish(this.codec), qualities);
|
|
258
268
|
}
|
|
259
269
|
|
|
260
270
|
async restartTrack(options?: VideoCaptureOptions) {
|
|
@@ -491,7 +501,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
|
|
|
491
501
|
});
|
|
492
502
|
// only enable simulcast codec for preference codec setted
|
|
493
503
|
if (!this.codec && codecs.length > 0) {
|
|
494
|
-
await this.setPublishingLayers(
|
|
504
|
+
await this.setPublishingLayers(this.isSvcPublish(codecs[0].codec), codecs[0].qualities);
|
|
495
505
|
|
|
496
506
|
return [];
|
|
497
507
|
}
|
|
@@ -501,7 +511,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
|
|
|
501
511
|
const newCodecs: VideoCodec[] = [];
|
|
502
512
|
for await (const codec of codecs) {
|
|
503
513
|
if (!this.codec || this.codec === codec.codec) {
|
|
504
|
-
await this.setPublishingLayers(
|
|
514
|
+
await this.setPublishingLayers(this.isSvcPublish(codec.codec), codec.qualities);
|
|
505
515
|
} else {
|
|
506
516
|
const simulcastCodecInfo = this.simulcastCodecs.get(codec.codec as VideoCodec);
|
|
507
517
|
this.log.debug(`try setPublishingCodec for ${codec.codec}`, {
|
|
@@ -522,7 +532,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
|
|
|
522
532
|
simulcastCodecInfo.encodings!,
|
|
523
533
|
codec.qualities,
|
|
524
534
|
this.senderLock,
|
|
525
|
-
|
|
535
|
+
this.isSvcPublish(codec.codec),
|
|
526
536
|
this.log,
|
|
527
537
|
this.logContext,
|
|
528
538
|
);
|
|
@@ -79,7 +79,7 @@ export interface TrackPublishDefaults {
|
|
|
79
79
|
|
|
80
80
|
/**
|
|
81
81
|
* scalability mode for svc codecs, defaults to 'L3T3_KEY'.
|
|
82
|
-
* for svc codecs, simulcast is disabled.
|
|
82
|
+
* for svc codecs, simulcast is disabled if more than one spatial layer is used ('L2Tx' or 'L3Tx').
|
|
83
83
|
*/
|
|
84
84
|
scalabilityMode?: ScalabilityMode;
|
|
85
85
|
|
package/src/room/utils.test.ts
CHANGED
|
@@ -4,10 +4,13 @@ import {
|
|
|
4
4
|
ddExtensionURI,
|
|
5
5
|
extractMaxAgeFromRequestHeaders,
|
|
6
6
|
getClientInfo,
|
|
7
|
+
isSVCSimulcast,
|
|
8
|
+
isSVCSimulcastSupportedByServer,
|
|
7
9
|
negotiateDependencyDescriptor,
|
|
8
10
|
splitUtf8,
|
|
9
11
|
supportsAdaptiveStream,
|
|
10
12
|
toWebsocketUrl,
|
|
13
|
+
usesLegacySVCEncodings,
|
|
11
14
|
} from './utils';
|
|
12
15
|
|
|
13
16
|
describe('toWebsocketUrl', () => {
|
|
@@ -280,3 +283,87 @@ describe('supportsAdaptiveStream', () => {
|
|
|
280
283
|
expect(supportsAdaptiveStream()).toBe(false);
|
|
281
284
|
});
|
|
282
285
|
});
|
|
286
|
+
|
|
287
|
+
describe('isSVCSimulcast', () => {
|
|
288
|
+
it('requires an svc capable codec, simulcast and a single spatial layer mode', () => {
|
|
289
|
+
expect(isSVCSimulcast('vp9', { simulcast: true, scalabilityMode: 'L1T2' })).toBe(true);
|
|
290
|
+
expect(isSVCSimulcast('av1', { simulcast: true, scalabilityMode: 'L1T3' })).toBe(true);
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
it('stays on svc without the opt in', () => {
|
|
294
|
+
expect(isSVCSimulcast('vp9', { simulcast: false, scalabilityMode: 'L1T2' })).toBe(false);
|
|
295
|
+
expect(isSVCSimulcast('vp9', { simulcast: true })).toBe(false);
|
|
296
|
+
// a multi spatial layer mode is svc by definition
|
|
297
|
+
expect(isSVCSimulcast('vp9', { simulcast: true, scalabilityMode: 'L3T3_KEY' })).toBe(false);
|
|
298
|
+
expect(isSVCSimulcast('vp9', undefined)).toBe(false);
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
it('does not apply to non svc codecs', () => {
|
|
302
|
+
expect(isSVCSimulcast('vp8', { simulcast: true, scalabilityMode: 'L1T2' })).toBe(false);
|
|
303
|
+
expect(isSVCSimulcast('h264', { simulcast: true, scalabilityMode: 'L1T2' })).toBe(false);
|
|
304
|
+
expect(isSVCSimulcast(undefined, { simulcast: true, scalabilityMode: 'L1T2' })).toBe(false);
|
|
305
|
+
});
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
describe('isSVCSimulcastSupportedByServer', () => {
|
|
309
|
+
it('requires a server newer than 1.13.6', () => {
|
|
310
|
+
expect(isSVCSimulcastSupportedByServer('1.13.7')).toBe(true);
|
|
311
|
+
expect(isSVCSimulcastSupportedByServer('1.14.0')).toBe(true);
|
|
312
|
+
expect(isSVCSimulcastSupportedByServer('2.0.0')).toBe(true);
|
|
313
|
+
});
|
|
314
|
+
|
|
315
|
+
it('rejects 1.13.6 and older', () => {
|
|
316
|
+
expect(isSVCSimulcastSupportedByServer('1.13.6')).toBe(false);
|
|
317
|
+
expect(isSVCSimulcastSupportedByServer('1.13.5')).toBe(false);
|
|
318
|
+
expect(isSVCSimulcastSupportedByServer('1.9.0')).toBe(false);
|
|
319
|
+
expect(isSVCSimulcastSupportedByServer('0.15.1')).toBe(false);
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
it('treats an unknown version as unsupported', () => {
|
|
323
|
+
expect(isSVCSimulcastSupportedByServer(undefined)).toBe(false);
|
|
324
|
+
expect(isSVCSimulcastSupportedByServer('')).toBe(false);
|
|
325
|
+
});
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
describe('usesLegacySVCEncodings', () => {
|
|
329
|
+
const stubUserAgent = (userAgent: string) =>
|
|
330
|
+
vi.stubGlobal('navigator', { userAgent, product: 'Gecko' });
|
|
331
|
+
|
|
332
|
+
afterEach(() => {
|
|
333
|
+
vi.unstubAllGlobals();
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
const chrome = (v: string) =>
|
|
337
|
+
`Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${v} Safari/537.36`;
|
|
338
|
+
|
|
339
|
+
it('is legacy on chrome before M113', () => {
|
|
340
|
+
stubUserAgent(chrome('112.0.0.0'));
|
|
341
|
+
expect(usesLegacySVCEncodings()).toBe(true);
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
it('is not legacy from chrome M113 onwards', () => {
|
|
345
|
+
stubUserAgent(chrome('113.0.0.0'));
|
|
346
|
+
expect(usesLegacySVCEncodings()).toBe(false);
|
|
347
|
+
stubUserAgent(chrome('120.0.0.0'));
|
|
348
|
+
expect(usesLegacySVCEncodings()).toBe(false);
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
it('is legacy on safari, which has no rid based svc simulcast', () => {
|
|
352
|
+
stubUserAgent(
|
|
353
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
|
|
354
|
+
);
|
|
355
|
+
expect(usesLegacySVCEncodings()).toBe(true);
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
it('is legacy on iOS, where every browser is webkit', () => {
|
|
359
|
+
stubUserAgent(
|
|
360
|
+
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.0.0 Mobile/15E148 Safari/604.1',
|
|
361
|
+
);
|
|
362
|
+
expect(usesLegacySVCEncodings()).toBe(true);
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
it('is legacy on react native regardless of chrome version', () => {
|
|
366
|
+
vi.stubGlobal('navigator', { userAgent: chrome('120.0.0.0'), product: 'ReactNative' });
|
|
367
|
+
expect(usesLegacySVCEncodings()).toBe(true);
|
|
368
|
+
});
|
|
369
|
+
});
|