livekit-client 1.9.6 → 1.10.0
Sign up to get free protection for your applications and to get access to all the features.
- package/dist/livekit-client.esm.mjs +1318 -885
- package/dist/livekit-client.esm.mjs.map +1 -1
- package/dist/livekit-client.umd.js +1 -1
- package/dist/livekit-client.umd.js.map +1 -1
- package/dist/src/api/SignalClient.d.ts +2 -1
- package/dist/src/api/SignalClient.d.ts.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/proto/livekit_models.d.ts +108 -10
- package/dist/src/proto/livekit_models.d.ts.map +1 -1
- package/dist/src/proto/livekit_rtc.d.ts +513 -194
- package/dist/src/proto/livekit_rtc.d.ts.map +1 -1
- package/dist/src/room/Room.d.ts +3 -2
- package/dist/src/room/Room.d.ts.map +1 -1
- package/dist/src/room/events.d.ts +5 -1
- package/dist/src/room/events.d.ts.map +1 -1
- package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
- package/dist/src/room/participant/Participant.d.ts +2 -2
- package/dist/src/room/participant/Participant.d.ts.map +1 -1
- package/dist/src/room/participant/RemoteParticipant.d.ts.map +1 -1
- package/dist/src/room/participant/publishUtils.d.ts +8 -0
- package/dist/src/room/participant/publishUtils.d.ts.map +1 -1
- package/dist/src/room/track/LocalTrack.d.ts +32 -0
- package/dist/src/room/track/LocalTrack.d.ts.map +1 -1
- package/dist/src/room/track/LocalVideoTrack.d.ts.map +1 -1
- package/dist/src/room/track/RemoteTrackPublication.d.ts +4 -1
- package/dist/src/room/track/RemoteTrackPublication.d.ts.map +1 -1
- package/dist/src/room/track/TrackPublication.d.ts +2 -1
- package/dist/src/room/track/TrackPublication.d.ts.map +1 -1
- package/dist/src/room/track/options.d.ts +1 -1
- package/dist/src/room/track/options.d.ts.map +1 -1
- package/dist/src/room/track/processor/types.d.ts +19 -0
- package/dist/src/room/track/processor/types.d.ts.map +1 -0
- package/dist/src/utils/browserParser.d.ts.map +1 -1
- package/dist/ts4.2/src/api/SignalClient.d.ts +2 -1
- package/dist/ts4.2/src/index.d.ts +1 -0
- package/dist/ts4.2/src/proto/livekit_models.d.ts +126 -12
- package/dist/ts4.2/src/proto/livekit_rtc.d.ts +617 -254
- package/dist/ts4.2/src/room/Room.d.ts +3 -2
- package/dist/ts4.2/src/room/events.d.ts +5 -1
- package/dist/ts4.2/src/room/participant/Participant.d.ts +2 -2
- package/dist/ts4.2/src/room/participant/publishUtils.d.ts +8 -0
- package/dist/ts4.2/src/room/track/LocalTrack.d.ts +32 -0
- package/dist/ts4.2/src/room/track/RemoteTrackPublication.d.ts +4 -1
- package/dist/ts4.2/src/room/track/TrackPublication.d.ts +2 -1
- package/dist/ts4.2/src/room/track/options.d.ts +1 -1
- package/dist/ts4.2/src/room/track/processor/types.d.ts +19 -0
- package/package.json +14 -13
- package/src/api/SignalClient.ts +8 -1
- package/src/index.ts +1 -0
- package/src/proto/google/protobuf/timestamp.ts +3 -3
- package/src/proto/livekit_models.ts +254 -161
- package/src/proto/livekit_rtc.ts +334 -180
- package/src/room/Room.ts +26 -1
- package/src/room/events.ts +4 -0
- package/src/room/participant/LocalParticipant.ts +23 -3
- package/src/room/participant/Participant.ts +2 -1
- package/src/room/participant/RemoteParticipant.ts +4 -1
- package/src/room/participant/publishUtils.ts +68 -12
- package/src/room/track/LocalTrack.ts +120 -16
- package/src/room/track/LocalVideoTrack.ts +96 -33
- package/src/room/track/RemoteTrackPublication.ts +8 -1
- package/src/room/track/Track.ts +3 -3
- package/src/room/track/TrackPublication.ts +2 -1
- package/src/room/track/options.ts +1 -1
- package/src/room/track/processor/types.ts +20 -0
- package/src/utils/browserParser.ts +1 -4
package/src/room/Room.ts
CHANGED
@@ -18,6 +18,7 @@ import {
|
|
18
18
|
Room as RoomModel,
|
19
19
|
ServerInfo,
|
20
20
|
SpeakerInfo,
|
21
|
+
SubscriptionError,
|
21
22
|
TrackInfo,
|
22
23
|
TrackSource,
|
23
24
|
TrackType,
|
@@ -29,6 +30,7 @@ import {
|
|
29
30
|
SimulateScenario,
|
30
31
|
StreamStateUpdate,
|
31
32
|
SubscriptionPermissionUpdate,
|
33
|
+
SubscriptionResponse,
|
32
34
|
} from '../proto/livekit_rtc';
|
33
35
|
import DeviceManager from './DeviceManager';
|
34
36
|
import RTCEngine from './RTCEngine';
|
@@ -207,6 +209,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
|
|
207
209
|
this.engine.client.onStreamStateUpdate = this.handleStreamStateUpdate;
|
208
210
|
this.engine.client.onSubscriptionPermissionUpdate = this.handleSubscriptionPermissionUpdate;
|
209
211
|
this.engine.client.onConnectionQuality = this.handleConnectionQualityUpdate;
|
212
|
+
this.engine.client.onSubscriptionError = this.handleSubscriptionError;
|
210
213
|
|
211
214
|
this.engine
|
212
215
|
.on(
|
@@ -1114,6 +1117,21 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
|
|
1114
1117
|
pub.setAllowed(update.allowed);
|
1115
1118
|
};
|
1116
1119
|
|
1120
|
+
private handleSubscriptionError = (update: SubscriptionResponse) => {
|
1121
|
+
const participant = Array.from(this.participants.values()).find((p) =>
|
1122
|
+
p.tracks.has(update.trackSid),
|
1123
|
+
);
|
1124
|
+
if (!participant) {
|
1125
|
+
return;
|
1126
|
+
}
|
1127
|
+
const pub = participant.getTrackPublication(update.trackSid);
|
1128
|
+
if (!pub) {
|
1129
|
+
return;
|
1130
|
+
}
|
1131
|
+
|
1132
|
+
pub.setSubscriptionError(update.err);
|
1133
|
+
};
|
1134
|
+
|
1117
1135
|
private handleDataPacket = (userPacket: UserPacket, kind: DataPacket_Kind) => {
|
1118
1136
|
// find the participant
|
1119
1137
|
const participant = this.participants.get(userPacket.participantSid);
|
@@ -1280,6 +1298,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
|
|
1280
1298
|
.on(ParticipantEvent.TrackSubscriptionStatusChanged, (pub, status) => {
|
1281
1299
|
this.emitWhenConnected(RoomEvent.TrackSubscriptionStatusChanged, pub, status, participant);
|
1282
1300
|
})
|
1301
|
+
.on(ParticipantEvent.TrackSubscriptionFailed, (trackSid, error) => {
|
1302
|
+
this.emit(RoomEvent.TrackSubscriptionFailed, trackSid, participant, error);
|
1303
|
+
})
|
1283
1304
|
.on(ParticipantEvent.TrackSubscriptionPermissionChanged, (pub, status) => {
|
1284
1305
|
this.emitWhenConnected(
|
1285
1306
|
RoomEvent.TrackSubscriptionPermissionChanged,
|
@@ -1609,7 +1630,11 @@ export type RoomEventCallbacks = {
|
|
1609
1630
|
publication: RemoteTrackPublication,
|
1610
1631
|
participant: RemoteParticipant,
|
1611
1632
|
) => void;
|
1612
|
-
trackSubscriptionFailed: (
|
1633
|
+
trackSubscriptionFailed: (
|
1634
|
+
trackSid: string,
|
1635
|
+
participant: RemoteParticipant,
|
1636
|
+
reason?: SubscriptionError,
|
1637
|
+
) => void;
|
1613
1638
|
trackUnpublished: (publication: RemoteTrackPublication, participant: RemoteParticipant) => void;
|
1614
1639
|
trackUnsubscribed: (
|
1615
1640
|
track: RemoteTrack,
|
package/src/room/events.ts
CHANGED
@@ -625,8 +625,8 @@ export default class LocalParticipant extends Participant {
|
|
625
625
|
// for svc codecs, disable simulcast and use vp8 for backup codec
|
626
626
|
if (track instanceof LocalVideoTrack) {
|
627
627
|
if (isSVCCodec(opts.videoCodec)) {
|
628
|
-
// set scalabilityMode to '
|
629
|
-
opts.scalabilityMode = opts.scalabilityMode ?? '
|
628
|
+
// set scalabilityMode to 'L3T3_KEY' by default
|
629
|
+
opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY';
|
630
630
|
}
|
631
631
|
|
632
632
|
// set up backup
|
@@ -647,6 +647,16 @@ export default class LocalParticipant extends Participant {
|
|
647
647
|
enableSimulcastLayers: true,
|
648
648
|
},
|
649
649
|
];
|
650
|
+
} else if (opts.videoCodec) {
|
651
|
+
// pass codec info to sfu so it can prefer codec for the client which don't support
|
652
|
+
// setCodecPreferences
|
653
|
+
req.simulcastCodecs = [
|
654
|
+
{
|
655
|
+
codec: opts.videoCodec,
|
656
|
+
cid: track.mediaStreamTrack.id,
|
657
|
+
enableSimulcastLayers: opts.simulcast ?? false,
|
658
|
+
},
|
659
|
+
];
|
650
660
|
}
|
651
661
|
}
|
652
662
|
|
@@ -656,7 +666,7 @@ export default class LocalParticipant extends Participant {
|
|
656
666
|
dims.height,
|
657
667
|
opts,
|
658
668
|
);
|
659
|
-
req.layers = videoLayersFromEncodings(req.width, req.height,
|
669
|
+
req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
|
660
670
|
} else if (track.kind === Track.Kind.Audio) {
|
661
671
|
encodings = [
|
662
672
|
{
|
@@ -850,6 +860,16 @@ export default class LocalParticipant extends Participant {
|
|
850
860
|
trackSender
|
851
861
|
) {
|
852
862
|
try {
|
863
|
+
for (const transceiver of this.engine.publisher.pc.getTransceivers()) {
|
864
|
+
// if sender is not currently sending (after replaceTrack(null))
|
865
|
+
// removeTrack would have no effect.
|
866
|
+
// to ensure we end up successfully removing the track, manually set
|
867
|
+
// the transceiver to inactive
|
868
|
+
if (transceiver.sender === trackSender) {
|
869
|
+
transceiver.direction = 'inactive';
|
870
|
+
negotiationNeeded = true;
|
871
|
+
}
|
872
|
+
}
|
853
873
|
if (this.engine.removeTrack(trackSender)) {
|
854
874
|
negotiationNeeded = true;
|
855
875
|
}
|
@@ -6,6 +6,7 @@ import {
|
|
6
6
|
ParticipantInfo,
|
7
7
|
ParticipantPermission,
|
8
8
|
ConnectionQuality as ProtoQuality,
|
9
|
+
SubscriptionError,
|
9
10
|
} from '../../proto/livekit_models';
|
10
11
|
import { ParticipantEvent, TrackEvent } from '../events';
|
11
12
|
import type LocalTrackPublication from '../track/LocalTrackPublication';
|
@@ -265,7 +266,7 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
|
|
265
266
|
export type ParticipantEventCallbacks = {
|
266
267
|
trackPublished: (publication: RemoteTrackPublication) => void;
|
267
268
|
trackSubscribed: (track: RemoteTrack, publication: RemoteTrackPublication) => void;
|
268
|
-
trackSubscriptionFailed: (trackSid: string) => void;
|
269
|
+
trackSubscriptionFailed: (trackSid: string, reason?: SubscriptionError) => void;
|
269
270
|
trackUnpublished: (publication: RemoteTrackPublication) => void;
|
270
271
|
trackUnsubscribed: (track: RemoteTrack, publication: RemoteTrackPublication) => void;
|
271
272
|
trackMuted: (publication: TrackPublication) => void;
|
@@ -1,6 +1,6 @@
|
|
1
1
|
import type { SignalClient } from '../../api/SignalClient';
|
2
2
|
import log from '../../logger';
|
3
|
-
import type { ParticipantInfo } from '../../proto/livekit_models';
|
3
|
+
import type { ParticipantInfo, SubscriptionError } from '../../proto/livekit_models';
|
4
4
|
import type { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
|
5
5
|
import { ParticipantEvent, TrackEvent } from '../events';
|
6
6
|
import RemoteAudioTrack from '../track/RemoteAudioTrack';
|
@@ -81,6 +81,9 @@ export default class RemoteParticipant extends Participant {
|
|
81
81
|
publication.on(TrackEvent.Unsubscribed, (previousTrack: RemoteTrack) => {
|
82
82
|
this.emit(ParticipantEvent.TrackUnsubscribed, previousTrack, publication);
|
83
83
|
});
|
84
|
+
publication.on(TrackEvent.SubscriptionFailed, (error: SubscriptionError) => {
|
85
|
+
this.emit(ParticipantEvent.TrackSubscriptionFailed, publication.trackSid, error);
|
86
|
+
});
|
84
87
|
}
|
85
88
|
|
86
89
|
getTrack(source: Track.Source): RemoteTrackPublication | undefined {
|
@@ -10,7 +10,7 @@ import type {
|
|
10
10
|
VideoCodec,
|
11
11
|
VideoEncoding,
|
12
12
|
} from '../track/options';
|
13
|
-
import { isSVCCodec } from '../utils';
|
13
|
+
import { getReactNativeOs, isReactNative, isSVCCodec } from '../utils';
|
14
14
|
|
15
15
|
/** @internal */
|
16
16
|
export function mediaTrackToLocalTrack(
|
@@ -128,17 +128,15 @@ export function computeVideoEncodings(
|
|
128
128
|
// svc use first encoding as the original, so we sort encoding from high to low
|
129
129
|
switch (scalabilityMode) {
|
130
130
|
case 'L3T3':
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
});
|
141
|
-
}
|
131
|
+
case 'L3T3_KEY':
|
132
|
+
encodings.push({
|
133
|
+
rid: videoRids[2],
|
134
|
+
maxBitrate: videoEncoding.maxBitrate,
|
135
|
+
/* @ts-ignore */
|
136
|
+
maxFramerate: original.encoding.maxFramerate,
|
137
|
+
/* @ts-ignore */
|
138
|
+
scalabilityMode: scalabilityMode,
|
139
|
+
});
|
142
140
|
log.debug('encodings', encodings);
|
143
141
|
return encodings;
|
144
142
|
|
@@ -321,6 +319,33 @@ function encodingsFromPresets(
|
|
321
319
|
}
|
322
320
|
encodings.push(encoding);
|
323
321
|
});
|
322
|
+
|
323
|
+
// RN ios simulcast requires all same framerates.
|
324
|
+
if (isReactNative() && getReactNativeOs() === 'ios') {
|
325
|
+
let topFramerate: number | undefined = undefined;
|
326
|
+
encodings.forEach((encoding) => {
|
327
|
+
if (!topFramerate) {
|
328
|
+
topFramerate = encoding.maxFramerate;
|
329
|
+
} else if (encoding.maxFramerate && encoding.maxFramerate > topFramerate) {
|
330
|
+
topFramerate = encoding.maxFramerate;
|
331
|
+
}
|
332
|
+
});
|
333
|
+
|
334
|
+
let notifyOnce = true;
|
335
|
+
encodings.forEach((encoding) => {
|
336
|
+
if (encoding.maxFramerate != topFramerate) {
|
337
|
+
if (notifyOnce) {
|
338
|
+
notifyOnce = false;
|
339
|
+
log.info(
|
340
|
+
`Simulcast on iOS React-Native requires all encodings to share the same framerate.`,
|
341
|
+
);
|
342
|
+
}
|
343
|
+
log.info(`Setting framerate of encoding \"${encoding.rid ?? ''}\" to ${topFramerate}`);
|
344
|
+
encoding.maxFramerate = topFramerate;
|
345
|
+
}
|
346
|
+
});
|
347
|
+
}
|
348
|
+
|
324
349
|
return encodings;
|
325
350
|
}
|
326
351
|
|
@@ -341,3 +366,34 @@ export function sortPresets(presets: Array<VideoPreset> | undefined) {
|
|
341
366
|
return 0;
|
342
367
|
});
|
343
368
|
}
|
369
|
+
|
370
|
+
/** @internal */
|
371
|
+
export class ScalabilityMode {
|
372
|
+
spatial: number;
|
373
|
+
|
374
|
+
temporal: number;
|
375
|
+
|
376
|
+
suffix: undefined | 'h' | '_KEY' | '_KEY_SHIFT';
|
377
|
+
|
378
|
+
constructor(scalabilityMode: string) {
|
379
|
+
const results = scalabilityMode.match(/^L(\d)T(\d)(h|_KEY|_KEY_SHIFT){0,1}$/);
|
380
|
+
if (!results) {
|
381
|
+
throw new Error('invalid scalability mode');
|
382
|
+
}
|
383
|
+
|
384
|
+
this.spatial = parseInt(results[1]);
|
385
|
+
this.temporal = parseInt(results[2]);
|
386
|
+
if (results.length > 3) {
|
387
|
+
switch (results[3]) {
|
388
|
+
case 'h':
|
389
|
+
case '_KEY':
|
390
|
+
case '_KEY_SHIFT':
|
391
|
+
this.suffix = results[3];
|
392
|
+
}
|
393
|
+
}
|
394
|
+
}
|
395
|
+
|
396
|
+
toString(): string {
|
397
|
+
return `L${this.spatial}T${this.temporal}${this.suffix ?? ''}`;
|
398
|
+
}
|
399
|
+
}
|
@@ -1,16 +1,12 @@
|
|
1
1
|
import log from '../../logger';
|
2
|
+
import { getBrowser } from '../../utils/browserParser';
|
2
3
|
import DeviceManager from '../DeviceManager';
|
3
|
-
import { TrackInvalidError } from '../errors';
|
4
|
+
import { DeviceUnsupportedError, TrackInvalidError } from '../errors';
|
4
5
|
import { TrackEvent } from '../events';
|
5
|
-
import {
|
6
|
-
Mutex,
|
7
|
-
getEmptyAudioStreamTrack,
|
8
|
-
getEmptyVideoStreamTrack,
|
9
|
-
isMobile,
|
10
|
-
sleep,
|
11
|
-
} from '../utils';
|
6
|
+
import { Mutex, compareVersions, isMobile, sleep } from '../utils';
|
12
7
|
import { Track, attachToElement, detachTrack } from './Track';
|
13
8
|
import type { VideoCodec } from './options';
|
9
|
+
import type { TrackProcessor } from './processor/types';
|
14
10
|
|
15
11
|
const defaultDimensionsTimeout = 1000;
|
16
12
|
|
@@ -31,6 +27,12 @@ export default abstract class LocalTrack extends Track {
|
|
31
27
|
|
32
28
|
protected pauseUpstreamLock: Mutex;
|
33
29
|
|
30
|
+
protected processorElement?: HTMLMediaElement;
|
31
|
+
|
32
|
+
protected processor?: TrackProcessor<typeof this.kind>;
|
33
|
+
|
34
|
+
protected isSettingUpProcessor: boolean = false;
|
35
|
+
|
34
36
|
/**
|
35
37
|
*
|
36
38
|
* @param mediaTrack
|
@@ -82,6 +84,10 @@ export default abstract class LocalTrack extends Track {
|
|
82
84
|
return this.providedByUser;
|
83
85
|
}
|
84
86
|
|
87
|
+
get mediaStreamTrack() {
|
88
|
+
return this.processor?.processedTrack ?? this._mediaStreamTrack;
|
89
|
+
}
|
90
|
+
|
85
91
|
async waitForDimensions(timeout = defaultDimensionsTimeout): Promise<Track.Dimensions> {
|
86
92
|
if (this.kind === Track.Kind.Audio) {
|
87
93
|
throw new Error('cannot get dimensions for audio tracks');
|
@@ -158,6 +164,9 @@ export default abstract class LocalTrack extends Track {
|
|
158
164
|
|
159
165
|
this.mediaStream = new MediaStream([track]);
|
160
166
|
this.providedByUser = userProvidedTrack;
|
167
|
+
if (this.processor) {
|
168
|
+
await this.stopProcessor();
|
169
|
+
}
|
161
170
|
return this;
|
162
171
|
}
|
163
172
|
|
@@ -180,7 +189,7 @@ export default abstract class LocalTrack extends Track {
|
|
180
189
|
|
181
190
|
// detach
|
182
191
|
this.attachedElements.forEach((el) => {
|
183
|
-
detachTrack(this.
|
192
|
+
detachTrack(this.mediaStreamTrack, el);
|
184
193
|
});
|
185
194
|
this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
|
186
195
|
// on Safari, the old audio track must be stopped before attempting to acquire
|
@@ -203,12 +212,16 @@ export default abstract class LocalTrack extends Track {
|
|
203
212
|
|
204
213
|
await this.resumeUpstream();
|
205
214
|
|
206
|
-
this.attachedElements.forEach((el) => {
|
207
|
-
attachToElement(newTrack, el);
|
208
|
-
});
|
209
|
-
|
210
215
|
this.mediaStream = mediaStream;
|
211
216
|
this.constraints = constraints;
|
217
|
+
if (this.processor) {
|
218
|
+
const processor = this.processor;
|
219
|
+
await this.setProcessor(processor);
|
220
|
+
} else {
|
221
|
+
this.attachedElements.forEach((el) => {
|
222
|
+
attachToElement(this._mediaStreamTrack, el);
|
223
|
+
});
|
224
|
+
}
|
212
225
|
this.emit(TrackEvent.Restarted, this);
|
213
226
|
return this;
|
214
227
|
}
|
@@ -253,6 +266,18 @@ export default abstract class LocalTrack extends Track {
|
|
253
266
|
this.emit(TrackEvent.Ended, this);
|
254
267
|
};
|
255
268
|
|
269
|
+
stop() {
|
270
|
+
super.stop();
|
271
|
+
this.processor?.destroy();
|
272
|
+
this.processor = undefined;
|
273
|
+
}
|
274
|
+
|
275
|
+
/**
|
276
|
+
* pauses publishing to the server without disabling the local MediaStreamTrack
|
277
|
+
* this is used to display a user's own video locally while pausing publishing to
|
278
|
+
* the server.
|
279
|
+
* this API is unsupported on Safari < 12 due to a bug
|
280
|
+
**/
|
256
281
|
async pauseUpstream() {
|
257
282
|
const unlock = await this.pauseUpstreamLock.lock();
|
258
283
|
try {
|
@@ -266,9 +291,12 @@ export default abstract class LocalTrack extends Track {
|
|
266
291
|
|
267
292
|
this._isUpstreamPaused = true;
|
268
293
|
this.emit(TrackEvent.UpstreamPaused, this);
|
269
|
-
const
|
270
|
-
|
271
|
-
|
294
|
+
const browser = getBrowser();
|
295
|
+
if (browser?.name === 'Safari' && compareVersions(browser.version, '12.0') < 0) {
|
296
|
+
// https://bugs.webkit.org/show_bug.cgi?id=184911
|
297
|
+
throw new DeviceUnsupportedError('pauseUpstream is not supported on Safari < 12.');
|
298
|
+
}
|
299
|
+
await this.sender.replaceTrack(null);
|
272
300
|
} finally {
|
273
301
|
unlock();
|
274
302
|
}
|
@@ -293,5 +321,81 @@ export default abstract class LocalTrack extends Track {
|
|
293
321
|
}
|
294
322
|
}
|
295
323
|
|
324
|
+
/**
|
325
|
+
* Sets a processor on this track.
|
326
|
+
* See https://github.com/livekit/track-processors-js for example usage
|
327
|
+
*
|
328
|
+
* @experimental
|
329
|
+
*
|
330
|
+
* @param processor
|
331
|
+
* @param showProcessedStreamLocally
|
332
|
+
* @returns
|
333
|
+
*/
|
334
|
+
async setProcessor(
|
335
|
+
processor: TrackProcessor<typeof this.kind>,
|
336
|
+
showProcessedStreamLocally = true,
|
337
|
+
) {
|
338
|
+
if (this.isSettingUpProcessor) {
|
339
|
+
log.warn('already trying to set up a processor');
|
340
|
+
return;
|
341
|
+
}
|
342
|
+
log.debug('setting up processor');
|
343
|
+
this.isSettingUpProcessor = true;
|
344
|
+
if (this.processor) {
|
345
|
+
await this.stopProcessor();
|
346
|
+
}
|
347
|
+
if (this.kind === 'unknown') {
|
348
|
+
throw TypeError('cannot set processor on track of unknown kind');
|
349
|
+
}
|
350
|
+
this.processorElement = this.processorElement ?? document.createElement(this.kind);
|
351
|
+
this.processorElement.muted = true;
|
352
|
+
|
353
|
+
attachToElement(this._mediaStreamTrack, this.processorElement);
|
354
|
+
this.processorElement.play().catch((e) => log.error(e));
|
355
|
+
|
356
|
+
const processorOptions = {
|
357
|
+
kind: this.kind,
|
358
|
+
track: this._mediaStreamTrack,
|
359
|
+
element: this.processorElement,
|
360
|
+
};
|
361
|
+
|
362
|
+
await processor.init(processorOptions);
|
363
|
+
this.processor = processor;
|
364
|
+
if (this.processor.processedTrack) {
|
365
|
+
for (const el of this.attachedElements) {
|
366
|
+
if (el !== this.processorElement && showProcessedStreamLocally) {
|
367
|
+
detachTrack(this._mediaStreamTrack, el);
|
368
|
+
attachToElement(this.processor.processedTrack, el);
|
369
|
+
}
|
370
|
+
}
|
371
|
+
await this.sender?.replaceTrack(this.processor.processedTrack);
|
372
|
+
}
|
373
|
+
this.isSettingUpProcessor = false;
|
374
|
+
}
|
375
|
+
|
376
|
+
getProcessor() {
|
377
|
+
return this.processor;
|
378
|
+
}
|
379
|
+
|
380
|
+
/**
|
381
|
+
* Stops the track processor
|
382
|
+
* See https://github.com/livekit/track-processors-js for example usage
|
383
|
+
*
|
384
|
+
* @experimental
|
385
|
+
* @returns
|
386
|
+
*/
|
387
|
+
async stopProcessor() {
|
388
|
+
if (!this.processor) return;
|
389
|
+
|
390
|
+
log.debug('stopping processor');
|
391
|
+
this.processor.processedTrack?.stop();
|
392
|
+
await this.processor.destroy();
|
393
|
+
this.processor = undefined;
|
394
|
+
this.processorElement?.remove();
|
395
|
+
this.processorElement = undefined;
|
396
|
+
|
397
|
+
await this.restart();
|
398
|
+
}
|
399
|
+
|
296
400
|
protected abstract monitorSender(): void;
|
297
401
|
}
|
@@ -2,6 +2,7 @@ import type { SignalClient } from '../../api/SignalClient';
|
|
2
2
|
import log from '../../logger';
|
3
3
|
import { VideoLayer, VideoQuality } from '../../proto/livekit_models';
|
4
4
|
import type { SubscribedCodec, SubscribedQuality } from '../../proto/livekit_rtc';
|
5
|
+
import { ScalabilityMode } from '../participant/publishUtils';
|
5
6
|
import { computeBitrate, monitorFrequency } from '../stats';
|
6
7
|
import type { VideoSenderStats } from '../stats';
|
7
8
|
import { Mutex, isFireFox, isMobile, isWeb } from '../utils';
|
@@ -349,45 +350,88 @@ async function setPublishingLayersForSender(
|
|
349
350
|
}
|
350
351
|
|
351
352
|
let hasChanged = false;
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
const
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
353
|
+
|
354
|
+
/* @ts-ignore */
|
355
|
+
if (encodings.length === 1 && encodings[0].scalabilityMode) {
|
356
|
+
// svc dynacast encodings
|
357
|
+
const encoding = encodings[0];
|
358
|
+
/* @ts-ignore */
|
359
|
+
// const mode = new ScalabilityMode(encoding.scalabilityMode);
|
360
|
+
let maxQuality = VideoQuality.OFF;
|
361
|
+
qualities.forEach((q) => {
|
362
|
+
if (q.enabled && (maxQuality === VideoQuality.OFF || q.quality > maxQuality)) {
|
363
|
+
maxQuality = q.quality;
|
364
|
+
}
|
365
|
+
});
|
366
|
+
|
367
|
+
if (maxQuality === VideoQuality.OFF) {
|
368
|
+
if (encoding.active) {
|
369
|
+
encoding.active = false;
|
370
|
+
hasChanged = true;
|
371
|
+
}
|
372
|
+
} else if (!encoding.active /* || mode.spatial !== maxQuality + 1*/) {
|
363
373
|
hasChanged = true;
|
364
|
-
encoding.active =
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
if (
|
374
|
-
|
375
|
-
|
376
|
-
encoding.maxBitrate = senderEncodings[idx].maxBitrate;
|
377
|
-
/* @ts-ignore */
|
378
|
-
encoding.maxFrameRate = senderEncodings[idx].maxFrameRate;
|
379
|
-
} else {
|
380
|
-
encoding.scaleResolutionDownBy = 4;
|
381
|
-
encoding.maxBitrate = 10;
|
382
|
-
/* @ts-ignore */
|
383
|
-
encoding.maxFrameRate = 2;
|
384
|
-
}
|
374
|
+
encoding.active = true;
|
375
|
+
/* disable closable spatial layer as it has video blur/frozen issue with current server/client
|
376
|
+
1. chrome 113: when switching to up layer with scalability Mode change, it will generate a
|
377
|
+
low resolution frame and recover very quickly, but noticable
|
378
|
+
2. livekit sfu: additional pli request cause video frozen for a few frames, also noticable
|
379
|
+
@ts-ignore
|
380
|
+
const originalMode = new ScalabilityMode(senderEncodings[0].scalabilityMode)
|
381
|
+
mode.spatial = maxQuality + 1;
|
382
|
+
mode.suffix = originalMode.suffix;
|
383
|
+
if (mode.spatial === 1) {
|
384
|
+
// no suffix for L1Tx
|
385
|
+
mode.suffix = undefined;
|
385
386
|
}
|
387
|
+
@ts-ignore
|
388
|
+
encoding.scalabilityMode = mode.toString();
|
389
|
+
encoding.scaleResolutionDownBy = 2 ** (2 - maxQuality);
|
390
|
+
*/
|
386
391
|
}
|
387
|
-
}
|
392
|
+
} else {
|
393
|
+
// simulcast dynacast encodings
|
394
|
+
encodings.forEach((encoding, idx) => {
|
395
|
+
let rid = encoding.rid ?? '';
|
396
|
+
if (rid === '') {
|
397
|
+
rid = 'q';
|
398
|
+
}
|
399
|
+
const quality = videoQualityForRid(rid);
|
400
|
+
const subscribedQuality = qualities.find((q) => q.quality === quality);
|
401
|
+
if (!subscribedQuality) {
|
402
|
+
return;
|
403
|
+
}
|
404
|
+
if (encoding.active !== subscribedQuality.enabled) {
|
405
|
+
hasChanged = true;
|
406
|
+
encoding.active = subscribedQuality.enabled;
|
407
|
+
log.debug(
|
408
|
+
`setting layer ${subscribedQuality.quality} to ${
|
409
|
+
encoding.active ? 'enabled' : 'disabled'
|
410
|
+
}`,
|
411
|
+
);
|
412
|
+
|
413
|
+
// FireFox does not support setting encoding.active to false, so we
|
414
|
+
// have a workaround of lowering its bitrate and resolution to the min.
|
415
|
+
if (isFireFox()) {
|
416
|
+
if (subscribedQuality.enabled) {
|
417
|
+
encoding.scaleResolutionDownBy = senderEncodings[idx].scaleResolutionDownBy;
|
418
|
+
encoding.maxBitrate = senderEncodings[idx].maxBitrate;
|
419
|
+
/* @ts-ignore */
|
420
|
+
encoding.maxFrameRate = senderEncodings[idx].maxFrameRate;
|
421
|
+
} else {
|
422
|
+
encoding.scaleResolutionDownBy = 4;
|
423
|
+
encoding.maxBitrate = 10;
|
424
|
+
/* @ts-ignore */
|
425
|
+
encoding.maxFrameRate = 2;
|
426
|
+
}
|
427
|
+
}
|
428
|
+
}
|
429
|
+
});
|
430
|
+
}
|
388
431
|
|
389
432
|
if (hasChanged) {
|
390
433
|
params.encodings = encodings;
|
434
|
+
log.debug(`setting encodings`, params.encodings);
|
391
435
|
await sender.setParameters(params);
|
392
436
|
}
|
393
437
|
} finally {
|
@@ -425,6 +469,25 @@ export function videoLayersFromEncodings(
|
|
425
469
|
},
|
426
470
|
];
|
427
471
|
}
|
472
|
+
|
473
|
+
/* @ts-ignore */
|
474
|
+
if (encodings.length === 1 && encodings[0].scalabilityMode) {
|
475
|
+
// svc layers
|
476
|
+
/* @ts-ignore */
|
477
|
+
const sm = new ScalabilityMode(encodings[0].scalabilityMode);
|
478
|
+
const layers = [];
|
479
|
+
for (let i = 0; i < sm.spatial; i += 1) {
|
480
|
+
layers.push({
|
481
|
+
quality: VideoQuality.HIGH - i,
|
482
|
+
width: width / 2 ** i,
|
483
|
+
height: height / 2 ** i,
|
484
|
+
bitrate: encodings[0].maxBitrate ? encodings[0].maxBitrate / 3 ** i : 0,
|
485
|
+
ssrc: 0,
|
486
|
+
});
|
487
|
+
}
|
488
|
+
return layers;
|
489
|
+
}
|
490
|
+
|
428
491
|
return encodings.map((encoding) => {
|
429
492
|
const scale = encoding.scaleResolutionDownBy ?? 1;
|
430
493
|
let quality = videoQualityForRid(encoding.rid ?? '');
|
@@ -1,5 +1,5 @@
|
|
1
1
|
import log from '../../logger';
|
2
|
-
import { TrackInfo, VideoQuality } from '../../proto/livekit_models';
|
2
|
+
import { SubscriptionError, TrackInfo, VideoQuality } from '../../proto/livekit_models';
|
3
3
|
import { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
|
4
4
|
import { TrackEvent } from '../events';
|
5
5
|
import type RemoteTrack from './RemoteTrack';
|
@@ -24,6 +24,8 @@ export default class RemoteTrackPublication extends TrackPublication {
|
|
24
24
|
|
25
25
|
protected fps?: number;
|
26
26
|
|
27
|
+
protected subscriptionError?: SubscriptionError;
|
28
|
+
|
27
29
|
constructor(kind: Track.Kind, ti: TrackInfo, autoSubscribe: boolean | undefined) {
|
28
30
|
super(kind, ti.sid, ti.name);
|
29
31
|
this.subscribed = autoSubscribe;
|
@@ -205,6 +207,11 @@ export default class RemoteTrackPublication extends TrackPublication {
|
|
205
207
|
this.emitSubscriptionUpdateIfChanged(prevStatus);
|
206
208
|
}
|
207
209
|
|
210
|
+
/** @internal */
|
211
|
+
setSubscriptionError(error: SubscriptionError) {
|
212
|
+
this.emit(TrackEvent.SubscriptionFailed, error);
|
213
|
+
}
|
214
|
+
|
208
215
|
/** @internal */
|
209
216
|
updateInfo(info: TrackInfo) {
|
210
217
|
super.updateInfo(info);
|
package/src/room/track/Track.ts
CHANGED
@@ -118,7 +118,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
|
|
118
118
|
// even if we believe it's already attached to the element, it's possible
|
119
119
|
// the element's srcObject was set to something else out of band.
|
120
120
|
// we'll want to re-attach it in that case
|
121
|
-
attachToElement(this.
|
121
|
+
attachToElement(this.mediaStreamTrack, element);
|
122
122
|
|
123
123
|
// handle auto playback failures
|
124
124
|
const allMediaStreamTracks = (element.srcObject as MediaStream).getTracks();
|
@@ -167,7 +167,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
|
|
167
167
|
try {
|
168
168
|
// detach from a single element
|
169
169
|
if (element) {
|
170
|
-
detachTrack(this.
|
170
|
+
detachTrack(this.mediaStreamTrack, element);
|
171
171
|
const idx = this.attachedElements.indexOf(element);
|
172
172
|
if (idx >= 0) {
|
173
173
|
this.attachedElements.splice(idx, 1);
|
@@ -179,7 +179,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
|
|
179
179
|
|
180
180
|
const detached: HTMLMediaElement[] = [];
|
181
181
|
this.attachedElements.forEach((elm) => {
|
182
|
-
detachTrack(this.
|
182
|
+
detachTrack(this.mediaStreamTrack, elm);
|
183
183
|
detached.push(elm);
|
184
184
|
this.recycleElement(elm);
|
185
185
|
this.emit(TrackEvent.ElementDetached, elm);
|