livekit-client 2.19.1 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/livekit-client.e2ee.worker.js +1 -1
- package/dist/livekit-client.e2ee.worker.js.map +1 -1
- package/dist/livekit-client.e2ee.worker.mjs +204 -145
- package/dist/livekit-client.e2ee.worker.mjs.map +1 -1
- package/dist/livekit-client.esm.mjs +664 -526
- package/dist/livekit-client.esm.mjs.map +1 -1
- package/dist/livekit-client.fm.worker.js +2 -0
- package/dist/{livekit-client.pt.worker.js.map → livekit-client.fm.worker.js.map} +1 -1
- package/dist/{livekit-client.pt.worker.mjs → livekit-client.fm.worker.mjs} +133 -99
- package/dist/livekit-client.fm.worker.mjs.map +1 -0
- package/dist/livekit-client.umd.js +1 -1
- package/dist/livekit-client.umd.js.map +1 -1
- package/dist/src/api/SignalClient.d.ts.map +1 -1
- package/dist/src/e2ee/E2eeManager.d.ts +1 -1
- package/dist/src/e2ee/E2eeManager.d.ts.map +1 -1
- package/dist/src/e2ee/types.d.ts +5 -5
- package/dist/src/e2ee/types.d.ts.map +1 -1
- package/dist/src/e2ee/worker/FrameCryptor.d.ts +20 -10
- package/dist/src/e2ee/worker/FrameCryptor.d.ts.map +1 -1
- package/dist/src/{packetTrailer/PacketTrailerManager.d.ts → frameMetadata/FrameMetadataManager.d.ts} +8 -4
- package/dist/src/{packetTrailer/PacketTrailerManager.d.ts.map → frameMetadata/FrameMetadataManager.d.ts.map} +1 -1
- package/dist/src/{packetTrailer/packetTrailer.d.ts → frameMetadata/frameMetadata.d.ts} +7 -7
- package/dist/src/frameMetadata/frameMetadata.d.ts.map +1 -0
- package/dist/src/{packetTrailer → frameMetadata}/types.d.ts +10 -6
- package/dist/src/frameMetadata/types.d.ts.map +1 -0
- package/dist/src/frameMetadata/utils.d.ts +9 -0
- package/dist/src/{packetTrailer → frameMetadata}/utils.d.ts.map +1 -1
- package/dist/src/frameMetadata/worker/frameMetadata.worker.d.ts +2 -0
- package/dist/src/frameMetadata/worker/frameMetadata.worker.d.ts.map +1 -0
- package/dist/src/index.d.ts +10 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/options.d.ts +8 -4
- package/dist/src/options.d.ts.map +1 -1
- package/dist/src/room/PCTransport.d.ts +1 -0
- package/dist/src/room/PCTransport.d.ts.map +1 -1
- package/dist/src/room/PCTransportManager.d.ts +1 -0
- package/dist/src/room/PCTransportManager.d.ts.map +1 -1
- package/dist/src/room/RTCEngine.d.ts +11 -1
- package/dist/src/room/RTCEngine.d.ts.map +1 -1
- package/dist/src/room/Room.d.ts +2 -2
- package/dist/src/room/Room.d.ts.map +1 -1
- package/dist/src/room/participant/LocalParticipant.d.ts +2 -2
- package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
- package/dist/src/room/participant/publishUtils.d.ts.map +1 -1
- package/dist/src/room/rpc/client/RpcClientManager.d.ts.map +1 -1
- package/dist/src/room/token-source/TokenSource.d.ts.map +1 -1
- package/dist/src/room/token-source/types.d.ts +1 -0
- package/dist/src/room/token-source/types.d.ts.map +1 -1
- package/dist/src/room/token-source/utils.d.ts.map +1 -1
- package/dist/src/room/track/FrameMetadataExtractor.d.ts +19 -0
- package/dist/src/room/track/FrameMetadataExtractor.d.ts.map +1 -0
- package/dist/src/room/track/RemoteVideoTrack.d.ts +7 -7
- package/dist/src/room/track/RemoteVideoTrack.d.ts.map +1 -1
- package/dist/src/room/track/options.d.ts +17 -5
- package/dist/src/room/track/options.d.ts.map +1 -1
- package/dist/ts4.2/e2ee/E2eeManager.d.ts +1 -1
- package/dist/ts4.2/e2ee/types.d.ts +5 -5
- package/dist/ts4.2/e2ee/worker/FrameCryptor.d.ts +19 -9
- package/dist/ts4.2/{packetTrailer/PacketTrailerManager.d.ts → frameMetadata/FrameMetadataManager.d.ts} +8 -4
- package/dist/ts4.2/{packetTrailer/packetTrailer.d.ts → frameMetadata/frameMetadata.d.ts} +7 -7
- package/dist/ts4.2/{packetTrailer → frameMetadata}/types.d.ts +10 -6
- package/dist/ts4.2/frameMetadata/utils.d.ts +9 -0
- package/dist/ts4.2/frameMetadata/worker/frameMetadata.worker.d.ts +2 -0
- package/dist/ts4.2/index.d.ts +9 -3
- package/dist/ts4.2/options.d.ts +8 -4
- package/dist/ts4.2/room/PCTransport.d.ts +1 -0
- package/dist/ts4.2/room/PCTransportManager.d.ts +1 -0
- package/dist/ts4.2/room/RTCEngine.d.ts +11 -1
- package/dist/ts4.2/room/Room.d.ts +2 -2
- package/dist/ts4.2/room/participant/LocalParticipant.d.ts +2 -2
- package/dist/ts4.2/room/token-source/types.d.ts +1 -0
- package/dist/ts4.2/room/track/FrameMetadataExtractor.d.ts +19 -0
- package/dist/ts4.2/room/track/RemoteVideoTrack.d.ts +7 -7
- package/dist/ts4.2/room/track/options.d.ts +17 -5
- package/package.json +25 -20
- package/src/api/SignalClient.test.ts +4 -4
- package/src/api/SignalClient.ts +7 -3
- package/src/api/WebSocketStream.test.ts +7 -7
- package/src/e2ee/E2eeManager.ts +13 -11
- package/src/e2ee/types.ts +5 -5
- package/src/e2ee/worker/FrameCryptor.test.ts +5 -5
- package/src/e2ee/worker/FrameCryptor.ts +69 -30
- package/src/e2ee/worker/e2ee.worker.ts +5 -5
- package/src/e2ee/worker/tsconfig.json +4 -1
- package/src/{packetTrailer/PacketTrailerManager.test.ts → frameMetadata/FrameMetadataManager.test.ts} +10 -8
- package/src/{packetTrailer/PacketTrailerManager.ts → frameMetadata/FrameMetadataManager.ts} +21 -15
- package/src/{packetTrailer/packetTrailer.test.ts → frameMetadata/frameMetadata.test.ts} +1 -1
- package/src/{packetTrailer/packetTrailer.ts → frameMetadata/frameMetadata.ts} +9 -9
- package/src/{packetTrailer → frameMetadata}/types.ts +12 -6
- package/src/{packetTrailer → frameMetadata}/utils.test.ts +19 -19
- package/src/{packetTrailer → frameMetadata}/utils.ts +11 -11
- package/src/{packetTrailer/worker/packetTrailer.worker.ts → frameMetadata/worker/frameMetadata.worker.ts} +5 -5
- package/src/{packetTrailer → frameMetadata}/worker/tsconfig.json +4 -1
- package/src/index.ts +13 -2
- package/src/options.ts +9 -4
- package/src/room/PCTransport.ts +4 -0
- package/src/room/PCTransportManager.ts +4 -0
- package/src/room/RTCEngine.test.ts +143 -9
- package/src/room/RTCEngine.ts +156 -48
- package/src/room/Room.ts +10 -10
- package/src/room/participant/LocalParticipant.test.ts +19 -19
- package/src/room/participant/LocalParticipant.ts +21 -16
- package/src/room/participant/publishUtils.test.ts +6 -0
- package/src/room/participant/publishUtils.ts +4 -1
- package/src/room/rpc/client/RpcClientManager.ts +6 -3
- package/src/room/rpc/server/RpcServerManager.ts +1 -1
- package/src/room/token-source/TokenSource.ts +9 -23
- package/src/room/token-source/types.ts +1 -0
- package/src/room/token-source/utils.ts +1 -0
- package/src/room/track/{PacketTrailerExtractor.ts → FrameMetadataExtractor.ts} +8 -8
- package/src/room/track/RemoteVideoTrack.ts +7 -11
- package/src/room/track/options.ts +19 -5
- package/dist/livekit-client.pt.worker.js +0 -2
- package/dist/livekit-client.pt.worker.mjs.map +0 -1
- package/dist/src/packetTrailer/packetTrailer.d.ts.map +0 -1
- package/dist/src/packetTrailer/types.d.ts.map +0 -1
- package/dist/src/packetTrailer/utils.d.ts +0 -9
- package/dist/src/packetTrailer/worker/packetTrailer.worker.d.ts +0 -2
- package/dist/src/packetTrailer/worker/packetTrailer.worker.d.ts.map +0 -1
- package/dist/src/room/track/PacketTrailerExtractor.d.ts +0 -19
- package/dist/src/room/track/PacketTrailerExtractor.d.ts.map +0 -1
- package/dist/ts4.2/packetTrailer/utils.d.ts +0 -9
- package/dist/ts4.2/packetTrailer/worker/packetTrailer.worker.d.ts +0 -2
- package/dist/ts4.2/room/track/PacketTrailerExtractor.d.ts +0 -19
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
// TODO code inspired by https://github.com/webrtc/samples/blob/gh-pages/src/content/insertable-streams/endtoend-encryption/js/worker.js
|
|
2
2
|
import { EventEmitter } from 'events';
|
|
3
3
|
import type TypedEventEmitter from 'typed-emitter';
|
|
4
|
-
import { workerLogger } from '../../logger';
|
|
5
4
|
import {
|
|
6
5
|
appendPacketTrailerToEncodedFrame,
|
|
7
6
|
processPacketTrailer,
|
|
8
|
-
} from '../../
|
|
9
|
-
import type {
|
|
10
|
-
import {
|
|
11
|
-
import
|
|
7
|
+
} from '../../frameMetadata/frameMetadata';
|
|
8
|
+
import type { FrameMetadataPublishOptions } from '../../frameMetadata/types';
|
|
9
|
+
import { hasFrameMetadataPublishOptions } from '../../frameMetadata/utils';
|
|
10
|
+
import { workerLogger } from '../../logger';
|
|
11
|
+
import { type VideoCodec, videoCodecs } from '../../room/track/options';
|
|
12
|
+
import { mimeTypeToVideoCodecString } from '../../room/track/utils';
|
|
12
13
|
import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants';
|
|
13
14
|
import { CryptorError, CryptorErrorReason } from '../errors';
|
|
14
15
|
import { type CryptorCallbacks, CryptorEvent } from '../events';
|
|
@@ -87,11 +88,11 @@ export class FrameCryptor extends BaseFrameCryptor {
|
|
|
87
88
|
* When false, we skip the per-frame trailer extraction path entirely
|
|
88
89
|
* on decode to avoid unnecessary work on tracks that don't use it.
|
|
89
90
|
*/
|
|
90
|
-
private
|
|
91
|
+
private hasFrameMetadata: boolean = false;
|
|
91
92
|
|
|
92
|
-
private
|
|
93
|
+
private frameMetadataOpts?: FrameMetadataPublishOptions;
|
|
93
94
|
|
|
94
|
-
private
|
|
95
|
+
private frameMetadataFrameId = 0;
|
|
95
96
|
|
|
96
97
|
/**
|
|
97
98
|
* Throttling mechanism for decryption errors to prevent memory leaks
|
|
@@ -106,6 +107,12 @@ export class FrameCryptor extends BaseFrameCryptor {
|
|
|
106
107
|
|
|
107
108
|
private readonly ERROR_WINDOW_MS = 60000; // 1 minute window
|
|
108
109
|
|
|
110
|
+
/**
|
|
111
|
+
* Tracks (participant, trackId, payloadType) tuples for which we've already logged a NALU
|
|
112
|
+
* fallback, so a persistent bad state doesn't flood the console (Firefox doesn't filter debug).
|
|
113
|
+
*/
|
|
114
|
+
private loggedNALUFallbacks: Set<string> = new Set();
|
|
115
|
+
|
|
109
116
|
constructor(opts: {
|
|
110
117
|
keys: ParticipantKeyHandler;
|
|
111
118
|
participantIdentity: string;
|
|
@@ -205,13 +212,13 @@ export class FrameCryptor extends BaseFrameCryptor {
|
|
|
205
212
|
* trailer data. When false, {@link decodeFunction} skips the per-frame
|
|
206
213
|
* trailer extraction branch entirely.
|
|
207
214
|
*/
|
|
208
|
-
|
|
209
|
-
this.
|
|
215
|
+
setHasFrameMetadata(hasFrameMetadata: boolean) {
|
|
216
|
+
this.hasFrameMetadata = hasFrameMetadata;
|
|
210
217
|
}
|
|
211
218
|
|
|
212
|
-
|
|
213
|
-
this.
|
|
214
|
-
this.
|
|
219
|
+
setFrameMetadataOpts(frameMetadata?: FrameMetadataPublishOptions) {
|
|
220
|
+
this.frameMetadataOpts = frameMetadata;
|
|
221
|
+
this.frameMetadataFrameId = 0;
|
|
215
222
|
}
|
|
216
223
|
|
|
217
224
|
setupTransform(
|
|
@@ -221,14 +228,14 @@ export class FrameCryptor extends BaseFrameCryptor {
|
|
|
221
228
|
trackId: string,
|
|
222
229
|
isReuse: boolean,
|
|
223
230
|
codec?: VideoCodec,
|
|
224
|
-
|
|
231
|
+
frameMetadata?: FrameMetadataPublishOptions,
|
|
225
232
|
) {
|
|
226
233
|
if (codec) {
|
|
227
234
|
workerLogger.info('setting codec on cryptor to', { codec });
|
|
228
235
|
this.videoCodec = codec;
|
|
229
236
|
}
|
|
230
237
|
if (operation === 'encode') {
|
|
231
|
-
this.
|
|
238
|
+
this.setFrameMetadataOpts(frameMetadata);
|
|
232
239
|
}
|
|
233
240
|
|
|
234
241
|
workerLogger.debug('Setting up frame cryptor transform', {
|
|
@@ -399,7 +406,7 @@ export class FrameCryptor extends BaseFrameCryptor {
|
|
|
399
406
|
}
|
|
400
407
|
|
|
401
408
|
if (!this.isEnabled()) {
|
|
402
|
-
this.
|
|
409
|
+
this.appendFrameMetadata(encodedFrame);
|
|
403
410
|
return controller.enqueue(encodedFrame);
|
|
404
411
|
}
|
|
405
412
|
const keySet = this.keys.getKeySet();
|
|
@@ -468,7 +475,7 @@ export class FrameCryptor extends BaseFrameCryptor {
|
|
|
468
475
|
newData.set(newDataWithoutHeader, frameHeader.byteLength);
|
|
469
476
|
|
|
470
477
|
encodedFrame.data = newData.buffer;
|
|
471
|
-
this.
|
|
478
|
+
this.appendFrameMetadata(encodedFrame);
|
|
472
479
|
|
|
473
480
|
return controller.enqueue(encodedFrame);
|
|
474
481
|
} catch (e: any) {
|
|
@@ -487,16 +494,20 @@ export class FrameCryptor extends BaseFrameCryptor {
|
|
|
487
494
|
}
|
|
488
495
|
}
|
|
489
496
|
|
|
490
|
-
private
|
|
491
|
-
if (!
|
|
497
|
+
private appendFrameMetadata(encodedFrame: RTCEncodedVideoFrame | RTCEncodedAudioFrame) {
|
|
498
|
+
if (!hasFrameMetadataPublishOptions(this.frameMetadataOpts) || !isVideoFrame(encodedFrame)) {
|
|
492
499
|
return;
|
|
493
500
|
}
|
|
494
501
|
|
|
495
|
-
if (this.
|
|
496
|
-
this.
|
|
497
|
-
this.
|
|
502
|
+
if (this.frameMetadataOpts?.frameId) {
|
|
503
|
+
this.frameMetadataFrameId =
|
|
504
|
+
this.frameMetadataFrameId === 0xffffffff ? 1 : this.frameMetadataFrameId + 1;
|
|
498
505
|
}
|
|
499
|
-
appendPacketTrailerToEncodedFrame(
|
|
506
|
+
appendPacketTrailerToEncodedFrame(
|
|
507
|
+
encodedFrame,
|
|
508
|
+
this.frameMetadataOpts,
|
|
509
|
+
this.frameMetadataFrameId,
|
|
510
|
+
);
|
|
500
511
|
}
|
|
501
512
|
|
|
502
513
|
/**
|
|
@@ -509,7 +520,7 @@ export class FrameCryptor extends BaseFrameCryptor {
|
|
|
509
520
|
encodedFrame: RTCEncodedVideoFrame | RTCEncodedAudioFrame,
|
|
510
521
|
controller: TransformStreamDefaultController,
|
|
511
522
|
) {
|
|
512
|
-
if (this.
|
|
523
|
+
if (this.hasFrameMetadata && isVideoFrame(encodedFrame)) {
|
|
513
524
|
try {
|
|
514
525
|
const ptResult = processPacketTrailer(encodedFrame, this.trackId);
|
|
515
526
|
if (ptResult.data) {
|
|
@@ -795,22 +806,23 @@ export class FrameCryptor extends BaseFrameCryptor {
|
|
|
795
806
|
}
|
|
796
807
|
|
|
797
808
|
// Try NALU processing for H.264/H.265 codecs
|
|
809
|
+
const payloadType = frame.getMetadata().payloadType;
|
|
810
|
+
const fallbackKey = `${this.participantIdentity}-${this.trackId}-${payloadType}`;
|
|
798
811
|
try {
|
|
799
812
|
const knownCodec =
|
|
800
813
|
detectedCodec === 'h264' || detectedCodec === 'h265' ? detectedCodec : undefined;
|
|
801
814
|
const naluResult = processNALUsForEncryption(new Uint8Array(frame.data), knownCodec);
|
|
802
815
|
|
|
803
816
|
if (naluResult.requiresNALUProcessing) {
|
|
817
|
+
// Recovered for this tuple, allow a future failure to log again.
|
|
818
|
+
this.loggedNALUFallbacks.delete(fallbackKey);
|
|
804
819
|
return {
|
|
805
820
|
unencryptedBytes: naluResult.unencryptedBytes,
|
|
806
821
|
requiresNALUProcessing: true,
|
|
807
822
|
};
|
|
808
823
|
}
|
|
809
824
|
} catch (e) {
|
|
810
|
-
|
|
811
|
-
error: e,
|
|
812
|
-
...this.logContext,
|
|
813
|
-
});
|
|
825
|
+
this.logNALUFallbackOnce(fallbackKey, payloadType, e);
|
|
814
826
|
}
|
|
815
827
|
|
|
816
828
|
// Fallback to VP8 handling
|
|
@@ -818,13 +830,40 @@ export class FrameCryptor extends BaseFrameCryptor {
|
|
|
818
830
|
}
|
|
819
831
|
|
|
820
832
|
/**
|
|
821
|
-
*
|
|
833
|
+
* Logs a NALU processing fallback at most once per (participant, trackId, payloadType) tuple,
|
|
834
|
+
* so a persistent bad state doesn't flood the console (Firefox doesn't filter debug).
|
|
835
|
+
*/
|
|
836
|
+
private logNALUFallbackOnce(
|
|
837
|
+
fallbackKey: string,
|
|
838
|
+
payloadType: number | undefined,
|
|
839
|
+
error: unknown,
|
|
840
|
+
) {
|
|
841
|
+
if (this.loggedNALUFallbacks.has(fallbackKey)) {
|
|
842
|
+
return;
|
|
843
|
+
}
|
|
844
|
+
this.loggedNALUFallbacks.add(fallbackKey);
|
|
845
|
+
workerLogger.warn('NALU processing failed, falling back to VP8 handling', {
|
|
846
|
+
error,
|
|
847
|
+
payloadType,
|
|
848
|
+
...this.logContext,
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/**
|
|
853
|
+
* inspects frame mimetype if available. falls back to payloadtype and maps it to the codec specified in rtpMap
|
|
822
854
|
*/
|
|
823
855
|
private getVideoCodec(frame: RTCEncodedVideoFrame): VideoCodec | undefined {
|
|
856
|
+
const metadata = frame.getMetadata();
|
|
857
|
+
if (metadata.mimeType) {
|
|
858
|
+
const maybeKnownCodec = mimeTypeToVideoCodecString(metadata.mimeType);
|
|
859
|
+
if (videoCodecs.includes(maybeKnownCodec)) {
|
|
860
|
+
return maybeKnownCodec;
|
|
861
|
+
}
|
|
862
|
+
}
|
|
824
863
|
if (this.rtpMap.size === 0) {
|
|
825
864
|
return undefined;
|
|
826
865
|
}
|
|
827
|
-
const payloadType =
|
|
866
|
+
const payloadType = metadata.payloadType;
|
|
828
867
|
const codec = payloadType ? this.rtpMap.get(payloadType) : undefined;
|
|
829
868
|
return codec;
|
|
830
869
|
}
|
|
@@ -64,7 +64,7 @@ onmessage = (ev) => {
|
|
|
64
64
|
break;
|
|
65
65
|
case 'decode':
|
|
66
66
|
let cryptor = getTrackCryptor(data.participantIdentity, data.trackId);
|
|
67
|
-
cryptor.
|
|
67
|
+
cryptor.setHasFrameMetadata(data.hasPacketTrailer);
|
|
68
68
|
cryptor.setupTransform(
|
|
69
69
|
kind,
|
|
70
70
|
data.readableStream,
|
|
@@ -76,7 +76,7 @@ onmessage = (ev) => {
|
|
|
76
76
|
break;
|
|
77
77
|
case 'encode':
|
|
78
78
|
let pubCryptor = getTrackCryptor(data.participantIdentity, data.trackId);
|
|
79
|
-
pubCryptor.
|
|
79
|
+
pubCryptor.setHasFrameMetadata(data.hasPacketTrailer);
|
|
80
80
|
pubCryptor.setupTransform(
|
|
81
81
|
kind,
|
|
82
82
|
data.readableStream,
|
|
@@ -84,7 +84,7 @@ onmessage = (ev) => {
|
|
|
84
84
|
data.trackId,
|
|
85
85
|
data.isReuse,
|
|
86
86
|
data.codec,
|
|
87
|
-
data.packetTrailer,
|
|
87
|
+
data.packetTrailer, // wire format still uses 'packetTrailer' field name
|
|
88
88
|
);
|
|
89
89
|
break;
|
|
90
90
|
|
|
@@ -164,7 +164,7 @@ onmessage = (ev) => {
|
|
|
164
164
|
case 'updateCodec':
|
|
165
165
|
const trackCryptor = getTrackCryptor(data.participantIdentity, data.trackId);
|
|
166
166
|
trackCryptor.setVideoCodec(data.codec);
|
|
167
|
-
trackCryptor.
|
|
167
|
+
trackCryptor.setHasFrameMetadata(data.hasPacketTrailer);
|
|
168
168
|
workerLogger.info('updated codec', {
|
|
169
169
|
participantIdentity: data.participantIdentity,
|
|
170
170
|
trackId: data.trackId,
|
|
@@ -343,7 +343,7 @@ if (self.RTCTransformEvent) {
|
|
|
343
343
|
const { kind, participantIdentity, trackId, codec, hasPacketTrailer } = options;
|
|
344
344
|
messageQueue.run(async () => {
|
|
345
345
|
const cryptor = getTrackCryptor(participantIdentity, trackId);
|
|
346
|
-
cryptor.
|
|
346
|
+
cryptor.setHasFrameMetadata(hasPacketTrailer);
|
|
347
347
|
workerLogger.debug('onrtctransform setup', { participantIdentity, trackId, codec });
|
|
348
348
|
cryptor.setupTransform(
|
|
349
349
|
kind,
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
2
2
|
import type { TrackInfo } from '@livekit/protocol';
|
|
3
|
-
import {
|
|
3
|
+
import { FrameMetadataManager } from './FrameMetadataManager';
|
|
4
4
|
|
|
5
|
-
describe('
|
|
5
|
+
describe('FrameMetadataManager', () => {
|
|
6
6
|
const originalRTCRtpSender = window.RTCRtpSender;
|
|
7
7
|
const originalUserAgent = navigator.userAgent;
|
|
8
8
|
const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown })
|
|
@@ -63,7 +63,7 @@ describe('PacketTrailerManager', () => {
|
|
|
63
63
|
});
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
-
function setupWorkerReceiver(manager:
|
|
66
|
+
function setupWorkerReceiver(manager: FrameMetadataManager, receiver: RTCRtpReceiver) {
|
|
67
67
|
(
|
|
68
68
|
manager as unknown as {
|
|
69
69
|
setupWorkerReceiver: (receiver: RTCRtpReceiver, newTrackId: string) => void;
|
|
@@ -72,7 +72,7 @@ describe('PacketTrailerManager', () => {
|
|
|
72
72
|
}
|
|
73
73
|
|
|
74
74
|
function setupReceiver(
|
|
75
|
-
manager:
|
|
75
|
+
manager: FrameMetadataManager,
|
|
76
76
|
receiver: RTCRtpReceiver,
|
|
77
77
|
trackId: string,
|
|
78
78
|
trackInfo?: TrackInfo,
|
|
@@ -103,11 +103,13 @@ describe('PacketTrailerManager', () => {
|
|
|
103
103
|
it('uses RTCRtpScriptTransform for packet trailer extraction when supported', () => {
|
|
104
104
|
useSafariUserAgent();
|
|
105
105
|
const transform = {};
|
|
106
|
-
const RTCRtpScriptTransform = vi.fn(()
|
|
106
|
+
const RTCRtpScriptTransform = vi.fn(function () {
|
|
107
|
+
return transform;
|
|
108
|
+
});
|
|
107
109
|
setScriptTransform(RTCRtpScriptTransform);
|
|
108
110
|
|
|
109
111
|
const worker = {} as Worker;
|
|
110
|
-
const manager = new
|
|
112
|
+
const manager = new FrameMetadataManager({ worker });
|
|
111
113
|
const receiver = {
|
|
112
114
|
createEncodedStreams: vi.fn(),
|
|
113
115
|
} as unknown as RTCRtpReceiver;
|
|
@@ -129,7 +131,7 @@ describe('PacketTrailerManager', () => {
|
|
|
129
131
|
stubInsertableStreamsSupport();
|
|
130
132
|
|
|
131
133
|
const worker = { postMessage: vi.fn() } as unknown as Worker;
|
|
132
|
-
const manager = new
|
|
134
|
+
const manager = new FrameMetadataManager({ worker });
|
|
133
135
|
const { receiver, readable, writable, createEncodedStreams } = makeReceiver();
|
|
134
136
|
|
|
135
137
|
setupReceiver(manager, receiver, 'track-without-trailer');
|
|
@@ -153,7 +155,7 @@ describe('PacketTrailerManager', () => {
|
|
|
153
155
|
stubInsertableStreamsSupport();
|
|
154
156
|
|
|
155
157
|
const worker = { postMessage: vi.fn() } as unknown as Worker;
|
|
156
|
-
const manager = new
|
|
158
|
+
const manager = new FrameMetadataManager({ worker });
|
|
157
159
|
const { receiver } = makeReceiver();
|
|
158
160
|
const trackInfo = { packetTrailerFeatures: [1] } as unknown as TrackInfo;
|
|
159
161
|
|
|
@@ -2,13 +2,13 @@ import type { TrackInfo } from '@livekit/protocol';
|
|
|
2
2
|
import log from '../logger';
|
|
3
3
|
import type Room from '../room/Room';
|
|
4
4
|
import { RoomEvent } from '../room/events';
|
|
5
|
-
import {
|
|
5
|
+
import { FrameMetadataExtractor } from '../room/track/FrameMetadataExtractor';
|
|
6
6
|
import type RemoteTrack from '../room/track/RemoteTrack';
|
|
7
7
|
import RemoteVideoTrack from '../room/track/RemoteVideoTrack';
|
|
8
8
|
import type { PTDecodeMessage, PTUpdateTrackIdMessage, PTWorkerMessage } from './types';
|
|
9
|
-
import {
|
|
9
|
+
import { isFrameMetadataSupported, shouldUseFrameMetadataScriptTransform } from './utils';
|
|
10
10
|
|
|
11
|
-
export interface
|
|
11
|
+
export interface FrameMetadataOptions {
|
|
12
12
|
/**
|
|
13
13
|
* Dedicated worker for extracting packet trailers off the main thread.
|
|
14
14
|
*
|
|
@@ -34,12 +34,12 @@ export interface PacketTrailerOptions {
|
|
|
34
34
|
*
|
|
35
35
|
* @experimental
|
|
36
36
|
*/
|
|
37
|
-
export class
|
|
37
|
+
export class FrameMetadataManager {
|
|
38
38
|
private worker?: Worker;
|
|
39
39
|
|
|
40
40
|
private room?: Room;
|
|
41
41
|
|
|
42
|
-
private extractors = new Map<string,
|
|
42
|
+
private extractors = new Map<string, FrameMetadataExtractor>();
|
|
43
43
|
|
|
44
44
|
/**
|
|
45
45
|
* Tracks the trackId associated with each receiver that has had its
|
|
@@ -49,7 +49,7 @@ export class PacketTrailerManager {
|
|
|
49
49
|
*/
|
|
50
50
|
private workerPipelines = new Map<RTCRtpReceiver, string>();
|
|
51
51
|
|
|
52
|
-
constructor(options?:
|
|
52
|
+
constructor(options?: FrameMetadataOptions) {
|
|
53
53
|
this.worker = options?.worker;
|
|
54
54
|
}
|
|
55
55
|
|
|
@@ -100,18 +100,18 @@ export class PacketTrailerManager {
|
|
|
100
100
|
}
|
|
101
101
|
|
|
102
102
|
if (
|
|
103
|
-
!
|
|
103
|
+
!isFrameMetadataSupported(this.worker ? { worker: this.worker } : undefined) &&
|
|
104
104
|
!this.room?.hasE2EESetup
|
|
105
105
|
) {
|
|
106
|
-
log.warn('
|
|
106
|
+
log.warn('frame metadata transform not supported; skipping extraction');
|
|
107
107
|
return;
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
-
const extractor = new
|
|
110
|
+
const extractor = new FrameMetadataExtractor();
|
|
111
111
|
const trackId = track.mediaStreamID;
|
|
112
112
|
|
|
113
113
|
this.extractors.set(trackId, extractor);
|
|
114
|
-
track.
|
|
114
|
+
track.frameMetadataExtractor = extractor;
|
|
115
115
|
|
|
116
116
|
if (this.room?.hasE2EESetup) {
|
|
117
117
|
// E2EE worker strips the trailer and injects metadata directly into
|
|
@@ -123,7 +123,7 @@ export class PacketTrailerManager {
|
|
|
123
123
|
}
|
|
124
124
|
|
|
125
125
|
private setupPassthroughReceiver(receiver: RTCRtpReceiver, trackId: string) {
|
|
126
|
-
if (
|
|
126
|
+
if (shouldUseFrameMetadataScriptTransform()) {
|
|
127
127
|
if ('transform' in receiver) {
|
|
128
128
|
// @ts-ignore
|
|
129
129
|
receiver.transform = null;
|
|
@@ -133,7 +133,7 @@ export class PacketTrailerManager {
|
|
|
133
133
|
|
|
134
134
|
if (
|
|
135
135
|
this.worker &&
|
|
136
|
-
|
|
136
|
+
isFrameMetadataSupported({ worker: this.worker }) &&
|
|
137
137
|
!this.workerPipelines.has(receiver)
|
|
138
138
|
) {
|
|
139
139
|
this.setupWorkerReceiver(receiver, trackId, false);
|
|
@@ -155,7 +155,7 @@ export class PacketTrailerManager {
|
|
|
155
155
|
return;
|
|
156
156
|
}
|
|
157
157
|
|
|
158
|
-
if (
|
|
158
|
+
if (shouldUseFrameMetadataScriptTransform()) {
|
|
159
159
|
// @ts-ignore
|
|
160
160
|
receiver.transform = new RTCRtpScriptTransform(worker, {
|
|
161
161
|
kind: 'decode',
|
|
@@ -215,7 +215,7 @@ export class PacketTrailerManager {
|
|
|
215
215
|
}
|
|
216
216
|
|
|
217
217
|
if (track instanceof RemoteVideoTrack) {
|
|
218
|
-
track.
|
|
218
|
+
track.frameMetadataExtractor = undefined;
|
|
219
219
|
}
|
|
220
220
|
|
|
221
221
|
// The receiver pipeline is intentionally left running. If the receiver is
|
|
@@ -245,6 +245,12 @@ export class PacketTrailerManager {
|
|
|
245
245
|
};
|
|
246
246
|
|
|
247
247
|
private onWorkerError = (ev: ErrorEvent) => {
|
|
248
|
-
log.error('
|
|
248
|
+
log.error('frame metadata worker encountered an error:', { error: ev.error });
|
|
249
249
|
};
|
|
250
250
|
}
|
|
251
|
+
|
|
252
|
+
/** @deprecated Use {@link FrameMetadataManager} instead. */
|
|
253
|
+
export const PacketTrailerManager = FrameMetadataManager;
|
|
254
|
+
|
|
255
|
+
/** @deprecated Use {@link FrameMetadataOptions} instead. */
|
|
256
|
+
export type PacketTrailerOptions = FrameMetadataOptions;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import {
|
|
1
|
+
import type { FrameMetadata, FrameMetadataPublishOptions } from './types';
|
|
2
|
+
import { hasFrameMetadataPublishOptions } from './utils';
|
|
3
3
|
|
|
4
4
|
export const PACKET_TRAILER_MAGIC = Uint8Array.from([
|
|
5
5
|
'L'.charCodeAt(0),
|
|
@@ -17,7 +17,7 @@ const FRAME_ID_TLV_SIZE = 6;
|
|
|
17
17
|
|
|
18
18
|
export interface ExtractPacketTrailerResult {
|
|
19
19
|
data: Uint8Array;
|
|
20
|
-
metadata?:
|
|
20
|
+
metadata?: FrameMetadata;
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
export function appendPacketTrailer(
|
|
@@ -64,10 +64,10 @@ export function appendPacketTrailer(
|
|
|
64
64
|
|
|
65
65
|
export function appendPacketTrailerToEncodedFrame(
|
|
66
66
|
frame: RTCEncodedVideoFrame,
|
|
67
|
-
options:
|
|
67
|
+
options: FrameMetadataPublishOptions | undefined,
|
|
68
68
|
frameId: number,
|
|
69
69
|
): boolean {
|
|
70
|
-
if (!
|
|
70
|
+
if (!hasFrameMetadataPublishOptions(options) || frame.data.byteLength === 0) {
|
|
71
71
|
return false;
|
|
72
72
|
}
|
|
73
73
|
|
|
@@ -108,7 +108,7 @@ export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPac
|
|
|
108
108
|
const strippedData = bytes.subarray(0, trailerStart);
|
|
109
109
|
let offset = trailerStart;
|
|
110
110
|
let foundAny = false;
|
|
111
|
-
const metadata:
|
|
111
|
+
const metadata: FrameMetadata = {
|
|
112
112
|
userTimestamp: BigInt(0),
|
|
113
113
|
frameId: 0,
|
|
114
114
|
};
|
|
@@ -223,16 +223,16 @@ export function getFrameSsrc(frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame)
|
|
|
223
223
|
return 0;
|
|
224
224
|
}
|
|
225
225
|
|
|
226
|
-
export interface
|
|
226
|
+
export interface FrameMetadataPayload {
|
|
227
227
|
trackId: string;
|
|
228
228
|
rtpTimestamp: number;
|
|
229
229
|
ssrc: number;
|
|
230
|
-
metadata:
|
|
230
|
+
metadata: FrameMetadata;
|
|
231
231
|
}
|
|
232
232
|
|
|
233
233
|
export interface ProcessPacketTrailerResult {
|
|
234
234
|
data?: ArrayBuffer;
|
|
235
|
-
payload?:
|
|
235
|
+
payload?: FrameMetadataPayload;
|
|
236
236
|
}
|
|
237
237
|
|
|
238
238
|
/**
|
|
@@ -1,15 +1,21 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { FrameMetadataPayload } from './frameMetadata';
|
|
2
2
|
|
|
3
|
-
export interface
|
|
3
|
+
export interface FrameMetadata {
|
|
4
4
|
userTimestamp: bigint;
|
|
5
5
|
frameId: number;
|
|
6
6
|
}
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
/** @deprecated Use {@link FrameMetadata} instead. */
|
|
9
|
+
export type PacketTrailerMetadata = FrameMetadata;
|
|
10
|
+
|
|
11
|
+
export interface FrameMetadataPublishOptions {
|
|
9
12
|
timestamp?: boolean;
|
|
10
13
|
frameId?: boolean;
|
|
11
14
|
}
|
|
12
15
|
|
|
16
|
+
/** @deprecated Use {@link FrameMetadataPublishOptions} instead. */
|
|
17
|
+
export type PacketTrailerPublishOptions = FrameMetadataPublishOptions;
|
|
18
|
+
|
|
13
19
|
export interface PTBaseMessage {
|
|
14
20
|
kind: string;
|
|
15
21
|
data?: unknown;
|
|
@@ -38,7 +44,7 @@ export interface PTEncodeMessage extends PTBaseMessage {
|
|
|
38
44
|
data: {
|
|
39
45
|
readableStream: ReadableStream;
|
|
40
46
|
writableStream: WritableStream;
|
|
41
|
-
packetTrailer?:
|
|
47
|
+
packetTrailer?: FrameMetadataPublishOptions;
|
|
42
48
|
};
|
|
43
49
|
}
|
|
44
50
|
|
|
@@ -49,12 +55,12 @@ export type PTScriptTransformOptions =
|
|
|
49
55
|
}
|
|
50
56
|
| {
|
|
51
57
|
kind: 'encode';
|
|
52
|
-
packetTrailer?:
|
|
58
|
+
packetTrailer?: FrameMetadataPublishOptions;
|
|
53
59
|
};
|
|
54
60
|
|
|
55
61
|
export interface PTMetadataMessage extends PTBaseMessage {
|
|
56
62
|
kind: 'metadata';
|
|
57
|
-
data:
|
|
63
|
+
data: FrameMetadataPayload;
|
|
58
64
|
}
|
|
59
65
|
|
|
60
66
|
export interface PTUpdateTrackIdMessage extends PTBaseMessage {
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { PacketTrailerFeature } from '@livekit/protocol';
|
|
2
2
|
import { afterEach, describe, expect, it } from 'vitest';
|
|
3
3
|
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
getFrameMetadataFeatures,
|
|
5
|
+
getFrameMetadataPublishOptions,
|
|
6
|
+
isFrameMetadataSupported,
|
|
7
7
|
} from './utils';
|
|
8
8
|
|
|
9
|
-
describe('
|
|
9
|
+
describe('frame metadata support', () => {
|
|
10
10
|
const originalRTCRtpSender = window.RTCRtpSender;
|
|
11
11
|
const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown })
|
|
12
12
|
.RTCRtpScriptTransform;
|
|
@@ -46,20 +46,20 @@ describe('packet trailer support', () => {
|
|
|
46
46
|
});
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
-
it('supports
|
|
49
|
+
it('supports frame metadata with RTCRtpScriptTransform on Safari', () => {
|
|
50
50
|
stubScriptTransformSupport(
|
|
51
51
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
|
|
52
52
|
);
|
|
53
53
|
|
|
54
|
-
expect(
|
|
54
|
+
expect(isFrameMetadataSupported({ worker: {} as Worker })).toBe(true);
|
|
55
55
|
});
|
|
56
56
|
|
|
57
|
-
it('supports
|
|
57
|
+
it('supports frame metadata with RTCRtpScriptTransform on Firefox', () => {
|
|
58
58
|
stubScriptTransformSupport(
|
|
59
59
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 14.0; rv:144.0) Gecko/20100101 Firefox/144.0',
|
|
60
60
|
);
|
|
61
61
|
|
|
62
|
-
expect(
|
|
62
|
+
expect(isFrameMetadataSupported({ worker: {} as Worker })).toBe(true);
|
|
63
63
|
});
|
|
64
64
|
|
|
65
65
|
it('does not use RTCRtpScriptTransform support on Chromium-based browsers', () => {
|
|
@@ -67,39 +67,39 @@ describe('packet trailer support', () => {
|
|
|
67
67
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
|
|
68
68
|
);
|
|
69
69
|
|
|
70
|
-
expect(
|
|
70
|
+
expect(isFrameMetadataSupported({ worker: {} as Worker })).toBe(false);
|
|
71
71
|
});
|
|
72
72
|
});
|
|
73
73
|
|
|
74
|
-
describe('
|
|
74
|
+
describe('frame metadata publish features', () => {
|
|
75
75
|
it('maps publish options to protocol features', () => {
|
|
76
|
-
expect(
|
|
76
|
+
expect(getFrameMetadataFeatures({ timestamp: true, frameId: true })).toEqual([
|
|
77
77
|
PacketTrailerFeature.PTF_USER_TIMESTAMP,
|
|
78
78
|
PacketTrailerFeature.PTF_FRAME_ID,
|
|
79
79
|
]);
|
|
80
|
-
expect(
|
|
80
|
+
expect(getFrameMetadataFeatures({ timestamp: true })).toEqual([
|
|
81
81
|
PacketTrailerFeature.PTF_USER_TIMESTAMP,
|
|
82
82
|
]);
|
|
83
|
-
expect(
|
|
83
|
+
expect(getFrameMetadataFeatures({ frameId: true })).toEqual([
|
|
84
84
|
PacketTrailerFeature.PTF_FRAME_ID,
|
|
85
85
|
]);
|
|
86
|
-
expect(
|
|
86
|
+
expect(getFrameMetadataFeatures()).toEqual([]);
|
|
87
87
|
});
|
|
88
88
|
|
|
89
89
|
it('maps protocol features to publish options', () => {
|
|
90
90
|
expect(
|
|
91
|
-
|
|
91
|
+
getFrameMetadataPublishOptions([
|
|
92
92
|
PacketTrailerFeature.PTF_USER_TIMESTAMP,
|
|
93
93
|
PacketTrailerFeature.PTF_FRAME_ID,
|
|
94
94
|
]),
|
|
95
95
|
).toEqual({ timestamp: true, frameId: true });
|
|
96
|
-
expect(
|
|
96
|
+
expect(getFrameMetadataPublishOptions([PacketTrailerFeature.PTF_USER_TIMESTAMP])).toEqual({
|
|
97
97
|
timestamp: true,
|
|
98
98
|
});
|
|
99
|
-
expect(
|
|
99
|
+
expect(getFrameMetadataPublishOptions([PacketTrailerFeature.PTF_FRAME_ID])).toEqual({
|
|
100
100
|
frameId: true,
|
|
101
101
|
});
|
|
102
|
-
expect(
|
|
103
|
-
expect(
|
|
102
|
+
expect(getFrameMetadataPublishOptions()).toBeUndefined();
|
|
103
|
+
expect(getFrameMetadataPublishOptions([])).toBeUndefined();
|
|
104
104
|
});
|
|
105
105
|
});
|