livekit-client 2.15.4 → 2.15.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/dist/livekit-client.e2ee.worker.js +1 -1
  2. package/dist/livekit-client.e2ee.worker.js.map +1 -1
  3. package/dist/livekit-client.e2ee.worker.mjs +373 -164
  4. package/dist/livekit-client.e2ee.worker.mjs.map +1 -1
  5. package/dist/livekit-client.esm.mjs +982 -643
  6. package/dist/livekit-client.esm.mjs.map +1 -1
  7. package/dist/livekit-client.umd.js +1 -1
  8. package/dist/livekit-client.umd.js.map +1 -1
  9. package/dist/src/e2ee/E2eeManager.d.ts.map +1 -1
  10. package/dist/src/e2ee/worker/FrameCryptor.d.ts +0 -47
  11. package/dist/src/e2ee/worker/FrameCryptor.d.ts.map +1 -1
  12. package/dist/src/e2ee/worker/naluUtils.d.ts +27 -0
  13. package/dist/src/e2ee/worker/naluUtils.d.ts.map +1 -0
  14. package/dist/src/e2ee/worker/sifPayload.d.ts +22 -0
  15. package/dist/src/e2ee/worker/sifPayload.d.ts.map +1 -0
  16. package/dist/src/index.d.ts +2 -2
  17. package/dist/src/index.d.ts.map +1 -1
  18. package/dist/src/room/Room.d.ts +6 -10
  19. package/dist/src/room/Room.d.ts.map +1 -1
  20. package/dist/src/room/data-stream/incoming/IncomingDataStreamManager.d.ts +20 -0
  21. package/dist/src/room/data-stream/incoming/IncomingDataStreamManager.d.ts.map +1 -0
  22. package/dist/{ts4.2/src/room → src/room/data-stream/incoming}/StreamReader.d.ts +82 -56
  23. package/dist/src/room/data-stream/incoming/StreamReader.d.ts.map +1 -0
  24. package/dist/src/room/data-stream/outgoing/OutgoingDataStreamManager.d.ts +27 -0
  25. package/dist/src/room/data-stream/outgoing/OutgoingDataStreamManager.d.ts.map +1 -0
  26. package/dist/src/room/{StreamWriter.d.ts → data-stream/outgoing/StreamWriter.d.ts} +1 -1
  27. package/dist/src/room/data-stream/outgoing/StreamWriter.d.ts.map +1 -0
  28. package/dist/src/room/errors.d.ts +13 -0
  29. package/dist/src/room/errors.d.ts.map +1 -1
  30. package/dist/src/room/participant/LocalParticipant.d.ts +32 -19
  31. package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
  32. package/dist/src/room/track/LocalTrack.d.ts +7 -2
  33. package/dist/src/room/track/LocalTrack.d.ts.map +1 -1
  34. package/dist/src/room/track/RemoteVideoTrack.d.ts +1 -0
  35. package/dist/src/room/track/RemoteVideoTrack.d.ts.map +1 -1
  36. package/dist/src/room/track/Track.d.ts +4 -1
  37. package/dist/src/room/track/Track.d.ts.map +1 -1
  38. package/dist/src/room/types.d.ts +17 -1
  39. package/dist/src/room/types.d.ts.map +1 -1
  40. package/dist/src/room/utils.d.ts +8 -0
  41. package/dist/src/room/utils.d.ts.map +1 -1
  42. package/dist/ts4.2/src/e2ee/worker/FrameCryptor.d.ts +0 -47
  43. package/dist/ts4.2/src/e2ee/worker/naluUtils.d.ts +27 -0
  44. package/dist/ts4.2/src/e2ee/worker/sifPayload.d.ts +22 -0
  45. package/dist/ts4.2/src/index.d.ts +2 -2
  46. package/dist/ts4.2/src/room/Room.d.ts +6 -10
  47. package/dist/ts4.2/src/room/data-stream/incoming/IncomingDataStreamManager.d.ts +20 -0
  48. package/dist/{src/room → ts4.2/src/room/data-stream/incoming}/StreamReader.d.ts +82 -56
  49. package/dist/ts4.2/src/room/data-stream/outgoing/OutgoingDataStreamManager.d.ts +27 -0
  50. package/dist/ts4.2/src/room/{StreamWriter.d.ts → data-stream/outgoing/StreamWriter.d.ts} +1 -1
  51. package/dist/ts4.2/src/room/errors.d.ts +13 -0
  52. package/dist/ts4.2/src/room/participant/LocalParticipant.d.ts +32 -19
  53. package/dist/ts4.2/src/room/track/LocalTrack.d.ts +7 -2
  54. package/dist/ts4.2/src/room/track/RemoteVideoTrack.d.ts +1 -0
  55. package/dist/ts4.2/src/room/track/Track.d.ts +4 -1
  56. package/dist/ts4.2/src/room/types.d.ts +17 -1
  57. package/dist/ts4.2/src/room/utils.d.ts +8 -0
  58. package/package.json +7 -7
  59. package/src/e2ee/E2eeManager.ts +18 -1
  60. package/src/e2ee/worker/FrameCryptor.ts +56 -157
  61. package/src/e2ee/worker/e2ee.worker.ts +6 -1
  62. package/src/e2ee/worker/naluUtils.ts +328 -0
  63. package/src/e2ee/worker/sifPayload.ts +75 -0
  64. package/src/index.ts +2 -2
  65. package/src/room/Room.ts +104 -208
  66. package/src/room/data-stream/incoming/IncomingDataStreamManager.ts +247 -0
  67. package/src/room/data-stream/incoming/StreamReader.ts +317 -0
  68. package/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +316 -0
  69. package/src/room/{StreamWriter.ts → data-stream/outgoing/StreamWriter.ts} +1 -1
  70. package/src/room/errors.ts +34 -0
  71. package/src/room/participant/LocalParticipant.ts +39 -295
  72. package/src/room/track/LocalAudioTrack.ts +2 -2
  73. package/src/room/track/LocalTrack.ts +70 -50
  74. package/src/room/track/RemoteVideoTrack.ts +12 -2
  75. package/src/room/track/Track.ts +10 -1
  76. package/src/room/types.ts +22 -1
  77. package/src/room/utils.ts +14 -5
  78. package/dist/src/e2ee/worker/SifGuard.d.ts +0 -11
  79. package/dist/src/e2ee/worker/SifGuard.d.ts.map +0 -1
  80. package/dist/src/room/StreamReader.d.ts.map +0 -1
  81. package/dist/src/room/StreamWriter.d.ts.map +0 -1
  82. package/dist/ts4.2/src/e2ee/worker/SifGuard.d.ts +0 -11
  83. package/src/e2ee/worker/SifGuard.ts +0 -47
  84. package/src/room/StreamReader.ts +0 -170
@@ -1,13 +1,14 @@
1
1
  import { Codec, ParticipantInfo, ParticipantPermission } from '@livekit/protocol';
2
2
  import type { InternalRoomOptions } from '../../options';
3
3
  import type RTCEngine from '../RTCEngine';
4
- import { ByteStreamWriter, TextStreamWriter } from '../StreamWriter';
4
+ import type OutgoingDataStreamManager from '../data-stream/outgoing/OutgoingDataStreamManager';
5
+ import type { TextStreamWriter } from '../data-stream/outgoing/StreamWriter';
5
6
  import type { PerformRpcParams, RpcInvocationData } from '../rpc';
6
7
  import LocalTrack from '../track/LocalTrack';
7
8
  import LocalTrackPublication from '../track/LocalTrackPublication';
8
9
  import { Track } from '../track/Track';
9
10
  import type { AudioCaptureOptions, BackupVideoCodec, CreateLocalTracksOptions, ScreenShareCaptureOptions, TrackPublishOptions, VideoCaptureOptions } from '../track/options';
10
- import type { ChatMessage, DataPublishOptions, SendTextOptions, StreamTextOptions, TextStreamInfo } from '../types';
11
+ import type { ChatMessage, DataPublishOptions, SendFileOptions, SendTextOptions, StreamBytesOptions, StreamTextOptions, TextStreamInfo } from '../types';
11
12
  import Participant from './Participant';
12
13
  import type { ParticipantTrackPermission } from './ParticipantTrackPermission';
13
14
  import type RemoteParticipant from './RemoteParticipant';
@@ -34,12 +35,13 @@ export default class LocalParticipant extends Participant {
34
35
  private activeAgentFuture?;
35
36
  private firstActiveAgent?;
36
37
  private rpcHandlers;
38
+ private roomOutgoingDataStreamManager;
37
39
  private pendingSignalRequests;
38
40
  private enabledPublishVideoCodecs;
39
41
  private pendingAcks;
40
42
  private pendingResponses;
41
43
  /** @internal */
42
- constructor(sid: string, identity: string, engine: RTCEngine, options: InternalRoomOptions, roomRpcHandlers: Map<string, (data: RpcInvocationData) => Promise<string>>);
44
+ constructor(sid: string, identity: string, engine: RTCEngine, options: InternalRoomOptions, roomRpcHandlers: Map<string, (data: RpcInvocationData) => Promise<string>>, roomOutgoingDataStreamManager: OutgoingDataStreamManager);
43
45
  get lastCameraError(): Error | undefined;
44
46
  get lastMicrophoneError(): Error | undefined;
45
47
  get isE2EEEnabled(): boolean;
@@ -156,7 +158,9 @@ export default class LocalParticipant extends Participant {
156
158
  * @param digit DTMF digit
157
159
  */
158
160
  publishDtmf(code: number, digit: string): Promise<void>;
161
+ /** @deprecated Consider migrating to {@link sendText} */
159
162
  sendChatMessage(text: string, options?: SendTextOptions): Promise<ChatMessage>;
163
+ /** @deprecated Consider migrating to {@link sendText} */
160
164
  editChatMessage(editText: string, originalMessage: ChatMessage): Promise<{
161
165
  readonly message: string;
162
166
  readonly editTimestamp: number;
@@ -164,30 +168,39 @@ export default class LocalParticipant extends Participant {
164
168
  readonly timestamp: number;
165
169
  readonly attachedFiles?: Array<File>;
166
170
  }>;
171
+ /**
172
+ * Sends the given string to participants in the room via the data channel.
173
+ * For longer messages, consider using {@link streamText} instead.
174
+ *
175
+ * @param text The text payload
176
+ * @param options.topic Topic identifier used to route the stream to appropriate handlers.
177
+ */
167
178
  sendText(text: string, options?: SendTextOptions): Promise<TextStreamInfo>;
168
179
  /**
180
+ * Creates a new TextStreamWriter which can be used to stream text incrementally
181
+ * to participants in the room via the data channel.
182
+ *
183
+ * @param options.topic Topic identifier used to route the stream to appropriate handlers.
184
+ *
169
185
  * @internal
170
186
  * @experimental CAUTION, might get removed in a minor release
171
187
  */
172
188
  streamText(options?: StreamTextOptions): Promise<TextStreamWriter>;
173
- sendFile(file: File, options?: {
174
- mimeType?: string;
175
- topic?: string;
176
- destinationIdentities?: Array<string>;
177
- onProgress?: (progress: number) => void;
178
- }): Promise<{
189
+ /** Send a File to all participants in the room via the data channel.
190
+ * @param file The File object payload
191
+ * @param options.topic Topic identifier used to route the stream to appropriate handlers.
192
+ * @param options.onProgress A callback function used to monitor the upload progress percentage.
193
+ */
194
+ sendFile(file: File, options?: SendFileOptions): Promise<{
179
195
  id: string;
180
196
  }>;
181
- private _sendFile;
182
- streamBytes(options?: {
183
- name?: string;
184
- topic?: string;
185
- attributes?: Record<string, string>;
186
- destinationIdentities?: Array<string>;
187
- streamId?: string;
188
- mimeType?: string;
189
- totalSize?: number;
190
- }): Promise<ByteStreamWriter>;
197
+ /**
198
+ * Stream bytes incrementally to participants in the room via the data channel.
199
+ * For sending files, consider using {@link sendFile} instead.
200
+ *
201
+ * @param options.topic Topic identifier used to route the stream to appropriate handlers.
202
+ */
203
+ streamBytes(options?: StreamBytesOptions): Promise<import("../data-stream/outgoing/StreamWriter").ByteStreamWriter>;
191
204
  /**
192
205
  * Initiate an RPC call to a remote participant
193
206
  * @param params - Parameters for initiating the RPC call, see {@link PerformRpcParams}
@@ -23,11 +23,10 @@ export default abstract class LocalTrack<TrackKind extends Track.Kind = Track.Ki
23
23
  protected pauseUpstreamLock: Mutex;
24
24
  protected processorElement?: HTMLMediaElement;
25
25
  protected processor?: TrackProcessor<TrackKind, any>;
26
- protected processorLock: Mutex;
27
26
  protected audioContext?: AudioContext;
28
27
  protected manuallyStopped: boolean;
29
28
  protected localTrackRecorder: LocalTrackRecorder<typeof this> | undefined;
30
- private restartLock;
29
+ protected trackChangeLock: Mutex;
31
30
  /**
32
31
  *
33
32
  * @param mediaTrack
@@ -104,6 +103,12 @@ export default abstract class LocalTrack<TrackKind extends Track.Kind = Track.Ki
104
103
  * @returns
105
104
  */
106
105
  stopProcessor(keepElement?: boolean): Promise<void>;
106
+ /**
107
+ * @internal
108
+ * This method assumes the caller has acquired a trackChangeLock already.
109
+ * The public facing method for stopping the processor is `stopProcessor` and it wraps this method in the trackChangeLock.
110
+ */
111
+ protected internalStopProcessor(keepElement?: boolean): Promise<void>;
107
112
  /** @internal */
108
113
  startPreConnectBuffer(timeslice?: number): void;
109
114
  /** @internal */
@@ -11,6 +11,7 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
11
11
  private lastDimensions?;
12
12
  constructor(mediaTrack: MediaStreamTrack, sid: string, receiver: RTCRtpReceiver, adaptiveStreamSettings?: AdaptiveStreamSettings, loggerOptions?: LoggerOptions);
13
13
  get isAdaptiveStream(): boolean;
14
+ setStreamState(value: Track.StreamState): void;
14
15
  /**
15
16
  * Note: When using adaptiveStream, you need to use remoteVideoTrack.attach() to add the track to a HTMLVideoElement, otherwise your video tracks might never start
16
17
  */
@@ -15,6 +15,7 @@ export declare abstract class Track<TrackKind extends Track.Kind = Track.Kind> e
15
15
  attachedElements: HTMLMediaElement[];
16
16
  isMuted: boolean;
17
17
  source: Track.Source;
18
+ private _streamState;
18
19
  /**
19
20
  * sid is set after track is published to server, or if it's a remote track
20
21
  */
@@ -27,7 +28,9 @@ export declare abstract class Track<TrackKind extends Track.Kind = Track.Kind> e
27
28
  * indicates current state of stream, it'll indicate `paused` if the track
28
29
  * has been paused by congestion controller
29
30
  */
30
- streamState: Track.StreamState;
31
+ get streamState(): Track.StreamState;
32
+ /** @internal */
33
+ setStreamState(value: Track.StreamState): void;
31
34
  /** @internal */
32
35
  rtpTimestamp: number | undefined;
33
36
  protected _mediaStreamTrack: MediaStreamTrack;
@@ -1,4 +1,5 @@
1
- import type { DataStream_Chunk } from '@livekit/protocol';
1
+ import type { DataStream_Chunk, Encryption_Type } from '@livekit/protocol';
2
+ import type { Future } from './utils';
2
3
  export type SimulationOptions = {
3
4
  publish?: {
4
5
  audio?: boolean;
@@ -30,6 +31,19 @@ export interface StreamTextOptions {
30
31
  totalSize?: number;
31
32
  attributes?: Record<string, string>;
32
33
  }
34
+ export type StreamBytesOptions = {
35
+ name?: string;
36
+ topic?: string;
37
+ attributes?: Record<string, string>;
38
+ destinationIdentities?: Array<string>;
39
+ streamId?: string;
40
+ mimeType?: string;
41
+ totalSize?: number;
42
+ };
43
+ export type SendFileOptions = Pick<StreamBytesOptions, 'topic' | 'mimeType' | 'destinationIdentities'> & {
44
+ onProgress?: (progress: number) => void;
45
+ encryptionType?: Encryption_Type.NONE;
46
+ };
33
47
  export type DataPublishOptions = {
34
48
  /**
35
49
  * whether to send this as reliable or lossy.
@@ -76,6 +90,8 @@ export interface StreamController<T extends DataStream_Chunk> {
76
90
  controller: ReadableStreamDefaultController<T>;
77
91
  startTime: number;
78
92
  endTime?: number;
93
+ sendingParticipantIdentity: string;
94
+ outOfBandFailureRejectingFuture: Future<never>;
79
95
  }
80
96
  export interface BaseStreamInfo {
81
97
  id: string;
@@ -44,6 +44,14 @@ export declare function isReactNative(): boolean;
44
44
  export declare function isCloud(serverUrl: URL): boolean;
45
45
  export declare function getReactNativeOs(): string | undefined;
46
46
  export declare function getDevicePixelRatio(): number;
47
+ /**
48
+ * @param v1 - The first version string to compare.
49
+ * @param v2 - The second version string to compare.
50
+ * @returns A number indicating the order of the versions:
51
+ * - 1 if v1 is greater than v2
52
+ * - -1 if v1 is less than v2
53
+ * - 0 if v1 and v2 are equal
54
+ */
47
55
  export declare function compareVersions(v1: string, v2: string): number;
48
56
  export declare const getResizeObserver: () => ResizeObserver;
49
57
  export declare const getIntersectionObserver: () => IntersectionObserver;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "livekit-client",
3
- "version": "2.15.4",
3
+ "version": "2.15.6",
4
4
  "description": "JavaScript/TypeScript client SDK for LiveKit",
5
5
  "main": "./dist/livekit-client.umd.js",
6
6
  "unpkg": "./dist/livekit-client.umd.js",
@@ -50,8 +50,8 @@
50
50
  "@types/dom-mediacapture-record": "^1"
51
51
  },
52
52
  "devDependencies": {
53
- "@babel/core": "7.28.0",
54
- "@babel/preset-env": "7.28.0",
53
+ "@babel/core": "7.28.3",
54
+ "@babel/preset-env": "7.28.3",
55
55
  "@bufbuild/protoc-gen-es": "^1.10.0",
56
56
  "@changesets/cli": "2.29.5",
57
57
  "@livekit/changesets-changelog-github": "^0.0.4",
@@ -71,21 +71,21 @@
71
71
  "downlevel-dts": "^0.11.0",
72
72
  "eslint": "8.57.1",
73
73
  "eslint-config-airbnb-typescript": "18.0.0",
74
- "eslint-config-prettier": "10.1.5",
74
+ "eslint-config-prettier": "10.1.8",
75
75
  "eslint-plugin-ecmascript-compat": "^3.2.1",
76
76
  "eslint-plugin-import": "2.32.0",
77
77
  "gh-pages": "6.3.0",
78
78
  "happy-dom": "^17.2.0",
79
79
  "jsdom": "^26.1.0",
80
80
  "prettier": "^3.4.2",
81
- "rollup": "4.44.1",
81
+ "rollup": "4.46.2",
82
82
  "rollup-plugin-delete": "^2.1.0",
83
83
  "rollup-plugin-typescript2": "0.36.0",
84
84
  "size-limit": "^11.2.0",
85
- "typedoc": "0.28.7",
85
+ "typedoc": "0.28.10",
86
86
  "typedoc-plugin-no-inherit": "1.6.1",
87
87
  "typescript": "5.8.3",
88
- "vite": "7.0.0",
88
+ "vite": "7.1.2",
89
89
  "vitest": "^3.0.0"
90
90
  },
91
91
  "scripts": {
@@ -11,7 +11,7 @@ import type RemoteTrack from '../room/track/RemoteTrack';
11
11
  import type { Track } from '../room/track/Track';
12
12
  import type { VideoCodec } from '../room/track/options';
13
13
  import { mimeTypeToVideoCodecString } from '../room/track/utils';
14
- import { isLocalTrack } from '../room/utils';
14
+ import { isLocalTrack, isSafariBased, isVideoTrack } from '../room/utils';
15
15
  import type { BaseKeyProvider } from './KeyProvider';
16
16
  import { E2EE_FLAG } from './constants';
17
17
  import { type E2EEManagerCallbacks, EncryptionEvent, KeyProviderEvent } from './events';
@@ -226,6 +226,23 @@ export class E2EEManager
226
226
  this.setupE2EESender(track, sender);
227
227
  });
228
228
 
229
+ room.localParticipant.on(ParticipantEvent.LocalTrackPublished, (publication) => {
230
+ // Safari doesn't support retrieving payload information on RTCEncodedVideoFrame, so we need to update the codec manually once we have the trackInfo from the server
231
+ if (!isVideoTrack(publication.track) || !isSafariBased()) {
232
+ return;
233
+ }
234
+ const msg: UpdateCodecMessage = {
235
+ kind: 'updateCodec',
236
+ data: {
237
+ trackId: publication.track!.mediaStreamID,
238
+ codec: mimeTypeToVideoCodecString(publication.trackInfo!.codecs[0].mimeType),
239
+ participantIdentity: this.room!.localParticipant.identity,
240
+ },
241
+ };
242
+
243
+ this.worker.postMessage(msg);
244
+ });
245
+
229
246
  keyProvider
230
247
  .on(KeyProviderEvent.SetKey, (keyInfo) => this.postKey(keyInfo))
231
248
  .on(KeyProviderEvent.RatchetRequest, (participantId, keyIndex) =>
@@ -10,7 +10,8 @@ import { type CryptorCallbacks, CryptorEvent } from '../events';
10
10
  import type { DecodeRatchetOptions, KeyProviderOptions, KeySet, RatchetResult } from '../types';
11
11
  import { deriveKeys, isVideoFrame, needsRbspUnescaping, parseRbsp, writeRbsp } from '../utils';
12
12
  import type { ParticipantKeyHandler } from './ParticipantKeyHandler';
13
- import { SifGuard } from './SifGuard';
13
+ import { processNALUsForEncryption } from './naluUtils';
14
+ import { identifySifPayload } from './sifPayload';
14
15
 
15
16
  export const encryptionEnabledMap: Map<string, boolean> = new Map();
16
17
 
@@ -65,8 +66,6 @@ export class FrameCryptor extends BaseFrameCryptor {
65
66
  */
66
67
  private sifTrailer: Uint8Array;
67
68
 
68
- private sifGuard: SifGuard;
69
-
70
69
  private detectedCodec?: VideoCodec;
71
70
 
72
71
  private isTransformActive: boolean = false;
@@ -84,7 +83,6 @@ export class FrameCryptor extends BaseFrameCryptor {
84
83
  this.rtpMap = new Map();
85
84
  this.keyProviderOptions = opts.keyProviderOptions;
86
85
  this.sifTrailer = opts.sifTrailer ?? Uint8Array.from([]);
87
- this.sifGuard = new SifGuard();
88
86
  }
89
87
 
90
88
  private get logContext() {
@@ -116,7 +114,6 @@ export class FrameCryptor extends BaseFrameCryptor {
116
114
  }
117
115
  this.participantIdentity = id;
118
116
  this.keys = keys;
119
- this.sifGuard.reset();
120
117
  }
121
118
 
122
119
  unsetParticipant() {
@@ -304,7 +301,7 @@ export class FrameCryptor extends BaseFrameCryptor {
304
301
  newDataWithoutHeader.set(new Uint8Array(iv), cipherText.byteLength); // append IV.
305
302
  newDataWithoutHeader.set(frameTrailer, cipherText.byteLength + iv.byteLength); // append frame trailer.
306
303
 
307
- if (frameInfo.isH264) {
304
+ if (frameInfo.requiresNALUProcessing) {
308
305
  newDataWithoutHeader = writeRbsp(newDataWithoutHeader);
309
306
  }
310
307
 
@@ -347,27 +344,21 @@ export class FrameCryptor extends BaseFrameCryptor {
347
344
  // skip for decryption for empty dtx frames
348
345
  encodedFrame.data.byteLength === 0
349
346
  ) {
350
- workerLogger.debug('skipping empty frame', this.logContext);
351
- this.sifGuard.recordUserFrame();
352
347
  return controller.enqueue(encodedFrame);
353
348
  }
354
349
 
355
350
  if (isFrameServerInjected(encodedFrame.data, this.sifTrailer)) {
356
- workerLogger.debug('enqueue SIF', this.logContext);
357
- this.sifGuard.recordSif();
358
-
359
- if (this.sifGuard.isSifAllowed()) {
360
- encodedFrame.data = encodedFrame.data.slice(
361
- 0,
362
- encodedFrame.data.byteLength - this.sifTrailer.byteLength,
363
- );
351
+ encodedFrame.data = encodedFrame.data.slice(
352
+ 0,
353
+ encodedFrame.data.byteLength - this.sifTrailer.byteLength,
354
+ );
355
+ if (await identifySifPayload(encodedFrame.data)) {
356
+ workerLogger.debug('enqueue SIF', this.logContext);
364
357
  return controller.enqueue(encodedFrame);
365
358
  } else {
366
- workerLogger.warn('SIF limit reached, dropping frame');
359
+ workerLogger.warn('Unexpected SIF frame payload, dropping frame', this.logContext);
367
360
  return;
368
361
  }
369
- } else {
370
- this.sifGuard.recordUserFrame();
371
362
  }
372
363
  const data = new Uint8Array(encodedFrame.data);
373
364
  const keyIndex = data[encodedFrame.data.byteLength - 1];
@@ -441,7 +432,7 @@ export class FrameCryptor extends BaseFrameCryptor {
441
432
  frameHeader.length,
442
433
  encodedFrame.data.byteLength - frameHeader.length,
443
434
  );
444
- if (frameInfo.isH264 && needsRbspUnescaping(encryptedData)) {
435
+ if (frameInfo.requiresNALUProcessing && needsRbspUnescaping(encryptedData)) {
445
436
  encryptedData = parseRbsp(encryptedData);
446
437
  const newUint8 = new Uint8Array(frameHeader.byteLength + encryptedData.byteLength);
447
438
  newUint8.set(frameHeader);
@@ -584,66 +575,58 @@ export class FrameCryptor extends BaseFrameCryptor {
584
575
 
585
576
  private getUnencryptedBytes(frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame): {
586
577
  unencryptedBytes: number;
587
- isH264: boolean;
578
+ requiresNALUProcessing: boolean;
588
579
  } {
589
- var frameInfo = { unencryptedBytes: 0, isH264: false };
590
- if (isVideoFrame(frame)) {
591
- let detectedCodec = this.getVideoCodec(frame) ?? this.videoCodec;
592
- if (detectedCodec !== this.detectedCodec) {
593
- workerLogger.debug('detected different codec', {
594
- detectedCodec,
595
- oldCodec: this.detectedCodec,
596
- ...this.logContext,
597
- });
598
- this.detectedCodec = detectedCodec;
599
- }
600
-
601
- if (detectedCodec === 'av1') {
602
- throw new Error(`${detectedCodec} is not yet supported for end to end encryption`);
603
- }
580
+ // Handle audio frames
581
+ if (!isVideoFrame(frame)) {
582
+ return { unencryptedBytes: UNENCRYPTED_BYTES.audio, requiresNALUProcessing: false };
583
+ }
604
584
 
605
- if (detectedCodec === 'vp8') {
606
- frameInfo.unencryptedBytes = UNENCRYPTED_BYTES[frame.type];
607
- } else if (detectedCodec === 'vp9') {
608
- frameInfo.unencryptedBytes = 0;
609
- return frameInfo;
610
- }
585
+ // Detect and track codec changes
586
+ const detectedCodec = this.getVideoCodec(frame) ?? this.videoCodec;
587
+ if (detectedCodec !== this.detectedCodec) {
588
+ workerLogger.debug('detected different codec', {
589
+ detectedCodec,
590
+ oldCodec: this.detectedCodec,
591
+ ...this.logContext,
592
+ });
593
+ this.detectedCodec = detectedCodec;
594
+ }
611
595
 
612
- const data = new Uint8Array(frame.data);
613
- try {
614
- const naluIndices = findNALUIndices(data);
596
+ // Check for unsupported codecs
597
+ if (detectedCodec === 'av1') {
598
+ throw new Error(`${detectedCodec} is not yet supported for end to end encryption`);
599
+ }
615
600
 
616
- // if the detected codec is undefined we test whether it _looks_ like a h264 frame as a best guess
617
- frameInfo.isH264 =
618
- detectedCodec === 'h264' ||
619
- naluIndices.some((naluIndex) =>
620
- [NALUType.SLICE_IDR, NALUType.SLICE_NON_IDR].includes(parseNALUType(data[naluIndex])),
621
- );
601
+ // Handle VP8/VP9 codecs (no NALU processing needed)
602
+ if (detectedCodec === 'vp8') {
603
+ return { unencryptedBytes: UNENCRYPTED_BYTES[frame.type], requiresNALUProcessing: false };
604
+ }
605
+ if (detectedCodec === 'vp9') {
606
+ return { unencryptedBytes: 0, requiresNALUProcessing: false };
607
+ }
622
608
 
623
- if (frameInfo.isH264) {
624
- for (const index of naluIndices) {
625
- let type = parseNALUType(data[index]);
626
- switch (type) {
627
- case NALUType.SLICE_IDR:
628
- case NALUType.SLICE_NON_IDR:
629
- frameInfo.unencryptedBytes = index + 2;
630
- return frameInfo;
631
- default:
632
- break;
633
- }
634
- }
635
- throw new TypeError('Could not find NALU');
636
- }
637
- } catch (e) {
638
- // no op, we just continue and fallback to vp8
609
+ // Try NALU processing for H.264/H.265 codecs
610
+ try {
611
+ const knownCodec =
612
+ detectedCodec === 'h264' || detectedCodec === 'h265' ? detectedCodec : undefined;
613
+ const naluResult = processNALUsForEncryption(new Uint8Array(frame.data), knownCodec);
614
+
615
+ if (naluResult.requiresNALUProcessing) {
616
+ return {
617
+ unencryptedBytes: naluResult.unencryptedBytes,
618
+ requiresNALUProcessing: true,
619
+ };
639
620
  }
640
-
641
- frameInfo.unencryptedBytes = UNENCRYPTED_BYTES[frame.type];
642
- return frameInfo;
643
- } else {
644
- frameInfo.unencryptedBytes = UNENCRYPTED_BYTES.audio;
645
- return frameInfo;
621
+ } catch (e) {
622
+ workerLogger.debug('NALU processing failed, falling back to VP8 handling', {
623
+ error: e,
624
+ ...this.logContext,
625
+ });
646
626
  }
627
+
628
+ // Fallback to VP8 handling
629
+ return { unencryptedBytes: UNENCRYPTED_BYTES[frame.type], requiresNALUProcessing: false };
647
630
  }
648
631
 
649
632
  /**
@@ -659,90 +642,6 @@ export class FrameCryptor extends BaseFrameCryptor {
659
642
  }
660
643
  }
661
644
 
662
- /**
663
- * Slice the NALUs present in the supplied buffer, assuming it is already byte-aligned
664
- * code adapted from https://github.com/medooze/h264-frame-parser/blob/main/lib/NalUnits.ts to return indices only
665
- */
666
- export function findNALUIndices(stream: Uint8Array): number[] {
667
- const result: number[] = [];
668
- let start = 0,
669
- pos = 0,
670
- searchLength = stream.length - 2;
671
- while (pos < searchLength) {
672
- // skip until end of current NALU
673
- while (
674
- pos < searchLength &&
675
- !(stream[pos] === 0 && stream[pos + 1] === 0 && stream[pos + 2] === 1)
676
- )
677
- pos++;
678
- if (pos >= searchLength) pos = stream.length;
679
- // remove trailing zeros from current NALU
680
- let end = pos;
681
- while (end > start && stream[end - 1] === 0) end--;
682
- // save current NALU
683
- if (start === 0) {
684
- if (end !== start) throw TypeError('byte stream contains leading data');
685
- } else {
686
- result.push(start);
687
- }
688
- // begin new NALU
689
- start = pos = pos + 3;
690
- }
691
- return result;
692
- }
693
-
694
- export function parseNALUType(startByte: number): NALUType {
695
- return startByte & kNaluTypeMask;
696
- }
697
-
698
- const kNaluTypeMask = 0x1f;
699
-
700
- export enum NALUType {
701
- /** Coded slice of a non-IDR picture */
702
- SLICE_NON_IDR = 1,
703
- /** Coded slice data partition A */
704
- SLICE_PARTITION_A = 2,
705
- /** Coded slice data partition B */
706
- SLICE_PARTITION_B = 3,
707
- /** Coded slice data partition C */
708
- SLICE_PARTITION_C = 4,
709
- /** Coded slice of an IDR picture */
710
- SLICE_IDR = 5,
711
- /** Supplemental enhancement information */
712
- SEI = 6,
713
- /** Sequence parameter set */
714
- SPS = 7,
715
- /** Picture parameter set */
716
- PPS = 8,
717
- /** Access unit delimiter */
718
- AUD = 9,
719
- /** End of sequence */
720
- END_SEQ = 10,
721
- /** End of stream */
722
- END_STREAM = 11,
723
- /** Filler data */
724
- FILLER_DATA = 12,
725
- /** Sequence parameter set extension */
726
- SPS_EXT = 13,
727
- /** Prefix NAL unit */
728
- PREFIX_NALU = 14,
729
- /** Subset sequence parameter set */
730
- SUBSET_SPS = 15,
731
- /** Depth parameter set */
732
- DPS = 16,
733
-
734
- // 17, 18 reserved
735
-
736
- /** Coded slice of an auxiliary coded picture without partitioning */
737
- SLICE_AUX = 19,
738
- /** Coded slice extension */
739
- SLICE_EXT = 20,
740
- /** Coded slice extension for a depth view component or a 3D-AVC texture view component */
741
- SLICE_LAYER_EXT = 21,
742
-
743
- // 22, 23 reserved
744
- }
745
-
746
645
  /**
747
646
  * we use a magic frame trailer to detect whether a frame is injected
748
647
  * by the livekit server and thus to be treated as unencrypted
@@ -98,6 +98,11 @@ onmessage = (ev) => {
98
98
  break;
99
99
  case 'updateCodec':
100
100
  getTrackCryptor(data.participantIdentity, data.trackId).setVideoCodec(data.codec);
101
+ workerLogger.info('updated codec', {
102
+ participantIdentity: data.participantIdentity,
103
+ trackId: data.trackId,
104
+ codec: data.codec,
105
+ });
101
106
  break;
102
107
  case 'setRTPMap':
103
108
  // this is only used for the local participant
@@ -151,7 +156,7 @@ function getTrackCryptor(participantIdentity: string, trackId: string) {
151
156
  }
152
157
  let cryptor = cryptors[0];
153
158
  if (!cryptor) {
154
- workerLogger.info('creating new cryptor for', { participantIdentity });
159
+ workerLogger.info('creating new cryptor for', { participantIdentity, trackId });
155
160
  if (!keyProviderOptions) {
156
161
  throw Error('Missing keyProvider options');
157
162
  }