livekit-client 2.22.4 → 2.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/livekit-client.e2ee.worker.js +1 -1
- package/dist/livekit-client.e2ee.worker.js.map +1 -1
- package/dist/livekit-client.e2ee.worker.mjs +69 -7
- package/dist/livekit-client.e2ee.worker.mjs.map +1 -1
- package/dist/livekit-client.esm.mjs +1133 -289
- package/dist/livekit-client.esm.mjs.map +1 -1
- package/dist/livekit-client.fm.worker.js.map +1 -1
- package/dist/livekit-client.fm.worker.mjs.map +1 -1
- package/dist/livekit-client.umd.js +1 -1
- package/dist/livekit-client.umd.js.map +1 -1
- package/dist/src/api/SignalClient.d.ts.map +1 -1
- package/dist/src/api/WebSocketStream.d.ts +1 -0
- package/dist/src/api/WebSocketStream.d.ts.map +1 -1
- package/dist/src/api/utils.d.ts +1 -1
- package/dist/src/api/utils.d.ts.map +1 -1
- package/dist/src/e2ee/E2eeManager.d.ts.map +1 -1
- package/dist/src/e2ee/worker/FrameCryptor.d.ts.map +1 -1
- package/dist/src/e2ee/worker/naluUtils.d.ts +12 -1
- package/dist/src/e2ee/worker/naluUtils.d.ts.map +1 -1
- package/dist/src/index.d.ts +2 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/room/DefaultReconnectPolicy.d.ts.map +1 -1
- package/dist/src/room/PCTransport.d.ts +26 -4
- package/dist/src/room/PCTransport.d.ts.map +1 -1
- package/dist/src/room/RTCEngine.d.ts +8 -0
- package/dist/src/room/RTCEngine.d.ts.map +1 -1
- package/dist/src/room/Room.d.ts +53 -0
- package/dist/src/room/Room.d.ts.map +1 -1
- package/dist/src/room/data-channel/DataChannelManager.d.ts +9 -8
- package/dist/src/room/data-channel/DataChannelManager.d.ts.map +1 -1
- package/dist/src/room/data-channel/FlowControlledDataChannel.d.ts +25 -12
- package/dist/src/room/data-channel/FlowControlledDataChannel.d.ts.map +1 -1
- package/dist/src/room/data-channel/LossyDataChannel.d.ts.map +1 -1
- package/dist/src/room/data-channel/ReliableDataChannel.d.ts.map +1 -1
- package/dist/src/room/data-stream/incoming/IncomingDataStreamManager.d.ts.map +1 -1
- package/dist/src/room/participant/LocalParticipant.d.ts +4 -0
- package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
- package/dist/src/room/participant/Participant.d.ts +1 -1
- package/dist/src/room/participant/Participant.d.ts.map +1 -1
- package/dist/src/room/rpc/utils.d.ts.map +1 -1
- package/dist/src/room/track/LocalAudioTrack.d.ts +24 -1
- package/dist/src/room/track/LocalAudioTrack.d.ts.map +1 -1
- package/dist/src/room/track/LocalTrack.d.ts +18 -0
- package/dist/src/room/track/LocalTrack.d.ts.map +1 -1
- package/dist/src/room/track/LocalVideoTrack.d.ts +7 -0
- package/dist/src/room/track/LocalVideoTrack.d.ts.map +1 -1
- package/dist/src/room/track/RemoteAudioTrack.d.ts +6 -0
- package/dist/src/room/track/RemoteAudioTrack.d.ts.map +1 -1
- package/dist/src/room/track/processor/types.d.ts +11 -0
- package/dist/src/room/track/processor/types.d.ts.map +1 -1
- package/dist/src/room/track/utils.d.ts.map +1 -1
- package/dist/src/room/utils.d.ts +28 -1
- package/dist/src/room/utils.d.ts.map +1 -1
- package/dist/ts4.2/api/WebSocketStream.d.ts +1 -0
- package/dist/ts4.2/api/utils.d.ts +1 -1
- package/dist/ts4.2/e2ee/worker/naluUtils.d.ts +12 -1
- package/dist/ts4.2/index.d.ts +2 -2
- package/dist/ts4.2/room/PCTransport.d.ts +26 -4
- package/dist/ts4.2/room/RTCEngine.d.ts +8 -0
- package/dist/ts4.2/room/Room.d.ts +53 -0
- package/dist/ts4.2/room/data-channel/DataChannelManager.d.ts +9 -8
- package/dist/ts4.2/room/data-channel/FlowControlledDataChannel.d.ts +25 -12
- package/dist/ts4.2/room/participant/LocalParticipant.d.ts +4 -0
- package/dist/ts4.2/room/participant/Participant.d.ts +1 -1
- package/dist/ts4.2/room/track/LocalAudioTrack.d.ts +24 -1
- package/dist/ts4.2/room/track/LocalTrack.d.ts +18 -0
- package/dist/ts4.2/room/track/LocalVideoTrack.d.ts +7 -0
- package/dist/ts4.2/room/track/RemoteAudioTrack.d.ts +6 -0
- package/dist/ts4.2/room/track/processor/types.d.ts +11 -0
- package/dist/ts4.2/room/utils.d.ts +28 -1
- package/package.json +1 -1
- package/src/api/SignalClient.test.ts +234 -0
- package/src/api/SignalClient.ts +58 -23
- package/src/api/WebSocketStream.test.ts +11 -0
- package/src/api/WebSocketStream.ts +22 -1
- package/src/api/utils.ts +1 -1
- package/src/e2ee/E2eeManager.test.ts +43 -0
- package/src/e2ee/E2eeManager.ts +6 -2
- package/src/e2ee/worker/FrameCryptor.test.ts +212 -90
- package/src/e2ee/worker/FrameCryptor.ts +21 -4
- package/src/e2ee/worker/naluUtils.test.ts +123 -0
- package/src/e2ee/worker/naluUtils.ts +57 -2
- package/src/index.ts +2 -0
- package/src/room/DefaultReconnectPolicy.test.ts +58 -0
- package/src/room/DefaultReconnectPolicy.ts +14 -2
- package/src/room/PCTransport.test.ts +144 -2
- package/src/room/PCTransport.ts +151 -9
- package/src/room/RTCEngine.test.ts +234 -2
- package/src/room/RTCEngine.ts +102 -7
- package/src/room/Room.test.ts +116 -0
- package/src/room/Room.ts +286 -74
- package/src/room/data-channel/DataChannelManager.test.ts +24 -7
- package/src/room/data-channel/DataChannelManager.ts +43 -38
- package/src/room/data-channel/FlowControlledDataChannel.test.ts +60 -18
- package/src/room/data-channel/FlowControlledDataChannel.ts +40 -22
- package/src/room/data-channel/LossyDataChannel.test.ts +38 -0
- package/src/room/data-channel/LossyDataChannel.ts +10 -1
- package/src/room/data-channel/ReliableDataChannel.test.ts +72 -0
- package/src/room/data-channel/ReliableDataChannel.ts +16 -5
- package/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +31 -0
- package/src/room/data-stream/incoming/IncomingDataStreamManager.ts +11 -0
- package/src/room/participant/LocalParticipant.test.ts +136 -1
- package/src/room/participant/LocalParticipant.ts +66 -17
- package/src/room/participant/Participant.ts +6 -3
- package/src/room/participant/restartOnDefaultAudioDevice.test.ts +177 -0
- package/src/room/rpc/utils.test.ts +32 -0
- package/src/room/rpc/utils.ts +7 -0
- package/src/room/track/LocalAudioTrack.ts +101 -10
- package/src/room/track/LocalTrack.ts +70 -13
- package/src/room/track/LocalVideoTrack.ts +7 -0
- package/src/room/track/RemoteAudioTrack.ts +56 -11
- package/src/room/track/processor/types.ts +11 -0
- package/src/room/track/utils.test.ts +5 -0
- package/src/room/track/utils.ts +29 -5
- package/src/room/utils.test.ts +126 -0
- package/src/room/utils.ts +84 -21
|
@@ -5,7 +5,14 @@ import { debounce } from '../debounce';
|
|
|
5
5
|
import { DeviceUnsupportedError, TrackInvalidError } from '../errors';
|
|
6
6
|
import { TrackEvent } from '../events';
|
|
7
7
|
import type { LoggerOptions } from '../types';
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
compareVersions,
|
|
10
|
+
isAppleMobile,
|
|
11
|
+
isMobile,
|
|
12
|
+
markDeviceAcquisitionFailure,
|
|
13
|
+
sleep,
|
|
14
|
+
unwrapConstraint,
|
|
15
|
+
} from '../utils';
|
|
9
16
|
import { Track, attachToElement, detachTrack } from './Track';
|
|
10
17
|
import type { VideoCodec } from './options';
|
|
11
18
|
import type { TrackProcessor } from './processor/types';
|
|
@@ -132,6 +139,10 @@ export default abstract class LocalTrack<
|
|
|
132
139
|
return this.providedByUser;
|
|
133
140
|
}
|
|
134
141
|
|
|
142
|
+
get isStopped() {
|
|
143
|
+
return this.manuallyStopped;
|
|
144
|
+
}
|
|
145
|
+
|
|
135
146
|
get mediaStreamTrack() {
|
|
136
147
|
return this.processor?.processedTrack ?? this._mediaStreamTrack;
|
|
137
148
|
}
|
|
@@ -195,6 +206,9 @@ export default abstract class LocalTrack<
|
|
|
195
206
|
track: newTrack,
|
|
196
207
|
kind: this.kind,
|
|
197
208
|
element: this.processorElement,
|
|
209
|
+
// audio processors need the context they were initialised with in order to rebuild
|
|
210
|
+
// their graph, omitting it here would leave them without one
|
|
211
|
+
audioContext: this.audioContext,
|
|
198
212
|
localTrack: this,
|
|
199
213
|
});
|
|
200
214
|
processedTrack = this.processor.processedTrack;
|
|
@@ -218,6 +232,17 @@ export default abstract class LocalTrack<
|
|
|
218
232
|
attachToElement(processedTrack ?? newTrack, el);
|
|
219
233
|
});
|
|
220
234
|
}
|
|
235
|
+
await this.onMediaStreamTrackChanged();
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Hook invoked after the underlying MediaStreamTrack has been swapped and the processor has
|
|
240
|
+
* been restarted around it, so `processor.processedTrack` already holds the new track.
|
|
241
|
+
* Unlike `onSenderTrackSwapped` this also fires on the restart and unmute paths.
|
|
242
|
+
*/
|
|
243
|
+
protected async onMediaStreamTrackChanged(): Promise<void> {
|
|
244
|
+
// base implementation is a no-op; LocalAudioTrack overrides this to move the krisp
|
|
245
|
+
// feature listeners onto the new processed track.
|
|
221
246
|
}
|
|
222
247
|
|
|
223
248
|
async waitForDimensions(timeout = DEFAULT_DIMENSIONS_TIMEOUT): Promise<Track.Dimensions> {
|
|
@@ -255,7 +280,8 @@ export default abstract class LocalTrack<
|
|
|
255
280
|
async setDeviceId(deviceId: ConstrainDOMString): Promise<boolean> {
|
|
256
281
|
if (
|
|
257
282
|
this._constraints.deviceId === deviceId &&
|
|
258
|
-
this._mediaStreamTrack.getSettings().deviceId === unwrapConstraint(deviceId)
|
|
283
|
+
this._mediaStreamTrack.getSettings().deviceId === unwrapConstraint(deviceId) &&
|
|
284
|
+
this._mediaStreamTrack.readyState === 'live'
|
|
259
285
|
) {
|
|
260
286
|
return true;
|
|
261
287
|
}
|
|
@@ -386,19 +412,37 @@ export default abstract class LocalTrack<
|
|
|
386
412
|
this._mediaStreamTrack.stop();
|
|
387
413
|
|
|
388
414
|
// create new track and attach
|
|
389
|
-
|
|
415
|
+
let mediaStream: MediaStream;
|
|
416
|
+
try {
|
|
417
|
+
mediaStream = await navigator.mediaDevices.getUserMedia(streamConstraints);
|
|
418
|
+
} catch (e) {
|
|
419
|
+
markDeviceAcquisitionFailure(e);
|
|
420
|
+
throw e;
|
|
421
|
+
}
|
|
390
422
|
const newTrack = mediaStream.getTracks()[0];
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
423
|
+
try {
|
|
424
|
+
if (this.kind === Track.Kind.Video) {
|
|
425
|
+
// we already captured the audio track with the constraints, so we only need to apply the video constraints
|
|
426
|
+
await newTrack.applyConstraints(otherConstraints);
|
|
427
|
+
}
|
|
428
|
+
newTrack.addEventListener('ended', this.handleEnded);
|
|
429
|
+
this.log.debug('re-acquired MediaStreamTrack', this.logContext);
|
|
430
|
+
|
|
431
|
+
await this.setMediaStreamTrack(newTrack, false, isUnmuting);
|
|
432
|
+
} catch (e) {
|
|
433
|
+
// setMediaStreamTrack can throw after acquisition but before it adopts the new track,
|
|
434
|
+
// which would leave the capture device live with nothing referencing it
|
|
435
|
+
if (this._mediaStreamTrack !== newTrack) {
|
|
436
|
+
newTrack.stop();
|
|
437
|
+
}
|
|
438
|
+
throw e;
|
|
394
439
|
}
|
|
395
|
-
newTrack.addEventListener('ended', this.handleEnded);
|
|
396
|
-
this.log.debug('re-acquired MediaStreamTrack', this.logContext);
|
|
397
|
-
|
|
398
|
-
await this.setMediaStreamTrack(newTrack, false, isUnmuting);
|
|
399
440
|
this._constraints = constraints;
|
|
400
441
|
this.pendingDeviceChange = false;
|
|
401
442
|
this.emit(TrackEvent.Restarted, this);
|
|
443
|
+
return this;
|
|
444
|
+
} finally {
|
|
445
|
+
// in the finally because a throw can also leave the new track adopted, and so still live
|
|
402
446
|
if (this.manuallyStopped) {
|
|
403
447
|
this.log.warn(
|
|
404
448
|
'track was stopped during a restart, stopping restarted track',
|
|
@@ -406,8 +450,6 @@ export default abstract class LocalTrack<
|
|
|
406
450
|
);
|
|
407
451
|
this.stop();
|
|
408
452
|
}
|
|
409
|
-
return this;
|
|
410
|
-
} finally {
|
|
411
453
|
unlock();
|
|
412
454
|
}
|
|
413
455
|
}
|
|
@@ -475,7 +517,11 @@ export default abstract class LocalTrack<
|
|
|
475
517
|
this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
|
|
476
518
|
this._mediaStreamTrack.removeEventListener('mute', this.handleTrackMuteEvent);
|
|
477
519
|
this._mediaStreamTrack.removeEventListener('unmute', this.handleTrackUnmuteEvent);
|
|
478
|
-
|
|
520
|
+
// `stop` is synchronous, so we can't await the teardown of the processor here.
|
|
521
|
+
// make sure a failing teardown doesn't surface as an unhandled rejection.
|
|
522
|
+
this.processor?.destroy().catch((error) => {
|
|
523
|
+
this.log.error('failed to destroy processor', { ...this.logContext, error });
|
|
524
|
+
});
|
|
479
525
|
this.processor = undefined;
|
|
480
526
|
}
|
|
481
527
|
|
|
@@ -551,6 +597,11 @@ export default abstract class LocalTrack<
|
|
|
551
597
|
* Sets a processor on this track.
|
|
552
598
|
* See https://github.com/livekit/track-processors-js for example usage
|
|
553
599
|
*
|
|
600
|
+
* The processor lives on the track, so stopping the track destroys it. Unpublishing and
|
|
601
|
+
* disconnecting both stop the track by default, which means a processor has to be set again on
|
|
602
|
+
* the track published after a reconnect. See {@link TrackProcessor} for the full lifecycle,
|
|
603
|
+
* including what `stopLocalTrackOnUnpublish: false` changes.
|
|
604
|
+
*
|
|
554
605
|
* @param processor
|
|
555
606
|
* @param showProcessedStreamLocally
|
|
556
607
|
* @returns
|
|
@@ -617,6 +668,10 @@ export default abstract class LocalTrack<
|
|
|
617
668
|
await this.onSenderTrackSwapped();
|
|
618
669
|
}
|
|
619
670
|
|
|
671
|
+
/**
|
|
672
|
+
* Returns the processor currently set on this track, or `undefined` if there is none. A track
|
|
673
|
+
* that was stopped by an unpublish or a disconnect has none. See {@link TrackProcessor}.
|
|
674
|
+
*/
|
|
620
675
|
getProcessor() {
|
|
621
676
|
return this.processor;
|
|
622
677
|
}
|
|
@@ -625,6 +680,8 @@ export default abstract class LocalTrack<
|
|
|
625
680
|
* Stops the track processor
|
|
626
681
|
* See https://github.com/livekit/track-processors-js for example usage
|
|
627
682
|
*
|
|
683
|
+
* Only needed to remove a processor from a track that stays published. Unpublishing or
|
|
684
|
+
* disconnecting already destroys it. See {@link TrackProcessor}.
|
|
628
685
|
*/
|
|
629
686
|
async stopProcessor(keepElement = true) {
|
|
630
687
|
const unlock = await this.trackChangeLock.lock();
|
|
@@ -401,6 +401,13 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
|
|
|
401
401
|
await sender.setParameters(params);
|
|
402
402
|
}
|
|
403
403
|
|
|
404
|
+
/**
|
|
405
|
+
* Sets a processor on this track, and wires its processed track into the simulcast codecs.
|
|
406
|
+
*
|
|
407
|
+
* The processor is destroyed when the track is stopped, which unpublishing and disconnecting
|
|
408
|
+
* both do by default, so it has to be set again after reconnecting. See {@link TrackProcessor}
|
|
409
|
+
* for the full lifecycle.
|
|
410
|
+
*/
|
|
404
411
|
async setProcessor(
|
|
405
412
|
processor: TrackProcessor<Track.Kind.Video>,
|
|
406
413
|
showProcessedStreamLocally = true,
|
|
@@ -20,6 +20,13 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
|
|
|
20
20
|
|
|
21
21
|
private webAudioPluginNodes: AudioNode[];
|
|
22
22
|
|
|
23
|
+
/**
|
|
24
|
+
* the nodes of the graph currently wired up, in order, so teardown can undo exactly the edges
|
|
25
|
+
* `connectWebAudio` created. Reading `webAudioPluginNodes` instead would miss the old plugins
|
|
26
|
+
* once `setWebAudioPlugins` has replaced them.
|
|
27
|
+
*/
|
|
28
|
+
private connectedNodes: AudioNode[] = [];
|
|
29
|
+
|
|
23
30
|
private sinkId?: string;
|
|
24
31
|
|
|
25
32
|
constructor(
|
|
@@ -42,10 +49,11 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
|
|
|
42
49
|
* sets the volume for all attached audio elements
|
|
43
50
|
*/
|
|
44
51
|
setVolume(volume: number) {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
52
|
+
if (this.audioContext) {
|
|
53
|
+
// playback goes through the web audio graph, the attached elements themselves stay silent
|
|
54
|
+
this.gainNode?.gain.setTargetAtTime(volume, 0, 0.1);
|
|
55
|
+
} else {
|
|
56
|
+
for (const el of this.attachedElements) {
|
|
49
57
|
el.volume = volume;
|
|
50
58
|
}
|
|
51
59
|
}
|
|
@@ -96,7 +104,9 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
|
|
|
96
104
|
attach(): HTMLMediaElement;
|
|
97
105
|
attach(element: HTMLMediaElement): HTMLMediaElement;
|
|
98
106
|
attach(element?: HTMLMediaElement): HTMLMediaElement {
|
|
99
|
-
|
|
107
|
+
// the graph is fed by the track rather than by a specific element, so one connection covers
|
|
108
|
+
// all attached elements
|
|
109
|
+
const needsNewWebAudioConnection = !this.sourceNode;
|
|
100
110
|
if (!element) {
|
|
101
111
|
element = super.attach();
|
|
102
112
|
} else {
|
|
@@ -108,11 +118,15 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
|
|
|
108
118
|
this.log.error('Failed to set sink id on remote audio track', e, this.logContext);
|
|
109
119
|
});
|
|
110
120
|
}
|
|
111
|
-
if (this.audioContext
|
|
112
|
-
|
|
113
|
-
|
|
121
|
+
if (this.audioContext) {
|
|
122
|
+
// audio is routed through the web audio graph, so every attached element has to stay
|
|
123
|
+
// silent, no matter whether it is the one the graph was built from
|
|
114
124
|
element.volume = 0;
|
|
115
125
|
element.muted = true;
|
|
126
|
+
if (needsNewWebAudioConnection) {
|
|
127
|
+
this.log.debug('using audio context mapping', this.logContext);
|
|
128
|
+
this.connectWebAudio(this.audioContext, element);
|
|
129
|
+
}
|
|
116
130
|
}
|
|
117
131
|
|
|
118
132
|
if (this.elementVolume !== undefined) {
|
|
@@ -158,11 +172,26 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
|
|
|
158
172
|
* @experimental
|
|
159
173
|
*/
|
|
160
174
|
setAudioContext(audioContext: AudioContext | undefined) {
|
|
175
|
+
if (this.audioContext === audioContext) {
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
161
178
|
this.audioContext = audioContext;
|
|
162
|
-
if (audioContext
|
|
163
|
-
|
|
164
|
-
|
|
179
|
+
if (audioContext) {
|
|
180
|
+
// playback moves into the web audio graph, mute the elements to avoid double playback
|
|
181
|
+
for (const element of this.attachedElements) {
|
|
182
|
+
element.volume = 0;
|
|
183
|
+
element.muted = true;
|
|
184
|
+
}
|
|
185
|
+
if (this.attachedElements.length > 0) {
|
|
186
|
+
this.connectWebAudio(audioContext, this.attachedElements[0]);
|
|
187
|
+
}
|
|
188
|
+
} else {
|
|
165
189
|
this.disconnectWebAudio();
|
|
190
|
+
// playback moves back into the elements, which `attach` had muted in favour of the graph
|
|
191
|
+
for (const element of this.attachedElements) {
|
|
192
|
+
element.muted = false;
|
|
193
|
+
element.volume = this.elementVolume ?? 1;
|
|
194
|
+
}
|
|
166
195
|
}
|
|
167
196
|
}
|
|
168
197
|
|
|
@@ -183,12 +212,15 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
|
|
|
183
212
|
// @ts-ignore attached elements always have a srcObject set
|
|
184
213
|
this.sourceNode = context.createMediaStreamSource(element.srcObject);
|
|
185
214
|
let lastNode: AudioNode = this.sourceNode;
|
|
215
|
+
this.connectedNodes = [this.sourceNode];
|
|
186
216
|
this.webAudioPluginNodes.forEach((node) => {
|
|
187
217
|
lastNode.connect(node);
|
|
188
218
|
lastNode = node;
|
|
219
|
+
this.connectedNodes.push(node);
|
|
189
220
|
});
|
|
190
221
|
this.gainNode = context.createGain();
|
|
191
222
|
lastNode.connect(this.gainNode);
|
|
223
|
+
this.connectedNodes.push(this.gainNode);
|
|
192
224
|
this.gainNode.connect(context.destination);
|
|
193
225
|
|
|
194
226
|
if (this.elementVolume !== undefined) {
|
|
@@ -214,6 +246,19 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
|
|
|
214
246
|
}
|
|
215
247
|
|
|
216
248
|
private disconnectWebAudio() {
|
|
249
|
+
// the plugin nodes belong to whoever passed them in and may feed graphs of their own, so only
|
|
250
|
+
// the edges this track created come down. A bare `node.disconnect()` would drop the
|
|
251
|
+
// application's own routing with them
|
|
252
|
+
for (let i = 0; i < this.connectedNodes.length - 1; i += 1) {
|
|
253
|
+
try {
|
|
254
|
+
this.connectedNodes[i].disconnect(this.connectedNodes[i + 1]);
|
|
255
|
+
} catch {
|
|
256
|
+
// a plugin node the application already disconnected itself, the rest of the chain still
|
|
257
|
+
// has to come apart
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
this.connectedNodes = [];
|
|
261
|
+
// the source and gain nodes are ours alone, so every remaining edge of theirs is ours to drop
|
|
217
262
|
this.gainNode?.disconnect();
|
|
218
263
|
this.sourceNode?.disconnect();
|
|
219
264
|
this.gainNode = undefined;
|
|
@@ -22,6 +22,17 @@ export interface AudioProcessorOptions extends ProcessorOptions<Track.Kind.Audio
|
|
|
22
22
|
*/
|
|
23
23
|
export interface VideoProcessorOptions extends ProcessorOptions<Track.Kind.Video> {}
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* A processor is bound to the track it was set on, not to the room, and it does not survive the
|
|
27
|
+
* track being stopped: unpublishing a track or disconnecting from a room destroys its processor.
|
|
28
|
+
* Both paths stop the track by default, so after reconnecting or republishing, call
|
|
29
|
+
* `setProcessor` again on the new track to put the processor back.
|
|
30
|
+
*
|
|
31
|
+
* The exception is `stopLocalTrackOnUnpublish: false`, which keeps the track alive across an
|
|
32
|
+
* unpublish or disconnect. A video processor then survives with it. An audio processor only
|
|
33
|
+
* survives if the audio context came from the application, because a context the SDK created is
|
|
34
|
+
* closed on disconnect and the processor goes with it.
|
|
35
|
+
*/
|
|
25
36
|
export interface TrackProcessor<
|
|
26
37
|
T extends Track.Kind,
|
|
27
38
|
U extends ProcessorOptions<T> = ProcessorOptions<T>,
|
|
@@ -105,6 +105,11 @@ describe('constraintsForOptions', () => {
|
|
|
105
105
|
expect(videoOpts.frameRate).toEqual(VideoPresets.h720.resolution.frameRate);
|
|
106
106
|
expect(videoOpts.aspectRatio).toEqual(VideoPresets.h720.resolution.aspectRatio);
|
|
107
107
|
});
|
|
108
|
+
|
|
109
|
+
it('leaves an explicitly set exact deviceId untouched', () => {
|
|
110
|
+
const constraints = constraintsForOptions({ audio: { deviceId: { exact: 'default' } } });
|
|
111
|
+
expect((constraints.audio as MediaTrackConstraints).deviceId).toEqual({ exact: 'default' });
|
|
112
|
+
});
|
|
108
113
|
});
|
|
109
114
|
|
|
110
115
|
describe('diffAttributes', () => {
|
package/src/room/track/utils.ts
CHANGED
|
@@ -108,22 +108,46 @@ export function constraintsForOptions(options: CreateLocalTracksOptions): MediaS
|
|
|
108
108
|
*/
|
|
109
109
|
export async function detectSilence(track: AudioTrack, timeOffset = 200): Promise<boolean> {
|
|
110
110
|
const ctx = getNewAudioContext();
|
|
111
|
-
if (ctx) {
|
|
112
|
-
|
|
111
|
+
if (!ctx) {
|
|
112
|
+
return false;
|
|
113
|
+
}
|
|
114
|
+
let source: MediaStreamAudioSourceNode | undefined;
|
|
115
|
+
let analyser: AnalyserNode | undefined;
|
|
116
|
+
try {
|
|
117
|
+
analyser = ctx.createAnalyser();
|
|
113
118
|
analyser.fftSize = 2048;
|
|
114
119
|
|
|
115
120
|
const bufferLength = analyser.frequencyBinCount;
|
|
116
121
|
const dataArray = new Uint8Array(bufferLength);
|
|
117
|
-
|
|
122
|
+
source = ctx.createMediaStreamSource(new MediaStream([track.mediaStreamTrack]));
|
|
118
123
|
|
|
119
124
|
source.connect(analyser);
|
|
125
|
+
// read through a function so that `state` isn't narrowed to its value from before `resume`
|
|
126
|
+
const isRunning = () => ctx.state === 'running';
|
|
127
|
+
if (!isRunning()) {
|
|
128
|
+
try {
|
|
129
|
+
// `resume` can stay pending indefinitely on a blocked page rather than rejecting, which
|
|
130
|
+
// would keep the context and its nodes alive past the `finally` below
|
|
131
|
+
await Promise.race([ctx.resume(), sleep(200)]);
|
|
132
|
+
} catch {
|
|
133
|
+
// autoplay policies can keep us from resuming the context
|
|
134
|
+
}
|
|
135
|
+
if (!isRunning()) {
|
|
136
|
+
// a context that isn't running doesn't process any audio, so the analyser would only ever
|
|
137
|
+
// read silence. Report the track as not silent rather than raising a false positive.
|
|
138
|
+
return false;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
120
141
|
await sleep(timeOffset);
|
|
121
142
|
analyser.getByteTimeDomainData(dataArray);
|
|
122
143
|
const someNoise = dataArray.some((sample) => sample !== 128 && sample !== 0);
|
|
123
|
-
ctx.close();
|
|
124
144
|
return !someNoise;
|
|
145
|
+
} finally {
|
|
146
|
+
source?.disconnect();
|
|
147
|
+
analyser?.disconnect();
|
|
148
|
+
// don't let a failing teardown mask the detection result
|
|
149
|
+
await ctx.close().catch(() => {});
|
|
125
150
|
}
|
|
126
|
-
return false;
|
|
127
151
|
}
|
|
128
152
|
|
|
129
153
|
/**
|
package/src/room/utils.test.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
isCompressionStreamSupported,
|
|
14
14
|
isDeflateRawCompressionSupported,
|
|
15
15
|
isIPadOS,
|
|
16
|
+
isPermissionDeniedError,
|
|
16
17
|
isSVCSimulcast,
|
|
17
18
|
isSVCSimulcastSupportedByServer,
|
|
18
19
|
negotiateDependencyDescriptor,
|
|
@@ -491,3 +492,128 @@ describe('isDeflateRawCompressionSupported', () => {
|
|
|
491
492
|
}
|
|
492
493
|
});
|
|
493
494
|
});
|
|
495
|
+
|
|
496
|
+
describe('isPermissionDeniedError', () => {
|
|
497
|
+
function errorNamed(name: string) {
|
|
498
|
+
const e = new Error(name);
|
|
499
|
+
e.name = name;
|
|
500
|
+
return e;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
it('matches the permission denial names and their legacy aliases', () => {
|
|
504
|
+
expect(isPermissionDeniedError(errorNamed('NotAllowedError'))).toBe(true);
|
|
505
|
+
expect(isPermissionDeniedError(errorNamed('SecurityError'))).toBe(true);
|
|
506
|
+
expect(isPermissionDeniedError(errorNamed('PermissionDeniedError'))).toBe(true);
|
|
507
|
+
expect(isPermissionDeniedError(errorNamed('PermissionDismissedError'))).toBe(true);
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
it('does not match failures that a softer constraint could still satisfy', () => {
|
|
511
|
+
expect(isPermissionDeniedError(errorNamed('OverconstrainedError'))).toBe(false);
|
|
512
|
+
expect(isPermissionDeniedError(errorNamed('ConstraintNotSatisfiedError'))).toBe(false);
|
|
513
|
+
expect(isPermissionDeniedError(errorNamed('NotFoundError'))).toBe(false);
|
|
514
|
+
expect(isPermissionDeniedError(errorNamed('SomeNameNobodyHasShippedYet'))).toBe(false);
|
|
515
|
+
});
|
|
516
|
+
|
|
517
|
+
it('does not match a plain error, undefined or null', () => {
|
|
518
|
+
expect(isPermissionDeniedError(new Error('boom'))).toBe(false);
|
|
519
|
+
expect(isPermissionDeniedError(undefined)).toBe(false);
|
|
520
|
+
expect(isPermissionDeniedError(null)).toBe(false);
|
|
521
|
+
});
|
|
522
|
+
});
|
|
523
|
+
|
|
524
|
+
// `freshUtils` below calls `vi.resetModules()`, so a dynamic import of `./utils` from a later
|
|
525
|
+
// block would get a different module instance than the static import at the top of this file.
|
|
526
|
+
// Keep this block last.
|
|
527
|
+
describe('empty audio stream track refcounting', () => {
|
|
528
|
+
let closedContexts: number;
|
|
529
|
+
|
|
530
|
+
function makeTrack(): MediaStreamTrack {
|
|
531
|
+
const track = {
|
|
532
|
+
enabled: true,
|
|
533
|
+
stopped: false,
|
|
534
|
+
stop() {
|
|
535
|
+
track.stopped = true;
|
|
536
|
+
},
|
|
537
|
+
clone: () => makeTrack(),
|
|
538
|
+
};
|
|
539
|
+
return track as unknown as MediaStreamTrack;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/** reloads `./utils` so its module level context and refcount start empty */
|
|
543
|
+
async function freshUtils() {
|
|
544
|
+
closedContexts = 0;
|
|
545
|
+
vi.resetModules();
|
|
546
|
+
vi.stubGlobal(
|
|
547
|
+
'AudioContext',
|
|
548
|
+
class {
|
|
549
|
+
createOscillator() {
|
|
550
|
+
return { connect: () => {}, start: () => {} };
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
createGain() {
|
|
554
|
+
return { gain: { setValueAtTime: () => {} }, connect: () => {} };
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
createMediaStreamDestination() {
|
|
558
|
+
return { stream: { getAudioTracks: () => [makeTrack()] } };
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
async close() {
|
|
562
|
+
closedContexts += 1;
|
|
563
|
+
}
|
|
564
|
+
},
|
|
565
|
+
);
|
|
566
|
+
return import('./utils');
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
afterEach(() => {
|
|
570
|
+
vi.unstubAllGlobals();
|
|
571
|
+
});
|
|
572
|
+
|
|
573
|
+
it('closes the shared context once every track handed out is released', async () => {
|
|
574
|
+
const { getEmptyAudioStreamTrack, releaseEmptyAudioStreamTrack } = await freshUtils();
|
|
575
|
+
const first = getEmptyAudioStreamTrack();
|
|
576
|
+
const second = getEmptyAudioStreamTrack();
|
|
577
|
+
|
|
578
|
+
await releaseEmptyAudioStreamTrack(first);
|
|
579
|
+
expect(closedContexts).toBe(0);
|
|
580
|
+
|
|
581
|
+
await releaseEmptyAudioStreamTrack(second);
|
|
582
|
+
expect(closedContexts).toBe(1);
|
|
583
|
+
});
|
|
584
|
+
|
|
585
|
+
it('ignores a second release of the same track', async () => {
|
|
586
|
+
const { getEmptyAudioStreamTrack, releaseEmptyAudioStreamTrack } = await freshUtils();
|
|
587
|
+
const first = getEmptyAudioStreamTrack();
|
|
588
|
+
const second = getEmptyAudioStreamTrack();
|
|
589
|
+
|
|
590
|
+
await releaseEmptyAudioStreamTrack(first);
|
|
591
|
+
await releaseEmptyAudioStreamTrack(first);
|
|
592
|
+
expect(closedContexts).toBe(0);
|
|
593
|
+
|
|
594
|
+
await releaseEmptyAudioStreamTrack(second);
|
|
595
|
+
expect(closedContexts).toBe(1);
|
|
596
|
+
});
|
|
597
|
+
|
|
598
|
+
it('leaves a track it never handed out alone', async () => {
|
|
599
|
+
const { getEmptyAudioStreamTrack, releaseEmptyAudioStreamTrack } = await freshUtils();
|
|
600
|
+
const owned = getEmptyAudioStreamTrack();
|
|
601
|
+
const foreign = makeTrack();
|
|
602
|
+
|
|
603
|
+
await releaseEmptyAudioStreamTrack(foreign);
|
|
604
|
+
expect(closedContexts).toBe(0);
|
|
605
|
+
expect((foreign as unknown as { stopped: boolean }).stopped).toBe(false);
|
|
606
|
+
|
|
607
|
+
await releaseEmptyAudioStreamTrack(owned);
|
|
608
|
+
expect(closedContexts).toBe(1);
|
|
609
|
+
});
|
|
610
|
+
|
|
611
|
+
it('builds a fresh context for an acquire that arrives after the last release', async () => {
|
|
612
|
+
const { getEmptyAudioStreamTrack, releaseEmptyAudioStreamTrack } = await freshUtils();
|
|
613
|
+
await releaseEmptyAudioStreamTrack(getEmptyAudioStreamTrack());
|
|
614
|
+
expect(closedContexts).toBe(1);
|
|
615
|
+
|
|
616
|
+
await releaseEmptyAudioStreamTrack(getEmptyAudioStreamTrack());
|
|
617
|
+
expect(closedContexts).toBe(2);
|
|
618
|
+
});
|
|
619
|
+
});
|
package/src/room/utils.ts
CHANGED
|
@@ -559,6 +559,17 @@ export function createDummyVideoStreamTrack(
|
|
|
559
559
|
|
|
560
560
|
let emptyAudioStreamTrack: MediaStreamTrack | undefined;
|
|
561
561
|
|
|
562
|
+
let emptyAudioStreamContext: AudioContext | undefined;
|
|
563
|
+
|
|
564
|
+
/** tracks handed out by `getEmptyAudioStreamTrack` that haven't been released yet */
|
|
565
|
+
const outstandingEmptyAudioStreamTracks = new Set<MediaStreamTrack>();
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* Returns a silent audio track, cloned off a shared `AudioContext` created on first use. Hand the
|
|
569
|
+
* clone back to `releaseEmptyAudioStreamTrack` rather than stopping it, so the context can close.
|
|
570
|
+
* A clone that is only stopped stays counted as outstanding, which holds the shared context open
|
|
571
|
+
* for the lifetime of the page.
|
|
572
|
+
*/
|
|
562
573
|
export function getEmptyAudioStreamTrack() {
|
|
563
574
|
if (!emptyAudioStreamTrack) {
|
|
564
575
|
// implementation adapted from https://blog.mozilla.org/webrtc/warm-up-with-replacetrack/
|
|
@@ -572,31 +583,37 @@ export function getEmptyAudioStreamTrack() {
|
|
|
572
583
|
oscillator.start();
|
|
573
584
|
[emptyAudioStreamTrack] = dst.stream.getAudioTracks();
|
|
574
585
|
if (!emptyAudioStreamTrack) {
|
|
586
|
+
// don't hold on to a context we can't get a track out of
|
|
587
|
+
ctx.close().catch(() => {});
|
|
575
588
|
throw Error('Could not get empty media stream audio track');
|
|
576
589
|
}
|
|
577
590
|
emptyAudioStreamTrack.enabled = false;
|
|
591
|
+
emptyAudioStreamContext = ctx;
|
|
592
|
+
}
|
|
593
|
+
const track = emptyAudioStreamTrack.clone();
|
|
594
|
+
outstandingEmptyAudioStreamTracks.add(track);
|
|
595
|
+
return track;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
/**
|
|
599
|
+
* Stops a track obtained from `getEmptyAudioStreamTrack` and closes the shared `AudioContext`
|
|
600
|
+
* backing it once every track it handed out has been released. Does nothing for a track this
|
|
601
|
+
* module never handed out, or one that has been released already.
|
|
602
|
+
*/
|
|
603
|
+
export async function releaseEmptyAudioStreamTrack(track: MediaStreamTrack) {
|
|
604
|
+
if (!outstandingEmptyAudioStreamTracks.delete(track)) {
|
|
605
|
+
// the track wasn't obtained from `getEmptyAudioStreamTrack`, or has been released before
|
|
606
|
+
return;
|
|
578
607
|
}
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
export function getStereoAudioStreamTrack() {
|
|
583
|
-
const ctx = new AudioContext();
|
|
584
|
-
const oscLeft = ctx.createOscillator();
|
|
585
|
-
const oscRight = ctx.createOscillator();
|
|
586
|
-
oscLeft.frequency.value = 440;
|
|
587
|
-
oscRight.frequency.value = 220;
|
|
588
|
-
const merger = ctx.createChannelMerger(2);
|
|
589
|
-
oscLeft.connect(merger, 0, 0); // left channel
|
|
590
|
-
oscRight.connect(merger, 0, 1); // right channel
|
|
591
|
-
const dst = ctx.createMediaStreamDestination();
|
|
592
|
-
merger.connect(dst);
|
|
593
|
-
oscLeft.start();
|
|
594
|
-
oscRight.start();
|
|
595
|
-
const [stereoTrack] = dst.stream.getAudioTracks();
|
|
596
|
-
if (!stereoTrack) {
|
|
597
|
-
throw Error('Could not get stereo media stream audio track');
|
|
608
|
+
track.stop();
|
|
609
|
+
if (outstandingEmptyAudioStreamTracks.size > 0) {
|
|
610
|
+
return;
|
|
598
611
|
}
|
|
599
|
-
|
|
612
|
+
const context = emptyAudioStreamContext;
|
|
613
|
+
emptyAudioStreamTrack?.stop();
|
|
614
|
+
emptyAudioStreamTrack = undefined;
|
|
615
|
+
emptyAudioStreamContext = undefined;
|
|
616
|
+
await context?.close();
|
|
600
617
|
}
|
|
601
618
|
|
|
602
619
|
/** An object that represents a serialized version of a `new Promise((resolve, reject) => {})`
|
|
@@ -708,11 +725,20 @@ export function createAudioAnalyser(
|
|
|
708
725
|
return volume;
|
|
709
726
|
};
|
|
710
727
|
|
|
728
|
+
let isCleanedUp = false;
|
|
729
|
+
|
|
711
730
|
const cleanup = async () => {
|
|
712
|
-
|
|
731
|
+
if (isCleanedUp) {
|
|
732
|
+
// closing an already closed context would reject, make repeated cleanups a no-op instead
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
isCleanedUp = true;
|
|
736
|
+
mediaStreamSource.disconnect();
|
|
737
|
+
analyser.disconnect();
|
|
713
738
|
if (opts.cloneTrack) {
|
|
714
739
|
streamTrack.stop();
|
|
715
740
|
}
|
|
741
|
+
await audioContext.close();
|
|
716
742
|
};
|
|
717
743
|
|
|
718
744
|
return { calculateVolume, analyser, cleanup };
|
|
@@ -751,6 +777,43 @@ export function unwrapConstraint(constraint: ConstrainDOMString | ConstrainULong
|
|
|
751
777
|
throw Error('could not unwrap constraint');
|
|
752
778
|
}
|
|
753
779
|
|
|
780
|
+
/**
|
|
781
|
+
* The getUserMedia rejections that no softer constraint can recover from, because the user or the
|
|
782
|
+
* document refused capture outright. Retrying one of these would put a second prompt in front of
|
|
783
|
+
* someone who already said no. The last two names are legacy spellings still reported by older
|
|
784
|
+
* engines. Matched by `name` rather than with `instanceof`, because these are not reliably distinct
|
|
785
|
+
* constructors across environments. happy-dom, which the tests run under, aliases them to `Event`.
|
|
786
|
+
*/
|
|
787
|
+
export function isPermissionDeniedError(e: unknown): boolean {
|
|
788
|
+
return (
|
|
789
|
+
typeof e === 'object' &&
|
|
790
|
+
e !== null &&
|
|
791
|
+
'name' in e &&
|
|
792
|
+
(e.name === 'NotAllowedError' ||
|
|
793
|
+
e.name === 'SecurityError' ||
|
|
794
|
+
e.name === 'PermissionDeniedError' ||
|
|
795
|
+
e.name === 'PermissionDismissedError')
|
|
796
|
+
);
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
const deviceAcquisitionFailures = new WeakSet();
|
|
800
|
+
|
|
801
|
+
/** Records that `e` is the rejection getUserMedia itself produced. */
|
|
802
|
+
export function markDeviceAcquisitionFailure(e: unknown) {
|
|
803
|
+
if (typeof e === 'object' && e !== null) {
|
|
804
|
+
deviceAcquisitionFailures.add(e);
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
/**
|
|
809
|
+
* Whether a track restart failed at getUserMedia rather than in the work that adopts the new track.
|
|
810
|
+
* Recorded on the side rather than by wrapping, because callers of `restartTrack` match on the
|
|
811
|
+
* rejection's own name.
|
|
812
|
+
*/
|
|
813
|
+
export function isDeviceAcquisitionFailure(e: unknown): boolean {
|
|
814
|
+
return typeof e === 'object' && e !== null && deviceAcquisitionFailures.has(e);
|
|
815
|
+
}
|
|
816
|
+
|
|
754
817
|
export function toWebsocketUrl(url: string): string {
|
|
755
818
|
if (url.startsWith('http')) {
|
|
756
819
|
return url.replace(/^(http)/, 'ws');
|