@stream-io/video-client 1.44.3 → 1.44.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/index.browser.es.js +134 -45
- package/dist/index.browser.es.js.map +1 -1
- package/dist/index.cjs.js +134 -45
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.es.js +134 -45
- package/dist/index.es.js.map +1 -1
- package/dist/src/devices/BrowserPermission.d.ts +2 -0
- package/dist/src/devices/CameraManagerState.d.ts +2 -1
- package/dist/src/devices/MicrophoneManagerState.d.ts +2 -1
- package/dist/src/devices/devices.d.ts +2 -2
- package/dist/src/helpers/AudioBindingsWatchdog.d.ts +37 -0
- package/dist/src/helpers/DynascaleManager.d.ts +3 -1
- package/package.json +2 -2
- package/src/devices/BrowserPermission.ts +5 -0
- package/src/devices/CameraManager.ts +6 -1
- package/src/devices/CameraManagerState.ts +3 -2
- package/src/devices/MicrophoneManager.ts +1 -2
- package/src/devices/MicrophoneManagerState.ts +3 -2
- package/src/devices/devices.ts +26 -34
- package/src/helpers/AudioBindingsWatchdog.ts +118 -0
- package/src/helpers/DynascaleManager.ts +22 -24
- package/src/helpers/__tests__/AudioBindingsWatchdog.test.ts +325 -0
- package/src/helpers/__tests__/DynascaleManager.test.ts +64 -0
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { Tracer } from '../stats';
|
|
1
2
|
interface BrowserPermissionConfig {
|
|
2
3
|
constraints: DisplayMediaStreamOptions;
|
|
3
4
|
queryName: PermissionName;
|
|
5
|
+
tracer: Tracer | undefined;
|
|
4
6
|
}
|
|
5
7
|
export type BrowserPermissionState = PermissionState | 'prompting';
|
|
6
8
|
export declare class BrowserPermission {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DeviceManagerState } from './DeviceManagerState';
|
|
2
|
+
import { Tracer } from '../stats';
|
|
2
3
|
export type CameraDirection = 'front' | 'back' | undefined;
|
|
3
4
|
export declare class CameraManagerState extends DeviceManagerState {
|
|
4
5
|
private directionSubject;
|
|
@@ -8,7 +9,7 @@ export declare class CameraManagerState extends DeviceManagerState {
|
|
|
8
9
|
* back - means the camera facing the environment
|
|
9
10
|
*/
|
|
10
11
|
direction$: import("rxjs").Observable<CameraDirection>;
|
|
11
|
-
constructor();
|
|
12
|
+
constructor(tracer: Tracer | undefined);
|
|
12
13
|
/**
|
|
13
14
|
* The preferred camera direction
|
|
14
15
|
* front - means the camera facing the user
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { TrackDisableMode } from './DeviceManagerState';
|
|
2
2
|
import { AudioDeviceManagerState } from './AudioDeviceManagerState';
|
|
3
|
+
import { Tracer } from '../stats';
|
|
3
4
|
export declare class MicrophoneManagerState extends AudioDeviceManagerState<MediaTrackConstraints> {
|
|
4
5
|
private speakingWhileMutedSubject;
|
|
5
6
|
/**
|
|
6
7
|
* An Observable that emits `true` if the user's microphone is muted, but they're speaking.
|
|
7
8
|
*/
|
|
8
9
|
speakingWhileMuted$: import("rxjs").Observable<boolean>;
|
|
9
|
-
constructor(disableMode: TrackDisableMode);
|
|
10
|
+
constructor(disableMode: TrackDisableMode, tracer: Tracer | undefined);
|
|
10
11
|
/**
|
|
11
12
|
* `true` if the user's microphone is muted but they're speaking.
|
|
12
13
|
*/
|
|
@@ -9,12 +9,12 @@ export declare const checkIfAudioOutputChangeSupported: () => boolean;
|
|
|
9
9
|
* Keeps track of the browser permission to use microphone. This permission also
|
|
10
10
|
* affects an ability to enumerate audio devices.
|
|
11
11
|
*/
|
|
12
|
-
export declare const getAudioBrowserPermission: (...args:
|
|
12
|
+
export declare const getAudioBrowserPermission: (...args: (Tracer | undefined)[]) => BrowserPermission;
|
|
13
13
|
/**
|
|
14
14
|
* Keeps track of the browser permission to use camera. This permission also
|
|
15
15
|
* affects an ability to enumerate video devices.
|
|
16
16
|
*/
|
|
17
|
-
export declare const getVideoBrowserPermission: (...args:
|
|
17
|
+
export declare const getVideoBrowserPermission: (...args: (Tracer | undefined)[]) => BrowserPermission;
|
|
18
18
|
/**
|
|
19
19
|
* Prompts the user for a permission to use audio devices (if not already granted
|
|
20
20
|
* and was not prompted before) and lists the available 'audioinput' devices,
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { AudioTrackType } from '../types';
|
|
2
|
+
import { CallState } from '../store';
|
|
3
|
+
import { Tracer } from '../stats';
|
|
4
|
+
/**
|
|
5
|
+
* Tracks audio element bindings and periodically warns about
|
|
6
|
+
* remote participants whose audio streams have no bound element.
|
|
7
|
+
*/
|
|
8
|
+
export declare class AudioBindingsWatchdog {
|
|
9
|
+
private state;
|
|
10
|
+
private tracer;
|
|
11
|
+
private bindings;
|
|
12
|
+
private enabled;
|
|
13
|
+
private watchdogInterval?;
|
|
14
|
+
private readonly unsubscribeCallingState;
|
|
15
|
+
private logger;
|
|
16
|
+
constructor(state: CallState, tracer: Tracer);
|
|
17
|
+
/**
|
|
18
|
+
* Registers an audio element binding for the given session and track type.
|
|
19
|
+
* Warns if a different element is already bound to the same key.
|
|
20
|
+
*/
|
|
21
|
+
register: (audioElement: HTMLAudioElement, sessionId: string, trackType: AudioTrackType) => void;
|
|
22
|
+
/**
|
|
23
|
+
* Removes the audio element binding for the given session and track type.
|
|
24
|
+
*/
|
|
25
|
+
unregister: (sessionId: string, trackType: AudioTrackType) => void;
|
|
26
|
+
/**
|
|
27
|
+
* Enables or disables the watchdog.
|
|
28
|
+
* When disabled, the periodic check stops but bindings are still tracked.
|
|
29
|
+
*/
|
|
30
|
+
setEnabled: (enabled: boolean) => void;
|
|
31
|
+
/**
|
|
32
|
+
* Stops the watchdog and unsubscribes from callingState changes.
|
|
33
|
+
*/
|
|
34
|
+
dispose: () => void;
|
|
35
|
+
private start;
|
|
36
|
+
private stop;
|
|
37
|
+
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { AudioTrackType, DebounceType, VideoTrackType } from '../types';
|
|
2
2
|
import { VideoDimension } from '../gen/video/sfu/models/models';
|
|
3
3
|
import { ViewportTracker } from './ViewportTracker';
|
|
4
|
+
import { AudioBindingsWatchdog } from './AudioBindingsWatchdog';
|
|
4
5
|
import type { TrackSubscriptionDetails } from '../gen/video/sfu/signal_rpc/signal';
|
|
5
|
-
import
|
|
6
|
+
import { CallState } from '../store';
|
|
6
7
|
import type { StreamSfuClient } from '../StreamSfuClient';
|
|
7
8
|
import { SpeakerManager } from '../devices';
|
|
8
9
|
import { Tracer } from '../stats';
|
|
@@ -40,6 +41,7 @@ export declare class DynascaleManager {
|
|
|
40
41
|
private audioContext;
|
|
41
42
|
private sfuClient;
|
|
42
43
|
private pendingSubscriptionsUpdate;
|
|
44
|
+
readonly audioBindingsWatchdog: AudioBindingsWatchdog | undefined;
|
|
43
45
|
private videoTrackSubscriptionOverridesSubject;
|
|
44
46
|
videoTrackSubscriptionOverrides$: import("rxjs").Observable<VideoTrackSubscriptionOverrides>;
|
|
45
47
|
incomingVideoSettings$: import("rxjs").Observable<{
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@stream-io/video-client",
|
|
3
|
-
"version": "1.44.
|
|
3
|
+
"version": "1.44.5",
|
|
4
4
|
"main": "dist/index.cjs.js",
|
|
5
5
|
"module": "dist/index.es.js",
|
|
6
6
|
"browser": "dist/index.browser.es.js",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"@openapitools/openapi-generator-cli": "^2.25.0",
|
|
47
47
|
"@rollup/plugin-replace": "^6.0.2",
|
|
48
48
|
"@rollup/plugin-typescript": "^12.1.4",
|
|
49
|
-
"@stream-io/audio-filters-web": "^0.7.
|
|
49
|
+
"@stream-io/audio-filters-web": "^0.7.3",
|
|
50
50
|
"@stream-io/node-sdk": "^0.7.28",
|
|
51
51
|
"@total-typescript/shoehorn": "^0.1.2",
|
|
52
52
|
"@types/sdp-transform": "^2.15.0",
|
|
@@ -3,10 +3,12 @@ import { isReactNative } from '../helpers/platforms';
|
|
|
3
3
|
import { disposeOfMediaStream } from './utils';
|
|
4
4
|
import { withoutConcurrency } from '../helpers/concurrency';
|
|
5
5
|
import { videoLoggerSystem } from '../logger';
|
|
6
|
+
import { Tracer } from '../stats';
|
|
6
7
|
|
|
7
8
|
interface BrowserPermissionConfig {
|
|
8
9
|
constraints: DisplayMediaStreamOptions;
|
|
9
10
|
queryName: PermissionName;
|
|
11
|
+
tracer: Tracer | undefined;
|
|
10
12
|
}
|
|
11
13
|
|
|
12
14
|
export type BrowserPermissionState = PermissionState | 'prompting';
|
|
@@ -162,6 +164,9 @@ export class BrowserPermission {
|
|
|
162
164
|
|
|
163
165
|
private setState(state: BrowserPermissionState) {
|
|
164
166
|
if (this.state !== state) {
|
|
167
|
+
const { tracer, queryName } = this.permission;
|
|
168
|
+
const traceKey = `navigator.mediaDevices.${queryName}.permission`;
|
|
169
|
+
tracer?.trace(traceKey, { previous: this.state, state });
|
|
165
170
|
this.state = state;
|
|
166
171
|
this.listeners.forEach((listener) => listener(state));
|
|
167
172
|
}
|
|
@@ -25,7 +25,12 @@ export class CameraManager extends DeviceManager<CameraManagerState> {
|
|
|
25
25
|
call: Call,
|
|
26
26
|
devicePersistence: Required<DevicePersistenceOptions>,
|
|
27
27
|
) {
|
|
28
|
-
super(
|
|
28
|
+
super(
|
|
29
|
+
call,
|
|
30
|
+
new CameraManagerState(call.tracer),
|
|
31
|
+
TrackType.VIDEO,
|
|
32
|
+
devicePersistence,
|
|
33
|
+
);
|
|
29
34
|
}
|
|
30
35
|
|
|
31
36
|
private isDirectionSupportedByDevice() {
|
|
@@ -3,6 +3,7 @@ import { DeviceManagerState } from './DeviceManagerState';
|
|
|
3
3
|
import { isReactNative } from '../helpers/platforms';
|
|
4
4
|
import { getVideoBrowserPermission } from './devices';
|
|
5
5
|
import { RxUtils } from '../store';
|
|
6
|
+
import { Tracer } from '../stats';
|
|
6
7
|
|
|
7
8
|
export type CameraDirection = 'front' | 'back' | undefined;
|
|
8
9
|
|
|
@@ -18,8 +19,8 @@ export class CameraManagerState extends DeviceManagerState {
|
|
|
18
19
|
.asObservable()
|
|
19
20
|
.pipe(distinctUntilChanged());
|
|
20
21
|
|
|
21
|
-
constructor() {
|
|
22
|
-
super('stop-tracks', getVideoBrowserPermission());
|
|
22
|
+
constructor(tracer: Tracer | undefined) {
|
|
23
|
+
super('stop-tracks', getVideoBrowserPermission(tracer));
|
|
23
24
|
}
|
|
24
25
|
|
|
25
26
|
/**
|
|
@@ -51,7 +51,7 @@ export class MicrophoneManager extends AudioDeviceManager<MicrophoneManagerState
|
|
|
51
51
|
) {
|
|
52
52
|
super(
|
|
53
53
|
call,
|
|
54
|
-
new MicrophoneManagerState(disableMode),
|
|
54
|
+
new MicrophoneManagerState(disableMode, call.tracer),
|
|
55
55
|
TrackType.AUDIO,
|
|
56
56
|
devicePersistence,
|
|
57
57
|
);
|
|
@@ -169,7 +169,6 @@ export class MicrophoneManager extends AudioDeviceManager<MicrophoneManagerState
|
|
|
169
169
|
deviceId,
|
|
170
170
|
label,
|
|
171
171
|
};
|
|
172
|
-
console.log(event);
|
|
173
172
|
this.call.tracer.trace('mic.capture_report', event);
|
|
174
173
|
this.call.streamClient.dispatchEvent(event);
|
|
175
174
|
},
|
|
@@ -4,6 +4,7 @@ import { TrackDisableMode } from './DeviceManagerState';
|
|
|
4
4
|
import { AudioDeviceManagerState } from './AudioDeviceManagerState';
|
|
5
5
|
import { getAudioBrowserPermission, resolveDeviceId } from './devices';
|
|
6
6
|
import { AudioBitrateProfile } from '../gen/video/sfu/models/models';
|
|
7
|
+
import { Tracer } from '../stats';
|
|
7
8
|
|
|
8
9
|
export class MicrophoneManagerState extends AudioDeviceManagerState<MediaTrackConstraints> {
|
|
9
10
|
private speakingWhileMutedSubject = new BehaviorSubject<boolean>(false);
|
|
@@ -15,10 +16,10 @@ export class MicrophoneManagerState extends AudioDeviceManagerState<MediaTrackCo
|
|
|
15
16
|
.asObservable()
|
|
16
17
|
.pipe(distinctUntilChanged());
|
|
17
18
|
|
|
18
|
-
constructor(disableMode: TrackDisableMode) {
|
|
19
|
+
constructor(disableMode: TrackDisableMode, tracer: Tracer | undefined) {
|
|
19
20
|
super(
|
|
20
21
|
disableMode,
|
|
21
|
-
getAudioBrowserPermission(),
|
|
22
|
+
getAudioBrowserPermission(tracer),
|
|
22
23
|
AudioBitrateProfile.VOICE_STANDARD_UNSPECIFIED,
|
|
23
24
|
);
|
|
24
25
|
}
|
package/src/devices/devices.ts
CHANGED
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
} from 'rxjs';
|
|
12
12
|
import { BrowserPermission } from './BrowserPermission';
|
|
13
13
|
import { lazy } from '../helpers/lazy';
|
|
14
|
-
import { isFirefox
|
|
14
|
+
import { isFirefox } from '../helpers/browsers';
|
|
15
15
|
import { dumpStream, Tracer } from '../stats';
|
|
16
16
|
import { getCurrentValue } from '../store/rxUtils';
|
|
17
17
|
import { videoLoggerSystem } from '../logger';
|
|
@@ -61,8 +61,6 @@ const getDevices = (
|
|
|
61
61
|
*/
|
|
62
62
|
export const checkIfAudioOutputChangeSupported = () => {
|
|
63
63
|
if (typeof document === 'undefined') return false;
|
|
64
|
-
// Safari uses WebAudio API for playing audio, so we check the AudioContext prototype
|
|
65
|
-
if (isSafari()) return 'setSinkId' in AudioContext.prototype;
|
|
66
64
|
const element = document.createElement('audio');
|
|
67
65
|
return 'setSinkId' in element;
|
|
68
66
|
};
|
|
@@ -93,10 +91,11 @@ const videoDeviceConstraints = {
|
|
|
93
91
|
* affects an ability to enumerate audio devices.
|
|
94
92
|
*/
|
|
95
93
|
export const getAudioBrowserPermission = lazy(
|
|
96
|
-
() =>
|
|
94
|
+
(tracer: Tracer | undefined) =>
|
|
97
95
|
new BrowserPermission({
|
|
98
96
|
constraints: audioDeviceConstraints,
|
|
99
97
|
queryName: 'microphone' as PermissionName,
|
|
98
|
+
tracer,
|
|
100
99
|
}),
|
|
101
100
|
);
|
|
102
101
|
|
|
@@ -105,10 +104,11 @@ export const getAudioBrowserPermission = lazy(
|
|
|
105
104
|
* affects an ability to enumerate video devices.
|
|
106
105
|
*/
|
|
107
106
|
export const getVideoBrowserPermission = lazy(
|
|
108
|
-
() =>
|
|
107
|
+
(tracer: Tracer | undefined) =>
|
|
109
108
|
new BrowserPermission({
|
|
110
109
|
constraints: videoDeviceConstraints,
|
|
111
110
|
queryName: 'camera' as PermissionName,
|
|
111
|
+
tracer,
|
|
112
112
|
}),
|
|
113
113
|
);
|
|
114
114
|
|
|
@@ -132,11 +132,11 @@ const getDeviceChangeObserver = lazy((tracer: Tracer | undefined) => {
|
|
|
132
132
|
export const getAudioDevices = lazy((tracer?: Tracer) => {
|
|
133
133
|
return merge(
|
|
134
134
|
getDeviceChangeObserver(tracer),
|
|
135
|
-
getAudioBrowserPermission().asObservable(),
|
|
135
|
+
getAudioBrowserPermission(tracer).asObservable(),
|
|
136
136
|
).pipe(
|
|
137
137
|
startWith([]),
|
|
138
138
|
concatMap(() =>
|
|
139
|
-
getDevices(getAudioBrowserPermission(), 'audioinput', tracer),
|
|
139
|
+
getDevices(getAudioBrowserPermission(tracer), 'audioinput', tracer),
|
|
140
140
|
),
|
|
141
141
|
shareReplay(1),
|
|
142
142
|
);
|
|
@@ -151,11 +151,11 @@ export const getAudioDevices = lazy((tracer?: Tracer) => {
|
|
|
151
151
|
export const getVideoDevices = lazy((tracer?: Tracer) => {
|
|
152
152
|
return merge(
|
|
153
153
|
getDeviceChangeObserver(tracer),
|
|
154
|
-
getVideoBrowserPermission().asObservable(),
|
|
154
|
+
getVideoBrowserPermission(tracer).asObservable(),
|
|
155
155
|
).pipe(
|
|
156
156
|
startWith([]),
|
|
157
157
|
concatMap(() =>
|
|
158
|
-
getDevices(getVideoBrowserPermission(), 'videoinput', tracer),
|
|
158
|
+
getDevices(getVideoBrowserPermission(tracer), 'videoinput', tracer),
|
|
159
159
|
),
|
|
160
160
|
shareReplay(1),
|
|
161
161
|
);
|
|
@@ -170,11 +170,11 @@ export const getVideoDevices = lazy((tracer?: Tracer) => {
|
|
|
170
170
|
export const getAudioOutputDevices = lazy((tracer?: Tracer) => {
|
|
171
171
|
return merge(
|
|
172
172
|
getDeviceChangeObserver(tracer),
|
|
173
|
-
getAudioBrowserPermission().asObservable(),
|
|
173
|
+
getAudioBrowserPermission(tracer).asObservable(),
|
|
174
174
|
).pipe(
|
|
175
175
|
startWith([]),
|
|
176
176
|
concatMap(() =>
|
|
177
|
-
getDevices(getAudioBrowserPermission(), 'audiooutput', tracer),
|
|
177
|
+
getDevices(getAudioBrowserPermission(tracer), 'audiooutput', tracer),
|
|
178
178
|
),
|
|
179
179
|
shareReplay(1),
|
|
180
180
|
);
|
|
@@ -259,28 +259,24 @@ export const getAudioStream = async (
|
|
|
259
259
|
};
|
|
260
260
|
|
|
261
261
|
try {
|
|
262
|
-
await getAudioBrowserPermission().prompt({
|
|
262
|
+
await getAudioBrowserPermission(tracer).prompt({
|
|
263
263
|
throwOnNotAllowed: true,
|
|
264
264
|
forcePrompt: true,
|
|
265
265
|
});
|
|
266
266
|
return await getStream(constraints, tracer);
|
|
267
267
|
} catch (error) {
|
|
268
|
+
const logger = videoLoggerSystem.getLogger('devices');
|
|
268
269
|
if (isNotFoundOrOverconstrainedError(error) && trackConstraints?.deviceId) {
|
|
269
270
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
270
271
|
const { deviceId, ...relaxedConstraints } = trackConstraints;
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
{ error, constraints, relaxedConstraints },
|
|
276
|
-
);
|
|
272
|
+
logger.warn(
|
|
273
|
+
'Failed to get audio stream, will try again with relaxed constraints',
|
|
274
|
+
{ error, constraints, relaxedConstraints },
|
|
275
|
+
);
|
|
277
276
|
return getAudioStream(relaxedConstraints, tracer);
|
|
278
277
|
}
|
|
279
278
|
|
|
280
|
-
|
|
281
|
-
error,
|
|
282
|
-
constraints,
|
|
283
|
-
});
|
|
279
|
+
logger.error('Failed to get audio stream', { error, constraints });
|
|
284
280
|
throw error;
|
|
285
281
|
}
|
|
286
282
|
};
|
|
@@ -304,28 +300,24 @@ export const getVideoStream = async (
|
|
|
304
300
|
},
|
|
305
301
|
};
|
|
306
302
|
try {
|
|
307
|
-
await getVideoBrowserPermission().prompt({
|
|
303
|
+
await getVideoBrowserPermission(tracer).prompt({
|
|
308
304
|
throwOnNotAllowed: true,
|
|
309
305
|
forcePrompt: true,
|
|
310
306
|
});
|
|
311
307
|
return await getStream(constraints, tracer);
|
|
312
308
|
} catch (error) {
|
|
309
|
+
const logger = videoLoggerSystem.getLogger('devices');
|
|
313
310
|
if (isNotFoundOrOverconstrainedError(error) && trackConstraints?.deviceId) {
|
|
314
311
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
315
312
|
const { deviceId, ...relaxedConstraints } = trackConstraints;
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
);
|
|
322
|
-
return getVideoStream(relaxedConstraints);
|
|
313
|
+
logger.warn(
|
|
314
|
+
'Failed to get video stream, will try again with relaxed constraints',
|
|
315
|
+
{ error, constraints, relaxedConstraints },
|
|
316
|
+
);
|
|
317
|
+
return getVideoStream(relaxedConstraints, tracer);
|
|
323
318
|
}
|
|
324
319
|
|
|
325
|
-
|
|
326
|
-
error,
|
|
327
|
-
constraints,
|
|
328
|
-
});
|
|
320
|
+
logger.error('Failed to get video stream', { error, constraints });
|
|
329
321
|
throw error;
|
|
330
322
|
}
|
|
331
323
|
};
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import type { AudioTrackType } from '../types';
|
|
2
|
+
import { CallingState, CallState } from '../store';
|
|
3
|
+
import { createSubscription } from '../store/rxUtils';
|
|
4
|
+
import { videoLoggerSystem } from '../logger';
|
|
5
|
+
import { Tracer } from '../stats';
|
|
6
|
+
|
|
7
|
+
const toBindingKey = (
|
|
8
|
+
sessionId: string,
|
|
9
|
+
trackType: AudioTrackType = 'audioTrack',
|
|
10
|
+
) => `${sessionId}/${trackType}`;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Tracks audio element bindings and periodically warns about
|
|
14
|
+
* remote participants whose audio streams have no bound element.
|
|
15
|
+
*/
|
|
16
|
+
export class AudioBindingsWatchdog {
|
|
17
|
+
private bindings = new Map<string, HTMLAudioElement>();
|
|
18
|
+
private enabled = true;
|
|
19
|
+
private watchdogInterval?: NodeJS.Timeout;
|
|
20
|
+
private readonly unsubscribeCallingState: () => void;
|
|
21
|
+
private logger = videoLoggerSystem.getLogger('AudioBindingsWatchdog');
|
|
22
|
+
|
|
23
|
+
constructor(
|
|
24
|
+
private state: CallState,
|
|
25
|
+
private tracer: Tracer,
|
|
26
|
+
) {
|
|
27
|
+
this.unsubscribeCallingState = createSubscription(
|
|
28
|
+
state.callingState$,
|
|
29
|
+
(callingState) => {
|
|
30
|
+
if (!this.enabled) return;
|
|
31
|
+
if (callingState !== CallingState.JOINED) {
|
|
32
|
+
this.stop();
|
|
33
|
+
} else {
|
|
34
|
+
this.start();
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Registers an audio element binding for the given session and track type.
|
|
42
|
+
* Warns if a different element is already bound to the same key.
|
|
43
|
+
*/
|
|
44
|
+
register = (
|
|
45
|
+
audioElement: HTMLAudioElement,
|
|
46
|
+
sessionId: string,
|
|
47
|
+
trackType: AudioTrackType,
|
|
48
|
+
) => {
|
|
49
|
+
const key = toBindingKey(sessionId, trackType);
|
|
50
|
+
const existing = this.bindings.get(key);
|
|
51
|
+
if (existing && existing !== audioElement) {
|
|
52
|
+
this.logger.warn(
|
|
53
|
+
`Audio element already bound to ${sessionId} and ${trackType}`,
|
|
54
|
+
);
|
|
55
|
+
this.tracer.trace('audioBinding.alreadyBoundWarning', trackType);
|
|
56
|
+
}
|
|
57
|
+
this.bindings.set(key, audioElement);
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Removes the audio element binding for the given session and track type.
|
|
62
|
+
*/
|
|
63
|
+
unregister = (sessionId: string, trackType: AudioTrackType) => {
|
|
64
|
+
this.bindings.delete(toBindingKey(sessionId, trackType));
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Enables or disables the watchdog.
|
|
69
|
+
* When disabled, the periodic check stops but bindings are still tracked.
|
|
70
|
+
*/
|
|
71
|
+
setEnabled = (enabled: boolean) => {
|
|
72
|
+
this.enabled = enabled;
|
|
73
|
+
if (enabled) {
|
|
74
|
+
this.start();
|
|
75
|
+
} else {
|
|
76
|
+
this.stop();
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Stops the watchdog and unsubscribes from callingState changes.
|
|
82
|
+
*/
|
|
83
|
+
dispose = () => {
|
|
84
|
+
this.stop();
|
|
85
|
+
this.unsubscribeCallingState();
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
private start = () => {
|
|
89
|
+
clearInterval(this.watchdogInterval);
|
|
90
|
+
this.watchdogInterval = setInterval(() => {
|
|
91
|
+
const danglingUserIds: string[] = [];
|
|
92
|
+
for (const p of this.state.participants) {
|
|
93
|
+
if (p.isLocalParticipant) continue;
|
|
94
|
+
const { audioStream, screenShareAudioStream, sessionId, userId } = p;
|
|
95
|
+
if (audioStream && !this.bindings.has(toBindingKey(sessionId))) {
|
|
96
|
+
danglingUserIds.push(userId);
|
|
97
|
+
}
|
|
98
|
+
if (
|
|
99
|
+
screenShareAudioStream &&
|
|
100
|
+
!this.bindings.has(toBindingKey(sessionId, 'screenShareAudioTrack'))
|
|
101
|
+
) {
|
|
102
|
+
danglingUserIds.push(userId);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (danglingUserIds.length > 0) {
|
|
106
|
+
const key = 'audioBinding.danglingWarning';
|
|
107
|
+
this.tracer.traceOnce(key, key, danglingUserIds);
|
|
108
|
+
this.logger.warn(
|
|
109
|
+
`Dangling audio bindings detected. Did you forget to bind the audio element? user_ids: ${danglingUserIds}.`,
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
}, 3000);
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
private stop = () => {
|
|
116
|
+
clearInterval(this.watchdogInterval);
|
|
117
|
+
};
|
|
118
|
+
}
|
|
@@ -15,14 +15,16 @@ import {
|
|
|
15
15
|
takeWhile,
|
|
16
16
|
} from 'rxjs';
|
|
17
17
|
import { ViewportTracker } from './ViewportTracker';
|
|
18
|
+
import { AudioBindingsWatchdog } from './AudioBindingsWatchdog';
|
|
18
19
|
import { isFirefox, isSafari } from './browsers';
|
|
20
|
+
import { isReactNative } from './platforms';
|
|
19
21
|
import {
|
|
20
22
|
hasScreenShare,
|
|
21
23
|
hasScreenShareAudio,
|
|
22
24
|
hasVideo,
|
|
23
25
|
} from './participantUtils';
|
|
24
26
|
import type { TrackSubscriptionDetails } from '../gen/video/sfu/signal_rpc/signal';
|
|
25
|
-
import
|
|
27
|
+
import { CallState } from '../store';
|
|
26
28
|
import type { StreamSfuClient } from '../StreamSfuClient';
|
|
27
29
|
import { SpeakerManager } from '../devices';
|
|
28
30
|
import { getCurrentValue, setCurrentValue } from '../store/rxUtils';
|
|
@@ -71,10 +73,11 @@ export class DynascaleManager {
|
|
|
71
73
|
private callState: CallState;
|
|
72
74
|
private speaker: SpeakerManager;
|
|
73
75
|
private tracer: Tracer;
|
|
74
|
-
private useWebAudio =
|
|
76
|
+
private useWebAudio = false;
|
|
75
77
|
private audioContext: AudioContext | undefined;
|
|
76
78
|
private sfuClient: StreamSfuClient | undefined;
|
|
77
79
|
private pendingSubscriptionsUpdate: NodeJS.Timeout | null = null;
|
|
80
|
+
readonly audioBindingsWatchdog: AudioBindingsWatchdog | undefined;
|
|
78
81
|
|
|
79
82
|
private videoTrackSubscriptionOverridesSubject =
|
|
80
83
|
new BehaviorSubject<VideoTrackSubscriptionOverrides>({});
|
|
@@ -120,6 +123,9 @@ export class DynascaleManager {
|
|
|
120
123
|
this.callState = callState;
|
|
121
124
|
this.speaker = speaker;
|
|
122
125
|
this.tracer = tracer;
|
|
126
|
+
if (!isReactNative()) {
|
|
127
|
+
this.audioBindingsWatchdog = new AudioBindingsWatchdog(callState, tracer);
|
|
128
|
+
}
|
|
123
129
|
}
|
|
124
130
|
|
|
125
131
|
/**
|
|
@@ -129,7 +135,8 @@ export class DynascaleManager {
|
|
|
129
135
|
if (this.pendingSubscriptionsUpdate) {
|
|
130
136
|
clearTimeout(this.pendingSubscriptionsUpdate);
|
|
131
137
|
}
|
|
132
|
-
|
|
138
|
+
this.audioBindingsWatchdog?.dispose();
|
|
139
|
+
const context = this.audioContext;
|
|
133
140
|
if (context && context.state !== 'closed') {
|
|
134
141
|
document.removeEventListener('click', this.resumeAudioContext);
|
|
135
142
|
await context.close();
|
|
@@ -447,6 +454,7 @@ export class DynascaleManager {
|
|
|
447
454
|
});
|
|
448
455
|
resizeObserver?.observe(videoElement);
|
|
449
456
|
|
|
457
|
+
const isVideoTrack = trackType === 'videoTrack';
|
|
450
458
|
// element renders and gets bound - track subscription gets
|
|
451
459
|
// triggered first other ones get skipped on initial subscriptions
|
|
452
460
|
const publishedTracksSubscription = boundParticipant.isLocalParticipant
|
|
@@ -454,9 +462,7 @@ export class DynascaleManager {
|
|
|
454
462
|
: participant$
|
|
455
463
|
.pipe(
|
|
456
464
|
distinctUntilKeyChanged('publishedTracks'),
|
|
457
|
-
map((p) =>
|
|
458
|
-
trackType === 'videoTrack' ? hasVideo(p) : hasScreenShare(p),
|
|
459
|
-
),
|
|
465
|
+
map((p) => (isVideoTrack ? hasVideo(p) : hasScreenShare(p))),
|
|
460
466
|
distinctUntilChanged(),
|
|
461
467
|
)
|
|
462
468
|
.subscribe((isPublishing) => {
|
|
@@ -480,15 +486,11 @@ export class DynascaleManager {
|
|
|
480
486
|
// https://developer.mozilla.org/en-US/docs/Web/Media/Autoplay_guide
|
|
481
487
|
videoElement.muted = true;
|
|
482
488
|
|
|
489
|
+
const trackKey = isVideoTrack ? 'videoStream' : 'screenShareStream';
|
|
483
490
|
const streamSubscription = participant$
|
|
484
|
-
.pipe(
|
|
485
|
-
distinctUntilKeyChanged(
|
|
486
|
-
trackType === 'videoTrack' ? 'videoStream' : 'screenShareStream',
|
|
487
|
-
),
|
|
488
|
-
)
|
|
491
|
+
.pipe(distinctUntilKeyChanged(trackKey))
|
|
489
492
|
.subscribe((p) => {
|
|
490
|
-
const source =
|
|
491
|
-
trackType === 'videoTrack' ? p.videoStream : p.screenShareStream;
|
|
493
|
+
const source = isVideoTrack ? p.videoStream : p.screenShareStream;
|
|
492
494
|
if (videoElement.srcObject === source) return;
|
|
493
495
|
videoElement.srcObject = source ?? null;
|
|
494
496
|
if (isSafari() || isFirefox()) {
|
|
@@ -532,6 +534,8 @@ export class DynascaleManager {
|
|
|
532
534
|
const participant = this.callState.findParticipantBySessionId(sessionId);
|
|
533
535
|
if (!participant || participant.isLocalParticipant) return;
|
|
534
536
|
|
|
537
|
+
this.audioBindingsWatchdog?.register(audioElement, sessionId, trackType);
|
|
538
|
+
|
|
535
539
|
const participant$ = this.callState.participants$.pipe(
|
|
536
540
|
map((ps) => ps.find((p) => p.sessionId === sessionId)),
|
|
537
541
|
takeWhile((p) => !!p),
|
|
@@ -561,19 +565,12 @@ export class DynascaleManager {
|
|
|
561
565
|
let sourceNode: MediaStreamAudioSourceNode | undefined = undefined;
|
|
562
566
|
let gainNode: GainNode | undefined = undefined;
|
|
563
567
|
|
|
568
|
+
const isAudioTrack = trackType === 'audioTrack';
|
|
569
|
+
const trackKey = isAudioTrack ? 'audioStream' : 'screenShareAudioStream';
|
|
564
570
|
const updateMediaStreamSubscription = participant$
|
|
565
|
-
.pipe(
|
|
566
|
-
distinctUntilKeyChanged(
|
|
567
|
-
trackType === 'screenShareAudioTrack'
|
|
568
|
-
? 'screenShareAudioStream'
|
|
569
|
-
: 'audioStream',
|
|
570
|
-
),
|
|
571
|
-
)
|
|
571
|
+
.pipe(distinctUntilKeyChanged(trackKey))
|
|
572
572
|
.subscribe((p) => {
|
|
573
|
-
const source =
|
|
574
|
-
trackType === 'screenShareAudioTrack'
|
|
575
|
-
? p.screenShareAudioStream
|
|
576
|
-
: p.audioStream;
|
|
573
|
+
const source = isAudioTrack ? p.audioStream : p.screenShareAudioStream;
|
|
577
574
|
if (audioElement.srcObject === source) return;
|
|
578
575
|
|
|
579
576
|
setTimeout(() => {
|
|
@@ -630,6 +627,7 @@ export class DynascaleManager {
|
|
|
630
627
|
audioElement.autoplay = true;
|
|
631
628
|
|
|
632
629
|
return () => {
|
|
630
|
+
this.audioBindingsWatchdog?.unregister(sessionId, trackType);
|
|
633
631
|
sinkIdSubscription?.unsubscribe();
|
|
634
632
|
volumeSubscription.unsubscribe();
|
|
635
633
|
updateMediaStreamSubscription.unsubscribe();
|