@pexip/media 17.2.0 → 17.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/audioMixingProcessor.d.ts +18 -0
- package/dist/audioMixingProcessor.js +173 -0
- package/dist/audioProcessor.d.ts +86 -0
- package/dist/audioProcessor.js +310 -0
- package/dist/baseLogger.d.ts +37 -0
- package/dist/baseLogger.js +29 -0
- package/dist/displayMedia.d.ts +3 -0
- package/dist/displayMedia.js +31 -0
- package/dist/index.d.ts +11 -837
- package/dist/index.js +11 -0
- package/dist/logger.d.ts +11 -0
- package/dist/logger.js +44 -0
- package/dist/media.d.ts +8 -0
- package/dist/media.js +296 -0
- package/dist/previewController.d.ts +64 -0
- package/dist/previewController.js +375 -0
- package/dist/signals.d.ts +25 -0
- package/dist/signals.js +38 -0
- package/dist/status.d.ts +35 -0
- package/dist/status.js +199 -0
- package/dist/typeGuard.d.ts +9 -0
- package/dist/typeGuard.js +40 -0
- package/dist/types.d.ts +559 -0
- package/dist/types.js +278 -0
- package/dist/userMedia.d.ts +38 -0
- package/dist/userMedia.js +333 -0
- package/dist/utils.d.ts +108 -0
- package/dist/utils.js +360 -0
- package/dist/videoProcessor.d.ts +56 -0
- package/dist/videoProcessor.js +349 -0
- package/package.json +10 -10
- package/dist/index.mjs +0 -2900
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,44 @@
|
|
|
1
1
|
# @pexip/media
|
|
2
2
|
|
|
3
|
+
## 17.4.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 4a1908a: Use tsc to build packages
|
|
8
|
+
- fec1c90: Format following Prettier update
|
|
9
|
+
- f2a5e0e: Move trackProcessor param to be a functionn
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 99e25d2: correct `isInitialPermissionsNotGranted` validator
|
|
14
|
+
- Updated dependencies [4a1908a]
|
|
15
|
+
- Updated dependencies [fec1c90]
|
|
16
|
+
- Updated dependencies [ec134d1]
|
|
17
|
+
- Updated dependencies [24fe7e1]
|
|
18
|
+
- @pexip/signal@16.7.0
|
|
19
|
+
- @pexip/utils@16.10.0
|
|
20
|
+
- @pexip/media-control@17.4.0
|
|
21
|
+
- @pexip/media-processor@17.4.0
|
|
22
|
+
|
|
23
|
+
## 17.3.0
|
|
24
|
+
|
|
25
|
+
### Minor Changes
|
|
26
|
+
|
|
27
|
+
- 7abdb2332a: Allow force to applyChanges for preview controller
|
|
28
|
+
- 7b29d953: Update babel/traverse related deps
|
|
29
|
+
- acb51c1ee1: Resolution from the preview should not affect the main
|
|
30
|
+
|
|
31
|
+
### Patch Changes
|
|
32
|
+
|
|
33
|
+
- Updated dependencies [68c3ba1bb8]
|
|
34
|
+
- Updated dependencies [7b29d953]
|
|
35
|
+
- Updated dependencies [821d9a1ef8]
|
|
36
|
+
- Updated dependencies [96f3ce8802]
|
|
37
|
+
- @pexip/media-processor@17.3.0
|
|
38
|
+
- @pexip/media-control@17.3.0
|
|
39
|
+
- @pexip/utils@16.9.0
|
|
40
|
+
- @pexip/signal@16.6.0
|
|
41
|
+
|
|
3
42
|
## 17.2.0
|
|
4
43
|
|
|
5
44
|
### Minor Changes
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { MediaDeviceRequest } from '@pexip/media-control';
|
|
2
|
+
import type { AudioGraph, AudioNodeInit } from '@pexip/media-processor';
|
|
3
|
+
import type { Process, Media } from './types';
|
|
4
|
+
interface AudioStreamProcessorProps {
|
|
5
|
+
mixWithAdditionalMedia?: boolean;
|
|
6
|
+
merger?: AudioNodeInit<ChannelMergerNode, ChannelMergerNode>;
|
|
7
|
+
displaySource?: AudioNodeInit<MediaStreamAudioSourceNode, MediaStreamAudioSourceNode>;
|
|
8
|
+
audioGraph?: AudioGraph;
|
|
9
|
+
}
|
|
10
|
+
declare const FEATURE_KEYS: ['mixWithAdditionalMedia'];
|
|
11
|
+
type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
|
|
12
|
+
type FeatureProps = Pick<AudioStreamProcessorProps, FeaturePropKeys>;
|
|
13
|
+
export declare const updateFeatureProps: (constraints: MediaDeviceRequest['audio'], props: FeatureProps) => FeatureProps;
|
|
14
|
+
/**
|
|
15
|
+
* Create a Audio Mixing Processor and will own the stream passed-in
|
|
16
|
+
*/
|
|
17
|
+
export declare const createAudioMixingProcess: (getCurrrentMedia: () => MediaStream | undefined, scope?: string) => Process<Promise<Media>>;
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { muteStreamTrack, stopMediaStream, extractConstraintsWithKeys, } from '@pexip/media-control';
|
|
2
|
+
import { isEmpty } from '@pexip/utils';
|
|
3
|
+
import { createAudioGraph, createAudioGraphProxy, createChannelMergerGraphNode, createStreamDestinationGraphNode, createStreamSourceGraphNode, resumeAudioOnUnmute, } from '@pexip/media-processor';
|
|
4
|
+
import { logger } from './logger';
|
|
5
|
+
import { applyExtendedConstraints, shallowCopy, wrapToJSON } from './utils';
|
|
6
|
+
const FEATURE_KEYS = ['mixWithAdditionalMedia'];
|
|
7
|
+
const getAudioConstraints = extractConstraintsWithKeys(FEATURE_KEYS);
|
|
8
|
+
export const updateFeatureProps = (constraints, props) => {
|
|
9
|
+
const extracted = getAudioConstraints(constraints);
|
|
10
|
+
return FEATURE_KEYS.reduce((accm, key) => {
|
|
11
|
+
const [feature] = extracted[key];
|
|
12
|
+
if (feature !== undefined && props[key] !== feature) {
|
|
13
|
+
props[key] = feature;
|
|
14
|
+
return { ...accm, [key]: feature };
|
|
15
|
+
}
|
|
16
|
+
return accm;
|
|
17
|
+
}, {});
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Create a Audio Mixing Processor and will own the stream passed-in
|
|
21
|
+
*/
|
|
22
|
+
export const createAudioMixingProcess = (getCurrrentMedia, scope = 'mixer') => {
|
|
23
|
+
const props = {};
|
|
24
|
+
return async (mediaP) => {
|
|
25
|
+
const media = await mediaP;
|
|
26
|
+
updateFeatureProps(media.constraints?.audio, props);
|
|
27
|
+
const [mainTrack] = media.stream?.getAudioTracks() ?? [];
|
|
28
|
+
const displayStream = getCurrrentMedia();
|
|
29
|
+
const [displayTrack] = displayStream?.getAudioTracks?.() ?? [];
|
|
30
|
+
const applyConstraints = applyExtendedConstraints(media, async (constraints) => {
|
|
31
|
+
if (isEmpty(constraints.audio)) {
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
const features = updateFeatureProps(constraints.audio, props);
|
|
35
|
+
logger.debug({ scope, constraints: constraints.audio, features }, 'apply audio mixing constraints');
|
|
36
|
+
if (isEmpty(features) ||
|
|
37
|
+
(props.audioGraph &&
|
|
38
|
+
['closed', 'closing'].includes(props.audioGraph.state))) {
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
if (features.mixWithAdditionalMedia) {
|
|
42
|
+
const newStream = getCurrrentMedia();
|
|
43
|
+
const [newTrack] = getCurrrentMedia()?.getAudioTracks() ?? [];
|
|
44
|
+
if (!newTrack) {
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
if (!props.audioGraph || !props.merger) {
|
|
48
|
+
// TODO: Should update the media stream directly
|
|
49
|
+
// return replace(media.stream, newTrack);
|
|
50
|
+
logger.debug({ scope }, 'Should update the media stream directly');
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
if (props.displaySource && props.merger) {
|
|
54
|
+
if (newStream ===
|
|
55
|
+
props.displaySource.audioNode?.mediaStream) {
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
props.audioGraph.disconnect([
|
|
59
|
+
props.displaySource,
|
|
60
|
+
props.merger,
|
|
61
|
+
]);
|
|
62
|
+
}
|
|
63
|
+
if (newStream) {
|
|
64
|
+
props.displaySource =
|
|
65
|
+
createStreamSourceGraphNode(newStream);
|
|
66
|
+
props.audioGraph.connect([
|
|
67
|
+
props.displaySource,
|
|
68
|
+
props.merger,
|
|
69
|
+
]);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
else if (features.mixWithAdditionalMedia === false) {
|
|
73
|
+
// Unmixing
|
|
74
|
+
if (props.displaySource) {
|
|
75
|
+
props.displaySource.release();
|
|
76
|
+
props.audioGraph?.disconnect([props.displaySource]);
|
|
77
|
+
props.displaySource = undefined;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return Promise.resolve();
|
|
81
|
+
});
|
|
82
|
+
if (!media.stream || (!mainTrack && !displayTrack)) {
|
|
83
|
+
return media;
|
|
84
|
+
}
|
|
85
|
+
if (!mainTrack && displayTrack) {
|
|
86
|
+
media.stream?.addTrack(displayTrack);
|
|
87
|
+
return shallowCopy(media, {
|
|
88
|
+
muteAudio: () => {
|
|
89
|
+
/* Do Nothing */
|
|
90
|
+
},
|
|
91
|
+
applyConstraints,
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
try {
|
|
95
|
+
const mainSource = createStreamSourceGraphNode(media.stream);
|
|
96
|
+
props.displaySource =
|
|
97
|
+
displayStream && createStreamSourceGraphNode(displayStream);
|
|
98
|
+
props.merger = createChannelMergerGraphNode();
|
|
99
|
+
const destination = createStreamDestinationGraphNode({
|
|
100
|
+
channelCount: 1,
|
|
101
|
+
channelCountMode: 'explicit',
|
|
102
|
+
});
|
|
103
|
+
const initialAudioNodeConnection = [
|
|
104
|
+
[mainSource, props.merger],
|
|
105
|
+
[props.merger, destination],
|
|
106
|
+
];
|
|
107
|
+
logger.debug({ initialAudioNodeConnection, scope }, 'Initial AudioNodeConnection');
|
|
108
|
+
const audioGraph = createAudioGraphProxy(createAudioGraph(initialAudioNodeConnection), {
|
|
109
|
+
connect: (target, args) => {
|
|
110
|
+
logger.debug({ scope, target, args }, 'connect nodes');
|
|
111
|
+
},
|
|
112
|
+
disconnect: (target, args) => {
|
|
113
|
+
logger.debug({ scope, target, args }, 'disconnect nodes');
|
|
114
|
+
},
|
|
115
|
+
});
|
|
116
|
+
props.audioGraph = audioGraph;
|
|
117
|
+
const unsubscribes = media.rawStream
|
|
118
|
+
?.getAudioTracks()
|
|
119
|
+
.map(resumeAudioOnUnmute(audioGraph.context));
|
|
120
|
+
if (props.displaySource) {
|
|
121
|
+
props.audioGraph.connect([props.displaySource, props.merger]);
|
|
122
|
+
}
|
|
123
|
+
const tracks = [
|
|
124
|
+
...(destination?.node?.stream.getAudioTracks() ?? []),
|
|
125
|
+
...(media.stream?.getVideoTracks() ?? []),
|
|
126
|
+
];
|
|
127
|
+
const stream = new MediaStream(tracks);
|
|
128
|
+
const release = async () => {
|
|
129
|
+
logger.debug({ scope }, 'Release Media');
|
|
130
|
+
unsubscribes?.forEach(unsubscribe => unsubscribe());
|
|
131
|
+
stopMediaStream(stream);
|
|
132
|
+
// Release Props
|
|
133
|
+
await props.audioGraph?.release();
|
|
134
|
+
await media.release();
|
|
135
|
+
props.audioGraph = undefined;
|
|
136
|
+
props.merger = undefined;
|
|
137
|
+
props.displaySource = undefined;
|
|
138
|
+
};
|
|
139
|
+
const muteAudio = (mute) => {
|
|
140
|
+
media.muteAudio(mute);
|
|
141
|
+
// TODO: dbl check if this is safe
|
|
142
|
+
muteStreamTrack(mainSource.audioNode?.mediaStream)(mute, 'audio');
|
|
143
|
+
};
|
|
144
|
+
const prevGetSettings = media.getSettings;
|
|
145
|
+
return wrapToJSON(shallowCopy(media, {
|
|
146
|
+
stream,
|
|
147
|
+
applyConstraints,
|
|
148
|
+
muteAudio,
|
|
149
|
+
release,
|
|
150
|
+
getSettings: () => {
|
|
151
|
+
const { audio, video } = prevGetSettings();
|
|
152
|
+
const mixWithAdditionalMedia = !!props.mixWithAdditionalMedia;
|
|
153
|
+
const audioSettings = {
|
|
154
|
+
mixWithAdditionalMedia,
|
|
155
|
+
};
|
|
156
|
+
const settings = {
|
|
157
|
+
audio: audio.map(settings => ({
|
|
158
|
+
...settings,
|
|
159
|
+
...audioSettings,
|
|
160
|
+
})),
|
|
161
|
+
video,
|
|
162
|
+
};
|
|
163
|
+
logger.debug({ scope, settings: audioSettings }, 'get audio mixing processor settings');
|
|
164
|
+
return settings;
|
|
165
|
+
},
|
|
166
|
+
}));
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
logger.error({ scope, error }, 'Unable to use WebAudio, return the raw media instead');
|
|
170
|
+
return media;
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
};
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { AudioGraphOptions, AudioNodeInit, ThrottleOptions, DenoiseWorkletNodeInit, AnalyzerNodeInit, AudioGraph } from '@pexip/media-processor';
|
|
2
|
+
import type { MediaDeviceRequest } from '@pexip/media-control';
|
|
3
|
+
import type { Process, Media, DenoiseParams, AudioContentHint } from './types';
|
|
4
|
+
type AudioNodeInits = AudioNodeInit[];
|
|
5
|
+
/**
|
|
6
|
+
* A function to be called to create the AudioNodes needed for the graph
|
|
7
|
+
* creation
|
|
8
|
+
*
|
|
9
|
+
* @param media - Media to be used for the AudioGraph creation
|
|
10
|
+
*/
|
|
11
|
+
type CreateNodes = (media: Media) => AudioNodeInits;
|
|
12
|
+
interface AudioProcessOptions {
|
|
13
|
+
/**
|
|
14
|
+
* An option is being passed to AnalyserNode creation when used
|
|
15
|
+
* @see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/fftSize
|
|
16
|
+
*
|
|
17
|
+
* @defaultValue 2048
|
|
18
|
+
*/
|
|
19
|
+
fftSize?: number;
|
|
20
|
+
/**
|
|
21
|
+
* Params needed for setting up noise suppression WebAssembly and
|
|
22
|
+
* AudioWorklet
|
|
23
|
+
*/
|
|
24
|
+
denoiseParams?: DenoiseParams;
|
|
25
|
+
/**
|
|
26
|
+
* Update frequency for analyzer per second
|
|
27
|
+
*
|
|
28
|
+
* @defaultValue 0.5
|
|
29
|
+
*/
|
|
30
|
+
analyzerUpdateFrequency?: number;
|
|
31
|
+
/**
|
|
32
|
+
* Audio Signal Detection duration in second
|
|
33
|
+
*
|
|
34
|
+
* @defaultValue 4.0
|
|
35
|
+
*/
|
|
36
|
+
audioSignalDetectionDuration?: number;
|
|
37
|
+
/**
|
|
38
|
+
* Callback when Voice Activity detected
|
|
39
|
+
*/
|
|
40
|
+
onVoiceActivityDetected?: () => void;
|
|
41
|
+
/**
|
|
42
|
+
* Callback when Audio Signal detected
|
|
43
|
+
*/
|
|
44
|
+
onAudioSignalDetected?: (silent: boolean) => void;
|
|
45
|
+
/**
|
|
46
|
+
* @see AudioGraphOptions
|
|
47
|
+
*/
|
|
48
|
+
audioGraphOptions?: AudioGraphOptions;
|
|
49
|
+
/**
|
|
50
|
+
* Whether or to enable this processor
|
|
51
|
+
*/
|
|
52
|
+
shouldEnable: () => boolean;
|
|
53
|
+
/**
|
|
54
|
+
* Insert additional nodes between the source and destination
|
|
55
|
+
*/
|
|
56
|
+
createNodes?: CreateNodes;
|
|
57
|
+
/**
|
|
58
|
+
* Silent threshold, how large the value of the sample is considered as
|
|
59
|
+
* silent in FFTed time domain
|
|
60
|
+
*/
|
|
61
|
+
silentThreshold?: number;
|
|
62
|
+
scope?: string;
|
|
63
|
+
}
|
|
64
|
+
interface AudioStreamProcessorProps {
|
|
65
|
+
audioGraphOptions?: AudioGraphOptions;
|
|
66
|
+
denoiseWasm?: ArrayBuffer;
|
|
67
|
+
vad?: boolean;
|
|
68
|
+
asd?: boolean;
|
|
69
|
+
denoise?: boolean;
|
|
70
|
+
analyzerBuffer?: Float32Array;
|
|
71
|
+
denoiseNode?: DenoiseWorkletNodeInit;
|
|
72
|
+
additionalAudioSourceNode?: AudioNodeInit<MediaStreamAudioSourceNode, MediaStreamAudioSourceNode>;
|
|
73
|
+
mixerNode?: AudioNodeInit<ChannelMergerNode, ChannelMergerNode>;
|
|
74
|
+
analyzer?: AnalyzerNodeInit;
|
|
75
|
+
contentHint?: AudioContentHint;
|
|
76
|
+
audioGraph?: AudioGraph;
|
|
77
|
+
}
|
|
78
|
+
declare const FEATURE_KEYS: ['denoise', 'vad', 'asd', 'contentHint'];
|
|
79
|
+
type FeaturePropKeys = (typeof FEATURE_KEYS)[number];
|
|
80
|
+
type FeatureProps = Pick<AudioStreamProcessorProps, FeaturePropKeys>;
|
|
81
|
+
export declare const updateFeatureProps: (constraints: MediaDeviceRequest['audio'], props: FeatureProps) => FeatureProps;
|
|
82
|
+
/**
|
|
83
|
+
* Create a Audio Stream Processor and will own the stream passed-in
|
|
84
|
+
*/
|
|
85
|
+
export declare const createAudioStreamProcess: ({ analyzerUpdateFrequency, audioGraphOptions, audioSignalDetectionDuration, clock, createNodes, denoiseParams, fftSize, onAudioSignalDetected, onVoiceActivityDetected, shouldEnable, silentThreshold, throttleMs, scope, }: AudioProcessOptions & ThrottleOptions) => Process<Promise<Media>>;
|
|
86
|
+
export {};
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
import { stopMediaStream, extractConstraintsWithKeys, muteStreamTrack, } from '@pexip/media-control';
|
|
2
|
+
import { createQueue, isEmpty } from '@pexip/utils';
|
|
3
|
+
import { createAudioGraph, createAudioGraphProxy, createStreamSourceGraphNode, createStreamDestinationGraphNode, createAnalyzerSubscribableGraphNode, createDenoiseWorkletGraphNode, createAudioSignalDetector, createVADetector, createVoiceDetectorFromTimeData, createVoiceDetectorFromProbability, avg, } from '@pexip/media-processor';
|
|
4
|
+
import { logger } from './logger';
|
|
5
|
+
import { isAudioContentHint } from './typeGuard';
|
|
6
|
+
import { shallowCopy, wrapToJSON, applyExtendedConstraints } from './utils';
|
|
7
|
+
/**
|
|
8
|
+
* Fetch the wasm when it doesn't exist from the provided, otherwise do nothing
|
|
9
|
+
*
|
|
10
|
+
* @param denoiseWasm - The wasm if it exists
|
|
11
|
+
* @param wasmURL - The URL for the fetching
|
|
12
|
+
*/
|
|
13
|
+
const fetchDenoiseWasm = async (denoiseWasm, wasmURL) => {
|
|
14
|
+
if (!wasmURL || denoiseWasm) {
|
|
15
|
+
return denoiseWasm;
|
|
16
|
+
}
|
|
17
|
+
return await (await fetch(wasmURL)).arrayBuffer();
|
|
18
|
+
};
|
|
19
|
+
const FEATURE_KEYS = [
|
|
20
|
+
'denoise',
|
|
21
|
+
'vad',
|
|
22
|
+
'asd',
|
|
23
|
+
'contentHint',
|
|
24
|
+
];
|
|
25
|
+
const getAudioConstraints = extractConstraintsWithKeys(FEATURE_KEYS);
|
|
26
|
+
export const updateFeatureProps = (constraints, props) => {
|
|
27
|
+
const extracted = getAudioConstraints(constraints);
|
|
28
|
+
return FEATURE_KEYS.reduce((accm, key) => {
|
|
29
|
+
switch (key) {
|
|
30
|
+
case 'contentHint': {
|
|
31
|
+
const [[feature] = []] = extracted[key];
|
|
32
|
+
if (isAudioContentHint(feature) && props[key] !== feature) {
|
|
33
|
+
props[key] = feature;
|
|
34
|
+
return { ...accm, [key]: feature };
|
|
35
|
+
}
|
|
36
|
+
return accm;
|
|
37
|
+
}
|
|
38
|
+
case 'denoise':
|
|
39
|
+
case 'asd':
|
|
40
|
+
case 'vad': {
|
|
41
|
+
const [feature] = extracted[key];
|
|
42
|
+
if (feature !== undefined && props[key] !== feature) {
|
|
43
|
+
props[key] = feature;
|
|
44
|
+
return { ...accm, [key]: feature };
|
|
45
|
+
}
|
|
46
|
+
return accm;
|
|
47
|
+
}
|
|
48
|
+
default:
|
|
49
|
+
return accm;
|
|
50
|
+
}
|
|
51
|
+
}, {});
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Create a Audio Stream Processor and will own the stream passed-in
|
|
55
|
+
*/
|
|
56
|
+
export const createAudioStreamProcess = ({ analyzerUpdateFrequency = 0.5, // 0.5 Hz
|
|
57
|
+
audioGraphOptions, audioSignalDetectionDuration = 4.0, // 4 seconds
|
|
58
|
+
clock, createNodes, denoiseParams, fftSize = 2048, // FFT size
|
|
59
|
+
onAudioSignalDetected, onVoiceActivityDetected, shouldEnable, silentThreshold = 10.0 / 32767, // At least one LSB 16-bit data (compare is on absolute value).
|
|
60
|
+
throttleMs = 3000, // 3 seconds
|
|
61
|
+
scope = 'media', }) => {
|
|
62
|
+
const props = {
|
|
63
|
+
audioGraphOptions,
|
|
64
|
+
vad: false,
|
|
65
|
+
asd: false,
|
|
66
|
+
denoise: false,
|
|
67
|
+
};
|
|
68
|
+
const detectAudio = onAudioSignalDetected &&
|
|
69
|
+
createAudioSignalDetector(() => !!props.asd, onAudioSignalDetected);
|
|
70
|
+
const detectVA = onVoiceActivityDetected &&
|
|
71
|
+
createVADetector(onVoiceActivityDetected, () => !!props.vad, {
|
|
72
|
+
throttleMs,
|
|
73
|
+
clock,
|
|
74
|
+
});
|
|
75
|
+
const detectVAFromTimeData = detectVA?.(createVoiceDetectorFromTimeData());
|
|
76
|
+
const createDenoiseNode = async (denoise) => {
|
|
77
|
+
if (!denoise) {
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
if (props.denoiseNode) {
|
|
81
|
+
return props.denoiseNode;
|
|
82
|
+
}
|
|
83
|
+
if (denoiseParams?.workletModule) {
|
|
84
|
+
try {
|
|
85
|
+
await props.audioGraph?.addWorklet(denoiseParams?.workletModule, denoiseParams?.workletOptions);
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
logger.error({
|
|
89
|
+
scope,
|
|
90
|
+
error,
|
|
91
|
+
moduleURL: denoiseParams?.workletModule,
|
|
92
|
+
options: denoiseParams?.workletOptions,
|
|
93
|
+
}, 'Failed add worklet');
|
|
94
|
+
return undefined;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
props.denoiseWasm = await fetchDenoiseWasm(props.denoiseWasm, denoiseParams?.wasmURL);
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
logger.error({
|
|
102
|
+
scope,
|
|
103
|
+
error,
|
|
104
|
+
url: denoiseParams?.wasmURL,
|
|
105
|
+
prevWasm: props.denoiseWasm,
|
|
106
|
+
}, 'Failed to fetch denoise wasm');
|
|
107
|
+
return undefined;
|
|
108
|
+
}
|
|
109
|
+
const detectVAFromProbability = detectVA?.(createVoiceDetectorFromProbability());
|
|
110
|
+
if (props.denoiseWasm) {
|
|
111
|
+
const denoise = createDenoiseWorkletGraphNode(props.denoiseWasm, vads => {
|
|
112
|
+
detectVAFromProbability?.(avg(vads));
|
|
113
|
+
});
|
|
114
|
+
props.denoiseNode = denoise;
|
|
115
|
+
return denoise;
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
const createAnalyzer = () => {
|
|
119
|
+
const shouldUseAnalyzer = () => !!((props.vad && !props.denoise) || props.asd) &&
|
|
120
|
+
(detectAudio || detectVA);
|
|
121
|
+
if (!shouldUseAnalyzer()) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
if (props.analyzer) {
|
|
125
|
+
return props.analyzer;
|
|
126
|
+
}
|
|
127
|
+
const detectSilentAudio = detectAudio?.(createQueue(audioSignalDetectionDuration / analyzerUpdateFrequency), silentThreshold);
|
|
128
|
+
const analyzer = createAnalyzerSubscribableGraphNode({
|
|
129
|
+
updateFrequency: analyzerUpdateFrequency,
|
|
130
|
+
messageHandler: analyzer => {
|
|
131
|
+
if (shouldUseAnalyzer()) {
|
|
132
|
+
if (!props.analyzerBuffer) {
|
|
133
|
+
// Only Create the buffer when needed
|
|
134
|
+
props.analyzerBuffer = new Float32Array(fftSize);
|
|
135
|
+
}
|
|
136
|
+
analyzer.getFloatTimeDomainData(props.analyzerBuffer);
|
|
137
|
+
const data = Array.from(props.analyzerBuffer);
|
|
138
|
+
detectSilentAudio?.(data);
|
|
139
|
+
!props.denoise && detectVAFromTimeData?.(data);
|
|
140
|
+
}
|
|
141
|
+
},
|
|
142
|
+
fftSize,
|
|
143
|
+
});
|
|
144
|
+
props.analyzer = analyzer;
|
|
145
|
+
return analyzer;
|
|
146
|
+
};
|
|
147
|
+
return async (mediaP) => {
|
|
148
|
+
const media = await mediaP;
|
|
149
|
+
updateFeatureProps(media.constraints?.audio, props);
|
|
150
|
+
const shouldProcessAudio = shouldEnable() &&
|
|
151
|
+
!!media.stream?.getAudioTracks().length &&
|
|
152
|
+
(!!onVoiceActivityDetected ||
|
|
153
|
+
!!onAudioSignalDetected ||
|
|
154
|
+
props.asd ||
|
|
155
|
+
props.vad ||
|
|
156
|
+
props.denoise ||
|
|
157
|
+
!!createNodes);
|
|
158
|
+
if (!media.stream?.getAudioTracks().length || !shouldProcessAudio) {
|
|
159
|
+
return media;
|
|
160
|
+
}
|
|
161
|
+
try {
|
|
162
|
+
const source = createStreamSourceGraphNode(media.stream);
|
|
163
|
+
const destination = createStreamDestinationGraphNode();
|
|
164
|
+
const otherNodes = createNodes?.(media) ?? [];
|
|
165
|
+
const analyzer = createAnalyzer();
|
|
166
|
+
const initialAudioNodeConnection = [
|
|
167
|
+
[source, ...otherNodes, destination],
|
|
168
|
+
[source, analyzer],
|
|
169
|
+
];
|
|
170
|
+
logger.debug({ initialAudioNodeConnection, scope }, 'Initial AudioNodeConnection');
|
|
171
|
+
const audioGraph = createAudioGraphProxy(createAudioGraph(initialAudioNodeConnection, props.audioGraphOptions), {
|
|
172
|
+
connect: (target, args) => {
|
|
173
|
+
logger.debug({ scope, target, args }, 'connect nodes');
|
|
174
|
+
},
|
|
175
|
+
disconnect: (target, args) => {
|
|
176
|
+
logger.debug({ scope, target, args }, 'disconnect nodes');
|
|
177
|
+
},
|
|
178
|
+
});
|
|
179
|
+
props.audioGraph = audioGraph;
|
|
180
|
+
const denoiseNode = await createDenoiseNode(props.denoise);
|
|
181
|
+
const connectDenoise = (node) => {
|
|
182
|
+
if (node) {
|
|
183
|
+
audioGraph.disconnect([source, ...otherNodes, destination]);
|
|
184
|
+
audioGraph.connect([
|
|
185
|
+
source,
|
|
186
|
+
node,
|
|
187
|
+
...otherNodes,
|
|
188
|
+
destination,
|
|
189
|
+
]);
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
connectDenoise(denoiseNode);
|
|
193
|
+
const tracks = [
|
|
194
|
+
...(destination?.node?.stream.getAudioTracks() ?? []),
|
|
195
|
+
...(media.stream?.getVideoTracks().map(t => t.clone()) ?? []),
|
|
196
|
+
];
|
|
197
|
+
const stream = new MediaStream(tracks);
|
|
198
|
+
const applyConstraints = applyExtendedConstraints(media, async (constraints) => {
|
|
199
|
+
if (isEmpty(constraints.audio)) {
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
const features = updateFeatureProps(constraints.audio, props);
|
|
203
|
+
logger.debug({ scope, constraints: constraints.audio, features }, 'apply audio constraints');
|
|
204
|
+
if (isEmpty(features) ||
|
|
205
|
+
['closed', 'closing'].includes(audioGraph.state)) {
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
try {
|
|
209
|
+
const denoiseNode = await createDenoiseNode(props.denoise);
|
|
210
|
+
if (denoiseNode) {
|
|
211
|
+
if (!source.hasConnectedTo(denoiseNode)) {
|
|
212
|
+
connectDenoise(denoiseNode);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
if (props.denoiseNode) {
|
|
217
|
+
audioGraph.disconnect([
|
|
218
|
+
source,
|
|
219
|
+
props.denoiseNode,
|
|
220
|
+
...otherNodes,
|
|
221
|
+
destination,
|
|
222
|
+
]);
|
|
223
|
+
audioGraph.connect([
|
|
224
|
+
source,
|
|
225
|
+
...otherNodes,
|
|
226
|
+
destination,
|
|
227
|
+
]);
|
|
228
|
+
audioGraph.releaseInit(props.denoiseNode);
|
|
229
|
+
props.denoiseNode = undefined;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
const analyzer = createAnalyzer();
|
|
233
|
+
if (analyzer) {
|
|
234
|
+
audioGraph.connect([source, analyzer]);
|
|
235
|
+
}
|
|
236
|
+
else {
|
|
237
|
+
if (props.analyzer) {
|
|
238
|
+
audioGraph.disconnect([source, props.analyzer]);
|
|
239
|
+
audioGraph.releaseInit(props.analyzer);
|
|
240
|
+
props.analyzer = undefined;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
catch (error) {
|
|
245
|
+
if (error instanceof Error) {
|
|
246
|
+
logger.error({
|
|
247
|
+
scope,
|
|
248
|
+
constraints: constraints.audio,
|
|
249
|
+
features,
|
|
250
|
+
}, 'failed to apply audio constraints');
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
const release = async () => {
|
|
255
|
+
logger.debug({ scope }, 'Release Media');
|
|
256
|
+
stopMediaStream(stream);
|
|
257
|
+
// Release Props
|
|
258
|
+
await audioGraph.release();
|
|
259
|
+
await media.release();
|
|
260
|
+
props.denoiseNode = undefined;
|
|
261
|
+
props.analyzer = undefined;
|
|
262
|
+
props.audioGraph = undefined;
|
|
263
|
+
};
|
|
264
|
+
const muteAudio = (mute) => {
|
|
265
|
+
media.muteAudio(mute);
|
|
266
|
+
muteStreamTrack(stream)(mute, 'audio');
|
|
267
|
+
};
|
|
268
|
+
const muteVideo = (mute) => {
|
|
269
|
+
media.muteVideo(mute);
|
|
270
|
+
muteStreamTrack(stream)(mute, 'video');
|
|
271
|
+
};
|
|
272
|
+
const prevGetSettings = media.getSettings;
|
|
273
|
+
return wrapToJSON(shallowCopy(media, {
|
|
274
|
+
stream,
|
|
275
|
+
applyConstraints,
|
|
276
|
+
muteAudio,
|
|
277
|
+
muteVideo,
|
|
278
|
+
release,
|
|
279
|
+
getSettings: () => {
|
|
280
|
+
const { audio, video } = prevGetSettings();
|
|
281
|
+
const denoise = !!props.denoiseNode &&
|
|
282
|
+
source.hasConnectedTo(props.denoiseNode);
|
|
283
|
+
const asd = !!props.asd;
|
|
284
|
+
const vad = !!props.vad;
|
|
285
|
+
const contentHint = stream.getAudioTracks().at(0)
|
|
286
|
+
?.contentHint ?? '';
|
|
287
|
+
const audioSettings = {
|
|
288
|
+
denoise,
|
|
289
|
+
asd,
|
|
290
|
+
vad,
|
|
291
|
+
contentHint,
|
|
292
|
+
};
|
|
293
|
+
const settings = {
|
|
294
|
+
audio: audio.map(settings => ({
|
|
295
|
+
...settings,
|
|
296
|
+
...audioSettings,
|
|
297
|
+
})),
|
|
298
|
+
video,
|
|
299
|
+
};
|
|
300
|
+
logger.debug({ scope, settings: audioSettings }, 'get audio processor settings');
|
|
301
|
+
return settings;
|
|
302
|
+
},
|
|
303
|
+
}));
|
|
304
|
+
}
|
|
305
|
+
catch (error) {
|
|
306
|
+
logger.error({ scope, error }, 'Unable to use WebAudio, return the raw media instead');
|
|
307
|
+
return media;
|
|
308
|
+
}
|
|
309
|
+
};
|
|
310
|
+
};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Log meta and message with respective log level
|
|
3
|
+
*/
|
|
4
|
+
export type LogMethod = (meta: unknown, message?: string) => void;
|
|
5
|
+
export declare enum LogLevels {
|
|
6
|
+
trace = 10,
|
|
7
|
+
debug = 20,
|
|
8
|
+
info = 30,
|
|
9
|
+
warn = 40,
|
|
10
|
+
error = 50,
|
|
11
|
+
fatal = 60,
|
|
12
|
+
silent
|
|
13
|
+
}
|
|
14
|
+
export type LogLevelsString = keyof typeof LogLevels;
|
|
15
|
+
export type LogMethods = {
|
|
16
|
+
[key in LogLevelsString]: LogMethod;
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* Log Level from high to low, "fatal" | "error" | "warn" | "info" | "debug" | "trace"
|
|
20
|
+
* Typically, debug and trace logs are only valid for development, and not needed in production
|
|
21
|
+
*/
|
|
22
|
+
export interface Logger extends LogMethods {
|
|
23
|
+
/**
|
|
24
|
+
* Adds a value to the redaction set, which makes it replaced by [REDACTED] when logged to file.
|
|
25
|
+
*
|
|
26
|
+
* @remarks
|
|
27
|
+
* The redaction set is applied globally, and only applies to the log file, not console logs.
|
|
28
|
+
*
|
|
29
|
+
* @param value - the string to redact
|
|
30
|
+
*/
|
|
31
|
+
redact(value: string): void;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Create a logger with console API, and map fatal to error, skipping trace
|
|
35
|
+
* and silent, and there is no redaction
|
|
36
|
+
*/
|
|
37
|
+
export declare function createConsoleLogger(): Readonly<Logger>;
|