@pexip/media 16.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1079 -0
- package/LICENSE +188 -0
- package/README.md +350 -0
- package/dist/index.d.ts +739 -0
- package/dist/index.mjs +2698 -0
- package/package.json +50 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,739 @@
|
|
|
1
|
+
import * as _pexip_media_control from '@pexip/media-control';
|
|
2
|
+
import { MediaDeviceRequest, MediaDeviceInfoLike, InputConstraintSet } from '@pexip/media-control';
|
|
3
|
+
import { RenderEffects, SegmentationModel, RenderParams, Segmenter, ThrottleOptions, AudioGraphOptions, AudioNodeInit, VideoProcessor, SegmentationTransform } from '@pexip/media-processor';
|
|
4
|
+
import { Signal, SignalVariant } from '@pexip/signal';
|
|
5
|
+
import { AsyncQueueOptions } from '@pexip/utils';
|
|
6
|
+
|
|
7
|
+
type Unsubscribe = () => void;
|
|
8
|
+
interface ExtendedMediaTrackSettings extends MediaTrackSettings {
|
|
9
|
+
mixWithAdditionalMedia?: boolean;
|
|
10
|
+
channelCount?: number;
|
|
11
|
+
denoise?: boolean;
|
|
12
|
+
vad?: boolean;
|
|
13
|
+
asd?: boolean;
|
|
14
|
+
backgroundBlurAmount?: number;
|
|
15
|
+
bgImageUrl?: string;
|
|
16
|
+
edgeBlurAmount?: number;
|
|
17
|
+
flipHorizontal?: boolean;
|
|
18
|
+
foregroundThreshold?: number;
|
|
19
|
+
videoSegmentation?: RenderEffects;
|
|
20
|
+
videoSegmentationModel?: SegmentationModel;
|
|
21
|
+
}
|
|
22
|
+
type ExtendedMediaTrackSettingsKey = keyof ExtendedMediaTrackSettings;
|
|
23
|
+
interface MediaSettings {
|
|
24
|
+
audio: ExtendedMediaTrackSettings[];
|
|
25
|
+
video: ExtendedMediaTrackSettings[];
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* URL links needed for creating a denoise node for audio processing
|
|
29
|
+
*/
|
|
30
|
+
interface DenoiseParams {
|
|
31
|
+
/**
|
|
32
|
+
* wasm URL to get the denoise wasm
|
|
33
|
+
*/
|
|
34
|
+
wasmURL: string;
|
|
35
|
+
/**
|
|
36
|
+
* AudioWorklet module URL to get the worklet, @see AudioContext['audioWorklet']['addModule']
|
|
37
|
+
*/
|
|
38
|
+
workletModule: string;
|
|
39
|
+
/**
|
|
40
|
+
* AudioWorklet options to pass @see AudioContext['audioWorklet']['addModule']
|
|
41
|
+
*/
|
|
42
|
+
workletOptions?: WorkletOptions;
|
|
43
|
+
}
|
|
44
|
+
interface MediaAttributes {
|
|
45
|
+
/**
|
|
46
|
+
* The constraints used to request the media
|
|
47
|
+
*/
|
|
48
|
+
constraints?: MediaDeviceRequest;
|
|
49
|
+
/**
|
|
50
|
+
* The devices used for the media
|
|
51
|
+
*/
|
|
52
|
+
devices: MediaDeviceInfoLike[];
|
|
53
|
+
/**
|
|
54
|
+
* Media stream for the media
|
|
55
|
+
*/
|
|
56
|
+
stream?: MediaStream;
|
|
57
|
+
/**
|
|
58
|
+
* The raw stream obtained from the `getUserMedia` API
|
|
59
|
+
*/
|
|
60
|
+
rawStream?: MediaStream;
|
|
61
|
+
/**
|
|
62
|
+
* Audio input device is used for the audio track from the current stream
|
|
63
|
+
*/
|
|
64
|
+
audioInput?: MediaDeviceInfoLike;
|
|
65
|
+
/**
|
|
66
|
+
* Video input device is used for the video track from the current stream
|
|
67
|
+
*/
|
|
68
|
+
videoInput?: MediaDeviceInfoLike;
|
|
69
|
+
/**
|
|
70
|
+
* The audio input device which is expected to be used for the current
|
|
71
|
+
* stream based on the provided constraints
|
|
72
|
+
*/
|
|
73
|
+
expectedAudioInput?: MediaDeviceInfoLike;
|
|
74
|
+
/**
|
|
75
|
+
* The video input device which is expected to be used for the current
|
|
76
|
+
* stream based on the provided constraints
|
|
77
|
+
*/
|
|
78
|
+
expectedVideoInput?: MediaDeviceInfoLike;
|
|
79
|
+
/**
|
|
80
|
+
* The status of the media
|
|
81
|
+
*/
|
|
82
|
+
status: UserMediaStatus;
|
|
83
|
+
/**
|
|
84
|
+
* Current mute state of audio track
|
|
85
|
+
* `undefined` means there is no such track from the stream
|
|
86
|
+
*/
|
|
87
|
+
audioMuted: boolean | undefined;
|
|
88
|
+
/**
|
|
89
|
+
* Current mute state of video track
|
|
90
|
+
* `undefined` means there is no such track from the stream
|
|
91
|
+
*/
|
|
92
|
+
videoMuted: boolean | undefined;
|
|
93
|
+
}
|
|
94
|
+
interface Media extends MediaAttributes {
|
|
95
|
+
/**
|
|
96
|
+
* mute/unmute the audio track
|
|
97
|
+
*/
|
|
98
|
+
muteAudio(mute: boolean): void;
|
|
99
|
+
/**
|
|
100
|
+
* mute/unmute the video track
|
|
101
|
+
*/
|
|
102
|
+
muteVideo(mute: boolean): void;
|
|
103
|
+
/**
|
|
104
|
+
* Apply the constraints to the current media
|
|
105
|
+
*/
|
|
106
|
+
applyConstraints(constraints: MediaDeviceRequest): Promise<void>;
|
|
107
|
+
/**
|
|
108
|
+
* Release the media resources, e.g. camera/microphone
|
|
109
|
+
*/
|
|
110
|
+
release(): Promise<void>;
|
|
111
|
+
getSettings(): MediaSettings;
|
|
112
|
+
toJSON?: () => unknown;
|
|
113
|
+
}
|
|
114
|
+
type Process<T> = (a: T) => Promise<Media>;
|
|
115
|
+
type MediaProcessor = Process<Promise<Media>>;
|
|
116
|
+
/**
|
|
117
|
+
* A media pipeline to get and process media
|
|
118
|
+
*/
|
|
119
|
+
type Pipeline<T = MediaDeviceRequest> = [
|
|
120
|
+
Process<T>,
|
|
121
|
+
...MediaProcessor[]
|
|
122
|
+
];
|
|
123
|
+
type ProcessMedia = (media: Media) => undefined | Media;
|
|
124
|
+
/**
|
|
125
|
+
* Use which processor API to process the stream track
|
|
126
|
+
* `stream` - Use MediaStreamTrackProcessor, when available
|
|
127
|
+
* `canvas` - Use Canvas
|
|
128
|
+
*/
|
|
129
|
+
type VideoStreamTrackProcessorAPIs = 'stream' | 'canvas';
|
|
130
|
+
declare enum UserMediaStatus {
|
|
131
|
+
/**
|
|
132
|
+
* The initial status
|
|
133
|
+
*/
|
|
134
|
+
Initial = "initial",
|
|
135
|
+
/**
|
|
136
|
+
* When we know the permissions were already granted from permission API
|
|
137
|
+
*/
|
|
138
|
+
InitialPermissionsGranted = "initial-permissions-granted",
|
|
139
|
+
/**
|
|
140
|
+
* When we do not know the permissions from permission API
|
|
141
|
+
*/
|
|
142
|
+
InitialPermissionsNotGranted = "initial-permissions-not-granted",
|
|
143
|
+
/**
|
|
144
|
+
* When video input permissions are initially denied in the browser and audio input permissions are unknown
|
|
145
|
+
*/
|
|
146
|
+
InitialPermissionsVideoInputDenied = "initial-permissions-videoinput-denied",
|
|
147
|
+
/**
|
|
148
|
+
* When audio input permissions are initially denied in the browser and video input permissions are unknown
|
|
149
|
+
*/
|
|
150
|
+
InitialPermissionsAudioInputDenied = "initial-permissions-audioinput-denied",
|
|
151
|
+
/**
|
|
152
|
+
* When video input permissions are granted in the browser and audio input permissions are unknown
|
|
153
|
+
*/
|
|
154
|
+
InitialPermissionsVideoInputGranted = "initial-permissions-videoinput-granted",
|
|
155
|
+
/**
|
|
156
|
+
* When audio input permissions are granted in the browser and video input permissions are unknown
|
|
157
|
+
*/
|
|
158
|
+
InitialPermissionsAudioInputGranted = "initial-permissions-audioinput-granted",
|
|
159
|
+
/**
|
|
160
|
+
* When audio input permissions are granted but video input permissions are initially denied
|
|
161
|
+
*/
|
|
162
|
+
InitialPermissionsGrantedVideoInputDenied = "initial-permissions-audioinput-granted-videoinput-denied",
|
|
163
|
+
/**
|
|
164
|
+
* When video input permissions are granted but audio input permissions are initially denied
|
|
165
|
+
*/
|
|
166
|
+
InitialPermissionsGrantedAudioInputDenied = "initial-permissions-videoinput-granted-audioinput-denied",
|
|
167
|
+
/**
|
|
168
|
+
* When there is no any kind of input devices
|
|
169
|
+
*/
|
|
170
|
+
NoDevicesFound = "no-devices-found",
|
|
171
|
+
/**
|
|
172
|
+
* When there is no any video input devices
|
|
173
|
+
*/
|
|
174
|
+
NoVideoDevicesFound = "no-video-devices-found",
|
|
175
|
+
/**
|
|
176
|
+
* When there is no any audio input devices
|
|
177
|
+
*/
|
|
178
|
+
NoAudioDevicesFound = "no-audio-devices-found",
|
|
179
|
+
/**
|
|
180
|
+
* Derived from `MediaDeviceFailure.AudioInputDeviceNotFoundError`
|
|
181
|
+
*/
|
|
182
|
+
AudioDeviceNotFound = "audio-device-not-found",
|
|
183
|
+
/**
|
|
184
|
+
* Derived from `MediaDeviceFailure.VideoInputDeviceNotFoundError`
|
|
185
|
+
*/
|
|
186
|
+
VideoDeviceNotFound = "video-device-not-found",
|
|
187
|
+
/**
|
|
188
|
+
* Derived from `MediaDeviceFailure.AudioAndVideoDeviceNotFoundError`
|
|
189
|
+
*/
|
|
190
|
+
AudioVideoDevicesNotFound = "audio-video-devices-not-found",
|
|
191
|
+
/**
|
|
192
|
+
* When Permission is granted by user for both video and audio, and both
|
|
193
|
+
* devices are exactly matched with the request constraints
|
|
194
|
+
*/
|
|
195
|
+
PermissionsGranted = "permissions-granted",
|
|
196
|
+
/**
|
|
197
|
+
* When Permission is granted by user for both video and audio, and both
|
|
198
|
+
* devices are NOT exactly matched with the request constraints
|
|
199
|
+
*/
|
|
200
|
+
PermissionsGrantedFallback = "permissions-granted-fallback-devices",
|
|
201
|
+
/**
|
|
202
|
+
* When Permission is granted by user for both video and audio, and audio
|
|
203
|
+
* input is NOT exactly matched with the request constraints
|
|
204
|
+
*/
|
|
205
|
+
PermissionsGrantedFallbackAudioinput = "permissions-granted-fallback-audioinput",
|
|
206
|
+
/**
|
|
207
|
+
* When Permission is granted by user for both video and audio, and video
|
|
208
|
+
* input is NOT exactly matched with the request constraints
|
|
209
|
+
*/
|
|
210
|
+
PermissionsGrantedFallbackVideoinput = "permissions-granted-fallback-videoinput",
|
|
211
|
+
/**
|
|
212
|
+
* When Permission for using both audio and video devices are rejected by the user
|
|
213
|
+
* from `PermissionDeniedError`
|
|
214
|
+
*/
|
|
215
|
+
PermissionsRejected = "permissions-rejected",
|
|
216
|
+
/**
|
|
217
|
+
* When Permission for using the audio device is rejected by the user
|
|
218
|
+
* from `PermissionDeniedError`
|
|
219
|
+
*/
|
|
220
|
+
PermissionsRejectedAudioInput = "permissions-rejected-audioinput",
|
|
221
|
+
/**
|
|
222
|
+
* When Permission for using the video device is rejected by the user
|
|
223
|
+
* from `PermissionDeniedError`
|
|
224
|
+
*/
|
|
225
|
+
PermissionsRejectedVideoInput = "permissions-rejected-videoinput",
|
|
226
|
+
/**
|
|
227
|
+
* When only request and return exact audio input device
|
|
228
|
+
*/
|
|
229
|
+
PermissionsOnlyAudioinput = "permissions-only-audioinput",
|
|
230
|
+
/**
|
|
231
|
+
* When only request audio input device because of no video devices
|
|
232
|
+
* available and returned exact audio input device
|
|
233
|
+
*/
|
|
234
|
+
PermissionsOnlyAudioinputNoVideoDevices = "permissions-only-audioinput-no-video-devices",
|
|
235
|
+
/**
|
|
236
|
+
* When only request and returned NOT exact audio input device
|
|
237
|
+
*/
|
|
238
|
+
PermissionsOnlyAudioinputFallback = "permissions-only-fallback-audioinput",
|
|
239
|
+
/**
|
|
240
|
+
* When only request audio input device because of no video devices
|
|
241
|
+
* available and returned NOT exact audio input device
|
|
242
|
+
*/
|
|
243
|
+
PermissionsOnlyAudioinputFallbackNoVideoDevices = "permissions-only-fallback-audioinput-no-video-devices",
|
|
244
|
+
/**
|
|
245
|
+
* When only request and return exact video input device
|
|
246
|
+
*/
|
|
247
|
+
PermissionsOnlyVideoinput = "permissions-only-videoinput",
|
|
248
|
+
/**
|
|
249
|
+
* When only request video input device because of no audio devices
|
|
250
|
+
* available and returned exact video input device
|
|
251
|
+
*/
|
|
252
|
+
PermissionsOnlyVideoinputNoAudioDevices = "permissions-only-videoinput-no-audio-devices",
|
|
253
|
+
/**
|
|
254
|
+
* When only request and returned NOT exact video input device
|
|
255
|
+
*/
|
|
256
|
+
PermissionsOnlyVideoinputFallback = "permissions-only-fallback-videoinput",
|
|
257
|
+
/**
|
|
258
|
+
* When only request video input device because of no video devices
|
|
259
|
+
* available and returned NOT exact video input device
|
|
260
|
+
*/
|
|
261
|
+
PermissionsOnlyVideoinputFallbackNoAudioDevices = "permissions-only-fallback-videoinput-no-audio-devices",
|
|
262
|
+
/**
|
|
263
|
+
* When requesting the audio device is used by other application
|
|
264
|
+
*/
|
|
265
|
+
AudioDeviceInUse = "audio-device-in-use",
|
|
266
|
+
/**
|
|
267
|
+
* When requesting the video device is used by other application
|
|
268
|
+
*/
|
|
269
|
+
VideoDeviceInUse = "video-device-in-use",
|
|
270
|
+
/**
|
|
271
|
+
* When requesting both audio and video devices are used by other application
|
|
272
|
+
*/
|
|
273
|
+
DevicesInUse = "devices-in-use",
|
|
274
|
+
/**
|
|
275
|
+
* When requesting both audio and video with over-constrained
|
|
276
|
+
*/
|
|
277
|
+
Overconstrained = "overconstrained",
|
|
278
|
+
/**
|
|
279
|
+
* When requesting video with over-constrained
|
|
280
|
+
*/
|
|
281
|
+
VideoOverconstrained = "video-overconstrained",
|
|
282
|
+
/**
|
|
283
|
+
* When requesting audio with over-constrained
|
|
284
|
+
*/
|
|
285
|
+
AudioOverconstrained = "audio-overconstrained",
|
|
286
|
+
/**
|
|
287
|
+
* When requesting both audio and video with invalid constraints
|
|
288
|
+
*/
|
|
289
|
+
InvalidConstraints = "invalid-constraints",
|
|
290
|
+
/**
|
|
291
|
+
* When requesting video with invalid constraints
|
|
292
|
+
*/
|
|
293
|
+
InvalidVideoConstraints = "invalid-video-constraints",
|
|
294
|
+
/**
|
|
295
|
+
* When requesting audio with invalid constraints
|
|
296
|
+
*/
|
|
297
|
+
InvalidAudioConstraints = "invalid-audio-constraints",
|
|
298
|
+
/**
|
|
299
|
+
* When requesting both audio and video with NotSupportedError
|
|
300
|
+
*/
|
|
301
|
+
NotSupportedError = "not-supported-error",
|
|
302
|
+
/**
|
|
303
|
+
* When requesting video with NotSupportedError
|
|
304
|
+
*/
|
|
305
|
+
NotSupportedErrorOnlyVideoInput = "not-supported-error-only-video",
|
|
306
|
+
/**
|
|
307
|
+
* When requesting audio with NotSupportedError
|
|
308
|
+
*/
|
|
309
|
+
NotSupportedErrorOnlyAudioInput = "not-supported-error-only-audio",
|
|
310
|
+
/**
|
|
311
|
+
* Unknown error from both video and audio
|
|
312
|
+
*/
|
|
313
|
+
UnknownError = "unknown-error",
|
|
314
|
+
/**
|
|
315
|
+
* Unknown error from audio device
|
|
316
|
+
*/
|
|
317
|
+
UnknownErrorOnlyAudioinput = "unknown-error-only-audioinput",
|
|
318
|
+
/**
|
|
319
|
+
* Unknown error from video device
|
|
320
|
+
*/
|
|
321
|
+
UnknownErrorOnlyVideoinput = "unknown-error-only-videoinput"
|
|
322
|
+
}
|
|
323
|
+
interface AudioSignalDetectionOptions {
|
|
324
|
+
/**
|
|
325
|
+
* Audio Signal Detection Duration in second
|
|
326
|
+
*/
|
|
327
|
+
audioSignalDetectionDuration?: number;
|
|
328
|
+
/**
|
|
329
|
+
* Whether or not to detect audio signal for malfunctioning device
|
|
330
|
+
*/
|
|
331
|
+
shouldDetectAudio: () => boolean;
|
|
332
|
+
}
|
|
333
|
+
interface VoiceActivityDetectionOptions {
|
|
334
|
+
/**
|
|
335
|
+
* Voice Activity Detection in millisecond
|
|
336
|
+
*/
|
|
337
|
+
vadThrottleMS?: number;
|
|
338
|
+
/**
|
|
339
|
+
* Whether or not to detect voice activity
|
|
340
|
+
*/
|
|
341
|
+
shouldDetectVoiceActivity: () => boolean;
|
|
342
|
+
}
|
|
343
|
+
interface DeviceMuteState {
|
|
344
|
+
/**
|
|
345
|
+
* Indicating whether audio is muted
|
|
346
|
+
*/
|
|
347
|
+
audio: boolean;
|
|
348
|
+
/**
|
|
349
|
+
* Indicating whether video is muted
|
|
350
|
+
*/
|
|
351
|
+
video: boolean;
|
|
352
|
+
}
|
|
353
|
+
interface MediaOptions {
|
|
354
|
+
/**
|
|
355
|
+
* Media signals to be used for the module
|
|
356
|
+
*
|
|
357
|
+
* @see MediaSignals
|
|
358
|
+
*/
|
|
359
|
+
signals: MediaSignals;
|
|
360
|
+
/**
|
|
361
|
+
* Media Processors
|
|
362
|
+
*/
|
|
363
|
+
mediaProcessors: MediaProcessor[];
|
|
364
|
+
/**
|
|
365
|
+
* A function to get the devices' mute state
|
|
366
|
+
*/
|
|
367
|
+
getMuteState: () => DeviceMuteState;
|
|
368
|
+
/**
|
|
369
|
+
* Pass default constraints to use with get media wrappers
|
|
370
|
+
*/
|
|
371
|
+
getDefaultConstraints?: () => {
|
|
372
|
+
audio?: InputConstraintSet;
|
|
373
|
+
video?: InputConstraintSet;
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
interface MediaProps {
|
|
377
|
+
/**
|
|
378
|
+
* Current media
|
|
379
|
+
*/
|
|
380
|
+
media: Media;
|
|
381
|
+
/**
|
|
382
|
+
* Current device
|
|
383
|
+
*/
|
|
384
|
+
devices: MediaDeviceInfoLike[];
|
|
385
|
+
/**
|
|
386
|
+
* When should we discard the requested MediaStream
|
|
387
|
+
*/
|
|
388
|
+
discardMedia: boolean;
|
|
389
|
+
}
|
|
390
|
+
interface MediaController {
|
|
391
|
+
/**
|
|
392
|
+
* Current Media
|
|
393
|
+
*/
|
|
394
|
+
media: Media;
|
|
395
|
+
/**
|
|
396
|
+
* Current device list
|
|
397
|
+
*/
|
|
398
|
+
devices: MediaDeviceInfoLike[];
|
|
399
|
+
/**
|
|
400
|
+
* Execute the media pipeline immediately with provided constraints
|
|
401
|
+
*
|
|
402
|
+
* @param constraints - @see MediaDeviceRequest
|
|
403
|
+
*/
|
|
404
|
+
getUserMedia: (constraints: MediaDeviceRequest) => void;
|
|
405
|
+
/**
|
|
406
|
+
* Execute the media pipeline immediately with provided constraints. An
|
|
407
|
+
* explicit version of `getUserMedia` to make it possible to chain other
|
|
408
|
+
* async actions
|
|
409
|
+
*
|
|
410
|
+
* @param constraints - @see MediaDeviceRequest
|
|
411
|
+
*/
|
|
412
|
+
getUserMediaAsync: (constraints: MediaDeviceRequest) => Promise<void>;
|
|
413
|
+
/**
|
|
414
|
+
* Cross-check PermissionState and available devices before requesting user
|
|
415
|
+
* media, the request will be skipped if there is no granted device.
|
|
416
|
+
*/
|
|
417
|
+
tryAndGetUserMedia: (constraints: MediaDeviceRequest) => void;
|
|
418
|
+
}
|
|
419
|
+
type VideoRenderParams = Omit<RenderParams, 'backgroundImage' | 'effects'> & {
|
|
420
|
+
/**
|
|
421
|
+
* Target frame rate for the video segmentation
|
|
422
|
+
*/
|
|
423
|
+
frameRate: number;
|
|
424
|
+
/**
|
|
425
|
+
* Default background image URL for overlay effects
|
|
426
|
+
*/
|
|
427
|
+
bgImageUrl: string;
|
|
428
|
+
/**
|
|
429
|
+
* Render Effects
|
|
430
|
+
*/
|
|
431
|
+
videoSegmentation: RenderEffects;
|
|
432
|
+
};
|
|
433
|
+
interface StreamTrackSignals {
|
|
434
|
+
/**
|
|
435
|
+
* MediaStreamTrack events: mute
|
|
436
|
+
* https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack#events
|
|
437
|
+
*/
|
|
438
|
+
onStreamTrackMuted: Signal<MediaStreamTrack>;
|
|
439
|
+
/**
|
|
440
|
+
* MediaStreamTrack events: unmute
|
|
441
|
+
* https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack#events
|
|
442
|
+
*/
|
|
443
|
+
onStreamTrackUnmuted: Signal<MediaStreamTrack>;
|
|
444
|
+
/**
|
|
445
|
+
* MediaStreamTrack events: ended
|
|
446
|
+
* https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack#events
|
|
447
|
+
*/
|
|
448
|
+
onStreamTrackEnded: Signal<MediaStreamTrack>;
|
|
449
|
+
/**
|
|
450
|
+
* Emit `MediaStreamTrack` whenever a call to `Media['muteAudio']` or
|
|
451
|
+
* `Media['muteVideo']`
|
|
452
|
+
*/
|
|
453
|
+
onStreamTrackEnabled: Signal<MediaStreamTrack>;
|
|
454
|
+
}
|
|
455
|
+
interface MediaChangesSignals {
|
|
456
|
+
onDevicesChanged: Signal<MediaDeviceInfoLike[]>;
|
|
457
|
+
onStatusChanged: Signal<UserMediaStatus>;
|
|
458
|
+
onMediaChanged: Signal<Media>;
|
|
459
|
+
}
|
|
460
|
+
interface AudioDetectionSignals {
|
|
461
|
+
onVAD: Signal<undefined>;
|
|
462
|
+
onSilentDetected: Signal<boolean>;
|
|
463
|
+
}
|
|
464
|
+
type MediaSignalsOptional = Pick<Partial<MediaChangesSignals>, 'onDevicesChanged' | 'onStatusChanged'> & Partial<StreamTrackSignals>;
|
|
465
|
+
type MediaSignalsRequired = Pick<MediaChangesSignals, 'onMediaChanged'> & AudioDetectionSignals;
|
|
466
|
+
type MediaSignals = MediaSignalsRequired & MediaSignalsOptional;
|
|
467
|
+
declare enum DeniedDevices {
|
|
468
|
+
Microphone = "microphone",
|
|
469
|
+
Camera = "camera",
|
|
470
|
+
Both = "microphone-and-camera"
|
|
471
|
+
}
|
|
472
|
+
interface Segmenters {
|
|
473
|
+
mediapipeSelfie: Segmenter;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* Create an object to interact with the media scream, which is usually used for
|
|
478
|
+
* our main stream.
|
|
479
|
+
*
|
|
480
|
+
* @param options - @see MediaOptions
|
|
481
|
+
*/
|
|
482
|
+
declare const createMedia: ({ getMuteState, signals, mediaProcessors, getDefaultConstraints, }: MediaOptions) => MediaController;
|
|
483
|
+
|
|
484
|
+
type AudioNodeInits = AudioNodeInit[];
|
|
485
|
+
/**
|
|
486
|
+
* A function to be called to create the AudioNodes needed for the graph
|
|
487
|
+
* creation
|
|
488
|
+
*
|
|
489
|
+
* @param media - Media to be used for the AudioGraph creation
|
|
490
|
+
*/
|
|
491
|
+
type CreateNodes = (media: Media) => AudioNodeInits;
|
|
492
|
+
interface AudioProcessOptions {
|
|
493
|
+
/**
|
|
494
|
+
* An option is being passed to AnalyserNode creation when used
|
|
495
|
+
* @see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/fftSize
|
|
496
|
+
*
|
|
497
|
+
* @defaultValue 2048
|
|
498
|
+
*/
|
|
499
|
+
fftSize?: number;
|
|
500
|
+
/**
|
|
501
|
+
* Params needed for setting up noise suppression WebAssembly and
|
|
502
|
+
* AudioWorklet
|
|
503
|
+
*/
|
|
504
|
+
denoiseParams?: DenoiseParams;
|
|
505
|
+
/**
|
|
506
|
+
* Update frequency for analyzer per second
|
|
507
|
+
*
|
|
508
|
+
* @defaultValue 0.5
|
|
509
|
+
*/
|
|
510
|
+
analyzerUpdateFrequency?: number;
|
|
511
|
+
/**
|
|
512
|
+
* Audio Signal Detection duration in second
|
|
513
|
+
*
|
|
514
|
+
* @defaultValue 4.0
|
|
515
|
+
*/
|
|
516
|
+
audioSignalDetectionDuration?: number;
|
|
517
|
+
/**
|
|
518
|
+
* Callback when Voice Activity detected
|
|
519
|
+
*/
|
|
520
|
+
onVoiceActivityDetected?: () => void;
|
|
521
|
+
/**
|
|
522
|
+
* Callback when Audio Signal detected
|
|
523
|
+
*/
|
|
524
|
+
onAudioSignalDetected?: (silent: boolean) => void;
|
|
525
|
+
/**
|
|
526
|
+
* @see AudioGraphOptions
|
|
527
|
+
*/
|
|
528
|
+
audioGraphOptions?: AudioGraphOptions;
|
|
529
|
+
/**
|
|
530
|
+
* Whether or to enable this processor
|
|
531
|
+
*/
|
|
532
|
+
shouldEnable: () => boolean;
|
|
533
|
+
/**
|
|
534
|
+
* Insert additional nodes between the source and destination
|
|
535
|
+
*/
|
|
536
|
+
createNodes?: CreateNodes;
|
|
537
|
+
/**
|
|
538
|
+
* Silent threshold, how large the value of the sample is considered as
|
|
539
|
+
* silent in FFTed time domain
|
|
540
|
+
*/
|
|
541
|
+
silentThreshold?: number;
|
|
542
|
+
scope?: string;
|
|
543
|
+
}
|
|
544
|
+
/**
|
|
545
|
+
* Create a Audio Stream Processor and will own the stream passed-in
|
|
546
|
+
*/
|
|
547
|
+
declare const createAudioStreamProcess: ({ analyzerUpdateFrequency, audioGraphOptions, audioSignalDetectionDuration, clock, createNodes, denoiseParams, fftSize, onAudioSignalDetected, onVoiceActivityDetected, shouldEnable, silentThreshold, throttleMs, scope, }: AudioProcessOptions & ThrottleOptions) => Process<Promise<Media>>;
|
|
548
|
+
|
|
549
|
+
interface ProcessorDeps {
|
|
550
|
+
videoProcessor?: VideoProcessor;
|
|
551
|
+
transformer?: SegmentationTransform;
|
|
552
|
+
segmenters: Segmenters;
|
|
553
|
+
videoSegmentationModel?: SegmentationModel;
|
|
554
|
+
}
|
|
555
|
+
interface VideoStreamProcessOptions extends Partial<VideoRenderParams>, Omit<ProcessorDeps, 'videoProcessor'> {
|
|
556
|
+
/**
|
|
557
|
+
* What API to use for processing the MediaStreamTrack
|
|
558
|
+
* `stream` - Use MediaStreamTrackProcessor, when available
|
|
559
|
+
* `canvas` - Use Canvas
|
|
560
|
+
*/
|
|
561
|
+
trackProcessorAPI?: VideoStreamTrackProcessorAPIs;
|
|
562
|
+
/**
|
|
563
|
+
* Whether or to enable this processor
|
|
564
|
+
*/
|
|
565
|
+
shouldEnable: () => boolean;
|
|
566
|
+
/**
|
|
567
|
+
* Callback when error occurs
|
|
568
|
+
*/
|
|
569
|
+
onError?: (error: Error) => void;
|
|
570
|
+
processingWidth: number;
|
|
571
|
+
processingHeight: number;
|
|
572
|
+
hasInitializedDeps?: boolean;
|
|
573
|
+
width?: number;
|
|
574
|
+
height?: number;
|
|
575
|
+
scope?: string;
|
|
576
|
+
}
|
|
577
|
+
declare const createVideoStreamProcess: ({ trackProcessorAPI, processingWidth, processingHeight, shouldEnable, frameRate, videoSegmentation, foregroundThreshold, bgImageUrl, flipHorizontal, edgeBlurAmount, scope, ...options }: VideoStreamProcessOptions) => Process<Promise<Media>>;
|
|
578
|
+
|
|
579
|
+
/**
|
|
580
|
+
* Log meta and message with respective log level
|
|
581
|
+
*/
|
|
582
|
+
type LogMethod = (meta: unknown, message?: string) => void;
|
|
583
|
+
declare enum LogLevels {
|
|
584
|
+
trace = 10,
|
|
585
|
+
debug = 20,
|
|
586
|
+
info = 30,
|
|
587
|
+
warn = 40,
|
|
588
|
+
error = 50,
|
|
589
|
+
fatal = 60,
|
|
590
|
+
silent
|
|
591
|
+
}
|
|
592
|
+
type LogLevelsString = keyof typeof LogLevels;
|
|
593
|
+
type LogMethods = {
|
|
594
|
+
[key in LogLevelsString]: LogMethod;
|
|
595
|
+
};
|
|
596
|
+
/**
|
|
597
|
+
* Log Level from high to low, "fatal" | "error" | "warn" | "info" | "debug" | "trace"
|
|
598
|
+
* Typically, debug and trace logs are only valid for development, and not needed in production
|
|
599
|
+
*/
|
|
600
|
+
interface Logger extends LogMethods {
|
|
601
|
+
/**
|
|
602
|
+
* Adds a value to the redaction set, which makes it replaced by [REDACTED] when logged to file.
|
|
603
|
+
*
|
|
604
|
+
* @remarks
|
|
605
|
+
* The redaction set is applied globally, and only applies to the log file, not console logs.
|
|
606
|
+
*
|
|
607
|
+
* @param value - the string to redact
|
|
608
|
+
*/
|
|
609
|
+
redact(value: string): void;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
declare function setLogger(newLogger: Logger): void;
|
|
613
|
+
|
|
614
|
+
type EventCallback<T> = (event: T) => void;
|
|
615
|
+
type EventErrorCallback = (error: Error) => void;
|
|
616
|
+
type PreviewInput = MediaDeviceInfoLike | undefined;
|
|
617
|
+
interface PreviewEventHandler {
|
|
618
|
+
audioInput?: EventCallback<PreviewInput>;
|
|
619
|
+
videoInput?: EventCallback<PreviewInput>;
|
|
620
|
+
media?: EventCallback<Media>;
|
|
621
|
+
videoInputError?: EventErrorCallback;
|
|
622
|
+
audioInputError?: EventErrorCallback;
|
|
623
|
+
applyChangesError?: EventErrorCallback;
|
|
624
|
+
revertChangesError?: EventErrorCallback;
|
|
625
|
+
updatingPreview?: EventCallback<boolean>;
|
|
626
|
+
updatingMain?: EventCallback<boolean>;
|
|
627
|
+
unsubscribeMain?: Unsubscribe;
|
|
628
|
+
}
|
|
629
|
+
interface PreviewStreamParams {
|
|
630
|
+
getCurrentDevices: () => MediaDeviceInfoLike[];
|
|
631
|
+
getCurrentMedia: () => Media | undefined;
|
|
632
|
+
updateMainStream: (request: MediaDeviceRequest) => Promise<void>;
|
|
633
|
+
mediaSignal: MediaSignals['onMediaChanged'];
|
|
634
|
+
onEnded?: () => void;
|
|
635
|
+
fftSize?: number;
|
|
636
|
+
queueOptions?: Partial<AsyncQueueOptions>;
|
|
637
|
+
processors: MediaProcessor[];
|
|
638
|
+
}
|
|
639
|
+
interface PreviewControllerProps {
|
|
640
|
+
media: Media;
|
|
641
|
+
audioInput?: MediaDeviceInfoLike;
|
|
642
|
+
videoInput?: MediaDeviceInfoLike;
|
|
643
|
+
updatingPreview: boolean;
|
|
644
|
+
updatingMain: boolean;
|
|
645
|
+
originalMainAudioInput?: MediaDeviceInfoLike;
|
|
646
|
+
discardMedia: boolean;
|
|
647
|
+
initialized: boolean;
|
|
648
|
+
}
|
|
649
|
+
interface PreviewStreamController {
|
|
650
|
+
media: Media;
|
|
651
|
+
audioInputChanged: boolean;
|
|
652
|
+
videoInputChanged: boolean;
|
|
653
|
+
inputChanged: boolean;
|
|
654
|
+
audioInput: PreviewInput;
|
|
655
|
+
videoInput: PreviewInput;
|
|
656
|
+
updatingPreview: boolean;
|
|
657
|
+
updatingMain: boolean;
|
|
658
|
+
updateAudioInput(input: PreviewInput): void;
|
|
659
|
+
updateVideoInput(input: PreviewInput): void;
|
|
660
|
+
applyChanges(): Promise<void>;
|
|
661
|
+
revertChanges(): Promise<void>;
|
|
662
|
+
onMediaChanged(callback: EventCallback<Media>): Unsubscribe;
|
|
663
|
+
onAudioInputChanged(callback: EventCallback<PreviewInput>): Unsubscribe;
|
|
664
|
+
onVideoInputChanged(callback: EventCallback<PreviewInput>): Unsubscribe;
|
|
665
|
+
onUpdatingPreview(callback: EventCallback<boolean>): Unsubscribe;
|
|
666
|
+
onUpdatingMain(callback: EventCallback<boolean>): Unsubscribe;
|
|
667
|
+
onAudioInputError(callback: EventErrorCallback): Unsubscribe;
|
|
668
|
+
onVideoInputError(callback: EventErrorCallback): Unsubscribe;
|
|
669
|
+
onApplyChangesError(callback: EventErrorCallback): Unsubscribe;
|
|
670
|
+
onRevertChangesError(callback: EventErrorCallback): Unsubscribe;
|
|
671
|
+
}
|
|
672
|
+
declare const createPreviewStreamController: ({ getCurrentDevices, getCurrentMedia, updateMainStream, onEnded, mediaSignal, queueOptions, processors, }: PreviewStreamParams) => PreviewStreamController;
|
|
673
|
+
type CreatePreviewStreamController = typeof createPreviewStreamController;
|
|
674
|
+
|
|
675
|
+
type UserMediaValidator = (status: UserMediaStatus) => boolean;
|
|
676
|
+
declare const isFallbackVideo: UserMediaValidator;
|
|
677
|
+
declare const isFallbackAudio: UserMediaValidator;
|
|
678
|
+
declare const isFallback: UserMediaValidator;
|
|
679
|
+
declare const hasNoDevice: UserMediaValidator;
|
|
680
|
+
declare const hasNoAudioDevices: UserMediaValidator;
|
|
681
|
+
declare const hasNoVideoDevices: UserMediaValidator;
|
|
682
|
+
declare const isGrantedOnlyVideoNoAudioDevices: UserMediaValidator;
|
|
683
|
+
declare const isPromptAudio: UserMediaValidator;
|
|
684
|
+
declare const isGrantedOnlyVideo: UserMediaValidator;
|
|
685
|
+
declare const isGrantedOnlyAudioNoVideoDevices: UserMediaValidator;
|
|
686
|
+
declare const isPromptVideo: UserMediaValidator;
|
|
687
|
+
declare const isGrantedOnlyAudio: UserMediaValidator;
|
|
688
|
+
declare const areBothGranted: UserMediaValidator;
|
|
689
|
+
declare const isGrantedVideo: UserMediaValidator;
|
|
690
|
+
declare const isGrantedAudio: UserMediaValidator;
|
|
691
|
+
declare const isGranted: UserMediaValidator;
|
|
692
|
+
declare const isOnlyAudioError: UserMediaValidator;
|
|
693
|
+
declare const isOnlyVideoError: UserMediaValidator;
|
|
694
|
+
declare const isRejected: UserMediaValidator;
|
|
695
|
+
declare const isRejectedOnlyAudio: UserMediaValidator;
|
|
696
|
+
declare const isRejectedOnlyVideo: UserMediaValidator;
|
|
697
|
+
declare const isOverConstrained: UserMediaValidator;
|
|
698
|
+
declare const isUnknownError: UserMediaValidator;
|
|
699
|
+
declare const isInitial: UserMediaValidator;
|
|
700
|
+
declare const isInitialPermissions: UserMediaValidator;
|
|
701
|
+
declare const isInitialPermissionsNotGranted: UserMediaValidator;
|
|
702
|
+
declare const isInitialPermissionsGranted: UserMediaValidator;
|
|
703
|
+
declare const isAudioDeviceInUse: UserMediaValidator;
|
|
704
|
+
declare const isDeviceInUse: UserMediaValidator;
|
|
705
|
+
declare const isVideoDeviceInUse: UserMediaValidator;
|
|
706
|
+
declare const toDeniedDevices: (status?: UserMediaStatus) => DeniedDevices | undefined;
|
|
707
|
+
declare const getPermissionStatus: (getPermissionState?: (anyActiveStream?: boolean | undefined) => Promise<_pexip_media_control.InputDevicePermission>) => Promise<UserMediaStatus.Initial | UserMediaStatus.InitialPermissionsGranted | UserMediaStatus.InitialPermissionsNotGranted | UserMediaStatus.InitialPermissionsVideoInputDenied | UserMediaStatus.InitialPermissionsAudioInputDenied | UserMediaStatus.InitialPermissionsVideoInputGranted | UserMediaStatus.InitialPermissionsAudioInputGranted | UserMediaStatus.InitialPermissionsGrantedVideoInputDenied | UserMediaStatus.InitialPermissionsGrantedAudioInputDenied | UserMediaStatus.PermissionsRejected>;
|
|
708
|
+
declare const deriveInitialPermissionStatus: (prevStatus: UserMediaStatus, getPermissionState?: (anyActiveStream?: boolean | undefined) => Promise<_pexip_media_control.InputDevicePermission>) => Promise<UserMediaStatus>;
|
|
709
|
+
|
|
710
|
+
/**
|
|
711
|
+
* Create a general signal with consistent scoped name
|
|
712
|
+
*
|
|
713
|
+
* @param name - Signal name
|
|
714
|
+
* @param crucial - Signify if the signal is unmissable. @defaultValue true
|
|
715
|
+
* @param variant - The variant of the signal @see Signal @defaultValue 'generic'
|
|
716
|
+
*/
|
|
717
|
+
declare const createMediaSignal: <T = undefined>(name: string, crucial?: boolean, variant?: SignalVariant) => Signal<T>;
|
|
718
|
+
declare const REQUIRED_SIGNAL_KEYS: readonly ["onMediaChanged", "onVAD", "onSilentDetected"];
|
|
719
|
+
/**
|
|
720
|
+
* Create and return all required and optional (if specified with `more`),
|
|
721
|
+
* signals for media to work
|
|
722
|
+
*
|
|
723
|
+
* @param more - Keys from `MediaSignalsOptional`, @see MediaSignalsOptional
|
|
724
|
+
* @param scope - any scope prefix for the generated signal name, @see Signal
|
|
725
|
+
*
|
|
726
|
+
* The following signals created by default
|
|
727
|
+
* - 'onMediaChanged',
|
|
728
|
+
* - 'onVAD',
|
|
729
|
+
*
|
|
730
|
+
* @see REQUIRED_SIGNAL_KEYS
|
|
731
|
+
*/
|
|
732
|
+
declare const createMediaSignals: <K extends "onDevicesChanged" | "onStatusChanged" | keyof StreamTrackSignals>(more: K[], scope?: string) => Pick<Required<MediaSignals>, "onMediaChanged" | "onVAD" | "onSilentDetected" | K>;
|
|
733
|
+
|
|
734
|
+
/**
|
|
735
|
+
* Create a Audio Mixing Processor and will own the stream passed-in
|
|
736
|
+
*/
|
|
737
|
+
declare const createAudioMixingProcess: (getCurrrentMedia: () => MediaStream | undefined, scope?: string) => Process<Promise<Media>>;
|
|
738
|
+
|
|
739
|
+
export { AudioDetectionSignals, AudioSignalDetectionOptions, CreatePreviewStreamController, DeniedDevices, DenoiseParams, ExtendedMediaTrackSettings, ExtendedMediaTrackSettingsKey, Media, MediaAttributes, MediaChangesSignals, MediaController, MediaOptions, MediaProcessor, MediaProps, MediaSettings, MediaSignals, MediaSignalsOptional, MediaSignalsRequired, Pipeline, PreviewControllerProps, PreviewEventHandler, PreviewInput, PreviewStreamController, PreviewStreamParams, Process, ProcessMedia, REQUIRED_SIGNAL_KEYS, Segmenters, StreamTrackSignals, Unsubscribe, UserMediaStatus, UserMediaValidator, VideoRenderParams, VideoStreamTrackProcessorAPIs, VoiceActivityDetectionOptions, areBothGranted, createAudioMixingProcess, createAudioStreamProcess, createMedia, createMediaSignal, createMediaSignals, createPreviewStreamController, createVideoStreamProcess, deriveInitialPermissionStatus, getPermissionStatus, hasNoAudioDevices, hasNoDevice, hasNoVideoDevices, isAudioDeviceInUse, isDeviceInUse, isFallback, isFallbackAudio, isFallbackVideo, isGranted, isGrantedAudio, isGrantedOnlyAudio, isGrantedOnlyAudioNoVideoDevices, isGrantedOnlyVideo, isGrantedOnlyVideoNoAudioDevices, isGrantedVideo, isInitial, isInitialPermissions, isInitialPermissionsGranted, isInitialPermissionsNotGranted, isOnlyAudioError, isOnlyVideoError, isOverConstrained, isPromptAudio, isPromptVideo, isRejected, isRejectedOnlyAudio, isRejectedOnlyVideo, isUnknownError, isVideoDeviceInUse, setLogger, toDeniedDevices };
|