@micdrop/client 2.2.10 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -23
- package/dist/index.d.mts +581 -120
- package/dist/index.d.ts +581 -120
- package/dist/index.js +1100 -696
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1079 -694
- package/dist/index.mjs.map +1 -1
- package/package.json +10 -6
package/dist/index.d.mts
CHANGED
|
@@ -1,58 +1,279 @@
|
|
|
1
1
|
import { EventEmitter } from 'eventemitter3';
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
/**
|
|
4
|
+
* An input or output audio device offered by the system
|
|
5
|
+
*/
|
|
6
|
+
interface MicdropDevice {
|
|
7
|
+
/** Identifier to hand back to `changeMicDevice` or `changeSpeakerDevice` */
|
|
8
|
+
id: string;
|
|
9
|
+
/** Name to show, as the system spells it */
|
|
10
|
+
label: string;
|
|
11
|
+
kind: 'audioinput' | 'audiooutput';
|
|
5
12
|
}
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
13
|
+
interface MicEvents {
|
|
14
|
+
/** A batch of mono samples, in the -1..1 range, with its sample rate */
|
|
15
|
+
Frames: [Float32Array, number];
|
|
16
|
+
/** Level of the captured audio, emitted about ten times a second */
|
|
17
|
+
Volume: [number];
|
|
18
|
+
/** The system gained or lost an audio device */
|
|
19
|
+
DeviceChange: [];
|
|
20
|
+
Error: [Error];
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* What a {@link VAD} listens to. The `Mic` singleton implements it, and so can
|
|
24
|
+
* any other emitter, which makes voice activity detection testable on its own.
|
|
25
|
+
*/
|
|
26
|
+
type MicSource = Pick<EventEmitter<MicEvents>, 'on' | 'off'>;
|
|
27
|
+
interface MicDriverEvents {
|
|
28
|
+
/** A batch of mono samples, in the -1..1 range, with its sample rate */
|
|
29
|
+
Frames: [Float32Array, number];
|
|
30
|
+
/** The system gained or lost an audio device */
|
|
31
|
+
DeviceChange: [];
|
|
32
|
+
Error: [Error];
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Captures the microphone and pushes raw samples.
|
|
36
|
+
*
|
|
37
|
+
* `@micdrop/web` records with the Web Audio API and `@micdrop/react-native`
|
|
38
|
+
* with the native audio engine. Everything above this interface is shared.
|
|
39
|
+
*/
|
|
40
|
+
declare abstract class MicDriver extends EventEmitter<MicDriverEvents> {
|
|
41
|
+
abstract get isStarted(): boolean;
|
|
42
|
+
/** Identifier of the input device in use, when the system reports one */
|
|
43
|
+
abstract get deviceId(): string | undefined;
|
|
44
|
+
/**
|
|
45
|
+
* Asks for the microphone permission and starts capturing
|
|
46
|
+
* @param deviceId - Input device to record from, defaults to the system one
|
|
47
|
+
*/
|
|
48
|
+
abstract start(deviceId?: string): Promise<void>;
|
|
49
|
+
/** Stops capturing and releases the microphone */
|
|
50
|
+
abstract stop(): Promise<void>;
|
|
51
|
+
/** Lists the input devices the system offers */
|
|
52
|
+
abstract getDevices(): Promise<MicdropDevice[]>;
|
|
53
|
+
}
|
|
54
|
+
interface SpeakerDriverEvents {
|
|
55
|
+
StartPlaying: [];
|
|
56
|
+
StopPlaying: [];
|
|
57
|
+
/** Level of the audio being played */
|
|
58
|
+
Volume: [number];
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Plays the 16 kHz mono PCM sent by the server.
|
|
62
|
+
*/
|
|
63
|
+
declare abstract class SpeakerDriver extends EventEmitter<SpeakerDriverEvents> {
|
|
64
|
+
abstract get isPlaying(): boolean;
|
|
65
|
+
/** Identifier of the output device in use, when the system reports one */
|
|
66
|
+
abstract get deviceId(): string | undefined;
|
|
67
|
+
/** Prepares the audio graph, called before the first chunk arrives */
|
|
68
|
+
abstract start(): Promise<void>;
|
|
69
|
+
/**
|
|
70
|
+
* Queues samples for playback, right after the ones already queued
|
|
71
|
+
* @param pcm - Mono samples as 16 bits integers
|
|
72
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
73
|
+
*/
|
|
74
|
+
abstract play(pcm: Int16Array, sampleRate: number): void;
|
|
75
|
+
/** Drops everything queued and stops playing immediately */
|
|
76
|
+
abstract stopAudio(): void;
|
|
77
|
+
/** Releases the audio graph */
|
|
78
|
+
abstract stop(): Promise<void>;
|
|
79
|
+
/**
|
|
80
|
+
* Plays through another output
|
|
81
|
+
* @param deviceId - Output device, or route on a phone
|
|
82
|
+
*/
|
|
83
|
+
abstract changeDevice(deviceId: string): Promise<void>;
|
|
84
|
+
/** Lists the output devices the system offers */
|
|
85
|
+
abstract getDevices(): Promise<MicdropDevice[]>;
|
|
86
|
+
}
|
|
87
|
+
/** A buffer handed to an {@link AudioSink}, that can still be cancelled */
|
|
88
|
+
interface ScheduledAudio {
|
|
14
89
|
stop(): void;
|
|
15
|
-
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* The minimal slice of an audio engine needed to stream the assistant voice.
|
|
93
|
+
* Implemented on each platform, and faked in the test suite.
|
|
94
|
+
*/
|
|
95
|
+
interface AudioSink {
|
|
96
|
+
/** Sample rate of the underlying audio graph, in Hz */
|
|
97
|
+
readonly sampleRate: number;
|
|
98
|
+
/** Current time of the audio graph, in seconds */
|
|
99
|
+
readonly currentTime: number;
|
|
100
|
+
/**
|
|
101
|
+
* Plays samples at a given time on the audio graph clock
|
|
102
|
+
* @param samples - Mono samples, in the -1..1 range, at `sampleRate`
|
|
103
|
+
* @param when - When to start, in seconds on the audio graph clock
|
|
104
|
+
* @param onEnded - Called once the samples have been played
|
|
105
|
+
*/
|
|
106
|
+
schedule(samples: Float32Array, when: number, onEnded: () => void): ScheduledAudio;
|
|
16
107
|
}
|
|
17
108
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
109
|
+
/**
|
|
110
|
+
* The microphone, as a single object shared by the whole app.
|
|
111
|
+
*
|
|
112
|
+
* It wraps a {@link MicDriver}, the browser one or the native one, and turns
|
|
113
|
+
* the samples it captures into the level used by the VAD and by level meters.
|
|
114
|
+
* Nothing here knows which platform it runs on.
|
|
115
|
+
*/
|
|
116
|
+
declare class MicController extends EventEmitter<MicEvents> {
|
|
117
|
+
private driver;
|
|
118
|
+
private meter;
|
|
119
|
+
private pending;
|
|
120
|
+
private pendingLength;
|
|
121
|
+
private pendingRate;
|
|
122
|
+
private rateCheckedAt;
|
|
123
|
+
private rateCheckSamples;
|
|
124
|
+
private rateWarned;
|
|
125
|
+
/** Last level measured, in decibels, between -Infinity and 0 */
|
|
126
|
+
volume: number;
|
|
24
127
|
/**
|
|
25
|
-
*
|
|
26
|
-
* @param
|
|
27
|
-
* @returns The microphone stream
|
|
128
|
+
* Replaces the audio driver, to record with another library
|
|
129
|
+
* @param driver - The driver to record with
|
|
28
130
|
*/
|
|
29
|
-
|
|
131
|
+
setDriver(driver: MicDriver): void;
|
|
132
|
+
getDriver(): MicDriver;
|
|
133
|
+
get isStarted(): boolean;
|
|
134
|
+
get deviceId(): string | undefined;
|
|
30
135
|
/**
|
|
31
|
-
*
|
|
136
|
+
* Asks for the microphone permission and starts capturing
|
|
137
|
+
* @param deviceId - Input device to record from, defaults to the system one
|
|
32
138
|
*/
|
|
33
|
-
|
|
139
|
+
start(deviceId?: string): Promise<void>;
|
|
140
|
+
/** Stops capturing and releases the microphone */
|
|
141
|
+
stop(): Promise<void>;
|
|
142
|
+
/** Lists the input devices the system offers */
|
|
143
|
+
getDevices(): Promise<MicdropDevice[]>;
|
|
144
|
+
private onFrames;
|
|
145
|
+
/**
|
|
146
|
+
* Times the captured audio against the rate it claims to be at.
|
|
147
|
+
*
|
|
148
|
+
* A microphone that announces 16 kHz while delivering 48 kHz sends speech to
|
|
149
|
+
* the server three times too slow, which comes back as a short transcript in
|
|
150
|
+
* a random language. That is invisible otherwise, so it is worth saying out
|
|
151
|
+
* loud once.
|
|
152
|
+
* @param samples - How many samples just arrived
|
|
153
|
+
* @param sampleRate - The rate they are said to be at
|
|
154
|
+
*/
|
|
155
|
+
private checkRate;
|
|
156
|
+
private onDeviceChange;
|
|
157
|
+
private onError;
|
|
158
|
+
private resetVolume;
|
|
34
159
|
}
|
|
35
160
|
|
|
36
161
|
interface SpeakerEvents {
|
|
37
162
|
StartPlaying: void;
|
|
38
163
|
StopPlaying: void;
|
|
164
|
+
/** Level of the audio being played */
|
|
165
|
+
Volume: [number];
|
|
39
166
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
private
|
|
167
|
+
/**
|
|
168
|
+
* The speaker, as a single object shared by the whole app.
|
|
169
|
+
*
|
|
170
|
+
* It wraps a {@link SpeakerDriver} and plays the 16 kHz PCM sent by the server
|
|
171
|
+
* as it arrives.
|
|
172
|
+
*/
|
|
173
|
+
declare class SpeakerController extends EventEmitter<SpeakerEvents> {
|
|
174
|
+
private driver;
|
|
175
|
+
/** Last level measured, in decibels, between -Infinity and 0 */
|
|
176
|
+
volume: number;
|
|
177
|
+
/**
|
|
178
|
+
* Replaces the audio driver, to play sound with another library
|
|
179
|
+
* @param driver - The driver to play with
|
|
180
|
+
*/
|
|
181
|
+
setDriver(driver: SpeakerDriver): void;
|
|
182
|
+
getDriver(): SpeakerDriver;
|
|
183
|
+
get isPlaying(): boolean;
|
|
184
|
+
get deviceId(): string | undefined;
|
|
185
|
+
/** Prepares the audio graph, called before the first chunk arrives */
|
|
48
186
|
start(): Promise<void>;
|
|
49
|
-
|
|
50
|
-
|
|
187
|
+
/**
|
|
188
|
+
* Queues samples for playback, right after the ones already queued
|
|
189
|
+
* @param pcm - Mono samples as 16 bits integers, or the bytes holding them
|
|
190
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
191
|
+
*/
|
|
192
|
+
playAudio(pcm: Int16Array | ArrayBuffer, sampleRate?: number): void;
|
|
193
|
+
/** Drops everything queued and stops playing immediately */
|
|
194
|
+
stopAudio(): void;
|
|
195
|
+
/** Releases the audio graph */
|
|
196
|
+
stop(): Promise<void>;
|
|
197
|
+
/**
|
|
198
|
+
* Plays through another output
|
|
199
|
+
* @param deviceId - Output device, or route on a phone
|
|
200
|
+
*/
|
|
51
201
|
changeDevice(deviceId: string): Promise<void>;
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
private
|
|
202
|
+
/** Lists the output devices the system offers */
|
|
203
|
+
getDevices(): Promise<MicdropDevice[]>;
|
|
204
|
+
private onStartPlaying;
|
|
205
|
+
private onStopPlaying;
|
|
206
|
+
private onVolume;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
declare enum MicdropClientCommands {
|
|
210
|
+
StartSpeaking = "StartSpeaking",
|
|
211
|
+
StopSpeaking = "StopSpeaking",
|
|
212
|
+
Mute = "Mute"
|
|
213
|
+
}
|
|
214
|
+
declare enum MicdropServerCommands {
|
|
215
|
+
Message = "Message",
|
|
216
|
+
CancelLastUserMessage = "CancelLastUserMessage",
|
|
217
|
+
SkipAnswer = "SkipAnswer",
|
|
218
|
+
EndCall = "EndCall",
|
|
219
|
+
ToolCall = "ToolCall"
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Hears whether a sentence has landed, where voice activity detection only
|
|
223
|
+
* hears whether someone is speaking.
|
|
224
|
+
*
|
|
225
|
+
* `SmartTurn` from `@micdrop/smart-turn` implements it, and so can anything
|
|
226
|
+
* else, a call to a service included. Both sides of a call can hold one, the
|
|
227
|
+
* client to decide when its turn ends and the server to decide when to answer.
|
|
228
|
+
*/
|
|
229
|
+
interface TurnDetector {
|
|
230
|
+
/**
|
|
231
|
+
* Feeds the audio received since the last call
|
|
232
|
+
* @param samples - Mono samples, in the -1..1 range
|
|
233
|
+
* @param sampleRate - Sample rate of `samples`, in Hz
|
|
234
|
+
*/
|
|
235
|
+
push(samples: Float32Array, sampleRate?: number): void;
|
|
236
|
+
/** Answers whether the turn pushed so far sounds finished */
|
|
237
|
+
predict(): Promise<{
|
|
238
|
+
complete: boolean;
|
|
239
|
+
}>;
|
|
240
|
+
/** Starts a new turn, forgetting the previous one */
|
|
241
|
+
reset(): void;
|
|
242
|
+
}
|
|
243
|
+
interface MicdropCallSummary {
|
|
244
|
+
conversation: MicdropConversation;
|
|
245
|
+
duration: number;
|
|
246
|
+
}
|
|
247
|
+
type MicdropConversationItem = MicdropConversationMessage | MicdropConversationToolCall | MicdropConversationToolResult;
|
|
248
|
+
type MicdropConversation = Array<MicdropConversationItem>;
|
|
249
|
+
type MicdropAnswerMetadata = {
|
|
250
|
+
[key: string]: any;
|
|
251
|
+
};
|
|
252
|
+
interface MicdropConversationMessage<Data extends MicdropAnswerMetadata = MicdropAnswerMetadata> {
|
|
253
|
+
role: 'system' | 'user' | 'assistant';
|
|
254
|
+
content: string;
|
|
255
|
+
metadata?: Data;
|
|
256
|
+
}
|
|
257
|
+
interface MicdropConversationToolCall {
|
|
258
|
+
role: 'tool_call';
|
|
259
|
+
toolCallId: string;
|
|
260
|
+
toolName: string;
|
|
261
|
+
parameters: string;
|
|
262
|
+
}
|
|
263
|
+
interface MicdropConversationToolResult {
|
|
264
|
+
role: 'tool_result';
|
|
265
|
+
toolCallId: string;
|
|
266
|
+
toolName: string;
|
|
267
|
+
output: string;
|
|
55
268
|
}
|
|
269
|
+
interface MicdropToolCall {
|
|
270
|
+
name: string;
|
|
271
|
+
parameters: any;
|
|
272
|
+
output: any;
|
|
273
|
+
}
|
|
274
|
+
type DeepPartial<T> = T extends object ? {
|
|
275
|
+
[P in keyof T]?: DeepPartial<T[P]>;
|
|
276
|
+
} : T;
|
|
56
277
|
|
|
57
278
|
interface VADEvents {
|
|
58
279
|
StartSpeaking: void;
|
|
@@ -86,9 +307,9 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
|
|
|
86
307
|
abstract get isPaused(): boolean;
|
|
87
308
|
/**
|
|
88
309
|
* Starts the VAD
|
|
89
|
-
* @param
|
|
310
|
+
* @param mic - The microphone to listen to
|
|
90
311
|
*/
|
|
91
|
-
abstract start(
|
|
312
|
+
abstract start(mic: MicSource): Promise<void>;
|
|
92
313
|
/**
|
|
93
314
|
* Stops the VAD
|
|
94
315
|
*/
|
|
@@ -120,6 +341,13 @@ type VADConfig = VAD | VADConfigName | Array<VAD | VADConfigName>;
|
|
|
120
341
|
declare function getVAD(vad?: VADConfig): VAD;
|
|
121
342
|
declare function equalVADConfig(a?: VADConfig, b?: VADConfig): boolean;
|
|
122
343
|
|
|
344
|
+
/**
|
|
345
|
+
* How long a turn may stay open after the detector asked to wait.
|
|
346
|
+
*
|
|
347
|
+
* Long enough to cover the pause of someone looking for their words, and short
|
|
348
|
+
* enough that a wrong verdict costs one awkward silence rather than the call.
|
|
349
|
+
*/
|
|
350
|
+
declare const DEFAULT_TURN_MAX_WAIT = 4000;
|
|
123
351
|
interface MicRecorderState {
|
|
124
352
|
isStarting: boolean;
|
|
125
353
|
isStarted: boolean;
|
|
@@ -131,35 +359,167 @@ interface MicRecorderEvents {
|
|
|
131
359
|
StopSpeaking: void;
|
|
132
360
|
StateChange: [MicRecorderState];
|
|
133
361
|
}
|
|
362
|
+
/**
|
|
363
|
+
* Turns what the microphone captures into the chunks sent to the server.
|
|
364
|
+
*
|
|
365
|
+
* It only records while the VAD hears someone, and keeps a short reserve of
|
|
366
|
+
* audio so the first syllable, spoken before the VAD reacted, is sent too.
|
|
367
|
+
*
|
|
368
|
+
* Recording and the turn are two different things. The VAD decides what is
|
|
369
|
+
* worth sending, so the silences stay out of the stream either way. A
|
|
370
|
+
* {@link TurnDetector}, when there is one, decides when the turn is over, and
|
|
371
|
+
* a pause the speaker is about to fill leaves it open.
|
|
372
|
+
*/
|
|
134
373
|
declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
135
374
|
private vadConfig?;
|
|
375
|
+
private turnDetector?;
|
|
376
|
+
/**
|
|
377
|
+
* How long a turn stays open once the detector asked to wait.
|
|
378
|
+
*
|
|
379
|
+
* The way out of a detector that hears an unfinished sentence where there
|
|
380
|
+
* is none: the speaker who never comes back still gets an answer.
|
|
381
|
+
*/
|
|
382
|
+
turnMaxWait: number;
|
|
136
383
|
state: MicRecorderState;
|
|
137
384
|
vad: VAD;
|
|
138
|
-
private
|
|
139
|
-
private
|
|
140
|
-
private workletNode;
|
|
141
|
-
private delayedStream;
|
|
385
|
+
private mic;
|
|
386
|
+
private isRecording;
|
|
142
387
|
private speakingConfirmed;
|
|
388
|
+
private sourceSampleRate;
|
|
389
|
+
private reserve;
|
|
390
|
+
private reserveLength;
|
|
391
|
+
private buffer;
|
|
392
|
+
private bufferLength;
|
|
143
393
|
private queuedChunks;
|
|
144
|
-
|
|
145
|
-
|
|
394
|
+
private turnOpen;
|
|
395
|
+
private turnListening;
|
|
396
|
+
private turnTimer;
|
|
397
|
+
private turnRound;
|
|
398
|
+
constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
|
|
399
|
+
/**
|
|
400
|
+
* How long a turn stays open once the detector asked to wait.
|
|
401
|
+
*
|
|
402
|
+
* The way out of a detector that hears an unfinished sentence where there
|
|
403
|
+
* is none: the speaker who never comes back still gets an answer.
|
|
404
|
+
*/
|
|
405
|
+
turnMaxWait?: number);
|
|
406
|
+
start: (mic: MicSource) => Promise<void>;
|
|
146
407
|
stop: () => void;
|
|
408
|
+
/**
|
|
409
|
+
* Changes what decides when a turn is over
|
|
410
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
411
|
+
*/
|
|
412
|
+
changeTurnDetector: (turnDetector?: TurnDetector) => void;
|
|
147
413
|
changeVad: (vadConfig: VADConfig) => void;
|
|
414
|
+
private onFrames;
|
|
148
415
|
private onStartSpeaking;
|
|
149
416
|
private onConfirmSpeaking;
|
|
150
417
|
private onCancelSpeaking;
|
|
151
418
|
private onStopSpeaking;
|
|
152
|
-
|
|
419
|
+
/**
|
|
420
|
+
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
421
|
+
*
|
|
422
|
+
* A microphone that is no longer listening ends it straight away: a muted or
|
|
423
|
+
* paused call has nothing more to say, whatever the sentence sounded like.
|
|
424
|
+
*/
|
|
425
|
+
private closeTurn;
|
|
426
|
+
private endTurn;
|
|
427
|
+
/** Drops the pending close, both the timer and the answer still owed */
|
|
428
|
+
private cancelPendingClose;
|
|
429
|
+
private stopRecording;
|
|
430
|
+
/**
|
|
431
|
+
* Emits every complete chunk waiting in the buffer
|
|
432
|
+
* @param final - Also emit the incomplete last chunk
|
|
433
|
+
*/
|
|
434
|
+
private flushChunks;
|
|
435
|
+
private emitChunk;
|
|
436
|
+
private resetBuffers;
|
|
153
437
|
private changeState;
|
|
154
438
|
}
|
|
155
439
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
440
|
+
interface Pcm16AudioStreamEvents {
|
|
441
|
+
StartPlaying: [];
|
|
442
|
+
StopPlaying: [];
|
|
443
|
+
Volume: [number];
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* Turns the stream of PCM chunks sent by the server into continuous playback.
|
|
447
|
+
*
|
|
448
|
+
* Chunks arrive faster or slower than they are heard, so each one is scheduled
|
|
449
|
+
* right after the previous one on the audio graph clock instead of being played
|
|
450
|
+
* on arrival.
|
|
451
|
+
*/
|
|
452
|
+
declare class Pcm16AudioStream extends EventEmitter<Pcm16AudioStreamEvents> {
|
|
453
|
+
private sink;
|
|
454
|
+
isPlaying: boolean;
|
|
455
|
+
private sources;
|
|
456
|
+
private nextStartTime;
|
|
457
|
+
private prebuffering;
|
|
458
|
+
private prebuffer;
|
|
459
|
+
private prebufferDuration;
|
|
460
|
+
private quietTimer?;
|
|
461
|
+
private endTimer?;
|
|
462
|
+
private volumeTimers;
|
|
463
|
+
private meter;
|
|
464
|
+
constructor(sink: AudioSink);
|
|
465
|
+
/**
|
|
466
|
+
* Queues samples for playback
|
|
467
|
+
* @param pcm - Mono samples as 16 bits integers
|
|
468
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
469
|
+
*/
|
|
470
|
+
playAudio(pcm: Int16Array, sampleRate: number): void;
|
|
471
|
+
/** Drops everything queued and stops playing immediately */
|
|
472
|
+
stopAudio(): void;
|
|
473
|
+
private handleSamples;
|
|
474
|
+
private flushPrebuffer;
|
|
475
|
+
private resetPrebuffer;
|
|
476
|
+
private scheduleSamples;
|
|
477
|
+
private scheduleVolume;
|
|
478
|
+
private clearVolumeTimers;
|
|
479
|
+
private setIsPlaying;
|
|
161
480
|
}
|
|
162
481
|
|
|
482
|
+
/**
|
|
483
|
+
* PCM helpers shared by the microphone and the speaker.
|
|
484
|
+
*
|
|
485
|
+
* Everything here is pure so it can be exercised without a device: the same
|
|
486
|
+
* functions run on iOS, on Android and in the test suite.
|
|
487
|
+
*/
|
|
488
|
+
/**
|
|
489
|
+
* Resamples mono float samples with linear interpolation.
|
|
490
|
+
* @param input - Mono samples, in the -1..1 range
|
|
491
|
+
* @param inputRate - Sample rate of `input`, in Hz
|
|
492
|
+
* @param outputRate - Wanted sample rate, in Hz
|
|
493
|
+
* @returns The resampled samples, or `input` itself when both rates match
|
|
494
|
+
*/
|
|
495
|
+
declare function resample(input: Float32Array, inputRate: number, outputRate: number): Float32Array;
|
|
496
|
+
/**
|
|
497
|
+
* Converts float samples to signed 16 bits PCM
|
|
498
|
+
* @param input - Mono samples, in the -1..1 range
|
|
499
|
+
* @returns The same samples as 16 bits integers
|
|
500
|
+
*/
|
|
501
|
+
declare function floatToPcm16(input: Float32Array): Int16Array;
|
|
502
|
+
/**
|
|
503
|
+
* Converts signed 16 bits PCM to float samples
|
|
504
|
+
* @param input - Mono samples as 16 bits integers
|
|
505
|
+
* @returns The same samples in the -1..1 range
|
|
506
|
+
*/
|
|
507
|
+
declare function pcm16ToFloat(input: Int16Array): Float32Array;
|
|
508
|
+
/**
|
|
509
|
+
* Concatenates chunks of samples into a single buffer
|
|
510
|
+
* @param chunks - The chunks to concatenate, in order
|
|
511
|
+
* @returns One buffer holding every sample
|
|
512
|
+
*/
|
|
513
|
+
declare function concatFloat32(chunks: Float32Array[]): Float32Array;
|
|
514
|
+
/**
|
|
515
|
+
* Copies signed 16 bits PCM into a standalone ArrayBuffer, ready to be sent
|
|
516
|
+
* over a WebSocket. A typed array can be a view on a larger buffer, sending it
|
|
517
|
+
* as is would send the whole buffer.
|
|
518
|
+
* @param pcm - The samples to send
|
|
519
|
+
* @returns An ArrayBuffer holding exactly those samples
|
|
520
|
+
*/
|
|
521
|
+
declare function pcm16ToArrayBuffer(pcm: Int16Array): ArrayBuffer;
|
|
522
|
+
|
|
163
523
|
/**
|
|
164
524
|
* Combine an array of VADs
|
|
165
525
|
*/
|
|
@@ -169,121 +529,172 @@ declare class MultipleVAD extends VAD {
|
|
|
169
529
|
constructor(vads: Array<VAD | VADConfigName>);
|
|
170
530
|
get isStarted(): boolean;
|
|
171
531
|
get isPaused(): boolean;
|
|
172
|
-
start(
|
|
532
|
+
start(mic: MicSource): Promise<void>;
|
|
173
533
|
private onStatusChange;
|
|
174
534
|
stop(): Promise<void>;
|
|
175
535
|
pause(): Promise<void>;
|
|
176
536
|
resume(): Promise<void>;
|
|
177
537
|
}
|
|
178
538
|
|
|
539
|
+
/** Rate and window the Silero v5 model expects */
|
|
540
|
+
declare const SILERO_SAMPLE_RATE = 16000;
|
|
541
|
+
declare const SILERO_FRAME_SAMPLES = 512;
|
|
542
|
+
/**
|
|
543
|
+
* One loaded Silero model, whichever runtime holds it.
|
|
544
|
+
*
|
|
545
|
+
* `@micdrop/web` runs it with onnxruntime-web and `@micdrop/react-native` with
|
|
546
|
+
* onnxruntime-react-native. Both answer the same question: how likely is it
|
|
547
|
+
* that these 512 samples are speech.
|
|
548
|
+
*/
|
|
549
|
+
interface SileroModel {
|
|
550
|
+
/**
|
|
551
|
+
* Scores one window of audio
|
|
552
|
+
* @param frame - 512 mono samples at 16 kHz, in the -1..1 range
|
|
553
|
+
* @returns The probability that it is speech, between 0 and 1
|
|
554
|
+
*/
|
|
555
|
+
process(frame: Float32Array): Promise<number>;
|
|
556
|
+
/** Forgets what came before, at the start of a new call */
|
|
557
|
+
reset(): void;
|
|
558
|
+
/** Releases the runtime */
|
|
559
|
+
release(): Promise<void>;
|
|
560
|
+
}
|
|
561
|
+
/** Loads the model, which usually means fetching a couple of megabytes */
|
|
562
|
+
type SileroModelLoader = () => Promise<SileroModel>;
|
|
179
563
|
type SileroVADOptions = {
|
|
564
|
+
/** Above this probability, the window counts as speech */
|
|
180
565
|
positiveSpeechThreshold: number;
|
|
566
|
+
/** Below this probability, the window counts as silence */
|
|
181
567
|
negativeSpeechThreshold: number;
|
|
568
|
+
/** Windows of speech needed before a turn is confirmed */
|
|
182
569
|
minSpeechFrames: number;
|
|
570
|
+
/** Windows of silence tolerated inside a turn before it is closed */
|
|
183
571
|
redemptionFrames: number;
|
|
184
572
|
};
|
|
185
573
|
/**
|
|
186
|
-
* Silero
|
|
187
|
-
*
|
|
574
|
+
* Says how the Silero model is loaded on this platform. Called by
|
|
575
|
+
* `@micdrop/web` and `@micdrop/react-native` when they are imported.
|
|
576
|
+
* @param loader - Loads and returns a ready to use model
|
|
577
|
+
*/
|
|
578
|
+
declare function setSileroModelLoader(loader: SileroModelLoader): void;
|
|
579
|
+
/**
|
|
580
|
+
* Voice activity detection with the Silero model.
|
|
581
|
+
*
|
|
582
|
+
* It hears the difference between a voice and a noise, where
|
|
583
|
+
* {@link VolumeVAD} only hears how loud the room is. The model itself is
|
|
584
|
+
* loaded by the platform package, everything else runs the same everywhere.
|
|
585
|
+
*
|
|
586
|
+
* @see https://github.com/snakers4/silero-vad
|
|
188
587
|
*/
|
|
189
588
|
declare class SileroVAD extends VAD {
|
|
190
589
|
readonly name = "SileroVAD";
|
|
191
590
|
options: SileroVADOptions;
|
|
192
|
-
|
|
591
|
+
delay: number;
|
|
592
|
+
private model;
|
|
593
|
+
private mic;
|
|
193
594
|
private _isPaused;
|
|
595
|
+
private buffer;
|
|
596
|
+
private bufferLength;
|
|
597
|
+
private bufferRate;
|
|
598
|
+
private speaking;
|
|
599
|
+
private speechFrames;
|
|
600
|
+
private silenceFrames;
|
|
601
|
+
private processing;
|
|
194
602
|
constructor(options?: Partial<SileroVADOptions>);
|
|
195
603
|
get isStarted(): boolean;
|
|
196
604
|
get isPaused(): boolean;
|
|
197
|
-
start(
|
|
605
|
+
start(mic: MicSource): Promise<void>;
|
|
198
606
|
stop(): Promise<void>;
|
|
199
607
|
pause(): Promise<void>;
|
|
200
608
|
resume(): Promise<void>;
|
|
201
609
|
setOptions(options: Partial<SileroVADOptions>): void;
|
|
202
610
|
resetOptions(): void;
|
|
611
|
+
private onFrames;
|
|
612
|
+
private drain;
|
|
613
|
+
private onProbability;
|
|
614
|
+
private closeOpenTurn;
|
|
615
|
+
private resetState;
|
|
203
616
|
}
|
|
204
617
|
|
|
205
618
|
type VolumeVADOptions = {
|
|
619
|
+
/** Number of recent samples kept to decide whether someone is speaking */
|
|
206
620
|
history: number;
|
|
621
|
+
/** Level above which audio counts as speech, in decibels */
|
|
207
622
|
threshold: number;
|
|
208
623
|
};
|
|
209
624
|
/**
|
|
210
|
-
* Volume
|
|
211
|
-
*
|
|
625
|
+
* Volume based Voice Activity Detection.
|
|
626
|
+
*
|
|
627
|
+
* It follows the level reported by the microphone, and needs a few consecutive
|
|
628
|
+
* loud samples before confirming speech, so a door slam or a keyboard click
|
|
629
|
+
* does not start a turn.
|
|
212
630
|
*/
|
|
213
631
|
declare class VolumeVAD extends VAD {
|
|
214
632
|
readonly name = "VolumeVAD";
|
|
215
633
|
options: VolumeVADOptions;
|
|
216
|
-
|
|
217
|
-
private
|
|
218
|
-
private sourceNode;
|
|
219
|
-
private fftBins;
|
|
634
|
+
delay: number;
|
|
635
|
+
private mic;
|
|
220
636
|
private running;
|
|
637
|
+
private _isPaused;
|
|
221
638
|
private speaking;
|
|
222
639
|
private attemptSpeaking;
|
|
223
640
|
private speakingHistory;
|
|
224
641
|
constructor(options?: Partial<VolumeVADOptions>);
|
|
225
|
-
private getMaxVolume;
|
|
226
642
|
get isStarted(): boolean;
|
|
227
643
|
get isPaused(): boolean;
|
|
228
|
-
start(
|
|
229
|
-
private startLoop;
|
|
230
|
-
private resetHistory;
|
|
644
|
+
start(mic: MicSource): Promise<void>;
|
|
231
645
|
stop(): Promise<void>;
|
|
232
646
|
pause(): Promise<void>;
|
|
233
647
|
resume(): Promise<void>;
|
|
234
648
|
setOptions(options: Partial<VolumeVADOptions>): void;
|
|
235
649
|
resetOptions(): void;
|
|
650
|
+
private onVolume;
|
|
651
|
+
private reset;
|
|
236
652
|
}
|
|
237
653
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
}
|
|
273
|
-
interface MicdropConversationToolResult {
|
|
274
|
-
role: 'tool_result';
|
|
275
|
-
toolCallId: string;
|
|
276
|
-
toolName: string;
|
|
277
|
-
output: string;
|
|
278
|
-
}
|
|
279
|
-
interface MicdropToolCall {
|
|
280
|
-
name: string;
|
|
281
|
-
parameters: any;
|
|
282
|
-
output: any;
|
|
654
|
+
/**
|
|
655
|
+
* Measures how loud the microphone is, the way a browser AnalyserNode does.
|
|
656
|
+
*
|
|
657
|
+
* The level is the loudest frequency bin rather than the overall energy: a
|
|
658
|
+
* voice concentrates its power in a few bins while a fan or a room spreads it
|
|
659
|
+
* over all of them, so the two are told apart by about 25 dB instead of the 7 dB
|
|
660
|
+
* an RMS would give.
|
|
661
|
+
*
|
|
662
|
+
* Everything here is pure, so the same measurement runs in a browser, on a
|
|
663
|
+
* phone and in the test suite.
|
|
664
|
+
*/
|
|
665
|
+
/** Sample rate the measurement expects, so a bin means the same frequency everywhere */
|
|
666
|
+
declare const VOLUME_SAMPLE_RATE = 16000;
|
|
667
|
+
/**
|
|
668
|
+
* Follows the level of a stream of audio.
|
|
669
|
+
*
|
|
670
|
+
* It keeps the previous measurement, so a single quiet frame in the middle of a
|
|
671
|
+
* word does not read as a pause.
|
|
672
|
+
*/
|
|
673
|
+
declare class VolumeMeter {
|
|
674
|
+
private real;
|
|
675
|
+
private imaginary;
|
|
676
|
+
private magnitudes;
|
|
677
|
+
private smoothed;
|
|
678
|
+
private hasPrevious;
|
|
679
|
+
/**
|
|
680
|
+
* Measures a window of audio
|
|
681
|
+
* @param samples - At least 512 mono samples at 16 kHz, in the -1..1 range.
|
|
682
|
+
* Only the last 512 are looked at.
|
|
683
|
+
* @returns The level in decibels, between -Infinity (silence) and 0
|
|
684
|
+
*/
|
|
685
|
+
measure(samples: Float32Array): number;
|
|
686
|
+
/** Forgets the previous window, for when capture stops and starts again */
|
|
687
|
+
reset(): void;
|
|
283
688
|
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
689
|
+
/**
|
|
690
|
+
* Measures one window of audio, without following what came before
|
|
691
|
+
* @param samples - Mono samples at 16 kHz, in the -1..1 range
|
|
692
|
+
* @returns The level in decibels, between -Infinity (silence) and 0
|
|
693
|
+
*/
|
|
694
|
+
declare function getVolume(samples: Float32Array): number;
|
|
695
|
+
|
|
696
|
+
declare const Mic: MicController;
|
|
697
|
+
declare const Speaker: SpeakerController;
|
|
287
698
|
|
|
288
699
|
declare enum MicdropClientErrorCode {
|
|
289
700
|
MissingUrl = "MissingUrl",
|
|
@@ -299,7 +710,12 @@ declare class MicdropClientError extends Error {
|
|
|
299
710
|
code: MicdropClientErrorCode;
|
|
300
711
|
constructor(code?: MicdropClientErrorCode, message?: string);
|
|
301
712
|
}
|
|
302
|
-
|
|
713
|
+
/** What React Native gives on a WebSocket close */
|
|
714
|
+
interface WSCloseEvent {
|
|
715
|
+
code?: number;
|
|
716
|
+
reason?: string;
|
|
717
|
+
}
|
|
718
|
+
declare function getClientErrorFromWSCloseEvent(event: WSCloseEvent): MicdropClientError | undefined;
|
|
303
719
|
declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
|
|
304
720
|
declare function isRecoverableError(error: MicdropClientError): boolean;
|
|
305
721
|
|
|
@@ -318,6 +734,14 @@ interface MicdropOptions {
|
|
|
318
734
|
url?: string;
|
|
319
735
|
params?: Record<string, any>;
|
|
320
736
|
vad?: VADConfig;
|
|
737
|
+
/**
|
|
738
|
+
* Holds the turn open while the sentence sounds unfinished, so a hesitation
|
|
739
|
+
* no longer counts as the end of a question. Without one, a turn ends as
|
|
740
|
+
* soon as the VAD hears enough silence.
|
|
741
|
+
*/
|
|
742
|
+
turnDetector?: TurnDetector;
|
|
743
|
+
/** How long a turn may stay open after the detector asked to wait */
|
|
744
|
+
turnMaxWait?: number;
|
|
321
745
|
disableInterruption?: boolean;
|
|
322
746
|
debugLog?: boolean;
|
|
323
747
|
reconnect?: MicdropReconnectOptions;
|
|
@@ -336,8 +760,8 @@ interface MicdropState {
|
|
|
336
760
|
isMicMuted: boolean;
|
|
337
761
|
micDeviceId: string | undefined;
|
|
338
762
|
speakerDeviceId: string | undefined;
|
|
339
|
-
micDevices:
|
|
340
|
-
speakerDevices:
|
|
763
|
+
micDevices: MicdropDevice[];
|
|
764
|
+
speakerDevices: MicdropDevice[];
|
|
341
765
|
conversation: MicdropConversation;
|
|
342
766
|
error: MicdropClientError | undefined;
|
|
343
767
|
}
|
|
@@ -346,10 +770,9 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
346
770
|
micRecorder?: MicRecorder;
|
|
347
771
|
conversation: MicdropConversation;
|
|
348
772
|
error: MicdropClientError | undefined;
|
|
349
|
-
speakerDevices:
|
|
350
|
-
micDevices:
|
|
773
|
+
speakerDevices: MicdropDevice[];
|
|
774
|
+
micDevices: MicdropDevice[];
|
|
351
775
|
private ws?;
|
|
352
|
-
private micStream?;
|
|
353
776
|
private startTime;
|
|
354
777
|
private lastNotifiedState;
|
|
355
778
|
private _isProcessing;
|
|
@@ -387,6 +810,20 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
387
810
|
vad?: VADConfig;
|
|
388
811
|
deviceId?: string;
|
|
389
812
|
}) => Promise<void>;
|
|
813
|
+
/**
|
|
814
|
+
* Changes what decides when a turn is over, during a call or before one
|
|
815
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
816
|
+
*/
|
|
817
|
+
setTurnDetector: (turnDetector?: TurnDetector) => void;
|
|
818
|
+
/**
|
|
819
|
+
* Says how long a held turn waits for the rest of the sentence.
|
|
820
|
+
*
|
|
821
|
+
* The way out of a wrong verdict: a detector can hear an unfinished sentence
|
|
822
|
+
* where there is none, and the speaker who never comes back still gets an
|
|
823
|
+
* answer once this runs out.
|
|
824
|
+
* @param milliseconds - How long to wait past the detector asking to
|
|
825
|
+
*/
|
|
826
|
+
setTurnMaxWait: (milliseconds: number) => void;
|
|
390
827
|
changeMicDevice: (deviceId: string) => Promise<void>;
|
|
391
828
|
changeSpeakerDevice: (deviceId: string) => Promise<void>;
|
|
392
829
|
private stopMic;
|
|
@@ -408,4 +845,28 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
408
845
|
|
|
409
846
|
declare const Micdrop: MicdropClient;
|
|
410
847
|
|
|
411
|
-
|
|
848
|
+
/**
|
|
849
|
+
* Where settings that outlive a call are kept.
|
|
850
|
+
*
|
|
851
|
+
* A browser has `localStorage`, a phone has whatever the app installs. Reads
|
|
852
|
+
* are synchronous, so a VAD can be built with its saved options in hand.
|
|
853
|
+
*/
|
|
854
|
+
interface MicdropStorage {
|
|
855
|
+
getItem(key: string): string | null;
|
|
856
|
+
setItem(key: string, value: string): void;
|
|
857
|
+
removeItem(key: string): void;
|
|
858
|
+
}
|
|
859
|
+
declare enum MicdropStorageKeys {
|
|
860
|
+
MicDevice = "micdrop.micDevice",
|
|
861
|
+
SpeakerDevice = "micdrop.speakerDevice",
|
|
862
|
+
VolumeVADOptions = "micdrop.VolumeVADOptions",
|
|
863
|
+
SileroVADOptions = "micdrop.SileroVADOptions"
|
|
864
|
+
}
|
|
865
|
+
/**
|
|
866
|
+
* Chooses where settings are kept
|
|
867
|
+
* @param storage - Anything with the three `localStorage` methods
|
|
868
|
+
*/
|
|
869
|
+
declare function setMicdropStorage(storage: MicdropStorage): void;
|
|
870
|
+
declare const storage: MicdropStorage;
|
|
871
|
+
|
|
872
|
+
export { type AudioSink, DEFAULT_TURN_MAX_WAIT, type DeepPartial, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
|