@micdrop/client 2.2.10 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -1,58 +1,279 @@
1
1
  import { EventEmitter } from 'eventemitter3';
2
2
 
3
- interface AudioAnalyserEvents {
4
- volume: [number];
3
+ /**
4
+ * An input or output audio device offered by the system
5
+ */
6
+ interface MicdropDevice {
7
+ /** Identifier to hand back to `changeMicDevice` or `changeSpeakerDevice` */
8
+ id: string;
9
+ /** Name to show, as the system spells it */
10
+ label: string;
11
+ kind: 'audioinput' | 'audiooutput';
5
12
  }
6
- declare class AudioAnalyser extends EventEmitter<AudioAnalyserEvents> {
7
- volumeInterval: number;
8
- node: AnalyserNode;
9
- volume: number;
10
- private fftSize;
11
- private fftBins;
12
- private interval;
13
- constructor(audioContext: AudioContext, volumeInterval?: number);
13
+ interface MicEvents {
14
+ /** A batch of mono samples, in the -1..1 range, with its sample rate */
15
+ Frames: [Float32Array, number];
16
+ /** Level of the captured audio, emitted about ten times a second */
17
+ Volume: [number];
18
+ /** The system gained or lost an audio device */
19
+ DeviceChange: [];
20
+ Error: [Error];
21
+ }
22
+ /**
23
+ * What a {@link VAD} listens to. The `Mic` singleton implements it, and so can
24
+ * any other emitter, which makes voice activity detection testable on its own.
25
+ */
26
+ type MicSource = Pick<EventEmitter<MicEvents>, 'on' | 'off'>;
27
+ interface MicDriverEvents {
28
+ /** A batch of mono samples, in the -1..1 range, with its sample rate */
29
+ Frames: [Float32Array, number];
30
+ /** The system gained or lost an audio device */
31
+ DeviceChange: [];
32
+ Error: [Error];
33
+ }
34
+ /**
35
+ * Captures the microphone and pushes raw samples.
36
+ *
37
+ * `@micdrop/web` records with the Web Audio API and `@micdrop/react-native`
38
+ * with the native audio engine. Everything above this interface is shared.
39
+ */
40
+ declare abstract class MicDriver extends EventEmitter<MicDriverEvents> {
41
+ abstract get isStarted(): boolean;
42
+ /** Identifier of the input device in use, when the system reports one */
43
+ abstract get deviceId(): string | undefined;
44
+ /**
45
+ * Asks for the microphone permission and starts capturing
46
+ * @param deviceId - Input device to record from, defaults to the system one
47
+ */
48
+ abstract start(deviceId?: string): Promise<void>;
49
+ /** Stops capturing and releases the microphone */
50
+ abstract stop(): Promise<void>;
51
+ /** Lists the input devices the system offers */
52
+ abstract getDevices(): Promise<MicdropDevice[]>;
53
+ }
54
+ interface SpeakerDriverEvents {
55
+ StartPlaying: [];
56
+ StopPlaying: [];
57
+ /** Level of the audio being played */
58
+ Volume: [number];
59
+ }
60
+ /**
61
+ * Plays the 16 kHz mono PCM sent by the server.
62
+ */
63
+ declare abstract class SpeakerDriver extends EventEmitter<SpeakerDriverEvents> {
64
+ abstract get isPlaying(): boolean;
65
+ /** Identifier of the output device in use, when the system reports one */
66
+ abstract get deviceId(): string | undefined;
67
+ /** Prepares the audio graph, called before the first chunk arrives */
68
+ abstract start(): Promise<void>;
69
+ /**
70
+ * Queues samples for playback, right after the ones already queued
71
+ * @param pcm - Mono samples as 16 bits integers
72
+ * @param sampleRate - Sample rate of `pcm`, in Hz
73
+ */
74
+ abstract play(pcm: Int16Array, sampleRate: number): void;
75
+ /** Drops everything queued and stops playing immediately */
76
+ abstract stopAudio(): void;
77
+ /** Releases the audio graph */
78
+ abstract stop(): Promise<void>;
79
+ /**
80
+ * Plays through another output
81
+ * @param deviceId - Output device, or route on a phone
82
+ */
83
+ abstract changeDevice(deviceId: string): Promise<void>;
84
+ /** Lists the output devices the system offers */
85
+ abstract getDevices(): Promise<MicdropDevice[]>;
86
+ }
87
+ /** A buffer handed to an {@link AudioSink}, that can still be cancelled */
88
+ interface ScheduledAudio {
14
89
  stop(): void;
15
- private getMaxVolume;
90
+ }
91
+ /**
92
+ * The minimal slice of an audio engine needed to stream the assistant voice.
93
+ * Implemented on each platform, and faked in the test suite.
94
+ */
95
+ interface AudioSink {
96
+ /** Sample rate of the underlying audio graph, in Hz */
97
+ readonly sampleRate: number;
98
+ /** Current time of the audio graph, in seconds */
99
+ readonly currentTime: number;
100
+ /**
101
+ * Plays samples at a given time on the audio graph clock
102
+ * @param samples - Mono samples, in the -1..1 range, at `sampleRate`
103
+ * @param when - When to start, in seconds on the audio graph clock
104
+ * @param onEnded - Called once the samples have been played
105
+ */
106
+ schedule(samples: Float32Array, when: number, onEnded: () => void): ScheduledAudio;
16
107
  }
17
108
 
18
- declare class Mic$1 {
19
- analyser: AudioAnalyser;
20
- deviceId: string | undefined;
21
- private audioStream;
22
- private sourceNode;
23
- private getMicConstraints;
109
+ /**
110
+ * The microphone, as a single object shared by the whole app.
111
+ *
112
+ * It wraps a {@link MicDriver}, the browser one or the native one, and turns
113
+ * the samples it captures into the level used by the VAD and by level meters.
114
+ * Nothing here knows which platform it runs on.
115
+ */
116
+ declare class MicController extends EventEmitter<MicEvents> {
117
+ private driver;
118
+ private meter;
119
+ private pending;
120
+ private pendingLength;
121
+ private pendingRate;
122
+ private rateCheckedAt;
123
+ private rateCheckSamples;
124
+ private rateWarned;
125
+ /** Last level measured, in decibels, between -Infinity and 0 */
126
+ volume: number;
24
127
  /**
25
- * Starts the microphone
26
- * @param deviceId - The deviceId to use
27
- * @returns The microphone stream
128
+ * Replaces the audio driver, to record with another library
129
+ * @param driver - The driver to record with
28
130
  */
29
- start(deviceId?: string): Promise<MediaStream>;
131
+ setDriver(driver: MicDriver): void;
132
+ getDriver(): MicDriver;
133
+ get isStarted(): boolean;
134
+ get deviceId(): string | undefined;
30
135
  /**
31
- * Stops the microphone
136
+ * Asks for the microphone permission and starts capturing
137
+ * @param deviceId - Input device to record from, defaults to the system one
32
138
  */
33
- stop(): void;
139
+ start(deviceId?: string): Promise<void>;
140
+ /** Stops capturing and releases the microphone */
141
+ stop(): Promise<void>;
142
+ /** Lists the input devices the system offers */
143
+ getDevices(): Promise<MicdropDevice[]>;
144
+ private onFrames;
145
+ /**
146
+ * Times the captured audio against the rate it claims to be at.
147
+ *
148
+ * A microphone that announces 16 kHz while delivering 48 kHz sends speech to
149
+ * the server three times too slow, which comes back as a short transcript in
150
+ * a random language. That is invisible otherwise, so it is worth saying out
151
+ * loud once.
152
+ * @param samples - How many samples just arrived
153
+ * @param sampleRate - The rate they are said to be at
154
+ */
155
+ private checkRate;
156
+ private onDeviceChange;
157
+ private onError;
158
+ private resetVolume;
34
159
  }
35
160
 
36
161
  interface SpeakerEvents {
37
162
  StartPlaying: void;
38
163
  StopPlaying: void;
164
+ /** Level of the audio being played */
165
+ Volume: [number];
39
166
  }
40
- declare class Speaker$1 extends EventEmitter<SpeakerEvents> {
41
- isPlaying: boolean;
42
- analyser: AudioAnalyser;
43
- deviceId: string | undefined;
44
- devices: MediaDeviceInfo[];
45
- private audioStream?;
46
- private audioSourceNode?;
47
- private audioElementDestination?;
167
+ /**
168
+ * The speaker, as a single object shared by the whole app.
169
+ *
170
+ * It wraps a {@link SpeakerDriver} and plays the 16 kHz PCM sent by the server
171
+ * as it arrives.
172
+ */
173
+ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
174
+ private driver;
175
+ /** Last level measured, in decibels, between -Infinity and 0 */
176
+ volume: number;
177
+ /**
178
+ * Replaces the audio driver, to play sound with another library
179
+ * @param driver - The driver to play with
180
+ */
181
+ setDriver(driver: SpeakerDriver): void;
182
+ getDriver(): SpeakerDriver;
183
+ get isPlaying(): boolean;
184
+ get deviceId(): string | undefined;
185
+ /** Prepares the audio graph, called before the first chunk arrives */
48
186
  start(): Promise<void>;
49
- playAudio(blob: Blob): Promise<void>;
50
- stopAudio(): Promise<void>;
187
+ /**
188
+ * Queues samples for playback, right after the ones already queued
189
+ * @param pcm - Mono samples as 16 bits integers, or the bytes holding them
190
+ * @param sampleRate - Sample rate of `pcm`, in Hz
191
+ */
192
+ playAudio(pcm: Int16Array | ArrayBuffer, sampleRate?: number): void;
193
+ /** Drops everything queued and stops playing immediately */
194
+ stopAudio(): void;
195
+ /** Releases the audio graph */
196
+ stop(): Promise<void>;
197
+ /**
198
+ * Plays through another output
199
+ * @param deviceId - Output device, or route on a phone
200
+ */
51
201
  changeDevice(deviceId: string): Promise<void>;
52
- get isDefaultDevice(): boolean;
53
- private setIsPlaying;
54
- private setDevice;
202
+ /** Lists the output devices the system offers */
203
+ getDevices(): Promise<MicdropDevice[]>;
204
+ private onStartPlaying;
205
+ private onStopPlaying;
206
+ private onVolume;
207
+ }
208
+
209
+ declare enum MicdropClientCommands {
210
+ StartSpeaking = "StartSpeaking",
211
+ StopSpeaking = "StopSpeaking",
212
+ Mute = "Mute"
213
+ }
214
+ declare enum MicdropServerCommands {
215
+ Message = "Message",
216
+ CancelLastUserMessage = "CancelLastUserMessage",
217
+ SkipAnswer = "SkipAnswer",
218
+ EndCall = "EndCall",
219
+ ToolCall = "ToolCall"
220
+ }
221
+ /**
222
+ * Hears whether a sentence has landed, where voice activity detection only
223
+ * hears whether someone is speaking.
224
+ *
225
+ * `SmartTurn` from `@micdrop/smart-turn` implements it, and so can anything
226
+ * else, a call to a service included. Both sides of a call can hold one, the
227
+ * client to decide when its turn ends and the server to decide when to answer.
228
+ */
229
+ interface TurnDetector {
230
+ /**
231
+ * Feeds the audio received since the last call
232
+ * @param samples - Mono samples, in the -1..1 range
233
+ * @param sampleRate - Sample rate of `samples`, in Hz
234
+ */
235
+ push(samples: Float32Array, sampleRate?: number): void;
236
+ /** Answers whether the turn pushed so far sounds finished */
237
+ predict(): Promise<{
238
+ complete: boolean;
239
+ }>;
240
+ /** Starts a new turn, forgetting the previous one */
241
+ reset(): void;
242
+ }
243
+ interface MicdropCallSummary {
244
+ conversation: MicdropConversation;
245
+ duration: number;
246
+ }
247
+ type MicdropConversationItem = MicdropConversationMessage | MicdropConversationToolCall | MicdropConversationToolResult;
248
+ type MicdropConversation = Array<MicdropConversationItem>;
249
+ type MicdropAnswerMetadata = {
250
+ [key: string]: any;
251
+ };
252
+ interface MicdropConversationMessage<Data extends MicdropAnswerMetadata = MicdropAnswerMetadata> {
253
+ role: 'system' | 'user' | 'assistant';
254
+ content: string;
255
+ metadata?: Data;
256
+ }
257
+ interface MicdropConversationToolCall {
258
+ role: 'tool_call';
259
+ toolCallId: string;
260
+ toolName: string;
261
+ parameters: string;
262
+ }
263
+ interface MicdropConversationToolResult {
264
+ role: 'tool_result';
265
+ toolCallId: string;
266
+ toolName: string;
267
+ output: string;
55
268
  }
269
+ interface MicdropToolCall {
270
+ name: string;
271
+ parameters: any;
272
+ output: any;
273
+ }
274
+ type DeepPartial<T> = T extends object ? {
275
+ [P in keyof T]?: DeepPartial<T[P]>;
276
+ } : T;
56
277
 
57
278
  interface VADEvents {
58
279
  StartSpeaking: void;
@@ -86,9 +307,9 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
86
307
  abstract get isPaused(): boolean;
87
308
  /**
88
309
  * Starts the VAD
89
- * @param stream - The stream to start the VAD on
310
+ * @param mic - The microphone to listen to
90
311
  */
91
- abstract start(stream: MediaStream): Promise<void>;
312
+ abstract start(mic: MicSource): Promise<void>;
92
313
  /**
93
314
  * Stops the VAD
94
315
  */
@@ -120,6 +341,13 @@ type VADConfig = VAD | VADConfigName | Array<VAD | VADConfigName>;
120
341
  declare function getVAD(vad?: VADConfig): VAD;
121
342
  declare function equalVADConfig(a?: VADConfig, b?: VADConfig): boolean;
122
343
 
344
+ /**
345
+ * How long a turn may stay open after the detector asked to wait.
346
+ *
347
+ * Long enough to cover the pause of someone looking for their words, and short
348
+ * enough that a wrong verdict costs one awkward silence rather than the call.
349
+ */
350
+ declare const DEFAULT_TURN_MAX_WAIT = 4000;
123
351
  interface MicRecorderState {
124
352
  isStarting: boolean;
125
353
  isStarted: boolean;
@@ -131,35 +359,167 @@ interface MicRecorderEvents {
131
359
  StopSpeaking: void;
132
360
  StateChange: [MicRecorderState];
133
361
  }
362
+ /**
363
+ * Turns what the microphone captures into the chunks sent to the server.
364
+ *
365
+ * It only records while the VAD hears someone, and keeps a short reserve of
366
+ * audio so the first syllable, spoken before the VAD reacted, is sent too.
367
+ *
368
+ * Recording and the turn are two different things. The VAD decides what is
369
+ * worth sending, so the silences stay out of the stream either way. A
370
+ * {@link TurnDetector}, when there is one, decides when the turn is over, and
371
+ * a pause the speaker is about to fill leaves it open.
372
+ */
134
373
  declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
135
374
  private vadConfig?;
375
+ private turnDetector?;
376
+ /**
377
+ * How long a turn stays open once the detector asked to wait.
378
+ *
379
+ * The way out of a detector that hears an unfinished sentence where there
380
+ * is none: the speaker who never comes back still gets an answer.
381
+ */
382
+ turnMaxWait: number;
136
383
  state: MicRecorderState;
137
384
  vad: VAD;
138
- private stream;
139
- private source;
140
- private workletNode;
141
- private delayedStream;
385
+ private mic;
386
+ private isRecording;
142
387
  private speakingConfirmed;
388
+ private sourceSampleRate;
389
+ private reserve;
390
+ private reserveLength;
391
+ private buffer;
392
+ private bufferLength;
143
393
  private queuedChunks;
144
- constructor(vadConfig?: VADConfig | undefined);
145
- start: (stream: MediaStream) => Promise<void>;
394
+ private turnOpen;
395
+ private turnListening;
396
+ private turnTimer;
397
+ private turnRound;
398
+ constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
399
+ /**
400
+ * How long a turn stays open once the detector asked to wait.
401
+ *
402
+ * The way out of a detector that hears an unfinished sentence where there
403
+ * is none: the speaker who never comes back still gets an answer.
404
+ */
405
+ turnMaxWait?: number);
406
+ start: (mic: MicSource) => Promise<void>;
146
407
  stop: () => void;
408
+ /**
409
+ * Changes what decides when a turn is over
410
+ * @param turnDetector - The new detector, or nothing to leave it to the VAD
411
+ */
412
+ changeTurnDetector: (turnDetector?: TurnDetector) => void;
147
413
  changeVad: (vadConfig: VADConfig) => void;
414
+ private onFrames;
148
415
  private onStartSpeaking;
149
416
  private onConfirmSpeaking;
150
417
  private onCancelSpeaking;
151
418
  private onStopSpeaking;
152
- private onWorkletMessage;
419
+ /**
420
+ * Ends the turn, unless the detector hears an unfinished sentence.
421
+ *
422
+ * A microphone that is no longer listening ends it straight away: a muted or
423
+ * paused call has nothing more to say, whatever the sentence sounded like.
424
+ */
425
+ private closeTurn;
426
+ private endTurn;
427
+ /** Drops the pending close, both the timer and the answer still owed */
428
+ private cancelPendingClose;
429
+ private stopRecording;
430
+ /**
431
+ * Emits every complete chunk waiting in the buffer
432
+ * @param final - Also emit the incomplete last chunk
433
+ */
434
+ private flushChunks;
435
+ private emitChunk;
436
+ private resetBuffers;
153
437
  private changeState;
154
438
  }
155
439
 
156
- declare enum LocalStorageKeys {
157
- MicDevice = "micdrop.micDevice",
158
- SpeakerDevice = "micdrop.speakerDevice",
159
- VolumeVADOptions = "micdrop.VolumeVADOptions",
160
- SileroVADOptions = "micdrop.SileroVADOptions"
440
+ interface Pcm16AudioStreamEvents {
441
+ StartPlaying: [];
442
+ StopPlaying: [];
443
+ Volume: [number];
444
+ }
445
+ /**
446
+ * Turns the stream of PCM chunks sent by the server into continuous playback.
447
+ *
448
+ * Chunks arrive faster or slower than they are heard, so each one is scheduled
449
+ * right after the previous one on the audio graph clock instead of being played
450
+ * on arrival.
451
+ */
452
+ declare class Pcm16AudioStream extends EventEmitter<Pcm16AudioStreamEvents> {
453
+ private sink;
454
+ isPlaying: boolean;
455
+ private sources;
456
+ private nextStartTime;
457
+ private prebuffering;
458
+ private prebuffer;
459
+ private prebufferDuration;
460
+ private quietTimer?;
461
+ private endTimer?;
462
+ private volumeTimers;
463
+ private meter;
464
+ constructor(sink: AudioSink);
465
+ /**
466
+ * Queues samples for playback
467
+ * @param pcm - Mono samples as 16 bits integers
468
+ * @param sampleRate - Sample rate of `pcm`, in Hz
469
+ */
470
+ playAudio(pcm: Int16Array, sampleRate: number): void;
471
+ /** Drops everything queued and stops playing immediately */
472
+ stopAudio(): void;
473
+ private handleSamples;
474
+ private flushPrebuffer;
475
+ private resetPrebuffer;
476
+ private scheduleSamples;
477
+ private scheduleVolume;
478
+ private clearVolumeTimers;
479
+ private setIsPlaying;
161
480
  }
162
481
 
482
+ /**
483
+ * PCM helpers shared by the microphone and the speaker.
484
+ *
485
+ * Everything here is pure so it can be exercised without a device: the same
486
+ * functions run on iOS, on Android and in the test suite.
487
+ */
488
+ /**
489
+ * Resamples mono float samples with linear interpolation.
490
+ * @param input - Mono samples, in the -1..1 range
491
+ * @param inputRate - Sample rate of `input`, in Hz
492
+ * @param outputRate - Wanted sample rate, in Hz
493
+ * @returns The resampled samples, or `input` itself when both rates match
494
+ */
495
+ declare function resample(input: Float32Array, inputRate: number, outputRate: number): Float32Array;
496
+ /**
497
+ * Converts float samples to signed 16 bits PCM
498
+ * @param input - Mono samples, in the -1..1 range
499
+ * @returns The same samples as 16 bits integers
500
+ */
501
+ declare function floatToPcm16(input: Float32Array): Int16Array;
502
+ /**
503
+ * Converts signed 16 bits PCM to float samples
504
+ * @param input - Mono samples as 16 bits integers
505
+ * @returns The same samples in the -1..1 range
506
+ */
507
+ declare function pcm16ToFloat(input: Int16Array): Float32Array;
508
+ /**
509
+ * Concatenates chunks of samples into a single buffer
510
+ * @param chunks - The chunks to concatenate, in order
511
+ * @returns One buffer holding every sample
512
+ */
513
+ declare function concatFloat32(chunks: Float32Array[]): Float32Array;
514
+ /**
515
+ * Copies signed 16 bits PCM into a standalone ArrayBuffer, ready to be sent
516
+ * over a WebSocket. A typed array can be a view on a larger buffer, sending it
517
+ * as is would send the whole buffer.
518
+ * @param pcm - The samples to send
519
+ * @returns An ArrayBuffer holding exactly those samples
520
+ */
521
+ declare function pcm16ToArrayBuffer(pcm: Int16Array): ArrayBuffer;
522
+
163
523
  /**
164
524
  * Combine an array of VADs
165
525
  */
@@ -169,121 +529,172 @@ declare class MultipleVAD extends VAD {
169
529
  constructor(vads: Array<VAD | VADConfigName>);
170
530
  get isStarted(): boolean;
171
531
  get isPaused(): boolean;
172
- start(stream: MediaStream): Promise<void>;
532
+ start(mic: MicSource): Promise<void>;
173
533
  private onStatusChange;
174
534
  stop(): Promise<void>;
175
535
  pause(): Promise<void>;
176
536
  resume(): Promise<void>;
177
537
  }
178
538
 
539
+ /** Rate and window the Silero v5 model expects */
540
+ declare const SILERO_SAMPLE_RATE = 16000;
541
+ declare const SILERO_FRAME_SAMPLES = 512;
542
+ /**
543
+ * One loaded Silero model, whichever runtime holds it.
544
+ *
545
+ * `@micdrop/web` runs it with onnxruntime-web and `@micdrop/react-native` with
546
+ * onnxruntime-react-native. Both answer the same question: how likely is it
547
+ * that these 512 samples are speech.
548
+ */
549
+ interface SileroModel {
550
+ /**
551
+ * Scores one window of audio
552
+ * @param frame - 512 mono samples at 16 kHz, in the -1..1 range
553
+ * @returns The probability that it is speech, between 0 and 1
554
+ */
555
+ process(frame: Float32Array): Promise<number>;
556
+ /** Forgets what came before, at the start of a new call */
557
+ reset(): void;
558
+ /** Releases the runtime */
559
+ release(): Promise<void>;
560
+ }
561
+ /** Loads the model, which usually means fetching a couple of megabytes */
562
+ type SileroModelLoader = () => Promise<SileroModel>;
179
563
  type SileroVADOptions = {
564
+ /** Above this probability, the window counts as speech */
180
565
  positiveSpeechThreshold: number;
566
+ /** Below this probability, the window counts as silence */
181
567
  negativeSpeechThreshold: number;
568
+ /** Windows of speech needed before a turn is confirmed */
182
569
  minSpeechFrames: number;
570
+ /** Windows of silence tolerated inside a turn before it is closed */
183
571
  redemptionFrames: number;
184
572
  };
185
573
  /**
186
- * Silero VAD
187
- * @see https://github.com/ricky0123/vad
574
+ * Says how the Silero model is loaded on this platform. Called by
575
+ * `@micdrop/web` and `@micdrop/react-native` when they are imported.
576
+ * @param loader - Loads and returns a ready to use model
577
+ */
578
+ declare function setSileroModelLoader(loader: SileroModelLoader): void;
579
+ /**
580
+ * Voice activity detection with the Silero model.
581
+ *
582
+ * It hears the difference between a voice and a noise, where
583
+ * {@link VolumeVAD} only hears how loud the room is. The model itself is
584
+ * loaded by the platform package, everything else runs the same everywhere.
585
+ *
586
+ * @see https://github.com/snakers4/silero-vad
188
587
  */
189
588
  declare class SileroVAD extends VAD {
190
589
  readonly name = "SileroVAD";
191
590
  options: SileroVADOptions;
192
- private vad;
591
+ delay: number;
592
+ private model;
593
+ private mic;
193
594
  private _isPaused;
595
+ private buffer;
596
+ private bufferLength;
597
+ private bufferRate;
598
+ private speaking;
599
+ private speechFrames;
600
+ private silenceFrames;
601
+ private processing;
194
602
  constructor(options?: Partial<SileroVADOptions>);
195
603
  get isStarted(): boolean;
196
604
  get isPaused(): boolean;
197
- start(stream: MediaStream): Promise<void>;
605
+ start(mic: MicSource): Promise<void>;
198
606
  stop(): Promise<void>;
199
607
  pause(): Promise<void>;
200
608
  resume(): Promise<void>;
201
609
  setOptions(options: Partial<SileroVADOptions>): void;
202
610
  resetOptions(): void;
611
+ private onFrames;
612
+ private drain;
613
+ private onProbability;
614
+ private closeOpenTurn;
615
+ private resetState;
203
616
  }
204
617
 
205
618
  type VolumeVADOptions = {
619
+ /** Number of recent samples kept to decide whether someone is speaking */
206
620
  history: number;
621
+ /** Level above which audio counts as speech, in decibels */
207
622
  threshold: number;
208
623
  };
209
624
  /**
210
- * Volume-based Voice Activity Detection
211
- * Based on the hark library implementation
625
+ * Volume based Voice Activity Detection.
626
+ *
627
+ * It follows the level reported by the microphone, and needs a few consecutive
628
+ * loud samples before confirming speech, so a door slam or a keyboard click
629
+ * does not start a turn.
212
630
  */
213
631
  declare class VolumeVAD extends VAD {
214
632
  readonly name = "VolumeVAD";
215
633
  options: VolumeVADOptions;
216
- private _isPaused;
217
- private analyser;
218
- private sourceNode;
219
- private fftBins;
634
+ delay: number;
635
+ private mic;
220
636
  private running;
637
+ private _isPaused;
221
638
  private speaking;
222
639
  private attemptSpeaking;
223
640
  private speakingHistory;
224
641
  constructor(options?: Partial<VolumeVADOptions>);
225
- private getMaxVolume;
226
642
  get isStarted(): boolean;
227
643
  get isPaused(): boolean;
228
- start(stream: MediaStream): Promise<void>;
229
- private startLoop;
230
- private resetHistory;
644
+ start(mic: MicSource): Promise<void>;
231
645
  stop(): Promise<void>;
232
646
  pause(): Promise<void>;
233
647
  resume(): Promise<void>;
234
648
  setOptions(options: Partial<VolumeVADOptions>): void;
235
649
  resetOptions(): void;
650
+ private onVolume;
651
+ private reset;
236
652
  }
237
653
 
238
- declare const Mic: Mic$1;
239
- declare const Speaker: Speaker$1;
240
-
241
- declare enum MicdropClientCommands {
242
- StartSpeaking = "StartSpeaking",
243
- StopSpeaking = "StopSpeaking",
244
- Mute = "Mute"
245
- }
246
- declare enum MicdropServerCommands {
247
- Message = "Message",
248
- CancelLastUserMessage = "CancelLastUserMessage",
249
- SkipAnswer = "SkipAnswer",
250
- EndCall = "EndCall",
251
- ToolCall = "ToolCall"
252
- }
253
- interface MicdropCallSummary {
254
- conversation: MicdropConversation;
255
- duration: number;
256
- }
257
- type MicdropConversationItem = MicdropConversationMessage | MicdropConversationToolCall | MicdropConversationToolResult;
258
- type MicdropConversation = Array<MicdropConversationItem>;
259
- type MicdropAnswerMetadata = {
260
- [key: string]: any;
261
- };
262
- interface MicdropConversationMessage<Data extends MicdropAnswerMetadata = MicdropAnswerMetadata> {
263
- role: 'system' | 'user' | 'assistant';
264
- content: string;
265
- metadata?: Data;
266
- }
267
- interface MicdropConversationToolCall {
268
- role: 'tool_call';
269
- toolCallId: string;
270
- toolName: string;
271
- parameters: string;
272
- }
273
- interface MicdropConversationToolResult {
274
- role: 'tool_result';
275
- toolCallId: string;
276
- toolName: string;
277
- output: string;
278
- }
279
- interface MicdropToolCall {
280
- name: string;
281
- parameters: any;
282
- output: any;
654
+ /**
655
+ * Measures how loud the microphone is, the way a browser AnalyserNode does.
656
+ *
657
+ * The level is the loudest frequency bin rather than the overall energy: a
658
+ * voice concentrates its power in a few bins while a fan or a room spreads it
659
+ * over all of them, so the two are told apart by about 25 dB instead of the 7 dB
660
+ * an RMS would give.
661
+ *
662
+ * Everything here is pure, so the same measurement runs in a browser, on a
663
+ * phone and in the test suite.
664
+ */
665
+ /** Sample rate the measurement expects, so a bin means the same frequency everywhere */
666
+ declare const VOLUME_SAMPLE_RATE = 16000;
667
+ /**
668
+ * Follows the level of a stream of audio.
669
+ *
670
+ * It keeps the previous measurement, so a single quiet frame in the middle of a
671
+ * word does not read as a pause.
672
+ */
673
+ declare class VolumeMeter {
674
+ private real;
675
+ private imaginary;
676
+ private magnitudes;
677
+ private smoothed;
678
+ private hasPrevious;
679
+ /**
680
+ * Measures a window of audio
681
+ * @param samples - At least 512 mono samples at 16 kHz, in the -1..1 range.
682
+ * Only the last 512 are looked at.
683
+ * @returns The level in decibels, between -Infinity (silence) and 0
684
+ */
685
+ measure(samples: Float32Array): number;
686
+ /** Forgets the previous window, for when capture stops and starts again */
687
+ reset(): void;
283
688
  }
284
- type DeepPartial<T> = T extends object ? {
285
- [P in keyof T]?: DeepPartial<T[P]>;
286
- } : T;
689
+ /**
690
+ * Measures one window of audio, without following what came before
691
+ * @param samples - Mono samples at 16 kHz, in the -1..1 range
692
+ * @returns The level in decibels, between -Infinity (silence) and 0
693
+ */
694
+ declare function getVolume(samples: Float32Array): number;
695
+
696
+ declare const Mic: MicController;
697
+ declare const Speaker: SpeakerController;
287
698
 
288
699
  declare enum MicdropClientErrorCode {
289
700
  MissingUrl = "MissingUrl",
@@ -299,7 +710,12 @@ declare class MicdropClientError extends Error {
299
710
  code: MicdropClientErrorCode;
300
711
  constructor(code?: MicdropClientErrorCode, message?: string);
301
712
  }
302
- declare function getClientErrorFromWSCloseEvent(event: CloseEvent): MicdropClientError | undefined;
713
+ /** What React Native gives on a WebSocket close */
714
+ interface WSCloseEvent {
715
+ code?: number;
716
+ reason?: string;
717
+ }
718
+ declare function getClientErrorFromWSCloseEvent(event: WSCloseEvent): MicdropClientError | undefined;
303
719
  declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
304
720
  declare function isRecoverableError(error: MicdropClientError): boolean;
305
721
 
@@ -318,6 +734,14 @@ interface MicdropOptions {
318
734
  url?: string;
319
735
  params?: Record<string, any>;
320
736
  vad?: VADConfig;
737
+ /**
738
+ * Holds the turn open while the sentence sounds unfinished, so a hesitation
739
+ * no longer counts as the end of a question. Without one, a turn ends as
740
+ * soon as the VAD hears enough silence.
741
+ */
742
+ turnDetector?: TurnDetector;
743
+ /** How long a turn may stay open after the detector asked to wait */
744
+ turnMaxWait?: number;
321
745
  disableInterruption?: boolean;
322
746
  debugLog?: boolean;
323
747
  reconnect?: MicdropReconnectOptions;
@@ -336,8 +760,8 @@ interface MicdropState {
336
760
  isMicMuted: boolean;
337
761
  micDeviceId: string | undefined;
338
762
  speakerDeviceId: string | undefined;
339
- micDevices: MediaDeviceInfo[];
340
- speakerDevices: MediaDeviceInfo[];
763
+ micDevices: MicdropDevice[];
764
+ speakerDevices: MicdropDevice[];
341
765
  conversation: MicdropConversation;
342
766
  error: MicdropClientError | undefined;
343
767
  }
@@ -346,10 +770,9 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
346
770
  micRecorder?: MicRecorder;
347
771
  conversation: MicdropConversation;
348
772
  error: MicdropClientError | undefined;
349
- speakerDevices: MediaDeviceInfo[];
350
- micDevices: MediaDeviceInfo[];
773
+ speakerDevices: MicdropDevice[];
774
+ micDevices: MicdropDevice[];
351
775
  private ws?;
352
- private micStream?;
353
776
  private startTime;
354
777
  private lastNotifiedState;
355
778
  private _isProcessing;
@@ -387,6 +810,20 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
387
810
  vad?: VADConfig;
388
811
  deviceId?: string;
389
812
  }) => Promise<void>;
813
+ /**
814
+ * Changes what decides when a turn is over, during a call or before one
815
+ * @param turnDetector - The new detector, or nothing to leave it to the VAD
816
+ */
817
+ setTurnDetector: (turnDetector?: TurnDetector) => void;
818
+ /**
819
+ * Says how long a held turn waits for the rest of the sentence.
820
+ *
821
+ * The way out of a wrong verdict: a detector can hear an unfinished sentence
822
+ * where there is none, and the speaker who never comes back still gets an
823
+ * answer once this runs out.
824
+ * @param milliseconds - How long to wait past the detector asking to
825
+ */
826
+ setTurnMaxWait: (milliseconds: number) => void;
390
827
  changeMicDevice: (deviceId: string) => Promise<void>;
391
828
  changeSpeakerDevice: (deviceId: string) => Promise<void>;
392
829
  private stopMic;
@@ -408,4 +845,28 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
408
845
 
409
846
  declare const Micdrop: MicdropClient;
410
847
 
411
- export { type DeepPartial, LocalStorageKeys, Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropToolCall, MultipleVAD, SileroVAD, type SileroVADOptions, Speaker, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VolumeVAD, type VolumeVADOptions, equalVADConfig, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, isRecoverableError };
848
+ /**
849
+ * Where settings that outlive a call are kept.
850
+ *
851
+ * A browser has `localStorage`, a phone has whatever the app installs. Reads
852
+ * are synchronous, so a VAD can be built with its saved options in hand.
853
+ */
854
+ interface MicdropStorage {
855
+ getItem(key: string): string | null;
856
+ setItem(key: string, value: string): void;
857
+ removeItem(key: string): void;
858
+ }
859
+ declare enum MicdropStorageKeys {
860
+ MicDevice = "micdrop.micDevice",
861
+ SpeakerDevice = "micdrop.speakerDevice",
862
+ VolumeVADOptions = "micdrop.VolumeVADOptions",
863
+ SileroVADOptions = "micdrop.SileroVADOptions"
864
+ }
865
+ /**
866
+ * Chooses where settings are kept
867
+ * @param storage - Anything with the three `localStorage` methods
868
+ */
869
+ declare function setMicdropStorage(storage: MicdropStorage): void;
870
+ declare const storage: MicdropStorage;
871
+
872
+ export { type AudioSink, DEFAULT_TURN_MAX_WAIT, type DeepPartial, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };