@micdrop/client 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +152 -49
- package/dist/index.d.ts +152 -49
- package/dist/index.js +142 -6
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +141 -6
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -206,6 +206,76 @@ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
|
|
|
206
206
|
private onVolume;
|
|
207
207
|
}
|
|
208
208
|
|
|
209
|
+
declare enum MicdropClientCommands {
|
|
210
|
+
StartSpeaking = "StartSpeaking",
|
|
211
|
+
StopSpeaking = "StopSpeaking",
|
|
212
|
+
Mute = "Mute"
|
|
213
|
+
}
|
|
214
|
+
declare enum MicdropServerCommands {
|
|
215
|
+
Message = "Message",
|
|
216
|
+
PartialAssistantMessage = "PartialAssistantMessage",
|
|
217
|
+
CancelLastUserMessage = "CancelLastUserMessage",
|
|
218
|
+
SkipAnswer = "SkipAnswer",
|
|
219
|
+
EndCall = "EndCall",
|
|
220
|
+
ToolCall = "ToolCall"
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Hears whether a sentence has landed, where voice activity detection only
|
|
224
|
+
* hears whether someone is speaking.
|
|
225
|
+
*
|
|
226
|
+
* `SmartTurn` from `@micdrop/smart-turn` implements it, and so can anything
|
|
227
|
+
* else, a call to a service included. Both sides of a call can hold one, the
|
|
228
|
+
* client to decide when its turn ends and the server to decide when to answer.
|
|
229
|
+
*/
|
|
230
|
+
interface TurnDetector {
|
|
231
|
+
/**
|
|
232
|
+
* Feeds the audio received since the last call
|
|
233
|
+
* @param samples - Mono samples, in the -1..1 range
|
|
234
|
+
* @param sampleRate - Sample rate of `samples`, in Hz
|
|
235
|
+
*/
|
|
236
|
+
push(samples: Float32Array, sampleRate?: number): void;
|
|
237
|
+
/** Answers whether the turn pushed so far sounds finished */
|
|
238
|
+
predict(): Promise<{
|
|
239
|
+
complete: boolean;
|
|
240
|
+
}>;
|
|
241
|
+
/** Starts a new turn, forgetting the previous one */
|
|
242
|
+
reset(): void;
|
|
243
|
+
}
|
|
244
|
+
interface MicdropCallSummary {
|
|
245
|
+
conversation: MicdropConversation;
|
|
246
|
+
duration: number;
|
|
247
|
+
}
|
|
248
|
+
type MicdropConversationItem = MicdropConversationMessage | MicdropConversationToolCall | MicdropConversationToolResult;
|
|
249
|
+
type MicdropConversation = Array<MicdropConversationItem>;
|
|
250
|
+
type MicdropAnswerMetadata = {
|
|
251
|
+
[key: string]: any;
|
|
252
|
+
};
|
|
253
|
+
interface MicdropConversationMessage<Data extends MicdropAnswerMetadata = MicdropAnswerMetadata> {
|
|
254
|
+
role: 'system' | 'user' | 'assistant';
|
|
255
|
+
content: string;
|
|
256
|
+
metadata?: Data;
|
|
257
|
+
}
|
|
258
|
+
interface MicdropConversationToolCall {
|
|
259
|
+
role: 'tool_call';
|
|
260
|
+
toolCallId: string;
|
|
261
|
+
toolName: string;
|
|
262
|
+
parameters: string;
|
|
263
|
+
}
|
|
264
|
+
interface MicdropConversationToolResult {
|
|
265
|
+
role: 'tool_result';
|
|
266
|
+
toolCallId: string;
|
|
267
|
+
toolName: string;
|
|
268
|
+
output: string;
|
|
269
|
+
}
|
|
270
|
+
interface MicdropToolCall {
|
|
271
|
+
name: string;
|
|
272
|
+
parameters: any;
|
|
273
|
+
output: any;
|
|
274
|
+
}
|
|
275
|
+
type DeepPartial<T> = T extends object ? {
|
|
276
|
+
[P in keyof T]?: DeepPartial<T[P]>;
|
|
277
|
+
} : T;
|
|
278
|
+
|
|
209
279
|
interface VADEvents {
|
|
210
280
|
StartSpeaking: void;
|
|
211
281
|
ConfirmSpeaking: void;
|
|
@@ -272,6 +342,13 @@ type VADConfig = VAD | VADConfigName | Array<VAD | VADConfigName>;
|
|
|
272
342
|
declare function getVAD(vad?: VADConfig): VAD;
|
|
273
343
|
declare function equalVADConfig(a?: VADConfig, b?: VADConfig): boolean;
|
|
274
344
|
|
|
345
|
+
/**
|
|
346
|
+
* How long a turn may stay open after the detector asked to wait.
|
|
347
|
+
*
|
|
348
|
+
* Long enough to cover the pause of someone looking for their words, and short
|
|
349
|
+
* enough that a wrong verdict costs one awkward silence rather than the call.
|
|
350
|
+
*/
|
|
351
|
+
declare const DEFAULT_TURN_MAX_WAIT = 4000;
|
|
275
352
|
interface MicRecorderState {
|
|
276
353
|
isStarting: boolean;
|
|
277
354
|
isStarted: boolean;
|
|
@@ -288,9 +365,22 @@ interface MicRecorderEvents {
|
|
|
288
365
|
*
|
|
289
366
|
* It only records while the VAD hears someone, and keeps a short reserve of
|
|
290
367
|
* audio so the first syllable, spoken before the VAD reacted, is sent too.
|
|
368
|
+
*
|
|
369
|
+
* Recording and the turn are two different things. The VAD decides what is
|
|
370
|
+
* worth sending, so the silences stay out of the stream either way. A
|
|
371
|
+
* {@link TurnDetector}, when there is one, decides when the turn is over, and
|
|
372
|
+
* a pause the speaker is about to fill leaves it open.
|
|
291
373
|
*/
|
|
292
374
|
declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
293
375
|
private vadConfig?;
|
|
376
|
+
private turnDetector?;
|
|
377
|
+
/**
|
|
378
|
+
* How long a turn stays open once the detector asked to wait.
|
|
379
|
+
*
|
|
380
|
+
* The way out of a detector that hears an unfinished sentence where there
|
|
381
|
+
* is none: the speaker who never comes back still gets an answer.
|
|
382
|
+
*/
|
|
383
|
+
turnMaxWait: number;
|
|
294
384
|
state: MicRecorderState;
|
|
295
385
|
vad: VAD;
|
|
296
386
|
private mic;
|
|
@@ -302,15 +392,41 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
302
392
|
private buffer;
|
|
303
393
|
private bufferLength;
|
|
304
394
|
private queuedChunks;
|
|
305
|
-
|
|
395
|
+
private turnOpen;
|
|
396
|
+
private turnListening;
|
|
397
|
+
private turnTimer;
|
|
398
|
+
private turnRound;
|
|
399
|
+
constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
|
|
400
|
+
/**
|
|
401
|
+
* How long a turn stays open once the detector asked to wait.
|
|
402
|
+
*
|
|
403
|
+
* The way out of a detector that hears an unfinished sentence where there
|
|
404
|
+
* is none: the speaker who never comes back still gets an answer.
|
|
405
|
+
*/
|
|
406
|
+
turnMaxWait?: number);
|
|
306
407
|
start: (mic: MicSource) => Promise<void>;
|
|
307
408
|
stop: () => void;
|
|
409
|
+
/**
|
|
410
|
+
* Changes what decides when a turn is over
|
|
411
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
412
|
+
*/
|
|
413
|
+
changeTurnDetector: (turnDetector?: TurnDetector) => void;
|
|
308
414
|
changeVad: (vadConfig: VADConfig) => void;
|
|
309
415
|
private onFrames;
|
|
310
416
|
private onStartSpeaking;
|
|
311
417
|
private onConfirmSpeaking;
|
|
312
418
|
private onCancelSpeaking;
|
|
313
419
|
private onStopSpeaking;
|
|
420
|
+
/**
|
|
421
|
+
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
422
|
+
*
|
|
423
|
+
* A microphone that is no longer listening ends it straight away: a muted or
|
|
424
|
+
* paused call has nothing more to say, whatever the sentence sounded like.
|
|
425
|
+
*/
|
|
426
|
+
private closeTurn;
|
|
427
|
+
private endTurn;
|
|
428
|
+
/** Drops the pending close, both the timer and the answer still owed */
|
|
429
|
+
private cancelPendingClose;
|
|
314
430
|
private stopRecording;
|
|
315
431
|
/**
|
|
316
432
|
* Emits every complete chunk waiting in the buffer
|
|
@@ -581,53 +697,6 @@ declare function getVolume(samples: Float32Array): number;
|
|
|
581
697
|
declare const Mic: MicController;
|
|
582
698
|
declare const Speaker: SpeakerController;
|
|
583
699
|
|
|
584
|
-
declare enum MicdropClientCommands {
|
|
585
|
-
StartSpeaking = "StartSpeaking",
|
|
586
|
-
StopSpeaking = "StopSpeaking",
|
|
587
|
-
Mute = "Mute"
|
|
588
|
-
}
|
|
589
|
-
declare enum MicdropServerCommands {
|
|
590
|
-
Message = "Message",
|
|
591
|
-
CancelLastUserMessage = "CancelLastUserMessage",
|
|
592
|
-
SkipAnswer = "SkipAnswer",
|
|
593
|
-
EndCall = "EndCall",
|
|
594
|
-
ToolCall = "ToolCall"
|
|
595
|
-
}
|
|
596
|
-
interface MicdropCallSummary {
|
|
597
|
-
conversation: MicdropConversation;
|
|
598
|
-
duration: number;
|
|
599
|
-
}
|
|
600
|
-
type MicdropConversationItem = MicdropConversationMessage | MicdropConversationToolCall | MicdropConversationToolResult;
|
|
601
|
-
type MicdropConversation = Array<MicdropConversationItem>;
|
|
602
|
-
type MicdropAnswerMetadata = {
|
|
603
|
-
[key: string]: any;
|
|
604
|
-
};
|
|
605
|
-
interface MicdropConversationMessage<Data extends MicdropAnswerMetadata = MicdropAnswerMetadata> {
|
|
606
|
-
role: 'system' | 'user' | 'assistant';
|
|
607
|
-
content: string;
|
|
608
|
-
metadata?: Data;
|
|
609
|
-
}
|
|
610
|
-
interface MicdropConversationToolCall {
|
|
611
|
-
role: 'tool_call';
|
|
612
|
-
toolCallId: string;
|
|
613
|
-
toolName: string;
|
|
614
|
-
parameters: string;
|
|
615
|
-
}
|
|
616
|
-
interface MicdropConversationToolResult {
|
|
617
|
-
role: 'tool_result';
|
|
618
|
-
toolCallId: string;
|
|
619
|
-
toolName: string;
|
|
620
|
-
output: string;
|
|
621
|
-
}
|
|
622
|
-
interface MicdropToolCall {
|
|
623
|
-
name: string;
|
|
624
|
-
parameters: any;
|
|
625
|
-
output: any;
|
|
626
|
-
}
|
|
627
|
-
type DeepPartial<T> = T extends object ? {
|
|
628
|
-
[P in keyof T]?: DeepPartial<T[P]>;
|
|
629
|
-
} : T;
|
|
630
|
-
|
|
631
700
|
declare enum MicdropClientErrorCode {
|
|
632
701
|
MissingUrl = "MissingUrl",
|
|
633
702
|
Mic = "Mic",
|
|
@@ -656,6 +725,10 @@ interface MicdropEvents {
|
|
|
656
725
|
Error: [MicdropClientError];
|
|
657
726
|
StateChange: [MicdropState, MicdropState];
|
|
658
727
|
ToolCall: [MicdropToolCall];
|
|
728
|
+
/** A message was added to the conversation, by the user or the assistant */
|
|
729
|
+
Message: [MicdropConversationItem];
|
|
730
|
+
/** What the assistant is answering, before it is finished */
|
|
731
|
+
PartialAssistantMessage: [string];
|
|
659
732
|
}
|
|
660
733
|
interface MicdropReconnectOptions {
|
|
661
734
|
maxAttempts?: number;
|
|
@@ -666,6 +739,14 @@ interface MicdropOptions {
|
|
|
666
739
|
url?: string;
|
|
667
740
|
params?: Record<string, any>;
|
|
668
741
|
vad?: VADConfig;
|
|
742
|
+
/**
|
|
743
|
+
* Holds the turn open while the sentence sounds unfinished, so a hesitation
|
|
744
|
+
* no longer counts as the end of a question. Without one, a turn ends as
|
|
745
|
+
* soon as the VAD hears enough silence.
|
|
746
|
+
*/
|
|
747
|
+
turnDetector?: TurnDetector;
|
|
748
|
+
/** How long a turn may stay open after the detector asked to wait */
|
|
749
|
+
turnMaxWait?: number;
|
|
669
750
|
disableInterruption?: boolean;
|
|
670
751
|
debugLog?: boolean;
|
|
671
752
|
reconnect?: MicdropReconnectOptions;
|
|
@@ -687,12 +768,19 @@ interface MicdropState {
|
|
|
687
768
|
micDevices: MicdropDevice[];
|
|
688
769
|
speakerDevices: MicdropDevice[];
|
|
689
770
|
conversation: MicdropConversation;
|
|
771
|
+
/**
|
|
772
|
+
* Answer being written, sent by servers configured with `partialMessages`.
|
|
773
|
+
* Empty the rest of the time, and replaced by a conversation message once
|
|
774
|
+
* the answer is finished.
|
|
775
|
+
*/
|
|
776
|
+
partialAssistantMessage: string;
|
|
690
777
|
error: MicdropClientError | undefined;
|
|
691
778
|
}
|
|
692
779
|
declare class MicdropClient extends EventEmitter<MicdropEvents> implements MicdropState {
|
|
693
780
|
options: MicdropOptions;
|
|
694
781
|
micRecorder?: MicRecorder;
|
|
695
782
|
conversation: MicdropConversation;
|
|
783
|
+
partialAssistantMessage: string;
|
|
696
784
|
error: MicdropClientError | undefined;
|
|
697
785
|
speakerDevices: MicdropDevice[];
|
|
698
786
|
micDevices: MicdropDevice[];
|
|
@@ -734,6 +822,20 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
734
822
|
vad?: VADConfig;
|
|
735
823
|
deviceId?: string;
|
|
736
824
|
}) => Promise<void>;
|
|
825
|
+
/**
|
|
826
|
+
* Changes what decides when a turn is over, during a call or before one
|
|
827
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
828
|
+
*/
|
|
829
|
+
setTurnDetector: (turnDetector?: TurnDetector) => void;
|
|
830
|
+
/**
|
|
831
|
+
* Says how long a held turn waits for the rest of the sentence.
|
|
832
|
+
*
|
|
833
|
+
* The way out of a wrong verdict: a detector can hear an unfinished sentence
|
|
834
|
+
* where there is none, and the speaker who never comes back still gets an
|
|
835
|
+
* answer once this runs out.
|
|
836
|
+
* @param milliseconds - How long to wait past the detector asking to
|
|
837
|
+
*/
|
|
838
|
+
setTurnMaxWait: (milliseconds: number) => void;
|
|
737
839
|
changeMicDevice: (deviceId: string) => Promise<void>;
|
|
738
840
|
changeSpeakerDevice: (deviceId: string) => Promise<void>;
|
|
739
841
|
private stopMic;
|
|
@@ -748,6 +850,7 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
748
850
|
private onSpeakerStartPlaying;
|
|
749
851
|
private onSpeakerStopPlaying;
|
|
750
852
|
private addMessage;
|
|
853
|
+
private setPartialAssistantMessage;
|
|
751
854
|
private setError;
|
|
752
855
|
private notifyStateChange;
|
|
753
856
|
private log;
|
|
@@ -779,4 +882,4 @@ declare enum MicdropStorageKeys {
|
|
|
779
882
|
declare function setMicdropStorage(storage: MicdropStorage): void;
|
|
780
883
|
declare const storage: MicdropStorage;
|
|
781
884
|
|
|
782
|
-
export { type AudioSink, type DeepPartial, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
|
|
885
|
+
export { type AudioSink, DEFAULT_TURN_MAX_WAIT, type DeepPartial, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
|
package/dist/index.d.ts
CHANGED
|
@@ -206,6 +206,76 @@ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
|
|
|
206
206
|
private onVolume;
|
|
207
207
|
}
|
|
208
208
|
|
|
209
|
+
declare enum MicdropClientCommands {
|
|
210
|
+
StartSpeaking = "StartSpeaking",
|
|
211
|
+
StopSpeaking = "StopSpeaking",
|
|
212
|
+
Mute = "Mute"
|
|
213
|
+
}
|
|
214
|
+
declare enum MicdropServerCommands {
|
|
215
|
+
Message = "Message",
|
|
216
|
+
PartialAssistantMessage = "PartialAssistantMessage",
|
|
217
|
+
CancelLastUserMessage = "CancelLastUserMessage",
|
|
218
|
+
SkipAnswer = "SkipAnswer",
|
|
219
|
+
EndCall = "EndCall",
|
|
220
|
+
ToolCall = "ToolCall"
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Hears whether a sentence has landed, where voice activity detection only
|
|
224
|
+
* hears whether someone is speaking.
|
|
225
|
+
*
|
|
226
|
+
* `SmartTurn` from `@micdrop/smart-turn` implements it, and so can anything
|
|
227
|
+
* else, a call to a service included. Both sides of a call can hold one, the
|
|
228
|
+
* client to decide when its turn ends and the server to decide when to answer.
|
|
229
|
+
*/
|
|
230
|
+
interface TurnDetector {
|
|
231
|
+
/**
|
|
232
|
+
* Feeds the audio received since the last call
|
|
233
|
+
* @param samples - Mono samples, in the -1..1 range
|
|
234
|
+
* @param sampleRate - Sample rate of `samples`, in Hz
|
|
235
|
+
*/
|
|
236
|
+
push(samples: Float32Array, sampleRate?: number): void;
|
|
237
|
+
/** Answers whether the turn pushed so far sounds finished */
|
|
238
|
+
predict(): Promise<{
|
|
239
|
+
complete: boolean;
|
|
240
|
+
}>;
|
|
241
|
+
/** Starts a new turn, forgetting the previous one */
|
|
242
|
+
reset(): void;
|
|
243
|
+
}
|
|
244
|
+
interface MicdropCallSummary {
|
|
245
|
+
conversation: MicdropConversation;
|
|
246
|
+
duration: number;
|
|
247
|
+
}
|
|
248
|
+
type MicdropConversationItem = MicdropConversationMessage | MicdropConversationToolCall | MicdropConversationToolResult;
|
|
249
|
+
type MicdropConversation = Array<MicdropConversationItem>;
|
|
250
|
+
type MicdropAnswerMetadata = {
|
|
251
|
+
[key: string]: any;
|
|
252
|
+
};
|
|
253
|
+
interface MicdropConversationMessage<Data extends MicdropAnswerMetadata = MicdropAnswerMetadata> {
|
|
254
|
+
role: 'system' | 'user' | 'assistant';
|
|
255
|
+
content: string;
|
|
256
|
+
metadata?: Data;
|
|
257
|
+
}
|
|
258
|
+
interface MicdropConversationToolCall {
|
|
259
|
+
role: 'tool_call';
|
|
260
|
+
toolCallId: string;
|
|
261
|
+
toolName: string;
|
|
262
|
+
parameters: string;
|
|
263
|
+
}
|
|
264
|
+
interface MicdropConversationToolResult {
|
|
265
|
+
role: 'tool_result';
|
|
266
|
+
toolCallId: string;
|
|
267
|
+
toolName: string;
|
|
268
|
+
output: string;
|
|
269
|
+
}
|
|
270
|
+
interface MicdropToolCall {
|
|
271
|
+
name: string;
|
|
272
|
+
parameters: any;
|
|
273
|
+
output: any;
|
|
274
|
+
}
|
|
275
|
+
type DeepPartial<T> = T extends object ? {
|
|
276
|
+
[P in keyof T]?: DeepPartial<T[P]>;
|
|
277
|
+
} : T;
|
|
278
|
+
|
|
209
279
|
interface VADEvents {
|
|
210
280
|
StartSpeaking: void;
|
|
211
281
|
ConfirmSpeaking: void;
|
|
@@ -272,6 +342,13 @@ type VADConfig = VAD | VADConfigName | Array<VAD | VADConfigName>;
|
|
|
272
342
|
declare function getVAD(vad?: VADConfig): VAD;
|
|
273
343
|
declare function equalVADConfig(a?: VADConfig, b?: VADConfig): boolean;
|
|
274
344
|
|
|
345
|
+
/**
|
|
346
|
+
* How long a turn may stay open after the detector asked to wait.
|
|
347
|
+
*
|
|
348
|
+
* Long enough to cover the pause of someone looking for their words, and short
|
|
349
|
+
* enough that a wrong verdict costs one awkward silence rather than the call.
|
|
350
|
+
*/
|
|
351
|
+
declare const DEFAULT_TURN_MAX_WAIT = 4000;
|
|
275
352
|
interface MicRecorderState {
|
|
276
353
|
isStarting: boolean;
|
|
277
354
|
isStarted: boolean;
|
|
@@ -288,9 +365,22 @@ interface MicRecorderEvents {
|
|
|
288
365
|
*
|
|
289
366
|
* It only records while the VAD hears someone, and keeps a short reserve of
|
|
290
367
|
* audio so the first syllable, spoken before the VAD reacted, is sent too.
|
|
368
|
+
*
|
|
369
|
+
* Recording and the turn are two different things. The VAD decides what is
|
|
370
|
+
* worth sending, so the silences stay out of the stream either way. A
|
|
371
|
+
* {@link TurnDetector}, when there is one, decides when the turn is over, and
|
|
372
|
+
* a pause the speaker is about to fill leaves it open.
|
|
291
373
|
*/
|
|
292
374
|
declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
293
375
|
private vadConfig?;
|
|
376
|
+
private turnDetector?;
|
|
377
|
+
/**
|
|
378
|
+
* How long a turn stays open once the detector asked to wait.
|
|
379
|
+
*
|
|
380
|
+
* The way out of a detector that hears an unfinished sentence where there
|
|
381
|
+
* is none: the speaker who never comes back still gets an answer.
|
|
382
|
+
*/
|
|
383
|
+
turnMaxWait: number;
|
|
294
384
|
state: MicRecorderState;
|
|
295
385
|
vad: VAD;
|
|
296
386
|
private mic;
|
|
@@ -302,15 +392,41 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
302
392
|
private buffer;
|
|
303
393
|
private bufferLength;
|
|
304
394
|
private queuedChunks;
|
|
305
|
-
|
|
395
|
+
private turnOpen;
|
|
396
|
+
private turnListening;
|
|
397
|
+
private turnTimer;
|
|
398
|
+
private turnRound;
|
|
399
|
+
constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
|
|
400
|
+
/**
|
|
401
|
+
* How long a turn stays open once the detector asked to wait.
|
|
402
|
+
*
|
|
403
|
+
* The way out of a detector that hears an unfinished sentence where there
|
|
404
|
+
* is none: the speaker who never comes back still gets an answer.
|
|
405
|
+
*/
|
|
406
|
+
turnMaxWait?: number);
|
|
306
407
|
start: (mic: MicSource) => Promise<void>;
|
|
307
408
|
stop: () => void;
|
|
409
|
+
/**
|
|
410
|
+
* Changes what decides when a turn is over
|
|
411
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
412
|
+
*/
|
|
413
|
+
changeTurnDetector: (turnDetector?: TurnDetector) => void;
|
|
308
414
|
changeVad: (vadConfig: VADConfig) => void;
|
|
309
415
|
private onFrames;
|
|
310
416
|
private onStartSpeaking;
|
|
311
417
|
private onConfirmSpeaking;
|
|
312
418
|
private onCancelSpeaking;
|
|
313
419
|
private onStopSpeaking;
|
|
420
|
+
/**
|
|
421
|
+
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
422
|
+
*
|
|
423
|
+
* A microphone that is no longer listening ends it straight away: a muted or
|
|
424
|
+
* paused call has nothing more to say, whatever the sentence sounded like.
|
|
425
|
+
*/
|
|
426
|
+
private closeTurn;
|
|
427
|
+
private endTurn;
|
|
428
|
+
/** Drops the pending close, both the timer and the answer still owed */
|
|
429
|
+
private cancelPendingClose;
|
|
314
430
|
private stopRecording;
|
|
315
431
|
/**
|
|
316
432
|
* Emits every complete chunk waiting in the buffer
|
|
@@ -581,53 +697,6 @@ declare function getVolume(samples: Float32Array): number;
|
|
|
581
697
|
declare const Mic: MicController;
|
|
582
698
|
declare const Speaker: SpeakerController;
|
|
583
699
|
|
|
584
|
-
declare enum MicdropClientCommands {
|
|
585
|
-
StartSpeaking = "StartSpeaking",
|
|
586
|
-
StopSpeaking = "StopSpeaking",
|
|
587
|
-
Mute = "Mute"
|
|
588
|
-
}
|
|
589
|
-
declare enum MicdropServerCommands {
|
|
590
|
-
Message = "Message",
|
|
591
|
-
CancelLastUserMessage = "CancelLastUserMessage",
|
|
592
|
-
SkipAnswer = "SkipAnswer",
|
|
593
|
-
EndCall = "EndCall",
|
|
594
|
-
ToolCall = "ToolCall"
|
|
595
|
-
}
|
|
596
|
-
interface MicdropCallSummary {
|
|
597
|
-
conversation: MicdropConversation;
|
|
598
|
-
duration: number;
|
|
599
|
-
}
|
|
600
|
-
type MicdropConversationItem = MicdropConversationMessage | MicdropConversationToolCall | MicdropConversationToolResult;
|
|
601
|
-
type MicdropConversation = Array<MicdropConversationItem>;
|
|
602
|
-
type MicdropAnswerMetadata = {
|
|
603
|
-
[key: string]: any;
|
|
604
|
-
};
|
|
605
|
-
interface MicdropConversationMessage<Data extends MicdropAnswerMetadata = MicdropAnswerMetadata> {
|
|
606
|
-
role: 'system' | 'user' | 'assistant';
|
|
607
|
-
content: string;
|
|
608
|
-
metadata?: Data;
|
|
609
|
-
}
|
|
610
|
-
interface MicdropConversationToolCall {
|
|
611
|
-
role: 'tool_call';
|
|
612
|
-
toolCallId: string;
|
|
613
|
-
toolName: string;
|
|
614
|
-
parameters: string;
|
|
615
|
-
}
|
|
616
|
-
interface MicdropConversationToolResult {
|
|
617
|
-
role: 'tool_result';
|
|
618
|
-
toolCallId: string;
|
|
619
|
-
toolName: string;
|
|
620
|
-
output: string;
|
|
621
|
-
}
|
|
622
|
-
interface MicdropToolCall {
|
|
623
|
-
name: string;
|
|
624
|
-
parameters: any;
|
|
625
|
-
output: any;
|
|
626
|
-
}
|
|
627
|
-
type DeepPartial<T> = T extends object ? {
|
|
628
|
-
[P in keyof T]?: DeepPartial<T[P]>;
|
|
629
|
-
} : T;
|
|
630
|
-
|
|
631
700
|
declare enum MicdropClientErrorCode {
|
|
632
701
|
MissingUrl = "MissingUrl",
|
|
633
702
|
Mic = "Mic",
|
|
@@ -656,6 +725,10 @@ interface MicdropEvents {
|
|
|
656
725
|
Error: [MicdropClientError];
|
|
657
726
|
StateChange: [MicdropState, MicdropState];
|
|
658
727
|
ToolCall: [MicdropToolCall];
|
|
728
|
+
/** A message was added to the conversation, by the user or the assistant */
|
|
729
|
+
Message: [MicdropConversationItem];
|
|
730
|
+
/** What the assistant is answering, before it is finished */
|
|
731
|
+
PartialAssistantMessage: [string];
|
|
659
732
|
}
|
|
660
733
|
interface MicdropReconnectOptions {
|
|
661
734
|
maxAttempts?: number;
|
|
@@ -666,6 +739,14 @@ interface MicdropOptions {
|
|
|
666
739
|
url?: string;
|
|
667
740
|
params?: Record<string, any>;
|
|
668
741
|
vad?: VADConfig;
|
|
742
|
+
/**
|
|
743
|
+
* Holds the turn open while the sentence sounds unfinished, so a hesitation
|
|
744
|
+
* no longer counts as the end of a question. Without one, a turn ends as
|
|
745
|
+
* soon as the VAD hears enough silence.
|
|
746
|
+
*/
|
|
747
|
+
turnDetector?: TurnDetector;
|
|
748
|
+
/** How long a turn may stay open after the detector asked to wait */
|
|
749
|
+
turnMaxWait?: number;
|
|
669
750
|
disableInterruption?: boolean;
|
|
670
751
|
debugLog?: boolean;
|
|
671
752
|
reconnect?: MicdropReconnectOptions;
|
|
@@ -687,12 +768,19 @@ interface MicdropState {
|
|
|
687
768
|
micDevices: MicdropDevice[];
|
|
688
769
|
speakerDevices: MicdropDevice[];
|
|
689
770
|
conversation: MicdropConversation;
|
|
771
|
+
/**
|
|
772
|
+
* Answer being written, sent by servers configured with `partialMessages`.
|
|
773
|
+
* Empty the rest of the time, and replaced by a conversation message once
|
|
774
|
+
* the answer is finished.
|
|
775
|
+
*/
|
|
776
|
+
partialAssistantMessage: string;
|
|
690
777
|
error: MicdropClientError | undefined;
|
|
691
778
|
}
|
|
692
779
|
declare class MicdropClient extends EventEmitter<MicdropEvents> implements MicdropState {
|
|
693
780
|
options: MicdropOptions;
|
|
694
781
|
micRecorder?: MicRecorder;
|
|
695
782
|
conversation: MicdropConversation;
|
|
783
|
+
partialAssistantMessage: string;
|
|
696
784
|
error: MicdropClientError | undefined;
|
|
697
785
|
speakerDevices: MicdropDevice[];
|
|
698
786
|
micDevices: MicdropDevice[];
|
|
@@ -734,6 +822,20 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
734
822
|
vad?: VADConfig;
|
|
735
823
|
deviceId?: string;
|
|
736
824
|
}) => Promise<void>;
|
|
825
|
+
/**
|
|
826
|
+
* Changes what decides when a turn is over, during a call or before one
|
|
827
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
828
|
+
*/
|
|
829
|
+
setTurnDetector: (turnDetector?: TurnDetector) => void;
|
|
830
|
+
/**
|
|
831
|
+
* Says how long a held turn waits for the rest of the sentence.
|
|
832
|
+
*
|
|
833
|
+
* The way out of a wrong verdict: a detector can hear an unfinished sentence
|
|
834
|
+
* where there is none, and the speaker who never comes back still gets an
|
|
835
|
+
* answer once this runs out.
|
|
836
|
+
* @param milliseconds - How long to wait past the detector asking to
|
|
837
|
+
*/
|
|
838
|
+
setTurnMaxWait: (milliseconds: number) => void;
|
|
737
839
|
changeMicDevice: (deviceId: string) => Promise<void>;
|
|
738
840
|
changeSpeakerDevice: (deviceId: string) => Promise<void>;
|
|
739
841
|
private stopMic;
|
|
@@ -748,6 +850,7 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
748
850
|
private onSpeakerStartPlaying;
|
|
749
851
|
private onSpeakerStopPlaying;
|
|
750
852
|
private addMessage;
|
|
853
|
+
private setPartialAssistantMessage;
|
|
751
854
|
private setError;
|
|
752
855
|
private notifyStateChange;
|
|
753
856
|
private log;
|
|
@@ -779,4 +882,4 @@ declare enum MicdropStorageKeys {
|
|
|
779
882
|
declare function setMicdropStorage(storage: MicdropStorage): void;
|
|
780
883
|
declare const storage: MicdropStorage;
|
|
781
884
|
|
|
782
|
-
export { type AudioSink, type DeepPartial, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
|
|
885
|
+
export { type AudioSink, DEFAULT_TURN_MAX_WAIT, type DeepPartial, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
|