@micdrop/client 3.4.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/index.d.mts +158 -6
- package/dist/index.d.ts +158 -6
- package/dist/index.js +398 -78
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +396 -78
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -45,6 +45,8 @@ Speaker.setDriver(new MySpeaker())
|
|
|
45
45
|
|
|
46
46
|
`Pcm16AudioStream` does the playback scheduling for you if the platform offers something like Web Audio: give it an `AudioSink`, which is the small slice of it that Micdrop needs, and call its `destroy()` when the driver stops. When the output needs a moment once the microphone opens, as with Firefox and a Bluetooth headset switching to its call profile, call `warmUp(duration)`: the stream plays silence that long and holds the first answer until it is over.
|
|
47
47
|
|
|
48
|
+
When a speaker has a second way of playing, it can offer it through `useEchoFallback()` and `resetEchoFallback()`. A call moves there when the microphone hears the assistant [despite echo cancellation](https://micdrop.dev/docs/client/vad#echo-and-interruptions), and turns interruptions off when there is no such route.
|
|
49
|
+
|
|
48
50
|
For Silero, provide the inference and the state machine comes from here:
|
|
49
51
|
|
|
50
52
|
```ts
|
package/dist/index.d.mts
CHANGED
|
@@ -88,6 +88,13 @@ declare abstract class SpeakerDriver extends EventEmitter<SpeakerDriverEvents> {
|
|
|
88
88
|
abstract changeDevice(deviceId: string): Promise<void>;
|
|
89
89
|
/** Lists the output devices the system offers */
|
|
90
90
|
abstract getDevices(): Promise<MicdropDevice[]>;
|
|
91
|
+
/**
|
|
92
|
+
* Plays by another route, for when the microphone hears the current one
|
|
93
|
+
* @returns False if the platform has no other route to offer
|
|
94
|
+
*/
|
|
95
|
+
useEchoFallback(): Promise<boolean>;
|
|
96
|
+
/** Goes back to the usual route */
|
|
97
|
+
resetEchoFallback(): void;
|
|
91
98
|
}
|
|
92
99
|
/** A buffer handed to an {@link AudioSink}, that can still be cancelled */
|
|
93
100
|
interface ScheduledAudio {
|
|
@@ -208,6 +215,13 @@ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
|
|
|
208
215
|
changeDevice(deviceId: string): Promise<void>;
|
|
209
216
|
/** Lists the output devices the system offers */
|
|
210
217
|
getDevices(): Promise<MicdropDevice[]>;
|
|
218
|
+
/**
|
|
219
|
+
* Plays by another route, for when the microphone hears the current one
|
|
220
|
+
* @returns False if the platform has no other route to offer
|
|
221
|
+
*/
|
|
222
|
+
useEchoFallback(): Promise<boolean>;
|
|
223
|
+
/** Goes back to the usual route */
|
|
224
|
+
resetEchoFallback(): void;
|
|
211
225
|
private onStartPlaying;
|
|
212
226
|
private onStopPlaying;
|
|
213
227
|
private onVolume;
|
|
@@ -433,6 +447,11 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
433
447
|
turnMaxWait: number;
|
|
434
448
|
state: MicRecorderState;
|
|
435
449
|
vad: VAD;
|
|
450
|
+
/**
|
|
451
|
+
* Asked as speech is confirmed, before anyone hears of it. Saying it is echo
|
|
452
|
+
* drops the speech, and listens again in case the echo goes on.
|
|
453
|
+
*/
|
|
454
|
+
isEcho?: () => boolean;
|
|
436
455
|
private mic;
|
|
437
456
|
private isRecording;
|
|
438
457
|
private speakingConfirmed;
|
|
@@ -447,6 +466,7 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
447
466
|
private turnListening;
|
|
448
467
|
private turnTimer;
|
|
449
468
|
private turnRound;
|
|
469
|
+
private isDroppingSpeech;
|
|
450
470
|
constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
|
|
451
471
|
/**
|
|
452
472
|
* How long a turn stays open once the detector asked to wait.
|
|
@@ -477,6 +497,13 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
477
497
|
private onConfirmSpeaking;
|
|
478
498
|
private onCancelSpeaking;
|
|
479
499
|
private onStopSpeaking;
|
|
500
|
+
/**
|
|
501
|
+
* Forgets the speech heard so far.
|
|
502
|
+
*
|
|
503
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
504
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
505
|
+
*/
|
|
506
|
+
private dropSpeech;
|
|
480
507
|
/**
|
|
481
508
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
482
509
|
*
|
|
@@ -676,6 +703,7 @@ declare class SileroVAD extends VAD {
|
|
|
676
703
|
private silenceFrames;
|
|
677
704
|
private processing;
|
|
678
705
|
private round;
|
|
706
|
+
private startId;
|
|
679
707
|
constructor(options?: Partial<SileroVADOptions>);
|
|
680
708
|
get isStarted(): boolean;
|
|
681
709
|
get isPaused(): boolean;
|
|
@@ -728,6 +756,63 @@ declare class VolumeVAD extends VAD {
|
|
|
728
756
|
private reset;
|
|
729
757
|
}
|
|
730
758
|
|
|
759
|
+
/** Anything a runtime hands back that has to be released once done with */
|
|
760
|
+
interface Releasable {
|
|
761
|
+
release(): Promise<void>;
|
|
762
|
+
}
|
|
763
|
+
type SessionState<S> = {
|
|
764
|
+
status: 'unloaded';
|
|
765
|
+
} | {
|
|
766
|
+
status: 'loading';
|
|
767
|
+
promise: Promise<S>;
|
|
768
|
+
} | {
|
|
769
|
+
status: 'loaded';
|
|
770
|
+
session: S;
|
|
771
|
+
} | {
|
|
772
|
+
status: 'released';
|
|
773
|
+
};
|
|
774
|
+
/**
|
|
775
|
+
* An inference session created on first use, and released once.
|
|
776
|
+
*
|
|
777
|
+
* Every caller asking while it loads shares the same load, so two of them
|
|
778
|
+
* never create two sessions and leak one. A release during the load disposes
|
|
779
|
+
* of the session as soon as it arrives instead of letting it come back to life,
|
|
780
|
+
* and waits for the runs in flight before freeing the runtime under them. A
|
|
781
|
+
* failed load is forgotten, so the next call tries again.
|
|
782
|
+
*
|
|
783
|
+
* Shared by the Silero models of `@micdrop/web` and `@micdrop/react-native`.
|
|
784
|
+
* `@micdrop/smart-turn` holds the same few lines for its own models, as it
|
|
785
|
+
* depends on no other package.
|
|
786
|
+
*/
|
|
787
|
+
declare class LazySession<S extends Releasable> {
|
|
788
|
+
private name;
|
|
789
|
+
private create;
|
|
790
|
+
private state;
|
|
791
|
+
private runs;
|
|
792
|
+
/**
|
|
793
|
+
* @param name - What the session runs, to name it in errors
|
|
794
|
+
* @param create - Creates the session, called once unless it fails
|
|
795
|
+
*/
|
|
796
|
+
constructor(name: string, create: () => Promise<S>);
|
|
797
|
+
get status(): SessionState<S>['status'];
|
|
798
|
+
/** The session, loaded on the first call */
|
|
799
|
+
get(): Promise<S>;
|
|
800
|
+
/**
|
|
801
|
+
* Runs something on the session, which a release waits for
|
|
802
|
+
* @param run - Uses the session, and only until its promise settles
|
|
803
|
+
*/
|
|
804
|
+
use<T>(run: (session: S) => Promise<T>): Promise<T>;
|
|
805
|
+
/** Releases the session, for good */
|
|
806
|
+
release(): Promise<void>;
|
|
807
|
+
private isLoading;
|
|
808
|
+
private releasedError;
|
|
809
|
+
}
|
|
810
|
+
/**
|
|
811
|
+
* Fetches each address once for as long as the app runs, however many models
|
|
812
|
+
* ask for it. A failed download is forgotten, so a later call tries again.
|
|
813
|
+
*/
|
|
814
|
+
declare function createDownloadCache<T>(download: (url: string) => Promise<T>): (url: string) => Promise<T>;
|
|
815
|
+
|
|
731
816
|
/**
|
|
732
817
|
* Measures how loud the microphone is, the way a browser AnalyserNode does.
|
|
733
818
|
*
|
|
@@ -796,6 +881,15 @@ declare function getClientErrorFromWSCloseEvent(event: WSCloseEvent): MicdropCli
|
|
|
796
881
|
declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
|
|
797
882
|
declare function isRecoverableError(error: MicdropClientError): boolean;
|
|
798
883
|
|
|
884
|
+
/**
|
|
885
|
+
* How far a call has gone to keep the assistant out of the microphone.
|
|
886
|
+
*
|
|
887
|
+
* - `normal`: the speaker plays as usual
|
|
888
|
+
* - `fallback`: it plays by the other route the platform offers
|
|
889
|
+
* - `disabled`: the user may no longer interrupt, until the call ends
|
|
890
|
+
*/
|
|
891
|
+
type EchoStage = 'normal' | 'fallback' | 'disabled';
|
|
892
|
+
|
|
799
893
|
interface MicdropEvents {
|
|
800
894
|
EndCall: [];
|
|
801
895
|
Error: [MicdropClientError];
|
|
@@ -827,13 +921,37 @@ interface MicdropOptions {
|
|
|
827
921
|
turnMaxWait?: number;
|
|
828
922
|
/**
|
|
829
923
|
* Keeps the user from interrupting the assistant, whose voice they cannot
|
|
830
|
-
* speak over. Forced on when the microphone has no echo cancellation
|
|
924
|
+
* speak over. Forced on when the microphone has no echo cancellation, and
|
|
925
|
+
* by the call itself when the assistant keeps cutting itself off.
|
|
831
926
|
*/
|
|
832
927
|
disableInterruption?: boolean;
|
|
833
928
|
debugLog?: boolean;
|
|
834
929
|
reconnect?: MicdropReconnectOptions;
|
|
835
930
|
}
|
|
931
|
+
/**
|
|
932
|
+
* Where the call stands.
|
|
933
|
+
*
|
|
934
|
+
* - `stopped`: no call, the microphone may still be on for a test
|
|
935
|
+
* - `starting`: opening the microphone and the connection
|
|
936
|
+
* - `started`: the call is on
|
|
937
|
+
* - `reconnecting`: the connection dropped, the call waits for a new one
|
|
938
|
+
*/
|
|
939
|
+
type MicdropStatus = 'stopped' | 'starting' | 'started' | 'reconnecting';
|
|
940
|
+
/**
|
|
941
|
+
* Who has the floor, one at a time.
|
|
942
|
+
*
|
|
943
|
+
* - `idle`: nothing is heard nor awaited, when stopped, paused or muted
|
|
944
|
+
* - `listening`: waiting for the user to speak
|
|
945
|
+
* - `userSpeaking`: the user speaks
|
|
946
|
+
* - `processing`: the server prepares the answer
|
|
947
|
+
* - `assistantSpeaking`: the answer is playing
|
|
948
|
+
*/
|
|
949
|
+
type MicdropActivity = 'idle' | 'listening' | 'userSpeaking' | 'processing' | 'assistantSpeaking';
|
|
836
950
|
interface MicdropState {
|
|
951
|
+
/** Where the call stands, which `isStarting`, `isStarted` and `isReconnecting` spell out */
|
|
952
|
+
status: MicdropStatus;
|
|
953
|
+
/** Who has the floor, which `isListening`, `isUserSpeaking`, `isProcessing` and `isAssistantSpeaking` spell out */
|
|
954
|
+
activity: MicdropActivity;
|
|
837
955
|
isStarting: boolean;
|
|
838
956
|
isStarted: boolean;
|
|
839
957
|
isMuted: boolean;
|
|
@@ -872,13 +990,17 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
872
990
|
private _isProcessing;
|
|
873
991
|
private _isMuted;
|
|
874
992
|
private _isPaused;
|
|
875
|
-
private
|
|
876
|
-
private _isStartingCall;
|
|
993
|
+
private callStatus;
|
|
877
994
|
private reconnectAttempt;
|
|
878
995
|
private reconnectTimer?;
|
|
879
996
|
private connectionTimer?;
|
|
997
|
+
private echoGuard;
|
|
998
|
+
private pendingStop?;
|
|
880
999
|
constructor(options?: MicdropOptions);
|
|
881
1000
|
get vad(): VAD | undefined;
|
|
1001
|
+
get status(): MicdropStatus;
|
|
1002
|
+
get activity(): MicdropActivity;
|
|
1003
|
+
/** The call is on, reconnecting included */
|
|
882
1004
|
get isStarted(): boolean;
|
|
883
1005
|
get isStarting(): boolean;
|
|
884
1006
|
get isReconnecting(): boolean;
|
|
@@ -895,14 +1017,30 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
895
1017
|
/**
|
|
896
1018
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
897
1019
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
898
|
-
* itself and end up in the transcript.
|
|
1020
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
1021
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
899
1022
|
*/
|
|
900
1023
|
get isInterruptionDisabled(): boolean;
|
|
1024
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
1025
|
+
get echoStage(): EchoStage;
|
|
901
1026
|
get micDeviceId(): string | undefined;
|
|
902
1027
|
get speakerDeviceId(): string | undefined;
|
|
903
1028
|
get state(): MicdropState;
|
|
904
1029
|
start: (options: MicdropOptions) => Promise<void>;
|
|
905
1030
|
stop: () => Promise<void>;
|
|
1031
|
+
/** The assistant prepares or speaks an answer */
|
|
1032
|
+
private get isAnswering();
|
|
1033
|
+
/**
|
|
1034
|
+
* Stops the call once the assistant finished its answer, the one it is
|
|
1035
|
+
* still preparing included, such as the goodbye that follows an `EndCall`.
|
|
1036
|
+
* Stops right away when no answer is under way.
|
|
1037
|
+
*
|
|
1038
|
+
* A `start()` or a `stop()` meanwhile cancels the wait, so it never acts on
|
|
1039
|
+
* the next call.
|
|
1040
|
+
* @returns Resolves once the call is stopped, or the wait cancelled
|
|
1041
|
+
*/
|
|
1042
|
+
stopAfterAnswer: () => Promise<void>;
|
|
1043
|
+
private cancelPendingStop;
|
|
906
1044
|
mute: () => void;
|
|
907
1045
|
unmute: () => void;
|
|
908
1046
|
pause: () => void;
|
|
@@ -929,15 +1067,29 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
929
1067
|
changeSpeakerDevice: (deviceId: string) => Promise<void>;
|
|
930
1068
|
private stopMic;
|
|
931
1069
|
private updateDevices;
|
|
932
|
-
|
|
1070
|
+
/**
|
|
1071
|
+
* Opens the connection, unless one is already open or opening.
|
|
1072
|
+
*
|
|
1073
|
+
* The socket answers through handlers bound to it, so one left behind by a
|
|
1074
|
+
* stop or a reconnection never acts on the call that followed.
|
|
1075
|
+
*/
|
|
1076
|
+
private openWS;
|
|
933
1077
|
private onWSOpen;
|
|
934
1078
|
private onWSMessage;
|
|
935
1079
|
private onWSClose;
|
|
936
1080
|
private onWSError;
|
|
937
1081
|
private stopWS;
|
|
938
1082
|
private scheduleReconnect;
|
|
1083
|
+
private clearReconnectTimer;
|
|
939
1084
|
private onSpeakerStartPlaying;
|
|
940
1085
|
private onSpeakerStopPlaying;
|
|
1086
|
+
/**
|
|
1087
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
1088
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
1089
|
+
* neither the speaker nor the server hear of it.
|
|
1090
|
+
*/
|
|
1091
|
+
private isEchoOfAssistant;
|
|
1092
|
+
private onInterruptionsGivenUp;
|
|
941
1093
|
private addMessage;
|
|
942
1094
|
private setPartialAssistantMessage;
|
|
943
1095
|
private setError;
|
|
@@ -971,4 +1123,4 @@ declare enum MicdropStorageKeys {
|
|
|
971
1123
|
declare function setMicdropStorage(storage: MicdropStorage): void;
|
|
972
1124
|
declare const storage: MicdropStorage;
|
|
973
1125
|
|
|
974
|
-
export { type AudioSink, CONFIRM_MARGIN, DEFAULT_TURN_MAX_WAIT, type DeepPartial, ECHO_TAIL, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, currentTurn, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getTurnClassification, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
|
|
1126
|
+
export { type AudioSink, CONFIRM_MARGIN, DEFAULT_TURN_MAX_WAIT, type DeepPartial, ECHO_TAIL, LazySession, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropActivity, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStatus, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, type Releasable, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, createDownloadCache, currentTurn, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getTurnClassification, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
|
package/dist/index.d.ts
CHANGED
|
@@ -88,6 +88,13 @@ declare abstract class SpeakerDriver extends EventEmitter<SpeakerDriverEvents> {
|
|
|
88
88
|
abstract changeDevice(deviceId: string): Promise<void>;
|
|
89
89
|
/** Lists the output devices the system offers */
|
|
90
90
|
abstract getDevices(): Promise<MicdropDevice[]>;
|
|
91
|
+
/**
|
|
92
|
+
* Plays by another route, for when the microphone hears the current one
|
|
93
|
+
* @returns False if the platform has no other route to offer
|
|
94
|
+
*/
|
|
95
|
+
useEchoFallback(): Promise<boolean>;
|
|
96
|
+
/** Goes back to the usual route */
|
|
97
|
+
resetEchoFallback(): void;
|
|
91
98
|
}
|
|
92
99
|
/** A buffer handed to an {@link AudioSink}, that can still be cancelled */
|
|
93
100
|
interface ScheduledAudio {
|
|
@@ -208,6 +215,13 @@ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
|
|
|
208
215
|
changeDevice(deviceId: string): Promise<void>;
|
|
209
216
|
/** Lists the output devices the system offers */
|
|
210
217
|
getDevices(): Promise<MicdropDevice[]>;
|
|
218
|
+
/**
|
|
219
|
+
* Plays by another route, for when the microphone hears the current one
|
|
220
|
+
* @returns False if the platform has no other route to offer
|
|
221
|
+
*/
|
|
222
|
+
useEchoFallback(): Promise<boolean>;
|
|
223
|
+
/** Goes back to the usual route */
|
|
224
|
+
resetEchoFallback(): void;
|
|
211
225
|
private onStartPlaying;
|
|
212
226
|
private onStopPlaying;
|
|
213
227
|
private onVolume;
|
|
@@ -433,6 +447,11 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
433
447
|
turnMaxWait: number;
|
|
434
448
|
state: MicRecorderState;
|
|
435
449
|
vad: VAD;
|
|
450
|
+
/**
|
|
451
|
+
* Asked as speech is confirmed, before anyone hears of it. Saying it is echo
|
|
452
|
+
* drops the speech, and listens again in case the echo goes on.
|
|
453
|
+
*/
|
|
454
|
+
isEcho?: () => boolean;
|
|
436
455
|
private mic;
|
|
437
456
|
private isRecording;
|
|
438
457
|
private speakingConfirmed;
|
|
@@ -447,6 +466,7 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
447
466
|
private turnListening;
|
|
448
467
|
private turnTimer;
|
|
449
468
|
private turnRound;
|
|
469
|
+
private isDroppingSpeech;
|
|
450
470
|
constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
|
|
451
471
|
/**
|
|
452
472
|
* How long a turn stays open once the detector asked to wait.
|
|
@@ -477,6 +497,13 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
477
497
|
private onConfirmSpeaking;
|
|
478
498
|
private onCancelSpeaking;
|
|
479
499
|
private onStopSpeaking;
|
|
500
|
+
/**
|
|
501
|
+
* Forgets the speech heard so far.
|
|
502
|
+
*
|
|
503
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
504
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
505
|
+
*/
|
|
506
|
+
private dropSpeech;
|
|
480
507
|
/**
|
|
481
508
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
482
509
|
*
|
|
@@ -676,6 +703,7 @@ declare class SileroVAD extends VAD {
|
|
|
676
703
|
private silenceFrames;
|
|
677
704
|
private processing;
|
|
678
705
|
private round;
|
|
706
|
+
private startId;
|
|
679
707
|
constructor(options?: Partial<SileroVADOptions>);
|
|
680
708
|
get isStarted(): boolean;
|
|
681
709
|
get isPaused(): boolean;
|
|
@@ -728,6 +756,63 @@ declare class VolumeVAD extends VAD {
|
|
|
728
756
|
private reset;
|
|
729
757
|
}
|
|
730
758
|
|
|
759
|
+
/** Anything a runtime hands back that has to be released once done with */
|
|
760
|
+
interface Releasable {
|
|
761
|
+
release(): Promise<void>;
|
|
762
|
+
}
|
|
763
|
+
type SessionState<S> = {
|
|
764
|
+
status: 'unloaded';
|
|
765
|
+
} | {
|
|
766
|
+
status: 'loading';
|
|
767
|
+
promise: Promise<S>;
|
|
768
|
+
} | {
|
|
769
|
+
status: 'loaded';
|
|
770
|
+
session: S;
|
|
771
|
+
} | {
|
|
772
|
+
status: 'released';
|
|
773
|
+
};
|
|
774
|
+
/**
|
|
775
|
+
* An inference session created on first use, and released once.
|
|
776
|
+
*
|
|
777
|
+
* Every caller asking while it loads shares the same load, so two of them
|
|
778
|
+
* never create two sessions and leak one. A release during the load disposes
|
|
779
|
+
* of the session as soon as it arrives instead of letting it come back to life,
|
|
780
|
+
* and waits for the runs in flight before freeing the runtime under them. A
|
|
781
|
+
* failed load is forgotten, so the next call tries again.
|
|
782
|
+
*
|
|
783
|
+
* Shared by the Silero models of `@micdrop/web` and `@micdrop/react-native`.
|
|
784
|
+
* `@micdrop/smart-turn` holds the same few lines for its own models, as it
|
|
785
|
+
* depends on no other package.
|
|
786
|
+
*/
|
|
787
|
+
declare class LazySession<S extends Releasable> {
|
|
788
|
+
private name;
|
|
789
|
+
private create;
|
|
790
|
+
private state;
|
|
791
|
+
private runs;
|
|
792
|
+
/**
|
|
793
|
+
* @param name - What the session runs, to name it in errors
|
|
794
|
+
* @param create - Creates the session, called once unless it fails
|
|
795
|
+
*/
|
|
796
|
+
constructor(name: string, create: () => Promise<S>);
|
|
797
|
+
get status(): SessionState<S>['status'];
|
|
798
|
+
/** The session, loaded on the first call */
|
|
799
|
+
get(): Promise<S>;
|
|
800
|
+
/**
|
|
801
|
+
* Runs something on the session, which a release waits for
|
|
802
|
+
* @param run - Uses the session, and only until its promise settles
|
|
803
|
+
*/
|
|
804
|
+
use<T>(run: (session: S) => Promise<T>): Promise<T>;
|
|
805
|
+
/** Releases the session, for good */
|
|
806
|
+
release(): Promise<void>;
|
|
807
|
+
private isLoading;
|
|
808
|
+
private releasedError;
|
|
809
|
+
}
|
|
810
|
+
/**
|
|
811
|
+
* Fetches each address once for as long as the app runs, however many models
|
|
812
|
+
* ask for it. A failed download is forgotten, so a later call tries again.
|
|
813
|
+
*/
|
|
814
|
+
declare function createDownloadCache<T>(download: (url: string) => Promise<T>): (url: string) => Promise<T>;
|
|
815
|
+
|
|
731
816
|
/**
|
|
732
817
|
* Measures how loud the microphone is, the way a browser AnalyserNode does.
|
|
733
818
|
*
|
|
@@ -796,6 +881,15 @@ declare function getClientErrorFromWSCloseEvent(event: WSCloseEvent): MicdropCli
|
|
|
796
881
|
declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
|
|
797
882
|
declare function isRecoverableError(error: MicdropClientError): boolean;
|
|
798
883
|
|
|
884
|
+
/**
|
|
885
|
+
* How far a call has gone to keep the assistant out of the microphone.
|
|
886
|
+
*
|
|
887
|
+
* - `normal`: the speaker plays as usual
|
|
888
|
+
* - `fallback`: it plays by the other route the platform offers
|
|
889
|
+
* - `disabled`: the user may no longer interrupt, until the call ends
|
|
890
|
+
*/
|
|
891
|
+
type EchoStage = 'normal' | 'fallback' | 'disabled';
|
|
892
|
+
|
|
799
893
|
interface MicdropEvents {
|
|
800
894
|
EndCall: [];
|
|
801
895
|
Error: [MicdropClientError];
|
|
@@ -827,13 +921,37 @@ interface MicdropOptions {
|
|
|
827
921
|
turnMaxWait?: number;
|
|
828
922
|
/**
|
|
829
923
|
* Keeps the user from interrupting the assistant, whose voice they cannot
|
|
830
|
-
* speak over. Forced on when the microphone has no echo cancellation
|
|
924
|
+
* speak over. Forced on when the microphone has no echo cancellation, and
|
|
925
|
+
* by the call itself when the assistant keeps cutting itself off.
|
|
831
926
|
*/
|
|
832
927
|
disableInterruption?: boolean;
|
|
833
928
|
debugLog?: boolean;
|
|
834
929
|
reconnect?: MicdropReconnectOptions;
|
|
835
930
|
}
|
|
931
|
+
/**
|
|
932
|
+
* Where the call stands.
|
|
933
|
+
*
|
|
934
|
+
* - `stopped`: no call, the microphone may still be on for a test
|
|
935
|
+
* - `starting`: opening the microphone and the connection
|
|
936
|
+
* - `started`: the call is on
|
|
937
|
+
* - `reconnecting`: the connection dropped, the call waits for a new one
|
|
938
|
+
*/
|
|
939
|
+
type MicdropStatus = 'stopped' | 'starting' | 'started' | 'reconnecting';
|
|
940
|
+
/**
|
|
941
|
+
* Who has the floor, one at a time.
|
|
942
|
+
*
|
|
943
|
+
* - `idle`: nothing is heard nor awaited, when stopped, paused or muted
|
|
944
|
+
* - `listening`: waiting for the user to speak
|
|
945
|
+
* - `userSpeaking`: the user speaks
|
|
946
|
+
* - `processing`: the server prepares the answer
|
|
947
|
+
* - `assistantSpeaking`: the answer is playing
|
|
948
|
+
*/
|
|
949
|
+
type MicdropActivity = 'idle' | 'listening' | 'userSpeaking' | 'processing' | 'assistantSpeaking';
|
|
836
950
|
interface MicdropState {
|
|
951
|
+
/** Where the call stands, which `isStarting`, `isStarted` and `isReconnecting` spell out */
|
|
952
|
+
status: MicdropStatus;
|
|
953
|
+
/** Who has the floor, which `isListening`, `isUserSpeaking`, `isProcessing` and `isAssistantSpeaking` spell out */
|
|
954
|
+
activity: MicdropActivity;
|
|
837
955
|
isStarting: boolean;
|
|
838
956
|
isStarted: boolean;
|
|
839
957
|
isMuted: boolean;
|
|
@@ -872,13 +990,17 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
872
990
|
private _isProcessing;
|
|
873
991
|
private _isMuted;
|
|
874
992
|
private _isPaused;
|
|
875
|
-
private
|
|
876
|
-
private _isStartingCall;
|
|
993
|
+
private callStatus;
|
|
877
994
|
private reconnectAttempt;
|
|
878
995
|
private reconnectTimer?;
|
|
879
996
|
private connectionTimer?;
|
|
997
|
+
private echoGuard;
|
|
998
|
+
private pendingStop?;
|
|
880
999
|
constructor(options?: MicdropOptions);
|
|
881
1000
|
get vad(): VAD | undefined;
|
|
1001
|
+
get status(): MicdropStatus;
|
|
1002
|
+
get activity(): MicdropActivity;
|
|
1003
|
+
/** The call is on, reconnecting included */
|
|
882
1004
|
get isStarted(): boolean;
|
|
883
1005
|
get isStarting(): boolean;
|
|
884
1006
|
get isReconnecting(): boolean;
|
|
@@ -895,14 +1017,30 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
895
1017
|
/**
|
|
896
1018
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
897
1019
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
898
|
-
* itself and end up in the transcript.
|
|
1020
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
1021
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
899
1022
|
*/
|
|
900
1023
|
get isInterruptionDisabled(): boolean;
|
|
1024
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
1025
|
+
get echoStage(): EchoStage;
|
|
901
1026
|
get micDeviceId(): string | undefined;
|
|
902
1027
|
get speakerDeviceId(): string | undefined;
|
|
903
1028
|
get state(): MicdropState;
|
|
904
1029
|
start: (options: MicdropOptions) => Promise<void>;
|
|
905
1030
|
stop: () => Promise<void>;
|
|
1031
|
+
/** The assistant prepares or speaks an answer */
|
|
1032
|
+
private get isAnswering();
|
|
1033
|
+
/**
|
|
1034
|
+
* Stops the call once the assistant finished its answer, the one it is
|
|
1035
|
+
* still preparing included, such as the goodbye that follows an `EndCall`.
|
|
1036
|
+
* Stops right away when no answer is under way.
|
|
1037
|
+
*
|
|
1038
|
+
* A `start()` or a `stop()` meanwhile cancels the wait, so it never acts on
|
|
1039
|
+
* the next call.
|
|
1040
|
+
* @returns Resolves once the call is stopped, or the wait cancelled
|
|
1041
|
+
*/
|
|
1042
|
+
stopAfterAnswer: () => Promise<void>;
|
|
1043
|
+
private cancelPendingStop;
|
|
906
1044
|
mute: () => void;
|
|
907
1045
|
unmute: () => void;
|
|
908
1046
|
pause: () => void;
|
|
@@ -929,15 +1067,29 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
929
1067
|
changeSpeakerDevice: (deviceId: string) => Promise<void>;
|
|
930
1068
|
private stopMic;
|
|
931
1069
|
private updateDevices;
|
|
932
|
-
|
|
1070
|
+
/**
|
|
1071
|
+
* Opens the connection, unless one is already open or opening.
|
|
1072
|
+
*
|
|
1073
|
+
* The socket answers through handlers bound to it, so one left behind by a
|
|
1074
|
+
* stop or a reconnection never acts on the call that followed.
|
|
1075
|
+
*/
|
|
1076
|
+
private openWS;
|
|
933
1077
|
private onWSOpen;
|
|
934
1078
|
private onWSMessage;
|
|
935
1079
|
private onWSClose;
|
|
936
1080
|
private onWSError;
|
|
937
1081
|
private stopWS;
|
|
938
1082
|
private scheduleReconnect;
|
|
1083
|
+
private clearReconnectTimer;
|
|
939
1084
|
private onSpeakerStartPlaying;
|
|
940
1085
|
private onSpeakerStopPlaying;
|
|
1086
|
+
/**
|
|
1087
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
1088
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
1089
|
+
* neither the speaker nor the server hear of it.
|
|
1090
|
+
*/
|
|
1091
|
+
private isEchoOfAssistant;
|
|
1092
|
+
private onInterruptionsGivenUp;
|
|
941
1093
|
private addMessage;
|
|
942
1094
|
private setPartialAssistantMessage;
|
|
943
1095
|
private setError;
|
|
@@ -971,4 +1123,4 @@ declare enum MicdropStorageKeys {
|
|
|
971
1123
|
declare function setMicdropStorage(storage: MicdropStorage): void;
|
|
972
1124
|
declare const storage: MicdropStorage;
|
|
973
1125
|
|
|
974
|
-
export { type AudioSink, CONFIRM_MARGIN, DEFAULT_TURN_MAX_WAIT, type DeepPartial, ECHO_TAIL, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, currentTurn, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getTurnClassification, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
|
|
1126
|
+
export { type AudioSink, CONFIRM_MARGIN, DEFAULT_TURN_MAX_WAIT, type DeepPartial, ECHO_TAIL, LazySession, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropActivity, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStatus, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, type Releasable, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, createDownloadCache, currentTurn, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getTurnClassification, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
|