@micdrop/client 3.3.1 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/index.d.mts +65 -2
- package/dist/index.d.ts +65 -2
- package/dist/index.js +184 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +184 -2
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -43,7 +43,9 @@ Mic.setDriver(new MyMic())
|
|
|
43
43
|
Speaker.setDriver(new MySpeaker())
|
|
44
44
|
```
|
|
45
45
|
|
|
46
|
-
`Pcm16AudioStream` does the playback scheduling for you if the platform offers something like Web Audio: give it an `AudioSink`, which is the small slice of it that Micdrop needs.
|
|
46
|
+
`Pcm16AudioStream` does the playback scheduling for you if the platform offers something like Web Audio: give it an `AudioSink`, which is the small slice of it that Micdrop needs, and call its `destroy()` when the driver stops. When the output needs a moment once the microphone opens, as with Firefox and a Bluetooth headset switching to its call profile, call `warmUp(duration)`: the stream plays silence that long and holds the first answer until it is over.
|
|
47
|
+
|
|
48
|
+
When a speaker has a second way of playing, it can offer it through `useEchoFallback()` and `resetEchoFallback()`. A call moves there when the microphone hears the assistant [despite echo cancellation](https://micdrop.dev/docs/client/vad#echo-and-interruptions), and turns interruptions off when there is no such route.
|
|
47
49
|
|
|
48
50
|
For Silero, provide the inference and the state machine comes from here:
|
|
49
51
|
|
package/dist/index.d.mts
CHANGED
|
@@ -88,6 +88,13 @@ declare abstract class SpeakerDriver extends EventEmitter<SpeakerDriverEvents> {
|
|
|
88
88
|
abstract changeDevice(deviceId: string): Promise<void>;
|
|
89
89
|
/** Lists the output devices the system offers */
|
|
90
90
|
abstract getDevices(): Promise<MicdropDevice[]>;
|
|
91
|
+
/**
|
|
92
|
+
* Plays by another route, for when the microphone hears the current one
|
|
93
|
+
* @returns False if the platform has no other route to offer
|
|
94
|
+
*/
|
|
95
|
+
useEchoFallback(): Promise<boolean>;
|
|
96
|
+
/** Goes back to the usual route */
|
|
97
|
+
resetEchoFallback(): void;
|
|
91
98
|
}
|
|
92
99
|
/** A buffer handed to an {@link AudioSink}, that can still be cancelled */
|
|
93
100
|
interface ScheduledAudio {
|
|
@@ -208,6 +215,13 @@ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
|
|
|
208
215
|
changeDevice(deviceId: string): Promise<void>;
|
|
209
216
|
/** Lists the output devices the system offers */
|
|
210
217
|
getDevices(): Promise<MicdropDevice[]>;
|
|
218
|
+
/**
|
|
219
|
+
* Plays by another route, for when the microphone hears the current one
|
|
220
|
+
* @returns False if the platform has no other route to offer
|
|
221
|
+
*/
|
|
222
|
+
useEchoFallback(): Promise<boolean>;
|
|
223
|
+
/** Goes back to the usual route */
|
|
224
|
+
resetEchoFallback(): void;
|
|
211
225
|
private onStartPlaying;
|
|
212
226
|
private onStopPlaying;
|
|
213
227
|
private onVolume;
|
|
@@ -433,6 +447,11 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
433
447
|
turnMaxWait: number;
|
|
434
448
|
state: MicRecorderState;
|
|
435
449
|
vad: VAD;
|
|
450
|
+
/**
|
|
451
|
+
* Asked as speech is confirmed, before anyone hears of it. Saying it is echo
|
|
452
|
+
* drops the speech, and listens again in case the echo goes on.
|
|
453
|
+
*/
|
|
454
|
+
isEcho?: () => boolean;
|
|
436
455
|
private mic;
|
|
437
456
|
private isRecording;
|
|
438
457
|
private speakingConfirmed;
|
|
@@ -447,6 +466,7 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
447
466
|
private turnListening;
|
|
448
467
|
private turnTimer;
|
|
449
468
|
private turnRound;
|
|
469
|
+
private isDroppingSpeech;
|
|
450
470
|
constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
|
|
451
471
|
/**
|
|
452
472
|
* How long a turn stays open once the detector asked to wait.
|
|
@@ -477,6 +497,13 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
477
497
|
private onConfirmSpeaking;
|
|
478
498
|
private onCancelSpeaking;
|
|
479
499
|
private onStopSpeaking;
|
|
500
|
+
/**
|
|
501
|
+
* Forgets the speech heard so far.
|
|
502
|
+
*
|
|
503
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
504
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
505
|
+
*/
|
|
506
|
+
private dropSpeech;
|
|
480
507
|
/**
|
|
481
508
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
482
509
|
*
|
|
@@ -522,6 +549,9 @@ declare class Pcm16AudioStream extends EventEmitter<Pcm16AudioStreamEvents> {
|
|
|
522
549
|
private endTimer?;
|
|
523
550
|
private volumeTimers;
|
|
524
551
|
private meter;
|
|
552
|
+
private holdUntil;
|
|
553
|
+
private holdTimer?;
|
|
554
|
+
private warmUpAudio?;
|
|
525
555
|
constructor(sink: AudioSink);
|
|
526
556
|
/**
|
|
527
557
|
* Queues samples for playback
|
|
@@ -529,6 +559,18 @@ declare class Pcm16AudioStream extends EventEmitter<Pcm16AudioStreamEvents> {
|
|
|
529
559
|
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
530
560
|
*/
|
|
531
561
|
playAudio(pcm: Int16Array, sampleRate: number): void;
|
|
562
|
+
/**
|
|
563
|
+
* Plays silence for a while, and holds the next utterance until it is over.
|
|
564
|
+
*
|
|
565
|
+
* Firefox mangles what it plays while a Bluetooth headset switches to its
|
|
566
|
+
* call profile, which the headset does as soon as its microphone opens. The
|
|
567
|
+
* first answer then comes out all at once. Silence played through that switch
|
|
568
|
+
* goes unheard.
|
|
569
|
+
* @param duration - How long to play silence, in milliseconds
|
|
570
|
+
*/
|
|
571
|
+
warmUp(duration: number): void;
|
|
572
|
+
/** Stops playing, for a stream no longer used */
|
|
573
|
+
destroy(): void;
|
|
532
574
|
/** Drops everything queued and stops playing immediately */
|
|
533
575
|
stopAudio(): void;
|
|
534
576
|
private handleSamples;
|
|
@@ -781,6 +823,15 @@ declare function getClientErrorFromWSCloseEvent(event: WSCloseEvent): MicdropCli
|
|
|
781
823
|
declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
|
|
782
824
|
declare function isRecoverableError(error: MicdropClientError): boolean;
|
|
783
825
|
|
|
826
|
+
/**
|
|
827
|
+
* How far a call has gone to keep the assistant out of the microphone.
|
|
828
|
+
*
|
|
829
|
+
* - `normal`: the speaker plays as usual
|
|
830
|
+
* - `fallback`: it plays by the other route the platform offers
|
|
831
|
+
* - `disabled`: the user may no longer interrupt, until the call ends
|
|
832
|
+
*/
|
|
833
|
+
type EchoStage = 'normal' | 'fallback' | 'disabled';
|
|
834
|
+
|
|
784
835
|
interface MicdropEvents {
|
|
785
836
|
EndCall: [];
|
|
786
837
|
Error: [MicdropClientError];
|
|
@@ -812,7 +863,8 @@ interface MicdropOptions {
|
|
|
812
863
|
turnMaxWait?: number;
|
|
813
864
|
/**
|
|
814
865
|
* Keeps the user from interrupting the assistant, whose voice they cannot
|
|
815
|
-
* speak over. Forced on when the microphone has no echo cancellation
|
|
866
|
+
* speak over. Forced on when the microphone has no echo cancellation, and
|
|
867
|
+
* by the call itself when the assistant keeps cutting itself off.
|
|
816
868
|
*/
|
|
817
869
|
disableInterruption?: boolean;
|
|
818
870
|
debugLog?: boolean;
|
|
@@ -862,6 +914,7 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
862
914
|
private reconnectAttempt;
|
|
863
915
|
private reconnectTimer?;
|
|
864
916
|
private connectionTimer?;
|
|
917
|
+
private echoGuard;
|
|
865
918
|
constructor(options?: MicdropOptions);
|
|
866
919
|
get vad(): VAD | undefined;
|
|
867
920
|
get isStarted(): boolean;
|
|
@@ -880,9 +933,12 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
880
933
|
/**
|
|
881
934
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
882
935
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
883
|
-
* itself and end up in the transcript.
|
|
936
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
937
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
884
938
|
*/
|
|
885
939
|
get isInterruptionDisabled(): boolean;
|
|
940
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
941
|
+
get echoStage(): EchoStage;
|
|
886
942
|
get micDeviceId(): string | undefined;
|
|
887
943
|
get speakerDeviceId(): string | undefined;
|
|
888
944
|
get state(): MicdropState;
|
|
@@ -923,6 +979,13 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
923
979
|
private scheduleReconnect;
|
|
924
980
|
private onSpeakerStartPlaying;
|
|
925
981
|
private onSpeakerStopPlaying;
|
|
982
|
+
/**
|
|
983
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
984
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
985
|
+
* neither the speaker nor the server hear of it.
|
|
986
|
+
*/
|
|
987
|
+
private isEchoOfAssistant;
|
|
988
|
+
private onInterruptionsGivenUp;
|
|
926
989
|
private addMessage;
|
|
927
990
|
private setPartialAssistantMessage;
|
|
928
991
|
private setError;
|
package/dist/index.d.ts
CHANGED
|
@@ -88,6 +88,13 @@ declare abstract class SpeakerDriver extends EventEmitter<SpeakerDriverEvents> {
|
|
|
88
88
|
abstract changeDevice(deviceId: string): Promise<void>;
|
|
89
89
|
/** Lists the output devices the system offers */
|
|
90
90
|
abstract getDevices(): Promise<MicdropDevice[]>;
|
|
91
|
+
/**
|
|
92
|
+
* Plays by another route, for when the microphone hears the current one
|
|
93
|
+
* @returns False if the platform has no other route to offer
|
|
94
|
+
*/
|
|
95
|
+
useEchoFallback(): Promise<boolean>;
|
|
96
|
+
/** Goes back to the usual route */
|
|
97
|
+
resetEchoFallback(): void;
|
|
91
98
|
}
|
|
92
99
|
/** A buffer handed to an {@link AudioSink}, that can still be cancelled */
|
|
93
100
|
interface ScheduledAudio {
|
|
@@ -208,6 +215,13 @@ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
|
|
|
208
215
|
changeDevice(deviceId: string): Promise<void>;
|
|
209
216
|
/** Lists the output devices the system offers */
|
|
210
217
|
getDevices(): Promise<MicdropDevice[]>;
|
|
218
|
+
/**
|
|
219
|
+
* Plays by another route, for when the microphone hears the current one
|
|
220
|
+
* @returns False if the platform has no other route to offer
|
|
221
|
+
*/
|
|
222
|
+
useEchoFallback(): Promise<boolean>;
|
|
223
|
+
/** Goes back to the usual route */
|
|
224
|
+
resetEchoFallback(): void;
|
|
211
225
|
private onStartPlaying;
|
|
212
226
|
private onStopPlaying;
|
|
213
227
|
private onVolume;
|
|
@@ -433,6 +447,11 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
433
447
|
turnMaxWait: number;
|
|
434
448
|
state: MicRecorderState;
|
|
435
449
|
vad: VAD;
|
|
450
|
+
/**
|
|
451
|
+
* Asked as speech is confirmed, before anyone hears of it. Saying it is echo
|
|
452
|
+
* drops the speech, and listens again in case the echo goes on.
|
|
453
|
+
*/
|
|
454
|
+
isEcho?: () => boolean;
|
|
436
455
|
private mic;
|
|
437
456
|
private isRecording;
|
|
438
457
|
private speakingConfirmed;
|
|
@@ -447,6 +466,7 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
447
466
|
private turnListening;
|
|
448
467
|
private turnTimer;
|
|
449
468
|
private turnRound;
|
|
469
|
+
private isDroppingSpeech;
|
|
450
470
|
constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
|
|
451
471
|
/**
|
|
452
472
|
* How long a turn stays open once the detector asked to wait.
|
|
@@ -477,6 +497,13 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
477
497
|
private onConfirmSpeaking;
|
|
478
498
|
private onCancelSpeaking;
|
|
479
499
|
private onStopSpeaking;
|
|
500
|
+
/**
|
|
501
|
+
* Forgets the speech heard so far.
|
|
502
|
+
*
|
|
503
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
504
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
505
|
+
*/
|
|
506
|
+
private dropSpeech;
|
|
480
507
|
/**
|
|
481
508
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
482
509
|
*
|
|
@@ -522,6 +549,9 @@ declare class Pcm16AudioStream extends EventEmitter<Pcm16AudioStreamEvents> {
|
|
|
522
549
|
private endTimer?;
|
|
523
550
|
private volumeTimers;
|
|
524
551
|
private meter;
|
|
552
|
+
private holdUntil;
|
|
553
|
+
private holdTimer?;
|
|
554
|
+
private warmUpAudio?;
|
|
525
555
|
constructor(sink: AudioSink);
|
|
526
556
|
/**
|
|
527
557
|
* Queues samples for playback
|
|
@@ -529,6 +559,18 @@ declare class Pcm16AudioStream extends EventEmitter<Pcm16AudioStreamEvents> {
|
|
|
529
559
|
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
530
560
|
*/
|
|
531
561
|
playAudio(pcm: Int16Array, sampleRate: number): void;
|
|
562
|
+
/**
|
|
563
|
+
* Plays silence for a while, and holds the next utterance until it is over.
|
|
564
|
+
*
|
|
565
|
+
* Firefox mangles what it plays while a Bluetooth headset switches to its
|
|
566
|
+
* call profile, which the headset does as soon as its microphone opens. The
|
|
567
|
+
* first answer then comes out all at once. Silence played through that switch
|
|
568
|
+
* goes unheard.
|
|
569
|
+
* @param duration - How long to play silence, in milliseconds
|
|
570
|
+
*/
|
|
571
|
+
warmUp(duration: number): void;
|
|
572
|
+
/** Stops playing, for a stream no longer used */
|
|
573
|
+
destroy(): void;
|
|
532
574
|
/** Drops everything queued and stops playing immediately */
|
|
533
575
|
stopAudio(): void;
|
|
534
576
|
private handleSamples;
|
|
@@ -781,6 +823,15 @@ declare function getClientErrorFromWSCloseEvent(event: WSCloseEvent): MicdropCli
|
|
|
781
823
|
declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
|
|
782
824
|
declare function isRecoverableError(error: MicdropClientError): boolean;
|
|
783
825
|
|
|
826
|
+
/**
|
|
827
|
+
* How far a call has gone to keep the assistant out of the microphone.
|
|
828
|
+
*
|
|
829
|
+
* - `normal`: the speaker plays as usual
|
|
830
|
+
* - `fallback`: it plays by the other route the platform offers
|
|
831
|
+
* - `disabled`: the user may no longer interrupt, until the call ends
|
|
832
|
+
*/
|
|
833
|
+
type EchoStage = 'normal' | 'fallback' | 'disabled';
|
|
834
|
+
|
|
784
835
|
interface MicdropEvents {
|
|
785
836
|
EndCall: [];
|
|
786
837
|
Error: [MicdropClientError];
|
|
@@ -812,7 +863,8 @@ interface MicdropOptions {
|
|
|
812
863
|
turnMaxWait?: number;
|
|
813
864
|
/**
|
|
814
865
|
* Keeps the user from interrupting the assistant, whose voice they cannot
|
|
815
|
-
* speak over. Forced on when the microphone has no echo cancellation
|
|
866
|
+
* speak over. Forced on when the microphone has no echo cancellation, and
|
|
867
|
+
* by the call itself when the assistant keeps cutting itself off.
|
|
816
868
|
*/
|
|
817
869
|
disableInterruption?: boolean;
|
|
818
870
|
debugLog?: boolean;
|
|
@@ -862,6 +914,7 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
862
914
|
private reconnectAttempt;
|
|
863
915
|
private reconnectTimer?;
|
|
864
916
|
private connectionTimer?;
|
|
917
|
+
private echoGuard;
|
|
865
918
|
constructor(options?: MicdropOptions);
|
|
866
919
|
get vad(): VAD | undefined;
|
|
867
920
|
get isStarted(): boolean;
|
|
@@ -880,9 +933,12 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
880
933
|
/**
|
|
881
934
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
882
935
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
883
|
-
* itself and end up in the transcript.
|
|
936
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
937
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
884
938
|
*/
|
|
885
939
|
get isInterruptionDisabled(): boolean;
|
|
940
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
941
|
+
get echoStage(): EchoStage;
|
|
886
942
|
get micDeviceId(): string | undefined;
|
|
887
943
|
get speakerDeviceId(): string | undefined;
|
|
888
944
|
get state(): MicdropState;
|
|
@@ -923,6 +979,13 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
923
979
|
private scheduleReconnect;
|
|
924
980
|
private onSpeakerStartPlaying;
|
|
925
981
|
private onSpeakerStopPlaying;
|
|
982
|
+
/**
|
|
983
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
984
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
985
|
+
* neither the speaker nor the server hear of it.
|
|
986
|
+
*/
|
|
987
|
+
private isEchoOfAssistant;
|
|
988
|
+
private onInterruptionsGivenUp;
|
|
926
989
|
private addMessage;
|
|
927
990
|
private setPartialAssistantMessage;
|
|
928
991
|
private setError;
|
package/dist/index.js
CHANGED
|
@@ -497,6 +497,17 @@ var SpeakerController = class extends import_eventemitter32.EventEmitter {
|
|
|
497
497
|
if (!this.driver) return [];
|
|
498
498
|
return this.driver.getDevices();
|
|
499
499
|
}
|
|
500
|
+
/**
|
|
501
|
+
* Plays by another route, for when the microphone hears the current one
|
|
502
|
+
* @returns False if the platform has no other route to offer
|
|
503
|
+
*/
|
|
504
|
+
async useEchoFallback() {
|
|
505
|
+
return await this.driver?.useEchoFallback() ?? false;
|
|
506
|
+
}
|
|
507
|
+
/** Goes back to the usual route */
|
|
508
|
+
resetEchoFallback() {
|
|
509
|
+
this.driver?.resetEchoFallback();
|
|
510
|
+
}
|
|
500
511
|
};
|
|
501
512
|
|
|
502
513
|
// src/audio/MicRecorder.ts
|
|
@@ -999,6 +1010,8 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
999
1010
|
this.turnListening = false;
|
|
1000
1011
|
// Bumped whenever the turn moves on, so a late answer is dropped
|
|
1001
1012
|
this.turnRound = 0;
|
|
1013
|
+
// The VAD is being restarted, what it reports on the way is not speech
|
|
1014
|
+
this.isDroppingSpeech = false;
|
|
1002
1015
|
this.start = async (mic) => {
|
|
1003
1016
|
if (this.state.isStarted) {
|
|
1004
1017
|
throw new Error("MicRecorder is already started");
|
|
@@ -1122,6 +1135,7 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1122
1135
|
);
|
|
1123
1136
|
};
|
|
1124
1137
|
this.onStartSpeaking = () => {
|
|
1138
|
+
if (this.isDroppingSpeech) return;
|
|
1125
1139
|
this.speakingConfirmed = false;
|
|
1126
1140
|
this.isRecording = true;
|
|
1127
1141
|
this.buffer = [...this.reserve];
|
|
@@ -1135,6 +1149,11 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1135
1149
|
}
|
|
1136
1150
|
};
|
|
1137
1151
|
this.onConfirmSpeaking = () => {
|
|
1152
|
+
if (this.isDroppingSpeech) return;
|
|
1153
|
+
if (!this.turnOpen && this.isEcho?.()) {
|
|
1154
|
+
this.dropSpeech();
|
|
1155
|
+
return;
|
|
1156
|
+
}
|
|
1138
1157
|
this.speakingConfirmed = true;
|
|
1139
1158
|
this.changeState({ isSpeaking: true });
|
|
1140
1159
|
this.cancelPendingClose();
|
|
@@ -1145,12 +1164,14 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1145
1164
|
this.flushChunks();
|
|
1146
1165
|
};
|
|
1147
1166
|
this.onCancelSpeaking = () => {
|
|
1167
|
+
if (this.isDroppingSpeech) return;
|
|
1148
1168
|
this.stopRecording();
|
|
1149
1169
|
if (!this.turnOpen) {
|
|
1150
1170
|
this.turnListening = false;
|
|
1151
1171
|
}
|
|
1152
1172
|
};
|
|
1153
1173
|
this.onStopSpeaking = () => {
|
|
1174
|
+
if (this.isDroppingSpeech) return;
|
|
1154
1175
|
const wasConfirmed = this.speakingConfirmed;
|
|
1155
1176
|
this.changeState({ isSpeaking: false });
|
|
1156
1177
|
if (wasConfirmed) {
|
|
@@ -1162,6 +1183,26 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1162
1183
|
this.state = defaultMicRecorderState;
|
|
1163
1184
|
this.vad = getVAD(vadConfig);
|
|
1164
1185
|
}
|
|
1186
|
+
/**
|
|
1187
|
+
* Forgets the speech heard so far.
|
|
1188
|
+
*
|
|
1189
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
1190
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
1191
|
+
*/
|
|
1192
|
+
async dropSpeech() {
|
|
1193
|
+
this.isDroppingSpeech = true;
|
|
1194
|
+
this.stopRecording();
|
|
1195
|
+
this.turnListening = false;
|
|
1196
|
+
const vad = this.vad;
|
|
1197
|
+
try {
|
|
1198
|
+
await vad.pause();
|
|
1199
|
+
if (!this.speakerPlaying) await vad.resume();
|
|
1200
|
+
} catch (error) {
|
|
1201
|
+
console.error("[Micdrop] Restarting the VAD failed", error);
|
|
1202
|
+
} finally {
|
|
1203
|
+
this.isDroppingSpeech = false;
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1165
1206
|
/**
|
|
1166
1207
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
1167
1208
|
*
|
|
@@ -1289,6 +1330,8 @@ var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
|
|
|
1289
1330
|
this.prebufferDuration = 0;
|
|
1290
1331
|
this.volumeTimers = [];
|
|
1291
1332
|
this.meter = new VolumeMeter();
|
|
1333
|
+
// Wall time before which no utterance starts, see warmUp()
|
|
1334
|
+
this.holdUntil = 0;
|
|
1292
1335
|
}
|
|
1293
1336
|
/**
|
|
1294
1337
|
* Queues samples for playback
|
|
@@ -1304,6 +1347,34 @@ var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
|
|
|
1304
1347
|
);
|
|
1305
1348
|
this.handleSamples(samples);
|
|
1306
1349
|
}
|
|
1350
|
+
/**
|
|
1351
|
+
* Plays silence for a while, and holds the next utterance until it is over.
|
|
1352
|
+
*
|
|
1353
|
+
* Firefox mangles what it plays while a Bluetooth headset switches to its
|
|
1354
|
+
* call profile, which the headset does as soon as its microphone opens. The
|
|
1355
|
+
* first answer then comes out all at once. Silence played through that switch
|
|
1356
|
+
* goes unheard.
|
|
1357
|
+
* @param duration - How long to play silence, in milliseconds
|
|
1358
|
+
*/
|
|
1359
|
+
warmUp(duration) {
|
|
1360
|
+
this.holdUntil = Date.now() + duration;
|
|
1361
|
+
const samples = new Float32Array(
|
|
1362
|
+
Math.round(duration / 1e3 * this.sink.sampleRate)
|
|
1363
|
+
);
|
|
1364
|
+
this.warmUpAudio?.stop();
|
|
1365
|
+
this.warmUpAudio = this.sink.schedule(
|
|
1366
|
+
samples,
|
|
1367
|
+
this.sink.currentTime,
|
|
1368
|
+
() => this.warmUpAudio = void 0
|
|
1369
|
+
);
|
|
1370
|
+
}
|
|
1371
|
+
/** Stops playing, for a stream no longer used */
|
|
1372
|
+
destroy() {
|
|
1373
|
+
this.stopAudio();
|
|
1374
|
+
this.warmUpAudio?.stop();
|
|
1375
|
+
this.warmUpAudio = void 0;
|
|
1376
|
+
this.removeAllListeners();
|
|
1377
|
+
}
|
|
1307
1378
|
/** Drops everything queued and stops playing immediately */
|
|
1308
1379
|
stopAudio() {
|
|
1309
1380
|
for (const source of this.sources) {
|
|
@@ -1343,6 +1414,14 @@ var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
|
|
|
1343
1414
|
clearTimeout(this.quietTimer);
|
|
1344
1415
|
this.quietTimer = void 0;
|
|
1345
1416
|
}
|
|
1417
|
+
const hold = this.holdUntil - Date.now();
|
|
1418
|
+
if (hold > 0) {
|
|
1419
|
+
this.holdTimer ?? (this.holdTimer = setTimeout(() => {
|
|
1420
|
+
this.holdTimer = void 0;
|
|
1421
|
+
this.flushPrebuffer();
|
|
1422
|
+
}, hold));
|
|
1423
|
+
return;
|
|
1424
|
+
}
|
|
1346
1425
|
this.prebuffering = false;
|
|
1347
1426
|
const buffers = this.prebuffer;
|
|
1348
1427
|
this.prebuffer = [];
|
|
@@ -1358,6 +1437,10 @@ var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
|
|
|
1358
1437
|
clearTimeout(this.endTimer);
|
|
1359
1438
|
this.endTimer = void 0;
|
|
1360
1439
|
}
|
|
1440
|
+
if (this.holdTimer !== void 0) {
|
|
1441
|
+
clearTimeout(this.holdTimer);
|
|
1442
|
+
this.holdTimer = void 0;
|
|
1443
|
+
}
|
|
1361
1444
|
this.prebuffering = true;
|
|
1362
1445
|
this.prebuffer = [];
|
|
1363
1446
|
this.prebufferDuration = 0;
|
|
@@ -1439,6 +1522,16 @@ var MicDriver = class extends import_eventemitter36.EventEmitter {
|
|
|
1439
1522
|
}
|
|
1440
1523
|
};
|
|
1441
1524
|
var SpeakerDriver = class extends import_eventemitter36.EventEmitter {
|
|
1525
|
+
/**
|
|
1526
|
+
* Plays by another route, for when the microphone hears the current one
|
|
1527
|
+
* @returns False if the platform has no other route to offer
|
|
1528
|
+
*/
|
|
1529
|
+
async useEchoFallback() {
|
|
1530
|
+
return false;
|
|
1531
|
+
}
|
|
1532
|
+
/** Goes back to the usual route */
|
|
1533
|
+
resetEchoFallback() {
|
|
1534
|
+
}
|
|
1442
1535
|
};
|
|
1443
1536
|
|
|
1444
1537
|
// src/audio/index.ts
|
|
@@ -1538,6 +1631,51 @@ function isRecoverableError(error) {
|
|
|
1538
1631
|
return error.code === "Connection" /* Connection */ || error.code === "InternalServer" /* InternalServer */ || error.code === "Unknown" /* Unknown */;
|
|
1539
1632
|
}
|
|
1540
1633
|
|
|
1634
|
+
// src/client/EchoGuard.ts
|
|
1635
|
+
var ECHO_WATCH = 2e3;
|
|
1636
|
+
var EchoGuard = class {
|
|
1637
|
+
constructor() {
|
|
1638
|
+
this.stage = "normal";
|
|
1639
|
+
// The next sound opens a watch, the first of a call or after a new route
|
|
1640
|
+
this.isDue = true;
|
|
1641
|
+
}
|
|
1642
|
+
/** @param now - The current time, in milliseconds */
|
|
1643
|
+
speakerStarted(now) {
|
|
1644
|
+
if (!this.isDue) return;
|
|
1645
|
+
this.isDue = false;
|
|
1646
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1647
|
+
}
|
|
1648
|
+
/** @param now - The current time, in milliseconds */
|
|
1649
|
+
speakerStopped(now) {
|
|
1650
|
+
if (this.watchEnd === void 0) return;
|
|
1651
|
+
if (this.stage === "fallback" && now < this.watchEnd) {
|
|
1652
|
+
this.isDue = true;
|
|
1653
|
+
}
|
|
1654
|
+
this.watchEnd = void 0;
|
|
1655
|
+
}
|
|
1656
|
+
/**
|
|
1657
|
+
* Moves to the next stage if the speech heard can only be echo
|
|
1658
|
+
* @param now - The current time, in milliseconds
|
|
1659
|
+
* @returns True if the speech is to be ignored
|
|
1660
|
+
*/
|
|
1661
|
+
speechHeard(now) {
|
|
1662
|
+
if (this.watchEnd === void 0 || now >= this.watchEnd) return false;
|
|
1663
|
+
if (this.stage === "normal") {
|
|
1664
|
+
this.stage = "fallback";
|
|
1665
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1666
|
+
} else {
|
|
1667
|
+
this.stage = "disabled";
|
|
1668
|
+
this.watchEnd = void 0;
|
|
1669
|
+
}
|
|
1670
|
+
return true;
|
|
1671
|
+
}
|
|
1672
|
+
reset() {
|
|
1673
|
+
this.stage = "normal";
|
|
1674
|
+
this.isDue = true;
|
|
1675
|
+
this.watchEnd = void 0;
|
|
1676
|
+
}
|
|
1677
|
+
};
|
|
1678
|
+
|
|
1541
1679
|
// src/client/MicdropClient.ts
|
|
1542
1680
|
var DEFAULT_RECONNECT_OPTIONS = {
|
|
1543
1681
|
maxAttempts: Infinity,
|
|
@@ -1562,6 +1700,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1562
1700
|
// it can take a while, for the permission prompt or the browser itself
|
|
1563
1701
|
this._isStartingCall = false;
|
|
1564
1702
|
this.reconnectAttempt = 0;
|
|
1703
|
+
this.echoGuard = new EchoGuard();
|
|
1565
1704
|
this.start = async (options) => {
|
|
1566
1705
|
this.error = void 0;
|
|
1567
1706
|
this.options = { ...this.options, ...options };
|
|
@@ -1605,6 +1744,8 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1605
1744
|
console.error("[MicdropClient] Error stopping WebSocket", error);
|
|
1606
1745
|
}
|
|
1607
1746
|
Speaker.stopAudio();
|
|
1747
|
+
this.echoGuard.reset();
|
|
1748
|
+
Speaker.resetEchoFallback();
|
|
1608
1749
|
try {
|
|
1609
1750
|
await this.stopMic();
|
|
1610
1751
|
} catch (error) {
|
|
@@ -1661,6 +1802,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1661
1802
|
this.micRecorder.setSpeakerPlaying(
|
|
1662
1803
|
this.isInterruptionDisabled && Speaker.isPlaying
|
|
1663
1804
|
);
|
|
1805
|
+
this.micRecorder.isEcho = this.isEchoOfAssistant;
|
|
1664
1806
|
this.micRecorder.on("StateChange", () => {
|
|
1665
1807
|
this.notifyStateChange();
|
|
1666
1808
|
});
|
|
@@ -1852,6 +1994,9 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1852
1994
|
};
|
|
1853
1995
|
this.onSpeakerStartPlaying = () => {
|
|
1854
1996
|
this.log("Speaker started");
|
|
1997
|
+
if (this.isWSStarted) {
|
|
1998
|
+
this.echoGuard.speakerStarted(Date.now());
|
|
1999
|
+
}
|
|
1855
2000
|
if (this.isInterruptionDisabled) {
|
|
1856
2001
|
this.micRecorder?.setSpeakerPlaying(true);
|
|
1857
2002
|
this.vad?.pause();
|
|
@@ -1860,6 +2005,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1860
2005
|
};
|
|
1861
2006
|
this.onSpeakerStopPlaying = () => {
|
|
1862
2007
|
this.log("Speaker stopped");
|
|
2008
|
+
this.echoGuard.speakerStopped(Date.now());
|
|
1863
2009
|
this.micRecorder?.setSpeakerPlaying(false);
|
|
1864
2010
|
if (this.isInterruptionDisabled) {
|
|
1865
2011
|
setTimeout(() => {
|
|
@@ -1872,6 +2018,28 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1872
2018
|
this.notifyStateChange();
|
|
1873
2019
|
}
|
|
1874
2020
|
};
|
|
2021
|
+
/**
|
|
2022
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
2023
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
2024
|
+
* neither the speaker nor the server hear of it.
|
|
2025
|
+
*/
|
|
2026
|
+
this.isEchoOfAssistant = () => {
|
|
2027
|
+
if (this.isInterruptionDisabled || !Speaker.isPlaying) return false;
|
|
2028
|
+
if (!this.echoGuard.speechHeard(Date.now())) return false;
|
|
2029
|
+
this.log("Echo of the assistant, moving to", this.echoGuard.stage);
|
|
2030
|
+
if (this.echoGuard.stage === "fallback") {
|
|
2031
|
+
Speaker.useEchoFallback().then((moved) => {
|
|
2032
|
+
if (!moved && this.echoGuard.stage === "fallback") {
|
|
2033
|
+
this.echoGuard.stage = "disabled";
|
|
2034
|
+
this.onInterruptionsGivenUp();
|
|
2035
|
+
}
|
|
2036
|
+
});
|
|
2037
|
+
} else {
|
|
2038
|
+
this.onInterruptionsGivenUp();
|
|
2039
|
+
}
|
|
2040
|
+
this.notifyStateChange();
|
|
2041
|
+
return true;
|
|
2042
|
+
};
|
|
1875
2043
|
Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
|
|
1876
2044
|
Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
|
|
1877
2045
|
Mic.on("DeviceChange", this.updateDevices);
|
|
@@ -1921,10 +2089,15 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1921
2089
|
/**
|
|
1922
2090
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
1923
2091
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
1924
|
-
* itself and end up in the transcript.
|
|
2092
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
2093
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
1925
2094
|
*/
|
|
1926
2095
|
get isInterruptionDisabled() {
|
|
1927
|
-
return !!this.options.disableInterruption || Mic.echoCancellation === false;
|
|
2096
|
+
return !!this.options.disableInterruption || Mic.echoCancellation === false || this.echoGuard.stage === "disabled";
|
|
2097
|
+
}
|
|
2098
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
2099
|
+
get echoStage() {
|
|
2100
|
+
return this.echoGuard.stage;
|
|
1928
2101
|
}
|
|
1929
2102
|
get micDeviceId() {
|
|
1930
2103
|
return Mic.deviceId;
|
|
@@ -2033,6 +2206,15 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
2033
2206
|
}
|
|
2034
2207
|
}, config.delayMs);
|
|
2035
2208
|
}
|
|
2209
|
+
// The same as when the speaker starts on a call without interruptions
|
|
2210
|
+
onInterruptionsGivenUp() {
|
|
2211
|
+
this.log("Interruptions disabled until the end of the call");
|
|
2212
|
+
if (Speaker.isPlaying) {
|
|
2213
|
+
this.micRecorder?.setSpeakerPlaying(true);
|
|
2214
|
+
this.vad?.pause();
|
|
2215
|
+
}
|
|
2216
|
+
this.notifyStateChange();
|
|
2217
|
+
}
|
|
2036
2218
|
addMessage(message) {
|
|
2037
2219
|
if (message.role === "assistant") this.partialAssistantMessage = "";
|
|
2038
2220
|
this.conversation = [...this.conversation, message];
|