@micdrop/client 3.4.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/index.d.mts +50 -2
- package/dist/index.d.ts +50 -2
- package/dist/index.js +142 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +142 -2
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -45,6 +45,8 @@ Speaker.setDriver(new MySpeaker())
|
|
|
45
45
|
|
|
46
46
|
`Pcm16AudioStream` does the playback scheduling for you if the platform offers something like Web Audio: give it an `AudioSink`, which is the small slice of it that Micdrop needs, and call its `destroy()` when the driver stops. When the output needs a moment once the microphone opens, as with Firefox and a Bluetooth headset switching to its call profile, call `warmUp(duration)`: the stream plays silence that long and holds the first answer until it is over.
|
|
47
47
|
|
|
48
|
+
When a speaker has a second way of playing, it can offer it through `useEchoFallback()` and `resetEchoFallback()`. A call moves there when the microphone hears the assistant [despite echo cancellation](https://micdrop.dev/docs/client/vad#echo-and-interruptions), and turns interruptions off when there is no such route.
|
|
49
|
+
|
|
48
50
|
For Silero, provide the inference and the state machine comes from here:
|
|
49
51
|
|
|
50
52
|
```ts
|
package/dist/index.d.mts
CHANGED
|
@@ -88,6 +88,13 @@ declare abstract class SpeakerDriver extends EventEmitter<SpeakerDriverEvents> {
|
|
|
88
88
|
abstract changeDevice(deviceId: string): Promise<void>;
|
|
89
89
|
/** Lists the output devices the system offers */
|
|
90
90
|
abstract getDevices(): Promise<MicdropDevice[]>;
|
|
91
|
+
/**
|
|
92
|
+
* Plays by another route, for when the microphone hears the current one
|
|
93
|
+
* @returns False if the platform has no other route to offer
|
|
94
|
+
*/
|
|
95
|
+
useEchoFallback(): Promise<boolean>;
|
|
96
|
+
/** Goes back to the usual route */
|
|
97
|
+
resetEchoFallback(): void;
|
|
91
98
|
}
|
|
92
99
|
/** A buffer handed to an {@link AudioSink}, that can still be cancelled */
|
|
93
100
|
interface ScheduledAudio {
|
|
@@ -208,6 +215,13 @@ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
|
|
|
208
215
|
changeDevice(deviceId: string): Promise<void>;
|
|
209
216
|
/** Lists the output devices the system offers */
|
|
210
217
|
getDevices(): Promise<MicdropDevice[]>;
|
|
218
|
+
/**
|
|
219
|
+
* Plays by another route, for when the microphone hears the current one
|
|
220
|
+
* @returns False if the platform has no other route to offer
|
|
221
|
+
*/
|
|
222
|
+
useEchoFallback(): Promise<boolean>;
|
|
223
|
+
/** Goes back to the usual route */
|
|
224
|
+
resetEchoFallback(): void;
|
|
211
225
|
private onStartPlaying;
|
|
212
226
|
private onStopPlaying;
|
|
213
227
|
private onVolume;
|
|
@@ -433,6 +447,11 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
433
447
|
turnMaxWait: number;
|
|
434
448
|
state: MicRecorderState;
|
|
435
449
|
vad: VAD;
|
|
450
|
+
/**
|
|
451
|
+
* Asked as speech is confirmed, before anyone hears of it. Saying it is echo
|
|
452
|
+
* drops the speech, and listens again in case the echo goes on.
|
|
453
|
+
*/
|
|
454
|
+
isEcho?: () => boolean;
|
|
436
455
|
private mic;
|
|
437
456
|
private isRecording;
|
|
438
457
|
private speakingConfirmed;
|
|
@@ -447,6 +466,7 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
447
466
|
private turnListening;
|
|
448
467
|
private turnTimer;
|
|
449
468
|
private turnRound;
|
|
469
|
+
private isDroppingSpeech;
|
|
450
470
|
constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
|
|
451
471
|
/**
|
|
452
472
|
* How long a turn stays open once the detector asked to wait.
|
|
@@ -477,6 +497,13 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
477
497
|
private onConfirmSpeaking;
|
|
478
498
|
private onCancelSpeaking;
|
|
479
499
|
private onStopSpeaking;
|
|
500
|
+
/**
|
|
501
|
+
* Forgets the speech heard so far.
|
|
502
|
+
*
|
|
503
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
504
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
505
|
+
*/
|
|
506
|
+
private dropSpeech;
|
|
480
507
|
/**
|
|
481
508
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
482
509
|
*
|
|
@@ -796,6 +823,15 @@ declare function getClientErrorFromWSCloseEvent(event: WSCloseEvent): MicdropCli
|
|
|
796
823
|
declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
|
|
797
824
|
declare function isRecoverableError(error: MicdropClientError): boolean;
|
|
798
825
|
|
|
826
|
+
/**
|
|
827
|
+
* How far a call has gone to keep the assistant out of the microphone.
|
|
828
|
+
*
|
|
829
|
+
* - `normal`: the speaker plays as usual
|
|
830
|
+
* - `fallback`: it plays by the other route the platform offers
|
|
831
|
+
* - `disabled`: the user may no longer interrupt, until the call ends
|
|
832
|
+
*/
|
|
833
|
+
type EchoStage = 'normal' | 'fallback' | 'disabled';
|
|
834
|
+
|
|
799
835
|
interface MicdropEvents {
|
|
800
836
|
EndCall: [];
|
|
801
837
|
Error: [MicdropClientError];
|
|
@@ -827,7 +863,8 @@ interface MicdropOptions {
|
|
|
827
863
|
turnMaxWait?: number;
|
|
828
864
|
/**
|
|
829
865
|
* Keeps the user from interrupting the assistant, whose voice they cannot
|
|
830
|
-
* speak over. Forced on when the microphone has no echo cancellation
|
|
866
|
+
* speak over. Forced on when the microphone has no echo cancellation, and
|
|
867
|
+
* by the call itself when the assistant keeps cutting itself off.
|
|
831
868
|
*/
|
|
832
869
|
disableInterruption?: boolean;
|
|
833
870
|
debugLog?: boolean;
|
|
@@ -877,6 +914,7 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
877
914
|
private reconnectAttempt;
|
|
878
915
|
private reconnectTimer?;
|
|
879
916
|
private connectionTimer?;
|
|
917
|
+
private echoGuard;
|
|
880
918
|
constructor(options?: MicdropOptions);
|
|
881
919
|
get vad(): VAD | undefined;
|
|
882
920
|
get isStarted(): boolean;
|
|
@@ -895,9 +933,12 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
895
933
|
/**
|
|
896
934
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
897
935
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
898
|
-
* itself and end up in the transcript.
|
|
936
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
937
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
899
938
|
*/
|
|
900
939
|
get isInterruptionDisabled(): boolean;
|
|
940
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
941
|
+
get echoStage(): EchoStage;
|
|
901
942
|
get micDeviceId(): string | undefined;
|
|
902
943
|
get speakerDeviceId(): string | undefined;
|
|
903
944
|
get state(): MicdropState;
|
|
@@ -938,6 +979,13 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
938
979
|
private scheduleReconnect;
|
|
939
980
|
private onSpeakerStartPlaying;
|
|
940
981
|
private onSpeakerStopPlaying;
|
|
982
|
+
/**
|
|
983
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
984
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
985
|
+
* neither the speaker nor the server hear of it.
|
|
986
|
+
*/
|
|
987
|
+
private isEchoOfAssistant;
|
|
988
|
+
private onInterruptionsGivenUp;
|
|
941
989
|
private addMessage;
|
|
942
990
|
private setPartialAssistantMessage;
|
|
943
991
|
private setError;
|
package/dist/index.d.ts
CHANGED
|
@@ -88,6 +88,13 @@ declare abstract class SpeakerDriver extends EventEmitter<SpeakerDriverEvents> {
|
|
|
88
88
|
abstract changeDevice(deviceId: string): Promise<void>;
|
|
89
89
|
/** Lists the output devices the system offers */
|
|
90
90
|
abstract getDevices(): Promise<MicdropDevice[]>;
|
|
91
|
+
/**
|
|
92
|
+
* Plays by another route, for when the microphone hears the current one
|
|
93
|
+
* @returns False if the platform has no other route to offer
|
|
94
|
+
*/
|
|
95
|
+
useEchoFallback(): Promise<boolean>;
|
|
96
|
+
/** Goes back to the usual route */
|
|
97
|
+
resetEchoFallback(): void;
|
|
91
98
|
}
|
|
92
99
|
/** A buffer handed to an {@link AudioSink}, that can still be cancelled */
|
|
93
100
|
interface ScheduledAudio {
|
|
@@ -208,6 +215,13 @@ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
|
|
|
208
215
|
changeDevice(deviceId: string): Promise<void>;
|
|
209
216
|
/** Lists the output devices the system offers */
|
|
210
217
|
getDevices(): Promise<MicdropDevice[]>;
|
|
218
|
+
/**
|
|
219
|
+
* Plays by another route, for when the microphone hears the current one
|
|
220
|
+
* @returns False if the platform has no other route to offer
|
|
221
|
+
*/
|
|
222
|
+
useEchoFallback(): Promise<boolean>;
|
|
223
|
+
/** Goes back to the usual route */
|
|
224
|
+
resetEchoFallback(): void;
|
|
211
225
|
private onStartPlaying;
|
|
212
226
|
private onStopPlaying;
|
|
213
227
|
private onVolume;
|
|
@@ -433,6 +447,11 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
433
447
|
turnMaxWait: number;
|
|
434
448
|
state: MicRecorderState;
|
|
435
449
|
vad: VAD;
|
|
450
|
+
/**
|
|
451
|
+
* Asked as speech is confirmed, before anyone hears of it. Saying it is echo
|
|
452
|
+
* drops the speech, and listens again in case the echo goes on.
|
|
453
|
+
*/
|
|
454
|
+
isEcho?: () => boolean;
|
|
436
455
|
private mic;
|
|
437
456
|
private isRecording;
|
|
438
457
|
private speakingConfirmed;
|
|
@@ -447,6 +466,7 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
447
466
|
private turnListening;
|
|
448
467
|
private turnTimer;
|
|
449
468
|
private turnRound;
|
|
469
|
+
private isDroppingSpeech;
|
|
450
470
|
constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
|
|
451
471
|
/**
|
|
452
472
|
* How long a turn stays open once the detector asked to wait.
|
|
@@ -477,6 +497,13 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
477
497
|
private onConfirmSpeaking;
|
|
478
498
|
private onCancelSpeaking;
|
|
479
499
|
private onStopSpeaking;
|
|
500
|
+
/**
|
|
501
|
+
* Forgets the speech heard so far.
|
|
502
|
+
*
|
|
503
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
504
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
505
|
+
*/
|
|
506
|
+
private dropSpeech;
|
|
480
507
|
/**
|
|
481
508
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
482
509
|
*
|
|
@@ -796,6 +823,15 @@ declare function getClientErrorFromWSCloseEvent(event: WSCloseEvent): MicdropCli
|
|
|
796
823
|
declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
|
|
797
824
|
declare function isRecoverableError(error: MicdropClientError): boolean;
|
|
798
825
|
|
|
826
|
+
/**
|
|
827
|
+
* How far a call has gone to keep the assistant out of the microphone.
|
|
828
|
+
*
|
|
829
|
+
* - `normal`: the speaker plays as usual
|
|
830
|
+
* - `fallback`: it plays by the other route the platform offers
|
|
831
|
+
* - `disabled`: the user may no longer interrupt, until the call ends
|
|
832
|
+
*/
|
|
833
|
+
type EchoStage = 'normal' | 'fallback' | 'disabled';
|
|
834
|
+
|
|
799
835
|
interface MicdropEvents {
|
|
800
836
|
EndCall: [];
|
|
801
837
|
Error: [MicdropClientError];
|
|
@@ -827,7 +863,8 @@ interface MicdropOptions {
|
|
|
827
863
|
turnMaxWait?: number;
|
|
828
864
|
/**
|
|
829
865
|
* Keeps the user from interrupting the assistant, whose voice they cannot
|
|
830
|
-
* speak over. Forced on when the microphone has no echo cancellation
|
|
866
|
+
* speak over. Forced on when the microphone has no echo cancellation, and
|
|
867
|
+
* by the call itself when the assistant keeps cutting itself off.
|
|
831
868
|
*/
|
|
832
869
|
disableInterruption?: boolean;
|
|
833
870
|
debugLog?: boolean;
|
|
@@ -877,6 +914,7 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
877
914
|
private reconnectAttempt;
|
|
878
915
|
private reconnectTimer?;
|
|
879
916
|
private connectionTimer?;
|
|
917
|
+
private echoGuard;
|
|
880
918
|
constructor(options?: MicdropOptions);
|
|
881
919
|
get vad(): VAD | undefined;
|
|
882
920
|
get isStarted(): boolean;
|
|
@@ -895,9 +933,12 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
895
933
|
/**
|
|
896
934
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
897
935
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
898
|
-
* itself and end up in the transcript.
|
|
936
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
937
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
899
938
|
*/
|
|
900
939
|
get isInterruptionDisabled(): boolean;
|
|
940
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
941
|
+
get echoStage(): EchoStage;
|
|
901
942
|
get micDeviceId(): string | undefined;
|
|
902
943
|
get speakerDeviceId(): string | undefined;
|
|
903
944
|
get state(): MicdropState;
|
|
@@ -938,6 +979,13 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
938
979
|
private scheduleReconnect;
|
|
939
980
|
private onSpeakerStartPlaying;
|
|
940
981
|
private onSpeakerStopPlaying;
|
|
982
|
+
/**
|
|
983
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
984
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
985
|
+
* neither the speaker nor the server hear of it.
|
|
986
|
+
*/
|
|
987
|
+
private isEchoOfAssistant;
|
|
988
|
+
private onInterruptionsGivenUp;
|
|
941
989
|
private addMessage;
|
|
942
990
|
private setPartialAssistantMessage;
|
|
943
991
|
private setError;
|
package/dist/index.js
CHANGED
|
@@ -497,6 +497,17 @@ var SpeakerController = class extends import_eventemitter32.EventEmitter {
|
|
|
497
497
|
if (!this.driver) return [];
|
|
498
498
|
return this.driver.getDevices();
|
|
499
499
|
}
|
|
500
|
+
/**
|
|
501
|
+
* Plays by another route, for when the microphone hears the current one
|
|
502
|
+
* @returns False if the platform has no other route to offer
|
|
503
|
+
*/
|
|
504
|
+
async useEchoFallback() {
|
|
505
|
+
return await this.driver?.useEchoFallback() ?? false;
|
|
506
|
+
}
|
|
507
|
+
/** Goes back to the usual route */
|
|
508
|
+
resetEchoFallback() {
|
|
509
|
+
this.driver?.resetEchoFallback();
|
|
510
|
+
}
|
|
500
511
|
};
|
|
501
512
|
|
|
502
513
|
// src/audio/MicRecorder.ts
|
|
@@ -999,6 +1010,8 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
999
1010
|
this.turnListening = false;
|
|
1000
1011
|
// Bumped whenever the turn moves on, so a late answer is dropped
|
|
1001
1012
|
this.turnRound = 0;
|
|
1013
|
+
// The VAD is being restarted, what it reports on the way is not speech
|
|
1014
|
+
this.isDroppingSpeech = false;
|
|
1002
1015
|
this.start = async (mic) => {
|
|
1003
1016
|
if (this.state.isStarted) {
|
|
1004
1017
|
throw new Error("MicRecorder is already started");
|
|
@@ -1122,6 +1135,7 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1122
1135
|
);
|
|
1123
1136
|
};
|
|
1124
1137
|
this.onStartSpeaking = () => {
|
|
1138
|
+
if (this.isDroppingSpeech) return;
|
|
1125
1139
|
this.speakingConfirmed = false;
|
|
1126
1140
|
this.isRecording = true;
|
|
1127
1141
|
this.buffer = [...this.reserve];
|
|
@@ -1135,6 +1149,11 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1135
1149
|
}
|
|
1136
1150
|
};
|
|
1137
1151
|
this.onConfirmSpeaking = () => {
|
|
1152
|
+
if (this.isDroppingSpeech) return;
|
|
1153
|
+
if (!this.turnOpen && this.isEcho?.()) {
|
|
1154
|
+
this.dropSpeech();
|
|
1155
|
+
return;
|
|
1156
|
+
}
|
|
1138
1157
|
this.speakingConfirmed = true;
|
|
1139
1158
|
this.changeState({ isSpeaking: true });
|
|
1140
1159
|
this.cancelPendingClose();
|
|
@@ -1145,12 +1164,14 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1145
1164
|
this.flushChunks();
|
|
1146
1165
|
};
|
|
1147
1166
|
this.onCancelSpeaking = () => {
|
|
1167
|
+
if (this.isDroppingSpeech) return;
|
|
1148
1168
|
this.stopRecording();
|
|
1149
1169
|
if (!this.turnOpen) {
|
|
1150
1170
|
this.turnListening = false;
|
|
1151
1171
|
}
|
|
1152
1172
|
};
|
|
1153
1173
|
this.onStopSpeaking = () => {
|
|
1174
|
+
if (this.isDroppingSpeech) return;
|
|
1154
1175
|
const wasConfirmed = this.speakingConfirmed;
|
|
1155
1176
|
this.changeState({ isSpeaking: false });
|
|
1156
1177
|
if (wasConfirmed) {
|
|
@@ -1162,6 +1183,26 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1162
1183
|
this.state = defaultMicRecorderState;
|
|
1163
1184
|
this.vad = getVAD(vadConfig);
|
|
1164
1185
|
}
|
|
1186
|
+
/**
|
|
1187
|
+
* Forgets the speech heard so far.
|
|
1188
|
+
*
|
|
1189
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
1190
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
1191
|
+
*/
|
|
1192
|
+
async dropSpeech() {
|
|
1193
|
+
this.isDroppingSpeech = true;
|
|
1194
|
+
this.stopRecording();
|
|
1195
|
+
this.turnListening = false;
|
|
1196
|
+
const vad = this.vad;
|
|
1197
|
+
try {
|
|
1198
|
+
await vad.pause();
|
|
1199
|
+
if (!this.speakerPlaying) await vad.resume();
|
|
1200
|
+
} catch (error) {
|
|
1201
|
+
console.error("[Micdrop] Restarting the VAD failed", error);
|
|
1202
|
+
} finally {
|
|
1203
|
+
this.isDroppingSpeech = false;
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1165
1206
|
/**
|
|
1166
1207
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
1167
1208
|
*
|
|
@@ -1481,6 +1522,16 @@ var MicDriver = class extends import_eventemitter36.EventEmitter {
|
|
|
1481
1522
|
}
|
|
1482
1523
|
};
|
|
1483
1524
|
var SpeakerDriver = class extends import_eventemitter36.EventEmitter {
|
|
1525
|
+
/**
|
|
1526
|
+
* Plays by another route, for when the microphone hears the current one
|
|
1527
|
+
* @returns False if the platform has no other route to offer
|
|
1528
|
+
*/
|
|
1529
|
+
async useEchoFallback() {
|
|
1530
|
+
return false;
|
|
1531
|
+
}
|
|
1532
|
+
/** Goes back to the usual route */
|
|
1533
|
+
resetEchoFallback() {
|
|
1534
|
+
}
|
|
1484
1535
|
};
|
|
1485
1536
|
|
|
1486
1537
|
// src/audio/index.ts
|
|
@@ -1580,6 +1631,51 @@ function isRecoverableError(error) {
|
|
|
1580
1631
|
return error.code === "Connection" /* Connection */ || error.code === "InternalServer" /* InternalServer */ || error.code === "Unknown" /* Unknown */;
|
|
1581
1632
|
}
|
|
1582
1633
|
|
|
1634
|
+
// src/client/EchoGuard.ts
|
|
1635
|
+
var ECHO_WATCH = 2e3;
|
|
1636
|
+
var EchoGuard = class {
|
|
1637
|
+
constructor() {
|
|
1638
|
+
this.stage = "normal";
|
|
1639
|
+
// The next sound opens a watch, the first of a call or after a new route
|
|
1640
|
+
this.isDue = true;
|
|
1641
|
+
}
|
|
1642
|
+
/** @param now - The current time, in milliseconds */
|
|
1643
|
+
speakerStarted(now) {
|
|
1644
|
+
if (!this.isDue) return;
|
|
1645
|
+
this.isDue = false;
|
|
1646
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1647
|
+
}
|
|
1648
|
+
/** @param now - The current time, in milliseconds */
|
|
1649
|
+
speakerStopped(now) {
|
|
1650
|
+
if (this.watchEnd === void 0) return;
|
|
1651
|
+
if (this.stage === "fallback" && now < this.watchEnd) {
|
|
1652
|
+
this.isDue = true;
|
|
1653
|
+
}
|
|
1654
|
+
this.watchEnd = void 0;
|
|
1655
|
+
}
|
|
1656
|
+
/**
|
|
1657
|
+
* Moves to the next stage if the speech heard can only be echo
|
|
1658
|
+
* @param now - The current time, in milliseconds
|
|
1659
|
+
* @returns True if the speech is to be ignored
|
|
1660
|
+
*/
|
|
1661
|
+
speechHeard(now) {
|
|
1662
|
+
if (this.watchEnd === void 0 || now >= this.watchEnd) return false;
|
|
1663
|
+
if (this.stage === "normal") {
|
|
1664
|
+
this.stage = "fallback";
|
|
1665
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1666
|
+
} else {
|
|
1667
|
+
this.stage = "disabled";
|
|
1668
|
+
this.watchEnd = void 0;
|
|
1669
|
+
}
|
|
1670
|
+
return true;
|
|
1671
|
+
}
|
|
1672
|
+
reset() {
|
|
1673
|
+
this.stage = "normal";
|
|
1674
|
+
this.isDue = true;
|
|
1675
|
+
this.watchEnd = void 0;
|
|
1676
|
+
}
|
|
1677
|
+
};
|
|
1678
|
+
|
|
1583
1679
|
// src/client/MicdropClient.ts
|
|
1584
1680
|
var DEFAULT_RECONNECT_OPTIONS = {
|
|
1585
1681
|
maxAttempts: Infinity,
|
|
@@ -1604,6 +1700,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1604
1700
|
// it can take a while, for the permission prompt or the browser itself
|
|
1605
1701
|
this._isStartingCall = false;
|
|
1606
1702
|
this.reconnectAttempt = 0;
|
|
1703
|
+
this.echoGuard = new EchoGuard();
|
|
1607
1704
|
this.start = async (options) => {
|
|
1608
1705
|
this.error = void 0;
|
|
1609
1706
|
this.options = { ...this.options, ...options };
|
|
@@ -1647,6 +1744,8 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1647
1744
|
console.error("[MicdropClient] Error stopping WebSocket", error);
|
|
1648
1745
|
}
|
|
1649
1746
|
Speaker.stopAudio();
|
|
1747
|
+
this.echoGuard.reset();
|
|
1748
|
+
Speaker.resetEchoFallback();
|
|
1650
1749
|
try {
|
|
1651
1750
|
await this.stopMic();
|
|
1652
1751
|
} catch (error) {
|
|
@@ -1703,6 +1802,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1703
1802
|
this.micRecorder.setSpeakerPlaying(
|
|
1704
1803
|
this.isInterruptionDisabled && Speaker.isPlaying
|
|
1705
1804
|
);
|
|
1805
|
+
this.micRecorder.isEcho = this.isEchoOfAssistant;
|
|
1706
1806
|
this.micRecorder.on("StateChange", () => {
|
|
1707
1807
|
this.notifyStateChange();
|
|
1708
1808
|
});
|
|
@@ -1894,6 +1994,9 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1894
1994
|
};
|
|
1895
1995
|
this.onSpeakerStartPlaying = () => {
|
|
1896
1996
|
this.log("Speaker started");
|
|
1997
|
+
if (this.isWSStarted) {
|
|
1998
|
+
this.echoGuard.speakerStarted(Date.now());
|
|
1999
|
+
}
|
|
1897
2000
|
if (this.isInterruptionDisabled) {
|
|
1898
2001
|
this.micRecorder?.setSpeakerPlaying(true);
|
|
1899
2002
|
this.vad?.pause();
|
|
@@ -1902,6 +2005,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1902
2005
|
};
|
|
1903
2006
|
this.onSpeakerStopPlaying = () => {
|
|
1904
2007
|
this.log("Speaker stopped");
|
|
2008
|
+
this.echoGuard.speakerStopped(Date.now());
|
|
1905
2009
|
this.micRecorder?.setSpeakerPlaying(false);
|
|
1906
2010
|
if (this.isInterruptionDisabled) {
|
|
1907
2011
|
setTimeout(() => {
|
|
@@ -1914,6 +2018,28 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1914
2018
|
this.notifyStateChange();
|
|
1915
2019
|
}
|
|
1916
2020
|
};
|
|
2021
|
+
/**
|
|
2022
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
2023
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
2024
|
+
* neither the speaker nor the server hear of it.
|
|
2025
|
+
*/
|
|
2026
|
+
this.isEchoOfAssistant = () => {
|
|
2027
|
+
if (this.isInterruptionDisabled || !Speaker.isPlaying) return false;
|
|
2028
|
+
if (!this.echoGuard.speechHeard(Date.now())) return false;
|
|
2029
|
+
this.log("Echo of the assistant, moving to", this.echoGuard.stage);
|
|
2030
|
+
if (this.echoGuard.stage === "fallback") {
|
|
2031
|
+
Speaker.useEchoFallback().then((moved) => {
|
|
2032
|
+
if (!moved && this.echoGuard.stage === "fallback") {
|
|
2033
|
+
this.echoGuard.stage = "disabled";
|
|
2034
|
+
this.onInterruptionsGivenUp();
|
|
2035
|
+
}
|
|
2036
|
+
});
|
|
2037
|
+
} else {
|
|
2038
|
+
this.onInterruptionsGivenUp();
|
|
2039
|
+
}
|
|
2040
|
+
this.notifyStateChange();
|
|
2041
|
+
return true;
|
|
2042
|
+
};
|
|
1917
2043
|
Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
|
|
1918
2044
|
Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
|
|
1919
2045
|
Mic.on("DeviceChange", this.updateDevices);
|
|
@@ -1963,10 +2089,15 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1963
2089
|
/**
|
|
1964
2090
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
1965
2091
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
1966
|
-
* itself and end up in the transcript.
|
|
2092
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
2093
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
1967
2094
|
*/
|
|
1968
2095
|
get isInterruptionDisabled() {
|
|
1969
|
-
return !!this.options.disableInterruption || Mic.echoCancellation === false;
|
|
2096
|
+
return !!this.options.disableInterruption || Mic.echoCancellation === false || this.echoGuard.stage === "disabled";
|
|
2097
|
+
}
|
|
2098
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
2099
|
+
get echoStage() {
|
|
2100
|
+
return this.echoGuard.stage;
|
|
1970
2101
|
}
|
|
1971
2102
|
get micDeviceId() {
|
|
1972
2103
|
return Mic.deviceId;
|
|
@@ -2075,6 +2206,15 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
2075
2206
|
}
|
|
2076
2207
|
}, config.delayMs);
|
|
2077
2208
|
}
|
|
2209
|
+
// The same as when the speaker starts on a call without interruptions
|
|
2210
|
+
onInterruptionsGivenUp() {
|
|
2211
|
+
this.log("Interruptions disabled until the end of the call");
|
|
2212
|
+
if (Speaker.isPlaying) {
|
|
2213
|
+
this.micRecorder?.setSpeakerPlaying(true);
|
|
2214
|
+
this.vad?.pause();
|
|
2215
|
+
}
|
|
2216
|
+
this.notifyStateChange();
|
|
2217
|
+
}
|
|
2078
2218
|
addMessage(message) {
|
|
2079
2219
|
if (message.role === "assistant") this.partialAssistantMessage = "";
|
|
2080
2220
|
this.conversation = [...this.conversation, message];
|