@micdrop/client 3.3.1 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -43,7 +43,9 @@ Mic.setDriver(new MyMic())
43
43
  Speaker.setDriver(new MySpeaker())
44
44
  ```
45
45
 
46
- `Pcm16AudioStream` does the playback scheduling for you if the platform offers something like Web Audio: give it an `AudioSink`, which is the small slice of it that Micdrop needs.
46
+ `Pcm16AudioStream` does the playback scheduling for you if the platform offers something like Web Audio: give it an `AudioSink`, which is the small slice of it that Micdrop needs, and call its `destroy()` when the driver stops. When the output needs a moment once the microphone opens, as with Firefox and a Bluetooth headset switching to its call profile, call `warmUp(duration)`: the stream plays silence that long and holds the first answer until it is over.
47
+
48
+ When a speaker has a second way of playing, it can offer it through `useEchoFallback()` and `resetEchoFallback()`. A call moves there when the microphone hears the assistant [despite echo cancellation](https://micdrop.dev/docs/client/vad#echo-and-interruptions), and turns interruptions off when there is no such route.
47
49
 
48
50
  For Silero, provide the inference and the state machine comes from here:
49
51
 
package/dist/index.d.mts CHANGED
@@ -88,6 +88,13 @@ declare abstract class SpeakerDriver extends EventEmitter<SpeakerDriverEvents> {
88
88
  abstract changeDevice(deviceId: string): Promise<void>;
89
89
  /** Lists the output devices the system offers */
90
90
  abstract getDevices(): Promise<MicdropDevice[]>;
91
+ /**
92
+ * Plays by another route, for when the microphone hears the current one
93
+ * @returns False if the platform has no other route to offer
94
+ */
95
+ useEchoFallback(): Promise<boolean>;
96
+ /** Goes back to the usual route */
97
+ resetEchoFallback(): void;
91
98
  }
92
99
  /** A buffer handed to an {@link AudioSink}, that can still be cancelled */
93
100
  interface ScheduledAudio {
@@ -208,6 +215,13 @@ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
208
215
  changeDevice(deviceId: string): Promise<void>;
209
216
  /** Lists the output devices the system offers */
210
217
  getDevices(): Promise<MicdropDevice[]>;
218
+ /**
219
+ * Plays by another route, for when the microphone hears the current one
220
+ * @returns False if the platform has no other route to offer
221
+ */
222
+ useEchoFallback(): Promise<boolean>;
223
+ /** Goes back to the usual route */
224
+ resetEchoFallback(): void;
211
225
  private onStartPlaying;
212
226
  private onStopPlaying;
213
227
  private onVolume;
@@ -433,6 +447,11 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
433
447
  turnMaxWait: number;
434
448
  state: MicRecorderState;
435
449
  vad: VAD;
450
+ /**
451
+ * Asked as speech is confirmed, before anyone hears of it. Saying it is echo
452
+ * drops the speech, and listens again in case the echo goes on.
453
+ */
454
+ isEcho?: () => boolean;
436
455
  private mic;
437
456
  private isRecording;
438
457
  private speakingConfirmed;
@@ -447,6 +466,7 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
447
466
  private turnListening;
448
467
  private turnTimer;
449
468
  private turnRound;
469
+ private isDroppingSpeech;
450
470
  constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
451
471
  /**
452
472
  * How long a turn stays open once the detector asked to wait.
@@ -477,6 +497,13 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
477
497
  private onConfirmSpeaking;
478
498
  private onCancelSpeaking;
479
499
  private onStopSpeaking;
500
+ /**
501
+ * Forgets the speech heard so far.
502
+ *
503
+ * The VAD starts over rather than staying on it: an echo that carries on
504
+ * would keep it speaking past the moment it could be told from a person.
505
+ */
506
+ private dropSpeech;
480
507
  /**
481
508
  * Ends the turn, unless the detector hears an unfinished sentence.
482
509
  *
@@ -522,6 +549,9 @@ declare class Pcm16AudioStream extends EventEmitter<Pcm16AudioStreamEvents> {
522
549
  private endTimer?;
523
550
  private volumeTimers;
524
551
  private meter;
552
+ private holdUntil;
553
+ private holdTimer?;
554
+ private warmUpAudio?;
525
555
  constructor(sink: AudioSink);
526
556
  /**
527
557
  * Queues samples for playback
@@ -529,6 +559,18 @@ declare class Pcm16AudioStream extends EventEmitter<Pcm16AudioStreamEvents> {
529
559
  * @param sampleRate - Sample rate of `pcm`, in Hz
530
560
  */
531
561
  playAudio(pcm: Int16Array, sampleRate: number): void;
562
+ /**
563
+ * Plays silence for a while, and holds the next utterance until it is over.
564
+ *
565
+ * Firefox mangles what it plays while a Bluetooth headset switches to its
566
+ * call profile, which the headset does as soon as its microphone opens. The
567
+ * first answer then comes out all at once. Silence played through that switch
568
+ * goes unheard.
569
+ * @param duration - How long to play silence, in milliseconds
570
+ */
571
+ warmUp(duration: number): void;
572
+ /** Stops playing, for a stream no longer used */
573
+ destroy(): void;
532
574
  /** Drops everything queued and stops playing immediately */
533
575
  stopAudio(): void;
534
576
  private handleSamples;
@@ -781,6 +823,15 @@ declare function getClientErrorFromWSCloseEvent(event: WSCloseEvent): MicdropCli
781
823
  declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
782
824
  declare function isRecoverableError(error: MicdropClientError): boolean;
783
825
 
826
+ /**
827
+ * How far a call has gone to keep the assistant out of the microphone.
828
+ *
829
+ * - `normal`: the speaker plays as usual
830
+ * - `fallback`: it plays by the other route the platform offers
831
+ * - `disabled`: the user may no longer interrupt, until the call ends
832
+ */
833
+ type EchoStage = 'normal' | 'fallback' | 'disabled';
834
+
784
835
  interface MicdropEvents {
785
836
  EndCall: [];
786
837
  Error: [MicdropClientError];
@@ -812,7 +863,8 @@ interface MicdropOptions {
812
863
  turnMaxWait?: number;
813
864
  /**
814
865
  * Keeps the user from interrupting the assistant, whose voice they cannot
815
- * speak over. Forced on when the microphone has no echo cancellation.
866
+ * speak over. Forced on when the microphone has no echo cancellation, and
867
+ * by the call itself when the assistant keeps cutting itself off.
816
868
  */
817
869
  disableInterruption?: boolean;
818
870
  debugLog?: boolean;
@@ -862,6 +914,7 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
862
914
  private reconnectAttempt;
863
915
  private reconnectTimer?;
864
916
  private connectionTimer?;
917
+ private echoGuard;
865
918
  constructor(options?: MicdropOptions);
866
919
  get vad(): VAD | undefined;
867
920
  get isStarted(): boolean;
@@ -880,9 +933,12 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
880
933
  /**
881
934
  * Whether the user is kept from speaking over the assistant. Without echo
882
935
  * cancellation the microphone hears the assistant, which would interrupt
883
- * itself and end up in the transcript.
936
+ * itself and end up in the transcript. A call also gets here on its own when
937
+ * the assistant keeps cutting itself off, see {@link EchoGuard}.
884
938
  */
885
939
  get isInterruptionDisabled(): boolean;
940
+ /** How far this call has gone to keep the assistant out of the microphone */
941
+ get echoStage(): EchoStage;
886
942
  get micDeviceId(): string | undefined;
887
943
  get speakerDeviceId(): string | undefined;
888
944
  get state(): MicdropState;
@@ -923,6 +979,13 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
923
979
  private scheduleReconnect;
924
980
  private onSpeakerStartPlaying;
925
981
  private onSpeakerStopPlaying;
982
+ /**
983
+ * Treats speech heard just as the assistant starts as its echo, and steps up
984
+ * what keeps it out of the microphone. The interruption never happens, so
985
+ * neither the speaker nor the server hear of it.
986
+ */
987
+ private isEchoOfAssistant;
988
+ private onInterruptionsGivenUp;
926
989
  private addMessage;
927
990
  private setPartialAssistantMessage;
928
991
  private setError;
package/dist/index.d.ts CHANGED
@@ -88,6 +88,13 @@ declare abstract class SpeakerDriver extends EventEmitter<SpeakerDriverEvents> {
88
88
  abstract changeDevice(deviceId: string): Promise<void>;
89
89
  /** Lists the output devices the system offers */
90
90
  abstract getDevices(): Promise<MicdropDevice[]>;
91
+ /**
92
+ * Plays by another route, for when the microphone hears the current one
93
+ * @returns False if the platform has no other route to offer
94
+ */
95
+ useEchoFallback(): Promise<boolean>;
96
+ /** Goes back to the usual route */
97
+ resetEchoFallback(): void;
91
98
  }
92
99
  /** A buffer handed to an {@link AudioSink}, that can still be cancelled */
93
100
  interface ScheduledAudio {
@@ -208,6 +215,13 @@ declare class SpeakerController extends EventEmitter<SpeakerEvents> {
208
215
  changeDevice(deviceId: string): Promise<void>;
209
216
  /** Lists the output devices the system offers */
210
217
  getDevices(): Promise<MicdropDevice[]>;
218
+ /**
219
+ * Plays by another route, for when the microphone hears the current one
220
+ * @returns False if the platform has no other route to offer
221
+ */
222
+ useEchoFallback(): Promise<boolean>;
223
+ /** Goes back to the usual route */
224
+ resetEchoFallback(): void;
211
225
  private onStartPlaying;
212
226
  private onStopPlaying;
213
227
  private onVolume;
@@ -433,6 +447,11 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
433
447
  turnMaxWait: number;
434
448
  state: MicRecorderState;
435
449
  vad: VAD;
450
+ /**
451
+ * Asked as speech is confirmed, before anyone hears of it. Saying it is echo
452
+ * drops the speech, and listens again in case the echo goes on.
453
+ */
454
+ isEcho?: () => boolean;
436
455
  private mic;
437
456
  private isRecording;
438
457
  private speakingConfirmed;
@@ -447,6 +466,7 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
447
466
  private turnListening;
448
467
  private turnTimer;
449
468
  private turnRound;
469
+ private isDroppingSpeech;
450
470
  constructor(vadConfig?: VADConfig | undefined, turnDetector?: TurnDetector | undefined,
451
471
  /**
452
472
  * How long a turn stays open once the detector asked to wait.
@@ -477,6 +497,13 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
477
497
  private onConfirmSpeaking;
478
498
  private onCancelSpeaking;
479
499
  private onStopSpeaking;
500
+ /**
501
+ * Forgets the speech heard so far.
502
+ *
503
+ * The VAD starts over rather than staying on it: an echo that carries on
504
+ * would keep it speaking past the moment it could be told from a person.
505
+ */
506
+ private dropSpeech;
480
507
  /**
481
508
  * Ends the turn, unless the detector hears an unfinished sentence.
482
509
  *
@@ -522,6 +549,9 @@ declare class Pcm16AudioStream extends EventEmitter<Pcm16AudioStreamEvents> {
522
549
  private endTimer?;
523
550
  private volumeTimers;
524
551
  private meter;
552
+ private holdUntil;
553
+ private holdTimer?;
554
+ private warmUpAudio?;
525
555
  constructor(sink: AudioSink);
526
556
  /**
527
557
  * Queues samples for playback
@@ -529,6 +559,18 @@ declare class Pcm16AudioStream extends EventEmitter<Pcm16AudioStreamEvents> {
529
559
  * @param sampleRate - Sample rate of `pcm`, in Hz
530
560
  */
531
561
  playAudio(pcm: Int16Array, sampleRate: number): void;
562
+ /**
563
+ * Plays silence for a while, and holds the next utterance until it is over.
564
+ *
565
+ * Firefox mangles what it plays while a Bluetooth headset switches to its
566
+ * call profile, which the headset does as soon as its microphone opens. The
567
+ * first answer then comes out all at once. Silence played through that switch
568
+ * goes unheard.
569
+ * @param duration - How long to play silence, in milliseconds
570
+ */
571
+ warmUp(duration: number): void;
572
+ /** Stops playing, for a stream no longer used */
573
+ destroy(): void;
532
574
  /** Drops everything queued and stops playing immediately */
533
575
  stopAudio(): void;
534
576
  private handleSamples;
@@ -781,6 +823,15 @@ declare function getClientErrorFromWSCloseEvent(event: WSCloseEvent): MicdropCli
781
823
  declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
782
824
  declare function isRecoverableError(error: MicdropClientError): boolean;
783
825
 
826
+ /**
827
+ * How far a call has gone to keep the assistant out of the microphone.
828
+ *
829
+ * - `normal`: the speaker plays as usual
830
+ * - `fallback`: it plays by the other route the platform offers
831
+ * - `disabled`: the user may no longer interrupt, until the call ends
832
+ */
833
+ type EchoStage = 'normal' | 'fallback' | 'disabled';
834
+
784
835
  interface MicdropEvents {
785
836
  EndCall: [];
786
837
  Error: [MicdropClientError];
@@ -812,7 +863,8 @@ interface MicdropOptions {
812
863
  turnMaxWait?: number;
813
864
  /**
814
865
  * Keeps the user from interrupting the assistant, whose voice they cannot
815
- * speak over. Forced on when the microphone has no echo cancellation.
866
+ * speak over. Forced on when the microphone has no echo cancellation, and
867
+ * by the call itself when the assistant keeps cutting itself off.
816
868
  */
817
869
  disableInterruption?: boolean;
818
870
  debugLog?: boolean;
@@ -862,6 +914,7 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
862
914
  private reconnectAttempt;
863
915
  private reconnectTimer?;
864
916
  private connectionTimer?;
917
+ private echoGuard;
865
918
  constructor(options?: MicdropOptions);
866
919
  get vad(): VAD | undefined;
867
920
  get isStarted(): boolean;
@@ -880,9 +933,12 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
880
933
  /**
881
934
  * Whether the user is kept from speaking over the assistant. Without echo
882
935
  * cancellation the microphone hears the assistant, which would interrupt
883
- * itself and end up in the transcript.
936
+ * itself and end up in the transcript. A call also gets here on its own when
937
+ * the assistant keeps cutting itself off, see {@link EchoGuard}.
884
938
  */
885
939
  get isInterruptionDisabled(): boolean;
940
+ /** How far this call has gone to keep the assistant out of the microphone */
941
+ get echoStage(): EchoStage;
886
942
  get micDeviceId(): string | undefined;
887
943
  get speakerDeviceId(): string | undefined;
888
944
  get state(): MicdropState;
@@ -923,6 +979,13 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
923
979
  private scheduleReconnect;
924
980
  private onSpeakerStartPlaying;
925
981
  private onSpeakerStopPlaying;
982
+ /**
983
+ * Treats speech heard just as the assistant starts as its echo, and steps up
984
+ * what keeps it out of the microphone. The interruption never happens, so
985
+ * neither the speaker nor the server hear of it.
986
+ */
987
+ private isEchoOfAssistant;
988
+ private onInterruptionsGivenUp;
926
989
  private addMessage;
927
990
  private setPartialAssistantMessage;
928
991
  private setError;
package/dist/index.js CHANGED
@@ -497,6 +497,17 @@ var SpeakerController = class extends import_eventemitter32.EventEmitter {
497
497
  if (!this.driver) return [];
498
498
  return this.driver.getDevices();
499
499
  }
500
+ /**
501
+ * Plays by another route, for when the microphone hears the current one
502
+ * @returns False if the platform has no other route to offer
503
+ */
504
+ async useEchoFallback() {
505
+ return await this.driver?.useEchoFallback() ?? false;
506
+ }
507
+ /** Goes back to the usual route */
508
+ resetEchoFallback() {
509
+ this.driver?.resetEchoFallback();
510
+ }
500
511
  };
501
512
 
502
513
  // src/audio/MicRecorder.ts
@@ -999,6 +1010,8 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
999
1010
  this.turnListening = false;
1000
1011
  // Bumped whenever the turn moves on, so a late answer is dropped
1001
1012
  this.turnRound = 0;
1013
+ // The VAD is being restarted, what it reports on the way is not speech
1014
+ this.isDroppingSpeech = false;
1002
1015
  this.start = async (mic) => {
1003
1016
  if (this.state.isStarted) {
1004
1017
  throw new Error("MicRecorder is already started");
@@ -1122,6 +1135,7 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1122
1135
  );
1123
1136
  };
1124
1137
  this.onStartSpeaking = () => {
1138
+ if (this.isDroppingSpeech) return;
1125
1139
  this.speakingConfirmed = false;
1126
1140
  this.isRecording = true;
1127
1141
  this.buffer = [...this.reserve];
@@ -1135,6 +1149,11 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1135
1149
  }
1136
1150
  };
1137
1151
  this.onConfirmSpeaking = () => {
1152
+ if (this.isDroppingSpeech) return;
1153
+ if (!this.turnOpen && this.isEcho?.()) {
1154
+ this.dropSpeech();
1155
+ return;
1156
+ }
1138
1157
  this.speakingConfirmed = true;
1139
1158
  this.changeState({ isSpeaking: true });
1140
1159
  this.cancelPendingClose();
@@ -1145,12 +1164,14 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1145
1164
  this.flushChunks();
1146
1165
  };
1147
1166
  this.onCancelSpeaking = () => {
1167
+ if (this.isDroppingSpeech) return;
1148
1168
  this.stopRecording();
1149
1169
  if (!this.turnOpen) {
1150
1170
  this.turnListening = false;
1151
1171
  }
1152
1172
  };
1153
1173
  this.onStopSpeaking = () => {
1174
+ if (this.isDroppingSpeech) return;
1154
1175
  const wasConfirmed = this.speakingConfirmed;
1155
1176
  this.changeState({ isSpeaking: false });
1156
1177
  if (wasConfirmed) {
@@ -1162,6 +1183,26 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1162
1183
  this.state = defaultMicRecorderState;
1163
1184
  this.vad = getVAD(vadConfig);
1164
1185
  }
1186
+ /**
1187
+ * Forgets the speech heard so far.
1188
+ *
1189
+ * The VAD starts over rather than staying on it: an echo that carries on
1190
+ * would keep it speaking past the moment it could be told from a person.
1191
+ */
1192
+ async dropSpeech() {
1193
+ this.isDroppingSpeech = true;
1194
+ this.stopRecording();
1195
+ this.turnListening = false;
1196
+ const vad = this.vad;
1197
+ try {
1198
+ await vad.pause();
1199
+ if (!this.speakerPlaying) await vad.resume();
1200
+ } catch (error) {
1201
+ console.error("[Micdrop] Restarting the VAD failed", error);
1202
+ } finally {
1203
+ this.isDroppingSpeech = false;
1204
+ }
1205
+ }
1165
1206
  /**
1166
1207
  * Ends the turn, unless the detector hears an unfinished sentence.
1167
1208
  *
@@ -1289,6 +1330,8 @@ var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
1289
1330
  this.prebufferDuration = 0;
1290
1331
  this.volumeTimers = [];
1291
1332
  this.meter = new VolumeMeter();
1333
+ // Wall time before which no utterance starts, see warmUp()
1334
+ this.holdUntil = 0;
1292
1335
  }
1293
1336
  /**
1294
1337
  * Queues samples for playback
@@ -1304,6 +1347,34 @@ var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
1304
1347
  );
1305
1348
  this.handleSamples(samples);
1306
1349
  }
1350
+ /**
1351
+ * Plays silence for a while, and holds the next utterance until it is over.
1352
+ *
1353
+ * Firefox mangles what it plays while a Bluetooth headset switches to its
1354
+ * call profile, which the headset does as soon as its microphone opens. The
1355
+ * first answer then comes out all at once. Silence played through that switch
1356
+ * goes unheard.
1357
+ * @param duration - How long to play silence, in milliseconds
1358
+ */
1359
+ warmUp(duration) {
1360
+ this.holdUntil = Date.now() + duration;
1361
+ const samples = new Float32Array(
1362
+ Math.round(duration / 1e3 * this.sink.sampleRate)
1363
+ );
1364
+ this.warmUpAudio?.stop();
1365
+ this.warmUpAudio = this.sink.schedule(
1366
+ samples,
1367
+ this.sink.currentTime,
1368
+ () => this.warmUpAudio = void 0
1369
+ );
1370
+ }
1371
+ /** Stops playing, for a stream no longer used */
1372
+ destroy() {
1373
+ this.stopAudio();
1374
+ this.warmUpAudio?.stop();
1375
+ this.warmUpAudio = void 0;
1376
+ this.removeAllListeners();
1377
+ }
1307
1378
  /** Drops everything queued and stops playing immediately */
1308
1379
  stopAudio() {
1309
1380
  for (const source of this.sources) {
@@ -1343,6 +1414,14 @@ var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
1343
1414
  clearTimeout(this.quietTimer);
1344
1415
  this.quietTimer = void 0;
1345
1416
  }
1417
+ const hold = this.holdUntil - Date.now();
1418
+ if (hold > 0) {
1419
+ this.holdTimer ?? (this.holdTimer = setTimeout(() => {
1420
+ this.holdTimer = void 0;
1421
+ this.flushPrebuffer();
1422
+ }, hold));
1423
+ return;
1424
+ }
1346
1425
  this.prebuffering = false;
1347
1426
  const buffers = this.prebuffer;
1348
1427
  this.prebuffer = [];
@@ -1358,6 +1437,10 @@ var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
1358
1437
  clearTimeout(this.endTimer);
1359
1438
  this.endTimer = void 0;
1360
1439
  }
1440
+ if (this.holdTimer !== void 0) {
1441
+ clearTimeout(this.holdTimer);
1442
+ this.holdTimer = void 0;
1443
+ }
1361
1444
  this.prebuffering = true;
1362
1445
  this.prebuffer = [];
1363
1446
  this.prebufferDuration = 0;
@@ -1439,6 +1522,16 @@ var MicDriver = class extends import_eventemitter36.EventEmitter {
1439
1522
  }
1440
1523
  };
1441
1524
  var SpeakerDriver = class extends import_eventemitter36.EventEmitter {
1525
+ /**
1526
+ * Plays by another route, for when the microphone hears the current one
1527
+ * @returns False if the platform has no other route to offer
1528
+ */
1529
+ async useEchoFallback() {
1530
+ return false;
1531
+ }
1532
+ /** Goes back to the usual route */
1533
+ resetEchoFallback() {
1534
+ }
1442
1535
  };
1443
1536
 
1444
1537
  // src/audio/index.ts
@@ -1538,6 +1631,51 @@ function isRecoverableError(error) {
1538
1631
  return error.code === "Connection" /* Connection */ || error.code === "InternalServer" /* InternalServer */ || error.code === "Unknown" /* Unknown */;
1539
1632
  }
1540
1633
 
1634
+ // src/client/EchoGuard.ts
1635
+ var ECHO_WATCH = 2e3;
1636
+ var EchoGuard = class {
1637
+ constructor() {
1638
+ this.stage = "normal";
1639
+ // The next sound opens a watch, the first of a call or after a new route
1640
+ this.isDue = true;
1641
+ }
1642
+ /** @param now - The current time, in milliseconds */
1643
+ speakerStarted(now) {
1644
+ if (!this.isDue) return;
1645
+ this.isDue = false;
1646
+ this.watchEnd = now + ECHO_WATCH;
1647
+ }
1648
+ /** @param now - The current time, in milliseconds */
1649
+ speakerStopped(now) {
1650
+ if (this.watchEnd === void 0) return;
1651
+ if (this.stage === "fallback" && now < this.watchEnd) {
1652
+ this.isDue = true;
1653
+ }
1654
+ this.watchEnd = void 0;
1655
+ }
1656
+ /**
1657
+ * Moves to the next stage if the speech heard can only be echo
1658
+ * @param now - The current time, in milliseconds
1659
+ * @returns True if the speech is to be ignored
1660
+ */
1661
+ speechHeard(now) {
1662
+ if (this.watchEnd === void 0 || now >= this.watchEnd) return false;
1663
+ if (this.stage === "normal") {
1664
+ this.stage = "fallback";
1665
+ this.watchEnd = now + ECHO_WATCH;
1666
+ } else {
1667
+ this.stage = "disabled";
1668
+ this.watchEnd = void 0;
1669
+ }
1670
+ return true;
1671
+ }
1672
+ reset() {
1673
+ this.stage = "normal";
1674
+ this.isDue = true;
1675
+ this.watchEnd = void 0;
1676
+ }
1677
+ };
1678
+
1541
1679
  // src/client/MicdropClient.ts
1542
1680
  var DEFAULT_RECONNECT_OPTIONS = {
1543
1681
  maxAttempts: Infinity,
@@ -1562,6 +1700,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1562
1700
  // it can take a while, for the permission prompt or the browser itself
1563
1701
  this._isStartingCall = false;
1564
1702
  this.reconnectAttempt = 0;
1703
+ this.echoGuard = new EchoGuard();
1565
1704
  this.start = async (options) => {
1566
1705
  this.error = void 0;
1567
1706
  this.options = { ...this.options, ...options };
@@ -1605,6 +1744,8 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1605
1744
  console.error("[MicdropClient] Error stopping WebSocket", error);
1606
1745
  }
1607
1746
  Speaker.stopAudio();
1747
+ this.echoGuard.reset();
1748
+ Speaker.resetEchoFallback();
1608
1749
  try {
1609
1750
  await this.stopMic();
1610
1751
  } catch (error) {
@@ -1661,6 +1802,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1661
1802
  this.micRecorder.setSpeakerPlaying(
1662
1803
  this.isInterruptionDisabled && Speaker.isPlaying
1663
1804
  );
1805
+ this.micRecorder.isEcho = this.isEchoOfAssistant;
1664
1806
  this.micRecorder.on("StateChange", () => {
1665
1807
  this.notifyStateChange();
1666
1808
  });
@@ -1852,6 +1994,9 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1852
1994
  };
1853
1995
  this.onSpeakerStartPlaying = () => {
1854
1996
  this.log("Speaker started");
1997
+ if (this.isWSStarted) {
1998
+ this.echoGuard.speakerStarted(Date.now());
1999
+ }
1855
2000
  if (this.isInterruptionDisabled) {
1856
2001
  this.micRecorder?.setSpeakerPlaying(true);
1857
2002
  this.vad?.pause();
@@ -1860,6 +2005,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1860
2005
  };
1861
2006
  this.onSpeakerStopPlaying = () => {
1862
2007
  this.log("Speaker stopped");
2008
+ this.echoGuard.speakerStopped(Date.now());
1863
2009
  this.micRecorder?.setSpeakerPlaying(false);
1864
2010
  if (this.isInterruptionDisabled) {
1865
2011
  setTimeout(() => {
@@ -1872,6 +2018,28 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1872
2018
  this.notifyStateChange();
1873
2019
  }
1874
2020
  };
2021
+ /**
2022
+ * Treats speech heard just as the assistant starts as its echo, and steps up
2023
+ * what keeps it out of the microphone. The interruption never happens, so
2024
+ * neither the speaker nor the server hear of it.
2025
+ */
2026
+ this.isEchoOfAssistant = () => {
2027
+ if (this.isInterruptionDisabled || !Speaker.isPlaying) return false;
2028
+ if (!this.echoGuard.speechHeard(Date.now())) return false;
2029
+ this.log("Echo of the assistant, moving to", this.echoGuard.stage);
2030
+ if (this.echoGuard.stage === "fallback") {
2031
+ Speaker.useEchoFallback().then((moved) => {
2032
+ if (!moved && this.echoGuard.stage === "fallback") {
2033
+ this.echoGuard.stage = "disabled";
2034
+ this.onInterruptionsGivenUp();
2035
+ }
2036
+ });
2037
+ } else {
2038
+ this.onInterruptionsGivenUp();
2039
+ }
2040
+ this.notifyStateChange();
2041
+ return true;
2042
+ };
1875
2043
  Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
1876
2044
  Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
1877
2045
  Mic.on("DeviceChange", this.updateDevices);
@@ -1921,10 +2089,15 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1921
2089
  /**
1922
2090
  * Whether the user is kept from speaking over the assistant. Without echo
1923
2091
  * cancellation the microphone hears the assistant, which would interrupt
1924
- * itself and end up in the transcript.
2092
+ * itself and end up in the transcript. A call also gets here on its own when
2093
+ * the assistant keeps cutting itself off, see {@link EchoGuard}.
1925
2094
  */
1926
2095
  get isInterruptionDisabled() {
1927
- return !!this.options.disableInterruption || Mic.echoCancellation === false;
2096
+ return !!this.options.disableInterruption || Mic.echoCancellation === false || this.echoGuard.stage === "disabled";
2097
+ }
2098
+ /** How far this call has gone to keep the assistant out of the microphone */
2099
+ get echoStage() {
2100
+ return this.echoGuard.stage;
1928
2101
  }
1929
2102
  get micDeviceId() {
1930
2103
  return Mic.deviceId;
@@ -2033,6 +2206,15 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
2033
2206
  }
2034
2207
  }, config.delayMs);
2035
2208
  }
2209
+ // The same as when the speaker starts on a call without interruptions
2210
+ onInterruptionsGivenUp() {
2211
+ this.log("Interruptions disabled until the end of the call");
2212
+ if (Speaker.isPlaying) {
2213
+ this.micRecorder?.setSpeakerPlaying(true);
2214
+ this.vad?.pause();
2215
+ }
2216
+ this.notifyStateChange();
2217
+ }
2036
2218
  addMessage(message) {
2037
2219
  if (message.role === "assistant") this.partialAssistantMessage = "";
2038
2220
  this.conversation = [...this.conversation, message];