@micdrop/client 3.4.0 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -429,6 +429,17 @@ var SpeakerController = class extends EventEmitter2 {
429
429
  if (!this.driver) return [];
430
430
  return this.driver.getDevices();
431
431
  }
432
+ /**
433
+ * Plays by another route, for when the microphone hears the current one
434
+ * @returns False if the platform has no other route to offer
435
+ */
436
+ async useEchoFallback() {
437
+ return await this.driver?.useEchoFallback() ?? false;
438
+ }
439
+ /** Goes back to the usual route */
440
+ resetEchoFallback() {
441
+ this.driver?.resetEchoFallback();
442
+ }
432
443
  };
433
444
 
434
445
  // src/audio/MicRecorder.ts
@@ -931,6 +942,8 @@ var MicRecorder = class extends EventEmitter4 {
931
942
  this.turnListening = false;
932
943
  // Bumped whenever the turn moves on, so a late answer is dropped
933
944
  this.turnRound = 0;
945
+ // The VAD is being restarted, what it reports on the way is not speech
946
+ this.isDroppingSpeech = false;
934
947
  this.start = async (mic) => {
935
948
  if (this.state.isStarted) {
936
949
  throw new Error("MicRecorder is already started");
@@ -1054,6 +1067,7 @@ var MicRecorder = class extends EventEmitter4 {
1054
1067
  );
1055
1068
  };
1056
1069
  this.onStartSpeaking = () => {
1070
+ if (this.isDroppingSpeech) return;
1057
1071
  this.speakingConfirmed = false;
1058
1072
  this.isRecording = true;
1059
1073
  this.buffer = [...this.reserve];
@@ -1067,6 +1081,11 @@ var MicRecorder = class extends EventEmitter4 {
1067
1081
  }
1068
1082
  };
1069
1083
  this.onConfirmSpeaking = () => {
1084
+ if (this.isDroppingSpeech) return;
1085
+ if (!this.turnOpen && this.isEcho?.()) {
1086
+ this.dropSpeech();
1087
+ return;
1088
+ }
1070
1089
  this.speakingConfirmed = true;
1071
1090
  this.changeState({ isSpeaking: true });
1072
1091
  this.cancelPendingClose();
@@ -1077,12 +1096,14 @@ var MicRecorder = class extends EventEmitter4 {
1077
1096
  this.flushChunks();
1078
1097
  };
1079
1098
  this.onCancelSpeaking = () => {
1099
+ if (this.isDroppingSpeech) return;
1080
1100
  this.stopRecording();
1081
1101
  if (!this.turnOpen) {
1082
1102
  this.turnListening = false;
1083
1103
  }
1084
1104
  };
1085
1105
  this.onStopSpeaking = () => {
1106
+ if (this.isDroppingSpeech) return;
1086
1107
  const wasConfirmed = this.speakingConfirmed;
1087
1108
  this.changeState({ isSpeaking: false });
1088
1109
  if (wasConfirmed) {
@@ -1094,6 +1115,26 @@ var MicRecorder = class extends EventEmitter4 {
1094
1115
  this.state = defaultMicRecorderState;
1095
1116
  this.vad = getVAD(vadConfig);
1096
1117
  }
1118
+ /**
1119
+ * Forgets the speech heard so far.
1120
+ *
1121
+ * The VAD starts over rather than staying on it: an echo that carries on
1122
+ * would keep it speaking past the moment it could be told from a person.
1123
+ */
1124
+ async dropSpeech() {
1125
+ this.isDroppingSpeech = true;
1126
+ this.stopRecording();
1127
+ this.turnListening = false;
1128
+ const vad = this.vad;
1129
+ try {
1130
+ await vad.pause();
1131
+ if (!this.speakerPlaying) await vad.resume();
1132
+ } catch (error) {
1133
+ console.error("[Micdrop] Restarting the VAD failed", error);
1134
+ } finally {
1135
+ this.isDroppingSpeech = false;
1136
+ }
1137
+ }
1097
1138
  /**
1098
1139
  * Ends the turn, unless the detector hears an unfinished sentence.
1099
1140
  *
@@ -1413,6 +1454,16 @@ var MicDriver = class extends EventEmitter6 {
1413
1454
  }
1414
1455
  };
1415
1456
  var SpeakerDriver = class extends EventEmitter6 {
1457
+ /**
1458
+ * Plays by another route, for when the microphone hears the current one
1459
+ * @returns False if the platform has no other route to offer
1460
+ */
1461
+ async useEchoFallback() {
1462
+ return false;
1463
+ }
1464
+ /** Goes back to the usual route */
1465
+ resetEchoFallback() {
1466
+ }
1416
1467
  };
1417
1468
 
1418
1469
  // src/audio/index.ts
@@ -1512,6 +1563,51 @@ function isRecoverableError(error) {
1512
1563
  return error.code === "Connection" /* Connection */ || error.code === "InternalServer" /* InternalServer */ || error.code === "Unknown" /* Unknown */;
1513
1564
  }
1514
1565
 
1566
+ // src/client/EchoGuard.ts
1567
+ var ECHO_WATCH = 2e3;
1568
+ var EchoGuard = class {
1569
+ constructor() {
1570
+ this.stage = "normal";
1571
+ // The next sound opens a watch, the first of a call or after a new route
1572
+ this.isDue = true;
1573
+ }
1574
+ /** @param now - The current time, in milliseconds */
1575
+ speakerStarted(now) {
1576
+ if (!this.isDue) return;
1577
+ this.isDue = false;
1578
+ this.watchEnd = now + ECHO_WATCH;
1579
+ }
1580
+ /** @param now - The current time, in milliseconds */
1581
+ speakerStopped(now) {
1582
+ if (this.watchEnd === void 0) return;
1583
+ if (this.stage === "fallback" && now < this.watchEnd) {
1584
+ this.isDue = true;
1585
+ }
1586
+ this.watchEnd = void 0;
1587
+ }
1588
+ /**
1589
+ * Moves to the next stage if the speech heard can only be echo
1590
+ * @param now - The current time, in milliseconds
1591
+ * @returns True if the speech is to be ignored
1592
+ */
1593
+ speechHeard(now) {
1594
+ if (this.watchEnd === void 0 || now >= this.watchEnd) return false;
1595
+ if (this.stage === "normal") {
1596
+ this.stage = "fallback";
1597
+ this.watchEnd = now + ECHO_WATCH;
1598
+ } else {
1599
+ this.stage = "disabled";
1600
+ this.watchEnd = void 0;
1601
+ }
1602
+ return true;
1603
+ }
1604
+ reset() {
1605
+ this.stage = "normal";
1606
+ this.isDue = true;
1607
+ this.watchEnd = void 0;
1608
+ }
1609
+ };
1610
+
1515
1611
  // src/client/MicdropClient.ts
1516
1612
  var DEFAULT_RECONNECT_OPTIONS = {
1517
1613
  maxAttempts: Infinity,
@@ -1536,6 +1632,7 @@ var MicdropClient = class extends EventEmitter7 {
1536
1632
  // it can take a while, for the permission prompt or the browser itself
1537
1633
  this._isStartingCall = false;
1538
1634
  this.reconnectAttempt = 0;
1635
+ this.echoGuard = new EchoGuard();
1539
1636
  this.start = async (options) => {
1540
1637
  this.error = void 0;
1541
1638
  this.options = { ...this.options, ...options };
@@ -1579,6 +1676,8 @@ var MicdropClient = class extends EventEmitter7 {
1579
1676
  console.error("[MicdropClient] Error stopping WebSocket", error);
1580
1677
  }
1581
1678
  Speaker.stopAudio();
1679
+ this.echoGuard.reset();
1680
+ Speaker.resetEchoFallback();
1582
1681
  try {
1583
1682
  await this.stopMic();
1584
1683
  } catch (error) {
@@ -1635,6 +1734,7 @@ var MicdropClient = class extends EventEmitter7 {
1635
1734
  this.micRecorder.setSpeakerPlaying(
1636
1735
  this.isInterruptionDisabled && Speaker.isPlaying
1637
1736
  );
1737
+ this.micRecorder.isEcho = this.isEchoOfAssistant;
1638
1738
  this.micRecorder.on("StateChange", () => {
1639
1739
  this.notifyStateChange();
1640
1740
  });
@@ -1826,6 +1926,9 @@ var MicdropClient = class extends EventEmitter7 {
1826
1926
  };
1827
1927
  this.onSpeakerStartPlaying = () => {
1828
1928
  this.log("Speaker started");
1929
+ if (this.isWSStarted) {
1930
+ this.echoGuard.speakerStarted(Date.now());
1931
+ }
1829
1932
  if (this.isInterruptionDisabled) {
1830
1933
  this.micRecorder?.setSpeakerPlaying(true);
1831
1934
  this.vad?.pause();
@@ -1834,6 +1937,7 @@ var MicdropClient = class extends EventEmitter7 {
1834
1937
  };
1835
1938
  this.onSpeakerStopPlaying = () => {
1836
1939
  this.log("Speaker stopped");
1940
+ this.echoGuard.speakerStopped(Date.now());
1837
1941
  this.micRecorder?.setSpeakerPlaying(false);
1838
1942
  if (this.isInterruptionDisabled) {
1839
1943
  setTimeout(() => {
@@ -1846,6 +1950,28 @@ var MicdropClient = class extends EventEmitter7 {
1846
1950
  this.notifyStateChange();
1847
1951
  }
1848
1952
  };
1953
+ /**
1954
+ * Treats speech heard just as the assistant starts as its echo, and steps up
1955
+ * what keeps it out of the microphone. The interruption never happens, so
1956
+ * neither the speaker nor the server hear of it.
1957
+ */
1958
+ this.isEchoOfAssistant = () => {
1959
+ if (this.isInterruptionDisabled || !Speaker.isPlaying) return false;
1960
+ if (!this.echoGuard.speechHeard(Date.now())) return false;
1961
+ this.log("Echo of the assistant, moving to", this.echoGuard.stage);
1962
+ if (this.echoGuard.stage === "fallback") {
1963
+ Speaker.useEchoFallback().then((moved) => {
1964
+ if (!moved && this.echoGuard.stage === "fallback") {
1965
+ this.echoGuard.stage = "disabled";
1966
+ this.onInterruptionsGivenUp();
1967
+ }
1968
+ });
1969
+ } else {
1970
+ this.onInterruptionsGivenUp();
1971
+ }
1972
+ this.notifyStateChange();
1973
+ return true;
1974
+ };
1849
1975
  Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
1850
1976
  Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
1851
1977
  Mic.on("DeviceChange", this.updateDevices);
@@ -1895,10 +2021,15 @@ var MicdropClient = class extends EventEmitter7 {
1895
2021
  /**
1896
2022
  * Whether the user is kept from speaking over the assistant. Without echo
1897
2023
  * cancellation the microphone hears the assistant, which would interrupt
1898
- * itself and end up in the transcript.
2024
+ * itself and end up in the transcript. A call also gets here on its own when
2025
+ * the assistant keeps cutting itself off, see {@link EchoGuard}.
1899
2026
  */
1900
2027
  get isInterruptionDisabled() {
1901
- return !!this.options.disableInterruption || Mic.echoCancellation === false;
2028
+ return !!this.options.disableInterruption || Mic.echoCancellation === false || this.echoGuard.stage === "disabled";
2029
+ }
2030
+ /** How far this call has gone to keep the assistant out of the microphone */
2031
+ get echoStage() {
2032
+ return this.echoGuard.stage;
1902
2033
  }
1903
2034
  get micDeviceId() {
1904
2035
  return Mic.deviceId;
@@ -2007,6 +2138,15 @@ var MicdropClient = class extends EventEmitter7 {
2007
2138
  }
2008
2139
  }, config.delayMs);
2009
2140
  }
2141
+ // The same as when the speaker starts on a call without interruptions
2142
+ onInterruptionsGivenUp() {
2143
+ this.log("Interruptions disabled until the end of the call");
2144
+ if (Speaker.isPlaying) {
2145
+ this.micRecorder?.setSpeakerPlaying(true);
2146
+ this.vad?.pause();
2147
+ }
2148
+ this.notifyStateChange();
2149
+ }
2010
2150
  addMessage(message) {
2011
2151
  if (message.role === "assistant") this.partialAssistantMessage = "";
2012
2152
  this.conversation = [...this.conversation, message];