@micdrop/client 3.0.0 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -563,8 +563,8 @@ var MS_PER_FRAME = SILERO_FRAME_SAMPLES / SILERO_SAMPLE_RATE * 1e3;
563
563
  var defaultOptions = {
564
564
  positiveSpeechThreshold: 0.18,
565
565
  negativeSpeechThreshold: 0.11,
566
- minSpeechFrames: 8,
567
- redemptionFrames: 20
566
+ minSpeechFrames: 4,
567
+ redemptionFrames: 8
568
568
  };
569
569
  var loadModel;
570
570
  function setSileroModelLoader(loader) {
@@ -884,15 +884,18 @@ function equalVADConfig(a, b) {
884
884
  // src/audio/MicRecorder.ts
885
885
  var SAMPLE_RATE = 16e3;
886
886
  var CHUNK_INTERVAL = 100;
887
+ var DEFAULT_TURN_MAX_WAIT = 4e3;
887
888
  var defaultMicRecorderState = {
888
889
  isStarting: false,
889
890
  isStarted: false,
890
891
  isSpeaking: false
891
892
  };
892
893
  var MicRecorder = class extends EventEmitter4 {
893
- constructor(vadConfig) {
894
+ constructor(vadConfig, turnDetector, turnMaxWait = DEFAULT_TURN_MAX_WAIT) {
894
895
  super();
895
896
  this.vadConfig = vadConfig;
897
+ this.turnDetector = turnDetector;
898
+ this.turnMaxWait = turnMaxWait;
896
899
  this.isRecording = false;
897
900
  this.speakingConfirmed = false;
898
901
  this.sourceSampleRate = SAMPLE_RATE;
@@ -901,6 +904,12 @@ var MicRecorder = class extends EventEmitter4 {
901
904
  this.buffer = [];
902
905
  this.bufferLength = 0;
903
906
  this.queuedChunks = [];
907
+ // The turn is open between the first confirmed word and the moment the
908
+ // server is told to answer, which can span several pauses
909
+ this.turnOpen = false;
910
+ this.turnListening = false;
911
+ // Bumped whenever the turn moves on, so a late answer is dropped
912
+ this.turnRound = 0;
904
913
  this.start = async (mic) => {
905
914
  if (this.state.isStarted) {
906
915
  throw new Error("MicRecorder is already started");
@@ -933,6 +942,7 @@ var MicRecorder = class extends EventEmitter4 {
933
942
  isStarted: false,
934
943
  isSpeaking: false
935
944
  });
945
+ this.cancelPendingClose();
936
946
  try {
937
947
  this.mic?.off("Frames", this.onFrames);
938
948
  this.mic = void 0;
@@ -946,6 +956,17 @@ var MicRecorder = class extends EventEmitter4 {
946
956
  }
947
957
  this.resetBuffers();
948
958
  };
959
+ /**
960
+ * Changes what decides when a turn is over
961
+ * @param turnDetector - The new detector, or nothing to leave it to the VAD
962
+ */
963
+ this.changeTurnDetector = (turnDetector) => {
964
+ if (turnDetector === this.turnDetector) return;
965
+ if (this.turnOpen) {
966
+ this.endTurn();
967
+ }
968
+ this.turnDetector = turnDetector;
969
+ };
949
970
  this.changeVad = (vadConfig) => {
950
971
  if (equalVADConfig(vadConfig, this.vadConfig)) return;
951
972
  const mic = this.mic;
@@ -959,6 +980,9 @@ var MicRecorder = class extends EventEmitter4 {
959
980
  }
960
981
  };
961
982
  this.onFrames = (frames, sampleRate) => {
983
+ if (this.turnListening) {
984
+ this.turnDetector?.push(frames, sampleRate);
985
+ }
962
986
  if (sampleRate !== this.sourceSampleRate) {
963
987
  this.flushChunks();
964
988
  this.sourceSampleRate = sampleRate;
@@ -987,12 +1011,23 @@ var MicRecorder = class extends EventEmitter4 {
987
1011
  this.bufferLength = this.reserveLength;
988
1012
  this.reserve = [];
989
1013
  this.reserveLength = 0;
1014
+ if (this.turnDetector && !this.turnListening) {
1015
+ this.turnDetector.reset();
1016
+ this.turnListening = true;
1017
+ for (const frames of this.buffer) {
1018
+ this.turnDetector.push(frames, this.sourceSampleRate);
1019
+ }
1020
+ }
990
1021
  this.flushChunks();
991
1022
  };
992
1023
  this.onConfirmSpeaking = () => {
993
1024
  this.speakingConfirmed = true;
994
1025
  this.changeState({ isSpeaking: true });
995
- this.emit("StartSpeaking");
1026
+ this.cancelPendingClose();
1027
+ if (!this.turnOpen) {
1028
+ this.turnOpen = true;
1029
+ this.emit("StartSpeaking");
1030
+ }
996
1031
  for (const chunk of this.queuedChunks) {
997
1032
  this.emit("Chunk", chunk);
998
1033
  }
@@ -1000,6 +1035,9 @@ var MicRecorder = class extends EventEmitter4 {
1000
1035
  };
1001
1036
  this.onCancelSpeaking = () => {
1002
1037
  this.stopRecording();
1038
+ if (!this.turnOpen) {
1039
+ this.turnListening = false;
1040
+ }
1003
1041
  };
1004
1042
  this.onStopSpeaking = () => {
1005
1043
  const wasConfirmed = this.speakingConfirmed;
@@ -1008,11 +1046,55 @@ var MicRecorder = class extends EventEmitter4 {
1008
1046
  this.flushChunks(true);
1009
1047
  }
1010
1048
  this.stopRecording();
1011
- this.emit("StopSpeaking");
1049
+ this.closeTurn();
1012
1050
  };
1013
1051
  this.state = defaultMicRecorderState;
1014
1052
  this.vad = getVAD(vadConfig);
1015
1053
  }
1054
+ /**
1055
+ * Ends the turn, unless the detector hears an unfinished sentence.
1056
+ *
1057
+ * A microphone that is no longer listening ends it straight away: a muted or
1058
+ * paused call has nothing more to say, whatever the sentence sounded like.
1059
+ */
1060
+ async closeTurn() {
1061
+ if (!this.turnDetector || !this.turnOpen || !this.vad.isStarted || this.vad.isPaused) {
1062
+ this.endTurn();
1063
+ return;
1064
+ }
1065
+ const round = this.turnRound;
1066
+ try {
1067
+ const { complete } = await this.turnDetector.predict();
1068
+ if (round !== this.turnRound) return;
1069
+ if (complete) {
1070
+ this.endTurn();
1071
+ return;
1072
+ }
1073
+ } catch (error) {
1074
+ console.error("[Micdrop] Turn detection failed", error);
1075
+ this.endTurn();
1076
+ return;
1077
+ }
1078
+ this.cancelPendingClose();
1079
+ this.turnTimer = setTimeout(() => {
1080
+ this.turnTimer = void 0;
1081
+ this.endTurn();
1082
+ }, this.turnMaxWait);
1083
+ }
1084
+ endTurn() {
1085
+ this.cancelPendingClose();
1086
+ this.turnOpen = false;
1087
+ this.turnListening = false;
1088
+ this.emit("StopSpeaking");
1089
+ }
1090
+ /** Drops the pending close, both the timer and the answer still owed */
1091
+ cancelPendingClose() {
1092
+ this.turnRound++;
1093
+ if (this.turnTimer) {
1094
+ clearTimeout(this.turnTimer);
1095
+ this.turnTimer = void 0;
1096
+ }
1097
+ }
1016
1098
  stopRecording() {
1017
1099
  this.isRecording = false;
1018
1100
  this.speakingConfirmed = false;
@@ -1059,6 +1141,8 @@ var MicRecorder = class extends EventEmitter4 {
1059
1141
  resetBuffers() {
1060
1142
  this.isRecording = false;
1061
1143
  this.speakingConfirmed = false;
1144
+ this.turnOpen = false;
1145
+ this.turnListening = false;
1062
1146
  this.reserve = [];
1063
1147
  this.reserveLength = 0;
1064
1148
  this.buffer = [];
@@ -1261,6 +1345,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
1261
1345
  })(MicdropClientCommands || {});
1262
1346
  var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
1263
1347
  MicdropServerCommands2["Message"] = "Message";
1348
+ MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
1264
1349
  MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
1265
1350
  MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
1266
1351
  MicdropServerCommands2["EndCall"] = "EndCall";
@@ -1323,6 +1408,7 @@ var MicdropClient = class extends EventEmitter7 {
1323
1408
  super();
1324
1409
  this.options = options;
1325
1410
  this.conversation = [];
1411
+ this.partialAssistantMessage = "";
1326
1412
  this.speakerDevices = [];
1327
1413
  this.micDevices = [];
1328
1414
  this.startTime = 0;
@@ -1337,6 +1423,7 @@ var MicdropClient = class extends EventEmitter7 {
1337
1423
  this.options = { ...this.options, ...options };
1338
1424
  this.startTime = Date.now();
1339
1425
  this.conversation = [];
1426
+ this.partialAssistantMessage = "";
1340
1427
  this._isProcessing = true;
1341
1428
  this._isMuted = false;
1342
1429
  this._isPaused = false;
@@ -1352,6 +1439,7 @@ var MicdropClient = class extends EventEmitter7 {
1352
1439
  await this.startWS();
1353
1440
  };
1354
1441
  this.stop = async () => {
1442
+ this.partialAssistantMessage = "";
1355
1443
  this._isProcessing = false;
1356
1444
  this._isMuted = false;
1357
1445
  this._isPaused = false;
@@ -1414,7 +1502,11 @@ var MicdropClient = class extends EventEmitter7 {
1414
1502
  this.micRecorder.changeVad(options.vad);
1415
1503
  }
1416
1504
  } else {
1417
- this.micRecorder = new MicRecorder(this.options.vad);
1505
+ this.micRecorder = new MicRecorder(
1506
+ this.options.vad,
1507
+ this.options.turnDetector,
1508
+ this.options.turnMaxWait
1509
+ );
1418
1510
  this.micRecorder.on("StateChange", () => {
1419
1511
  this.notifyStateChange();
1420
1512
  });
@@ -1425,6 +1517,7 @@ var MicdropClient = class extends EventEmitter7 {
1425
1517
  this.micRecorder.on("StartSpeaking", () => {
1426
1518
  this.log("User start speaking");
1427
1519
  this.ws?.send("StartSpeaking" /* StartSpeaking */);
1520
+ this.partialAssistantMessage = "";
1428
1521
  this._isProcessing = false;
1429
1522
  this.notifyStateChange();
1430
1523
  Speaker.stopAudio();
@@ -1460,6 +1553,28 @@ var MicdropClient = class extends EventEmitter7 {
1460
1553
  throw error;
1461
1554
  }
1462
1555
  };
1556
+ /**
1557
+ * Changes what decides when a turn is over, during a call or before one
1558
+ * @param turnDetector - The new detector, or nothing to leave it to the VAD
1559
+ */
1560
+ this.setTurnDetector = (turnDetector) => {
1561
+ this.options.turnDetector = turnDetector;
1562
+ this.micRecorder?.changeTurnDetector(turnDetector);
1563
+ };
1564
+ /**
1565
+ * Says how long a held turn waits for the rest of the sentence.
1566
+ *
1567
+ * The way out of a wrong verdict: a detector can hear an unfinished sentence
1568
+ * where there is none, and the speaker who never comes back still gets an
1569
+ * answer once this runs out.
1570
+ * @param milliseconds - How long to wait past the detector asking to
1571
+ */
1572
+ this.setTurnMaxWait = (milliseconds) => {
1573
+ this.options.turnMaxWait = milliseconds;
1574
+ if (this.micRecorder) {
1575
+ this.micRecorder.turnMaxWait = milliseconds;
1576
+ }
1577
+ };
1463
1578
  this.changeMicDevice = async (deviceId) => {
1464
1579
  await this.startMic({ deviceId });
1465
1580
  };
@@ -1517,9 +1632,12 @@ var MicdropClient = class extends EventEmitter7 {
1517
1632
  } catch (error) {
1518
1633
  console.error("[MicdropClient] Error parsing message:", data, error);
1519
1634
  }
1635
+ } else if (data.startsWith("PartialAssistantMessage" /* PartialAssistantMessage */)) {
1636
+ this.setPartialAssistantMessage(data);
1520
1637
  } else if (data === "EndCall" /* EndCall */) {
1521
1638
  this.emit("EndCall");
1522
1639
  } else if (data === "SkipAnswer" /* SkipAnswer */) {
1640
+ this.partialAssistantMessage = "";
1523
1641
  this._isProcessing = false;
1524
1642
  this.notifyStateChange();
1525
1643
  } else if (data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
@@ -1647,6 +1765,7 @@ var MicdropClient = class extends EventEmitter7 {
1647
1765
  isMicStarted: this.isMicStarted,
1648
1766
  isMicMuted: this.isMicMuted,
1649
1767
  conversation: this.conversation,
1768
+ partialAssistantMessage: this.partialAssistantMessage,
1650
1769
  error: this.error,
1651
1770
  micDeviceId: this.micDeviceId,
1652
1771
  speakerDeviceId: this.speakerDeviceId,
@@ -1734,8 +1853,23 @@ var MicdropClient = class extends EventEmitter7 {
1734
1853
  }, config.delayMs);
1735
1854
  }
1736
1855
  addMessage(message) {
1856
+ if (message.role === "assistant") this.partialAssistantMessage = "";
1737
1857
  this.conversation = [...this.conversation, message];
1738
1858
  this.notifyStateChange();
1859
+ this.emit("Message", message);
1860
+ }
1861
+ setPartialAssistantMessage(data) {
1862
+ const command = "PartialAssistantMessage" /* PartialAssistantMessage */;
1863
+ let content;
1864
+ try {
1865
+ content = JSON.parse(data.substring(command.length + 1));
1866
+ } catch (error) {
1867
+ console.error("[MicdropClient] Error parsing partial answer:", data);
1868
+ return;
1869
+ }
1870
+ this.partialAssistantMessage = content;
1871
+ this.emit("PartialAssistantMessage", content);
1872
+ this.notifyStateChange();
1739
1873
  }
1740
1874
  setError(error) {
1741
1875
  console.error("[MicdropClient] Error:", error);
@@ -1761,6 +1895,7 @@ if (!globalScope2.micdropClient) {
1761
1895
  }
1762
1896
  var Micdrop = globalScope2.micdropClient;
1763
1897
  export {
1898
+ DEFAULT_TURN_MAX_WAIT,
1764
1899
  Mic,
1765
1900
  MicController,
1766
1901
  MicDriver,