@micdrop/client 3.0.0 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -20,6 +20,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
20
20
  // src/index.ts
21
21
  var index_exports = {};
22
22
  __export(index_exports, {
23
+ DEFAULT_TURN_MAX_WAIT: () => DEFAULT_TURN_MAX_WAIT,
23
24
  Mic: () => Mic,
24
25
  MicController: () => MicController,
25
26
  MicDriver: () => MicDriver,
@@ -626,8 +627,8 @@ var MS_PER_FRAME = SILERO_FRAME_SAMPLES / SILERO_SAMPLE_RATE * 1e3;
626
627
  var defaultOptions = {
627
628
  positiveSpeechThreshold: 0.18,
628
629
  negativeSpeechThreshold: 0.11,
629
- minSpeechFrames: 8,
630
- redemptionFrames: 20
630
+ minSpeechFrames: 4,
631
+ redemptionFrames: 8
631
632
  };
632
633
  var loadModel;
633
634
  function setSileroModelLoader(loader) {
@@ -947,15 +948,18 @@ function equalVADConfig(a, b) {
947
948
  // src/audio/MicRecorder.ts
948
949
  var SAMPLE_RATE = 16e3;
949
950
  var CHUNK_INTERVAL = 100;
951
+ var DEFAULT_TURN_MAX_WAIT = 4e3;
950
952
  var defaultMicRecorderState = {
951
953
  isStarting: false,
952
954
  isStarted: false,
953
955
  isSpeaking: false
954
956
  };
955
957
  var MicRecorder = class extends import_eventemitter34.EventEmitter {
956
- constructor(vadConfig) {
958
+ constructor(vadConfig, turnDetector, turnMaxWait = DEFAULT_TURN_MAX_WAIT) {
957
959
  super();
958
960
  this.vadConfig = vadConfig;
961
+ this.turnDetector = turnDetector;
962
+ this.turnMaxWait = turnMaxWait;
959
963
  this.isRecording = false;
960
964
  this.speakingConfirmed = false;
961
965
  this.sourceSampleRate = SAMPLE_RATE;
@@ -964,6 +968,12 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
964
968
  this.buffer = [];
965
969
  this.bufferLength = 0;
966
970
  this.queuedChunks = [];
971
+ // The turn is open between the first confirmed word and the moment the
972
+ // server is told to answer, which can span several pauses
973
+ this.turnOpen = false;
974
+ this.turnListening = false;
975
+ // Bumped whenever the turn moves on, so a late answer is dropped
976
+ this.turnRound = 0;
967
977
  this.start = async (mic) => {
968
978
  if (this.state.isStarted) {
969
979
  throw new Error("MicRecorder is already started");
@@ -996,6 +1006,7 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
996
1006
  isStarted: false,
997
1007
  isSpeaking: false
998
1008
  });
1009
+ this.cancelPendingClose();
999
1010
  try {
1000
1011
  this.mic?.off("Frames", this.onFrames);
1001
1012
  this.mic = void 0;
@@ -1009,6 +1020,17 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1009
1020
  }
1010
1021
  this.resetBuffers();
1011
1022
  };
1023
+ /**
1024
+ * Changes what decides when a turn is over
1025
+ * @param turnDetector - The new detector, or nothing to leave it to the VAD
1026
+ */
1027
+ this.changeTurnDetector = (turnDetector) => {
1028
+ if (turnDetector === this.turnDetector) return;
1029
+ if (this.turnOpen) {
1030
+ this.endTurn();
1031
+ }
1032
+ this.turnDetector = turnDetector;
1033
+ };
1012
1034
  this.changeVad = (vadConfig) => {
1013
1035
  if (equalVADConfig(vadConfig, this.vadConfig)) return;
1014
1036
  const mic = this.mic;
@@ -1022,6 +1044,9 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1022
1044
  }
1023
1045
  };
1024
1046
  this.onFrames = (frames, sampleRate) => {
1047
+ if (this.turnListening) {
1048
+ this.turnDetector?.push(frames, sampleRate);
1049
+ }
1025
1050
  if (sampleRate !== this.sourceSampleRate) {
1026
1051
  this.flushChunks();
1027
1052
  this.sourceSampleRate = sampleRate;
@@ -1050,12 +1075,23 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1050
1075
  this.bufferLength = this.reserveLength;
1051
1076
  this.reserve = [];
1052
1077
  this.reserveLength = 0;
1078
+ if (this.turnDetector && !this.turnListening) {
1079
+ this.turnDetector.reset();
1080
+ this.turnListening = true;
1081
+ for (const frames of this.buffer) {
1082
+ this.turnDetector.push(frames, this.sourceSampleRate);
1083
+ }
1084
+ }
1053
1085
  this.flushChunks();
1054
1086
  };
1055
1087
  this.onConfirmSpeaking = () => {
1056
1088
  this.speakingConfirmed = true;
1057
1089
  this.changeState({ isSpeaking: true });
1058
- this.emit("StartSpeaking");
1090
+ this.cancelPendingClose();
1091
+ if (!this.turnOpen) {
1092
+ this.turnOpen = true;
1093
+ this.emit("StartSpeaking");
1094
+ }
1059
1095
  for (const chunk of this.queuedChunks) {
1060
1096
  this.emit("Chunk", chunk);
1061
1097
  }
@@ -1063,6 +1099,9 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1063
1099
  };
1064
1100
  this.onCancelSpeaking = () => {
1065
1101
  this.stopRecording();
1102
+ if (!this.turnOpen) {
1103
+ this.turnListening = false;
1104
+ }
1066
1105
  };
1067
1106
  this.onStopSpeaking = () => {
1068
1107
  const wasConfirmed = this.speakingConfirmed;
@@ -1071,11 +1110,55 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1071
1110
  this.flushChunks(true);
1072
1111
  }
1073
1112
  this.stopRecording();
1074
- this.emit("StopSpeaking");
1113
+ this.closeTurn();
1075
1114
  };
1076
1115
  this.state = defaultMicRecorderState;
1077
1116
  this.vad = getVAD(vadConfig);
1078
1117
  }
1118
+ /**
1119
+ * Ends the turn, unless the detector hears an unfinished sentence.
1120
+ *
1121
+ * A microphone that is no longer listening ends it straight away: a muted or
1122
+ * paused call has nothing more to say, whatever the sentence sounded like.
1123
+ */
1124
+ async closeTurn() {
1125
+ if (!this.turnDetector || !this.turnOpen || !this.vad.isStarted || this.vad.isPaused) {
1126
+ this.endTurn();
1127
+ return;
1128
+ }
1129
+ const round = this.turnRound;
1130
+ try {
1131
+ const { complete } = await this.turnDetector.predict();
1132
+ if (round !== this.turnRound) return;
1133
+ if (complete) {
1134
+ this.endTurn();
1135
+ return;
1136
+ }
1137
+ } catch (error) {
1138
+ console.error("[Micdrop] Turn detection failed", error);
1139
+ this.endTurn();
1140
+ return;
1141
+ }
1142
+ this.cancelPendingClose();
1143
+ this.turnTimer = setTimeout(() => {
1144
+ this.turnTimer = void 0;
1145
+ this.endTurn();
1146
+ }, this.turnMaxWait);
1147
+ }
1148
+ endTurn() {
1149
+ this.cancelPendingClose();
1150
+ this.turnOpen = false;
1151
+ this.turnListening = false;
1152
+ this.emit("StopSpeaking");
1153
+ }
1154
+ /** Drops the pending close, both the timer and the answer still owed */
1155
+ cancelPendingClose() {
1156
+ this.turnRound++;
1157
+ if (this.turnTimer) {
1158
+ clearTimeout(this.turnTimer);
1159
+ this.turnTimer = void 0;
1160
+ }
1161
+ }
1079
1162
  stopRecording() {
1080
1163
  this.isRecording = false;
1081
1164
  this.speakingConfirmed = false;
@@ -1122,6 +1205,8 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1122
1205
  resetBuffers() {
1123
1206
  this.isRecording = false;
1124
1207
  this.speakingConfirmed = false;
1208
+ this.turnOpen = false;
1209
+ this.turnListening = false;
1125
1210
  this.reserve = [];
1126
1211
  this.reserveLength = 0;
1127
1212
  this.buffer = [];
@@ -1324,6 +1409,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
1324
1409
  })(MicdropClientCommands || {});
1325
1410
  var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
1326
1411
  MicdropServerCommands2["Message"] = "Message";
1412
+ MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
1327
1413
  MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
1328
1414
  MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
1329
1415
  MicdropServerCommands2["EndCall"] = "EndCall";
@@ -1386,6 +1472,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1386
1472
  super();
1387
1473
  this.options = options;
1388
1474
  this.conversation = [];
1475
+ this.partialAssistantMessage = "";
1389
1476
  this.speakerDevices = [];
1390
1477
  this.micDevices = [];
1391
1478
  this.startTime = 0;
@@ -1400,6 +1487,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1400
1487
  this.options = { ...this.options, ...options };
1401
1488
  this.startTime = Date.now();
1402
1489
  this.conversation = [];
1490
+ this.partialAssistantMessage = "";
1403
1491
  this._isProcessing = true;
1404
1492
  this._isMuted = false;
1405
1493
  this._isPaused = false;
@@ -1415,6 +1503,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1415
1503
  await this.startWS();
1416
1504
  };
1417
1505
  this.stop = async () => {
1506
+ this.partialAssistantMessage = "";
1418
1507
  this._isProcessing = false;
1419
1508
  this._isMuted = false;
1420
1509
  this._isPaused = false;
@@ -1477,7 +1566,11 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1477
1566
  this.micRecorder.changeVad(options.vad);
1478
1567
  }
1479
1568
  } else {
1480
- this.micRecorder = new MicRecorder(this.options.vad);
1569
+ this.micRecorder = new MicRecorder(
1570
+ this.options.vad,
1571
+ this.options.turnDetector,
1572
+ this.options.turnMaxWait
1573
+ );
1481
1574
  this.micRecorder.on("StateChange", () => {
1482
1575
  this.notifyStateChange();
1483
1576
  });
@@ -1488,6 +1581,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1488
1581
  this.micRecorder.on("StartSpeaking", () => {
1489
1582
  this.log("User start speaking");
1490
1583
  this.ws?.send("StartSpeaking" /* StartSpeaking */);
1584
+ this.partialAssistantMessage = "";
1491
1585
  this._isProcessing = false;
1492
1586
  this.notifyStateChange();
1493
1587
  Speaker.stopAudio();
@@ -1523,6 +1617,28 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1523
1617
  throw error;
1524
1618
  }
1525
1619
  };
1620
+ /**
1621
+ * Changes what decides when a turn is over, during a call or before one
1622
+ * @param turnDetector - The new detector, or nothing to leave it to the VAD
1623
+ */
1624
+ this.setTurnDetector = (turnDetector) => {
1625
+ this.options.turnDetector = turnDetector;
1626
+ this.micRecorder?.changeTurnDetector(turnDetector);
1627
+ };
1628
+ /**
1629
+ * Says how long a held turn waits for the rest of the sentence.
1630
+ *
1631
+ * The way out of a wrong verdict: a detector can hear an unfinished sentence
1632
+ * where there is none, and the speaker who never comes back still gets an
1633
+ * answer once this runs out.
1634
+ * @param milliseconds - How long to wait past the detector asking to
1635
+ */
1636
+ this.setTurnMaxWait = (milliseconds) => {
1637
+ this.options.turnMaxWait = milliseconds;
1638
+ if (this.micRecorder) {
1639
+ this.micRecorder.turnMaxWait = milliseconds;
1640
+ }
1641
+ };
1526
1642
  this.changeMicDevice = async (deviceId) => {
1527
1643
  await this.startMic({ deviceId });
1528
1644
  };
@@ -1580,9 +1696,12 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1580
1696
  } catch (error) {
1581
1697
  console.error("[MicdropClient] Error parsing message:", data, error);
1582
1698
  }
1699
+ } else if (data.startsWith("PartialAssistantMessage" /* PartialAssistantMessage */)) {
1700
+ this.setPartialAssistantMessage(data);
1583
1701
  } else if (data === "EndCall" /* EndCall */) {
1584
1702
  this.emit("EndCall");
1585
1703
  } else if (data === "SkipAnswer" /* SkipAnswer */) {
1704
+ this.partialAssistantMessage = "";
1586
1705
  this._isProcessing = false;
1587
1706
  this.notifyStateChange();
1588
1707
  } else if (data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
@@ -1710,6 +1829,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1710
1829
  isMicStarted: this.isMicStarted,
1711
1830
  isMicMuted: this.isMicMuted,
1712
1831
  conversation: this.conversation,
1832
+ partialAssistantMessage: this.partialAssistantMessage,
1713
1833
  error: this.error,
1714
1834
  micDeviceId: this.micDeviceId,
1715
1835
  speakerDeviceId: this.speakerDeviceId,
@@ -1797,8 +1917,23 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1797
1917
  }, config.delayMs);
1798
1918
  }
1799
1919
  addMessage(message) {
1920
+ if (message.role === "assistant") this.partialAssistantMessage = "";
1800
1921
  this.conversation = [...this.conversation, message];
1801
1922
  this.notifyStateChange();
1923
+ this.emit("Message", message);
1924
+ }
1925
+ setPartialAssistantMessage(data) {
1926
+ const command = "PartialAssistantMessage" /* PartialAssistantMessage */;
1927
+ let content;
1928
+ try {
1929
+ content = JSON.parse(data.substring(command.length + 1));
1930
+ } catch (error) {
1931
+ console.error("[MicdropClient] Error parsing partial answer:", data);
1932
+ return;
1933
+ }
1934
+ this.partialAssistantMessage = content;
1935
+ this.emit("PartialAssistantMessage", content);
1936
+ this.notifyStateChange();
1802
1937
  }
1803
1938
  setError(error) {
1804
1939
  console.error("[MicdropClient] Error:", error);
@@ -1825,6 +1960,7 @@ if (!globalScope2.micdropClient) {
1825
1960
  var Micdrop = globalScope2.micdropClient;
1826
1961
  // Annotate the CommonJS export names for ESM import in node:
1827
1962
  0 && (module.exports = {
1963
+ DEFAULT_TURN_MAX_WAIT,
1828
1964
  Mic,
1829
1965
  MicController,
1830
1966
  MicDriver,