@micdrop/client 3.2.1 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -223,6 +223,10 @@ var MicController = class extends EventEmitter {
223
223
  get deviceId() {
224
224
  return this.driver?.deviceId;
225
225
  }
226
+ /** Whether echo cancellation is on, undefined when nobody knows */
227
+ get echoCancellation() {
228
+ return this.driver?.echoCancellation;
229
+ }
226
230
  /**
227
231
  * Asks for the microphone permission and starts capturing
228
232
  * @param deviceId - Input device to record from, defaults to the system one
@@ -490,18 +494,19 @@ var MultipleVAD = class extends VAD {
490
494
  constructor(vads) {
491
495
  super();
492
496
  this.name = "MultipleVAD";
497
+ // Every VAD has to hear the speech, but only one has to be sure of it. A
498
+ // short word like "No" is over before the volume VAD gathers the loud
499
+ // reports it needs to confirm, while Silero confirms it right away.
493
500
  this.onStatusChange = () => {
494
501
  const isAllSilence = this.vads.every(
495
502
  (vad) => vad.status === 0 /* Silence */
496
503
  );
497
- const isAllSpeaking = this.vads.every(
498
- (vad) => vad.status === 2 /* Speaking */
499
- );
504
+ const isConfirmed = this.vads.every((vad) => vad.status !== 0 /* Silence */) && this.vads.some((vad) => vad.status === 2 /* Speaking */);
500
505
  switch (this.status) {
501
506
  case 0 /* Silence */:
502
507
  if (isAllSilence) break;
503
508
  this.emit("StartSpeaking");
504
- if (isAllSpeaking) {
509
+ if (isConfirmed) {
505
510
  this.emit("ConfirmSpeaking");
506
511
  }
507
512
  break;
@@ -510,7 +515,7 @@ var MultipleVAD = class extends VAD {
510
515
  this.emit("CancelSpeaking");
511
516
  break;
512
517
  }
513
- if (isAllSpeaking) {
518
+ if (isConfirmed) {
514
519
  this.emit("ConfirmSpeaking");
515
520
  }
516
521
  break;
@@ -522,6 +527,7 @@ var MultipleVAD = class extends VAD {
522
527
  }
523
528
  };
524
529
  this.vads = vads.map((vad) => getVAD(vad));
530
+ this.delay = Math.max(...this.vads.map((vad) => vad.delay));
525
531
  }
526
532
  get isStarted() {
527
533
  return this.vads.some((vad) => vad.isStarted);
@@ -530,18 +536,19 @@ var MultipleVAD = class extends VAD {
530
536
  return this.vads.every((vad) => vad.isPaused);
531
537
  }
532
538
  async start(mic) {
539
+ this.setStatus(0 /* Silence */);
533
540
  for (const vad of this.vads) {
534
- if (vad.isStarted) {
535
- continue;
536
- }
541
+ vad.off("ChangeStatus", this.onStatusChange);
537
542
  vad.on("ChangeStatus", this.onStatusChange);
538
- await vad.start(mic);
543
+ if (!vad.isStarted) {
544
+ await vad.start(mic);
545
+ }
539
546
  }
540
547
  }
541
548
  async stop() {
542
549
  for (const vad of this.vads) {
543
- vad.off("ChangeStatus", this.onStatusChange);
544
550
  await vad.stop();
551
+ vad.off("ChangeStatus", this.onStatusChange);
545
552
  }
546
553
  }
547
554
  async pause() {
@@ -575,10 +582,11 @@ var SileroVAD = class extends VAD {
575
582
  super();
576
583
  this.name = "SileroVAD";
577
584
  this.options = defaultOptions;
578
- // The model decides on the first window it hears speech in, so it is quick.
579
- // The reserve still covers a few windows, since scoring runs asynchronously
580
- // and may lag behind the audio on a busy device.
581
- this.delay = 3 * MS_PER_FRAME;
585
+ // The model scores a window as speech a few windows after the voice began:
586
+ // a soft consonant or a breathy onset stays under the threshold until the
587
+ // vowel comes. Scoring also runs asynchronously and may lag behind the audio
588
+ // on a busy device, so the reserve covers about a third of a second.
589
+ this.delay = 10 * MS_PER_FRAME;
582
590
  this._isPaused = false;
583
591
  this.buffer = [];
584
592
  this.bufferLength = 0;
@@ -747,9 +755,11 @@ var VolumeVAD = class extends VAD {
747
755
  this.options = defaultOptions2;
748
756
  // The microphone reports a level about every 100 ms, and two consecutive loud
749
757
  // ones are needed before a turn opens. Speech can therefore begin up to two
750
- // reports before it is noticed, and a third is kept as a margin: without that
751
- // reserve the first syllable of a sentence never reaches the server.
752
- this.delay = 300;
758
+ // reports before it is noticed. A quiet onset (a soft consonant, a word
759
+ // spoken low) stays under the threshold for a while too, so the reserve adds
760
+ // a margin: without it the first syllable of a sentence never reaches the
761
+ // server.
762
+ this.delay = 500;
753
763
  this.running = false;
754
764
  this._isPaused = false;
755
765
  this.speaking = false;
@@ -892,6 +902,8 @@ function equalVADConfig(a, b) {
892
902
  var SAMPLE_RATE = 16e3;
893
903
  var CHUNK_INTERVAL = 100;
894
904
  var DEFAULT_TURN_MAX_WAIT = 4e3;
905
+ var CONFIRM_MARGIN = 300;
906
+ var ECHO_TAIL = 300;
895
907
  var defaultMicRecorderState = {
896
908
  isStarting: false,
897
909
  isStarted: false,
@@ -910,7 +922,9 @@ var MicRecorder = class extends EventEmitter4 {
910
922
  this.reserveLength = 0;
911
923
  this.buffer = [];
912
924
  this.bufferLength = 0;
913
- this.queuedChunks = [];
925
+ // Samples left before the echo of the speaker has died out
926
+ this.echoSamples = 0;
927
+ this.speakerPlaying = false;
914
928
  // The turn is open between the first confirmed word and the moment the
915
929
  // server is told to answer, which can span several pauses
916
930
  this.turnOpen = false;
@@ -986,7 +1000,30 @@ var MicRecorder = class extends EventEmitter4 {
986
1000
  this.start(mic);
987
1001
  }
988
1002
  };
1003
+ /**
1004
+ * Leaves out what the microphone hears while the speaker plays, so the
1005
+ * assistant is never sent back to the server as if the user had said it.
1006
+ *
1007
+ * Only for calls the user may not interrupt: an interruption relies on echo
1008
+ * cancellation, and needs every word the user says over the assistant.
1009
+ * @param playing - True while the assistant is heard
1010
+ */
1011
+ this.setSpeakerPlaying = (playing) => {
1012
+ if (playing === this.speakerPlaying) return;
1013
+ this.speakerPlaying = playing;
1014
+ if (!playing) {
1015
+ this.echoSamples = Math.round(ECHO_TAIL / 1e3 * this.sourceSampleRate);
1016
+ }
1017
+ };
989
1018
  this.onFrames = (frames, sampleRate) => {
1019
+ if (this.speakerPlaying || this.echoSamples > 0) {
1020
+ this.echoSamples = Math.max(0, this.echoSamples - frames.length);
1021
+ this.reserve = [];
1022
+ this.reserveLength = 0;
1023
+ this.buffer = [];
1024
+ this.bufferLength = 0;
1025
+ return;
1026
+ }
990
1027
  if (this.turnListening) {
991
1028
  this.turnDetector?.push(frames, sampleRate);
992
1029
  }
@@ -996,28 +1033,31 @@ var MicRecorder = class extends EventEmitter4 {
996
1033
  this.reserve = [];
997
1034
  this.reserveLength = 0;
998
1035
  }
999
- if (this.isRecording) {
1000
- this.buffer.push(frames);
1001
- this.bufferLength += frames.length;
1002
- this.flushChunks();
1003
- return;
1004
- }
1005
1036
  this.reserve.push(frames);
1006
1037
  this.reserveLength += frames.length;
1007
- const reserveMax = Math.round(this.vad.delay / 1e3 * sampleRate);
1008
- while (this.reserve.length > 1 && this.reserveLength - this.reserve[0].length >= reserveMax) {
1009
- this.reserveLength -= this.reserve[0].length;
1010
- this.reserve.shift();
1038
+ this.reserveLength = trimFrames(
1039
+ this.reserve,
1040
+ this.reserveLength,
1041
+ Math.round(this.vad.delay / 1e3 * sampleRate)
1042
+ );
1043
+ if (!this.isRecording) return;
1044
+ this.buffer.push(frames);
1045
+ this.bufferLength += frames.length;
1046
+ if (this.speakingConfirmed) {
1047
+ this.flushChunks();
1048
+ return;
1011
1049
  }
1050
+ this.bufferLength = trimFrames(
1051
+ this.buffer,
1052
+ this.bufferLength,
1053
+ Math.round((this.vad.delay + CONFIRM_MARGIN) / 1e3 * sampleRate)
1054
+ );
1012
1055
  };
1013
1056
  this.onStartSpeaking = () => {
1014
1057
  this.speakingConfirmed = false;
1015
- this.queuedChunks.length = 0;
1016
1058
  this.isRecording = true;
1017
- this.buffer = this.reserve;
1059
+ this.buffer = [...this.reserve];
1018
1060
  this.bufferLength = this.reserveLength;
1019
- this.reserve = [];
1020
- this.reserveLength = 0;
1021
1061
  if (this.turnDetector && !this.turnListening) {
1022
1062
  this.turnDetector.reset();
1023
1063
  this.turnListening = true;
@@ -1025,7 +1065,6 @@ var MicRecorder = class extends EventEmitter4 {
1025
1065
  this.turnDetector.push(frames, this.sourceSampleRate);
1026
1066
  }
1027
1067
  }
1028
- this.flushChunks();
1029
1068
  };
1030
1069
  this.onConfirmSpeaking = () => {
1031
1070
  this.speakingConfirmed = true;
@@ -1035,10 +1074,7 @@ var MicRecorder = class extends EventEmitter4 {
1035
1074
  this.turnOpen = true;
1036
1075
  this.emit("StartSpeaking");
1037
1076
  }
1038
- for (const chunk of this.queuedChunks) {
1039
- this.emit("Chunk", chunk);
1040
- }
1041
- this.queuedChunks.length = 0;
1077
+ this.flushChunks();
1042
1078
  };
1043
1079
  this.onCancelSpeaking = () => {
1044
1080
  this.stopRecording();
@@ -1105,7 +1141,6 @@ var MicRecorder = class extends EventEmitter4 {
1105
1141
  stopRecording() {
1106
1142
  this.isRecording = false;
1107
1143
  this.speakingConfirmed = false;
1108
- this.queuedChunks.length = 0;
1109
1144
  this.buffer = [];
1110
1145
  this.bufferLength = 0;
1111
1146
  }
@@ -1117,7 +1152,7 @@ var MicRecorder = class extends EventEmitter4 {
1117
1152
  const chunkLength = Math.round(
1118
1153
  CHUNK_INTERVAL / 1e3 * this.sourceSampleRate
1119
1154
  );
1120
- if (this.bufferLength === 0) return;
1155
+ if (!this.speakingConfirmed || this.bufferLength === 0) return;
1121
1156
  if (this.bufferLength < chunkLength && !final) return;
1122
1157
  const merged = concatFloat32(this.buffer);
1123
1158
  let offset = 0;
@@ -1139,10 +1174,6 @@ var MicRecorder = class extends EventEmitter4 {
1139
1174
  const pcm = floatToPcm16(
1140
1175
  resample(samples, this.sourceSampleRate, SAMPLE_RATE)
1141
1176
  );
1142
- if (!this.speakingConfirmed) {
1143
- this.queuedChunks.push(pcm);
1144
- return;
1145
- }
1146
1177
  this.emit("Chunk", pcm);
1147
1178
  }
1148
1179
  resetBuffers() {
@@ -1154,7 +1185,6 @@ var MicRecorder = class extends EventEmitter4 {
1154
1185
  this.reserveLength = 0;
1155
1186
  this.buffer = [];
1156
1187
  this.bufferLength = 0;
1157
- this.queuedChunks.length = 0;
1158
1188
  }
1159
1189
  changeState(state) {
1160
1190
  const hasChanged = Object.keys(state).some(
@@ -1165,6 +1195,13 @@ var MicRecorder = class extends EventEmitter4 {
1165
1195
  this.emit("StateChange", this.state);
1166
1196
  }
1167
1197
  };
1198
+ function trimFrames(frames, length, max) {
1199
+ while (frames.length > 1 && length - frames[0].length >= max) {
1200
+ length -= frames[0].length;
1201
+ frames.shift();
1202
+ }
1203
+ return length;
1204
+ }
1168
1205
 
1169
1206
  // src/audio/Pcm16AudioStream.ts
1170
1207
  import { EventEmitter as EventEmitter5 } from "eventemitter3";
@@ -1325,6 +1362,13 @@ var Pcm16AudioStream = class extends EventEmitter5 {
1325
1362
  // src/audio/types.ts
1326
1363
  import { EventEmitter as EventEmitter6 } from "eventemitter3";
1327
1364
  var MicDriver = class extends EventEmitter6 {
1365
+ /**
1366
+ * Whether the system removes the speaker from what the microphone hears.
1367
+ * False when it is known to be off, undefined when the system does not say.
1368
+ */
1369
+ get echoCancellation() {
1370
+ return void 0;
1371
+ }
1328
1372
  };
1329
1373
  var SpeakerDriver = class extends EventEmitter6 {
1330
1374
  };
@@ -1357,8 +1401,30 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
1357
1401
  MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
1358
1402
  MicdropServerCommands2["EndCall"] = "EndCall";
1359
1403
  MicdropServerCommands2["ToolCall"] = "ToolCall";
1404
+ MicdropServerCommands2["Classification"] = "Classification";
1360
1405
  return MicdropServerCommands2;
1361
1406
  })(MicdropServerCommands || {});
1407
+ function currentTurn(conversation) {
1408
+ let start = conversation.length;
1409
+ while (start > 0) {
1410
+ const item = conversation[start - 1];
1411
+ if (item.role !== "user" || item.metadata?.classification) break;
1412
+ start--;
1413
+ }
1414
+ const messages = conversation.slice(start);
1415
+ return {
1416
+ transcript: messages.map((message) => message.content).join(" "),
1417
+ messages,
1418
+ before: conversation.slice(0, start)
1419
+ };
1420
+ }
1421
+ function getTurnClassification(conversation) {
1422
+ for (let index = conversation.length - 1; index >= 0; index--) {
1423
+ const item = conversation[index];
1424
+ if (item.role === "user") return item.metadata?.classification;
1425
+ }
1426
+ return void 0;
1427
+ }
1362
1428
 
1363
1429
  // src/client/MicdropClientError.ts
1364
1430
  var MicdropClientErrorCode = /* @__PURE__ */ ((MicdropClientErrorCode2) => {
@@ -1483,7 +1549,7 @@ var MicdropClient = class extends EventEmitter7 {
1483
1549
  this.notifyStateChange();
1484
1550
  };
1485
1551
  this.unmute = () => {
1486
- if (!this.isPaused && !(this.options.disableInterruption && Speaker.isPlaying)) {
1552
+ if (!this.isPaused && !(this.isInterruptionDisabled && Speaker.isPlaying)) {
1487
1553
  this.vad?.resume();
1488
1554
  }
1489
1555
  this._isMuted = false;
@@ -1524,6 +1590,9 @@ var MicdropClient = class extends EventEmitter7 {
1524
1590
  this.options.turnDetector,
1525
1591
  this.options.turnMaxWait
1526
1592
  );
1593
+ this.micRecorder.setSpeakerPlaying(
1594
+ this.isInterruptionDisabled && Speaker.isPlaying
1595
+ );
1527
1596
  this.micRecorder.on("StateChange", () => {
1528
1597
  this.notifyStateChange();
1529
1598
  });
@@ -1555,6 +1624,9 @@ var MicdropClient = class extends EventEmitter7 {
1555
1624
  } else {
1556
1625
  storage.removeItem("micdrop.micDevice" /* MicDevice */);
1557
1626
  }
1627
+ if (Mic.echoCancellation === false && !this.options.disableInterruption) {
1628
+ this.log("No echo cancellation, interruptions are disabled");
1629
+ }
1558
1630
  await this.micRecorder.start(Mic);
1559
1631
  await Speaker.start();
1560
1632
  await this.updateDevices();
@@ -1676,6 +1748,19 @@ var MicdropClient = class extends EventEmitter7 {
1676
1748
  } catch (error) {
1677
1749
  console.error("[MicdropClient] Error parsing tool call:", data, error);
1678
1750
  }
1751
+ } else if (data.startsWith("Classification" /* Classification */)) {
1752
+ try {
1753
+ const classification = JSON.parse(
1754
+ data.substring("Classification" /* Classification */.length + 1)
1755
+ );
1756
+ this.emit("Classification", classification);
1757
+ } catch (error) {
1758
+ console.error(
1759
+ "[MicdropClient] Error parsing classification:",
1760
+ data,
1761
+ error
1762
+ );
1763
+ }
1679
1764
  }
1680
1765
  };
1681
1766
  this.onWSClose = (event) => {
@@ -1699,20 +1784,22 @@ var MicdropClient = class extends EventEmitter7 {
1699
1784
  };
1700
1785
  this.onSpeakerStartPlaying = () => {
1701
1786
  this.log("Speaker started");
1702
- if (this.options.disableInterruption) {
1787
+ if (this.isInterruptionDisabled) {
1788
+ this.micRecorder?.setSpeakerPlaying(true);
1703
1789
  this.vad?.pause();
1704
1790
  }
1705
1791
  this.notifyStateChange();
1706
1792
  };
1707
1793
  this.onSpeakerStopPlaying = () => {
1708
1794
  this.log("Speaker stopped");
1709
- if (this.options.disableInterruption) {
1795
+ this.micRecorder?.setSpeakerPlaying(false);
1796
+ if (this.isInterruptionDisabled) {
1710
1797
  setTimeout(() => {
1711
1798
  if (!this.isMuted) {
1712
1799
  this.vad?.resume();
1713
1800
  this.notifyStateChange();
1714
1801
  }
1715
- }, 200);
1802
+ }, ECHO_TAIL);
1716
1803
  } else {
1717
1804
  this.notifyStateChange();
1718
1805
  }
@@ -1763,6 +1850,14 @@ var MicdropClient = class extends EventEmitter7 {
1763
1850
  get isAssistantSpeaking() {
1764
1851
  return Speaker.isPlaying;
1765
1852
  }
1853
+ /**
1854
+ * Whether the user is kept from speaking over the assistant. Without echo
1855
+ * cancellation the microphone hears the assistant, which would interrupt
1856
+ * itself and end up in the transcript.
1857
+ */
1858
+ get isInterruptionDisabled() {
1859
+ return !!this.options.disableInterruption || Mic.echoCancellation === false;
1860
+ }
1766
1861
  get micDeviceId() {
1767
1862
  return Mic.deviceId;
1768
1863
  }
@@ -1913,7 +2008,9 @@ if (!globalScope2.micdropClient) {
1913
2008
  }
1914
2009
  var Micdrop = globalScope2.micdropClient;
1915
2010
  export {
2011
+ CONFIRM_MARGIN,
1916
2012
  DEFAULT_TURN_MAX_WAIT,
2013
+ ECHO_TAIL,
1917
2014
  Mic,
1918
2015
  MicController,
1919
2016
  MicDriver,
@@ -1939,10 +2036,12 @@ export {
1939
2036
  VolumeMeter,
1940
2037
  VolumeVAD,
1941
2038
  concatFloat32,
2039
+ currentTurn,
1942
2040
  equalVADConfig,
1943
2041
  floatToPcm16,
1944
2042
  getClientErrorFromWSCloseEvent,
1945
2043
  getClientErrorFromWSCloseEventCode,
2044
+ getTurnClassification,
1946
2045
  getVAD,
1947
2046
  getVolume,
1948
2047
  isRecoverableError,