@micdrop/client 3.2.0 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -223,6 +223,10 @@ var MicController = class extends EventEmitter {
223
223
  get deviceId() {
224
224
  return this.driver?.deviceId;
225
225
  }
226
+ /** Whether echo cancellation is on, undefined when nobody knows */
227
+ get echoCancellation() {
228
+ return this.driver?.echoCancellation;
229
+ }
226
230
  /**
227
231
  * Asks for the microphone permission and starts capturing
228
232
  * @param deviceId - Input device to record from, defaults to the system one
@@ -490,18 +494,19 @@ var MultipleVAD = class extends VAD {
490
494
  constructor(vads) {
491
495
  super();
492
496
  this.name = "MultipleVAD";
497
+ // Every VAD has to hear the speech, but only one has to be sure of it. A
498
+ // short word like "No" is over before the volume VAD gathers the loud
499
+ // reports it needs to confirm, while Silero confirms it right away.
493
500
  this.onStatusChange = () => {
494
501
  const isAllSilence = this.vads.every(
495
502
  (vad) => vad.status === 0 /* Silence */
496
503
  );
497
- const isAllSpeaking = this.vads.every(
498
- (vad) => vad.status === 2 /* Speaking */
499
- );
504
+ const isConfirmed = this.vads.every((vad) => vad.status !== 0 /* Silence */) && this.vads.some((vad) => vad.status === 2 /* Speaking */);
500
505
  switch (this.status) {
501
506
  case 0 /* Silence */:
502
507
  if (isAllSilence) break;
503
508
  this.emit("StartSpeaking");
504
- if (isAllSpeaking) {
509
+ if (isConfirmed) {
505
510
  this.emit("ConfirmSpeaking");
506
511
  }
507
512
  break;
@@ -510,7 +515,7 @@ var MultipleVAD = class extends VAD {
510
515
  this.emit("CancelSpeaking");
511
516
  break;
512
517
  }
513
- if (isAllSpeaking) {
518
+ if (isConfirmed) {
514
519
  this.emit("ConfirmSpeaking");
515
520
  }
516
521
  break;
@@ -522,6 +527,7 @@ var MultipleVAD = class extends VAD {
522
527
  }
523
528
  };
524
529
  this.vads = vads.map((vad) => getVAD(vad));
530
+ this.delay = Math.max(...this.vads.map((vad) => vad.delay));
525
531
  }
526
532
  get isStarted() {
527
533
  return this.vads.some((vad) => vad.isStarted);
@@ -530,18 +536,19 @@ var MultipleVAD = class extends VAD {
530
536
  return this.vads.every((vad) => vad.isPaused);
531
537
  }
532
538
  async start(mic) {
539
+ this.setStatus(0 /* Silence */);
533
540
  for (const vad of this.vads) {
534
- if (vad.isStarted) {
535
- continue;
536
- }
541
+ vad.off("ChangeStatus", this.onStatusChange);
537
542
  vad.on("ChangeStatus", this.onStatusChange);
538
- await vad.start(mic);
543
+ if (!vad.isStarted) {
544
+ await vad.start(mic);
545
+ }
539
546
  }
540
547
  }
541
548
  async stop() {
542
549
  for (const vad of this.vads) {
543
- vad.off("ChangeStatus", this.onStatusChange);
544
550
  await vad.stop();
551
+ vad.off("ChangeStatus", this.onStatusChange);
545
552
  }
546
553
  }
547
554
  async pause() {
@@ -575,10 +582,11 @@ var SileroVAD = class extends VAD {
575
582
  super();
576
583
  this.name = "SileroVAD";
577
584
  this.options = defaultOptions;
578
- // The model decides on the first window it hears speech in, so it is quick.
579
- // The reserve still covers a few windows, since scoring runs asynchronously
580
- // and may lag behind the audio on a busy device.
581
- this.delay = 3 * MS_PER_FRAME;
585
+ // The model scores a window as speech a few windows after the voice began:
586
+ // a soft consonant or a breathy onset stays under the threshold until the
587
+ // vowel comes. Scoring also runs asynchronously and may lag behind the audio
588
+ // on a busy device, so the reserve covers about a third of a second.
589
+ this.delay = 10 * MS_PER_FRAME;
582
590
  this._isPaused = false;
583
591
  this.buffer = [];
584
592
  this.bufferLength = 0;
@@ -587,6 +595,9 @@ var SileroVAD = class extends VAD {
587
595
  this.speechFrames = 0;
588
596
  this.silenceFrames = 0;
589
597
  this.processing = false;
598
+ // Bumped whenever the state is reset, so a window scored before that is
599
+ // dropped rather than reopening a turn
600
+ this.round = 0;
590
601
  this.onFrames = (frames, sampleRate) => {
591
602
  if (this._isPaused || !this.model) return;
592
603
  if (sampleRate !== this.bufferRate) {
@@ -624,6 +635,7 @@ var SileroVAD = class extends VAD {
624
635
  }
625
636
  this.model = await loadModel();
626
637
  this.resetState();
638
+ this.setStatus(0 /* Silence */);
627
639
  this.mic = mic;
628
640
  mic.on("Frames", this.onFrames);
629
641
  }
@@ -670,9 +682,11 @@ var SileroVAD = class extends VAD {
670
682
  this.buffer = rest.length ? [rest] : [];
671
683
  this.bufferLength = rest.length;
672
684
  const frame = resample(window, this.bufferRate, SILERO_SAMPLE_RATE);
685
+ const round = this.round;
673
686
  const probability = await this.model.process(
674
687
  frame.length === SILERO_FRAME_SAMPLES ? frame : fit(frame, SILERO_FRAME_SAMPLES)
675
688
  );
689
+ if (round !== this.round) break;
676
690
  this.onProbability(probability);
677
691
  }
678
692
  } catch (error) {
@@ -713,6 +727,7 @@ var SileroVAD = class extends VAD {
713
727
  this.emit(confirmed ? "StopSpeaking" : "CancelSpeaking");
714
728
  }
715
729
  resetState() {
730
+ this.round++;
716
731
  this.model?.reset();
717
732
  this.buffer = [];
718
733
  this.bufferLength = 0;
@@ -740,9 +755,11 @@ var VolumeVAD = class extends VAD {
740
755
  this.options = defaultOptions2;
741
756
  // The microphone reports a level about every 100 ms, and two consecutive loud
742
757
  // ones are needed before a turn opens. Speech can therefore begin up to two
743
- // reports before it is noticed, and a third is kept as a margin: without that
744
- // reserve the first syllable of a sentence never reaches the server.
745
- this.delay = 300;
758
+ // reports before it is noticed. A quiet onset (a soft consonant, a word
759
+ // spoken low) stays under the threshold for a while too, so the reserve adds
760
+ // a margin: without it the first syllable of a sentence never reaches the
761
+ // server.
762
+ this.delay = 500;
746
763
  this.running = false;
747
764
  this._isPaused = false;
748
765
  this.speaking = false;
@@ -885,6 +902,8 @@ function equalVADConfig(a, b) {
885
902
  var SAMPLE_RATE = 16e3;
886
903
  var CHUNK_INTERVAL = 100;
887
904
  var DEFAULT_TURN_MAX_WAIT = 4e3;
905
+ var CONFIRM_MARGIN = 300;
906
+ var ECHO_TAIL = 300;
888
907
  var defaultMicRecorderState = {
889
908
  isStarting: false,
890
909
  isStarted: false,
@@ -903,7 +922,9 @@ var MicRecorder = class extends EventEmitter4 {
903
922
  this.reserveLength = 0;
904
923
  this.buffer = [];
905
924
  this.bufferLength = 0;
906
- this.queuedChunks = [];
925
+ // Samples left before the echo of the speaker has died out
926
+ this.echoSamples = 0;
927
+ this.speakerPlaying = false;
907
928
  // The turn is open between the first confirmed word and the moment the
908
929
  // server is told to answer, which can span several pauses
909
930
  this.turnOpen = false;
@@ -979,7 +1000,30 @@ var MicRecorder = class extends EventEmitter4 {
979
1000
  this.start(mic);
980
1001
  }
981
1002
  };
1003
+ /**
1004
+ * Leaves out what the microphone hears while the speaker plays, so the
1005
+ * assistant is never sent back to the server as if the user had said it.
1006
+ *
1007
+ * Only for calls the user may not interrupt: an interruption relies on echo
1008
+ * cancellation, and needs every word the user says over the assistant.
1009
+ * @param playing - True while the assistant is heard
1010
+ */
1011
+ this.setSpeakerPlaying = (playing) => {
1012
+ if (playing === this.speakerPlaying) return;
1013
+ this.speakerPlaying = playing;
1014
+ if (!playing) {
1015
+ this.echoSamples = Math.round(ECHO_TAIL / 1e3 * this.sourceSampleRate);
1016
+ }
1017
+ };
982
1018
  this.onFrames = (frames, sampleRate) => {
1019
+ if (this.speakerPlaying || this.echoSamples > 0) {
1020
+ this.echoSamples = Math.max(0, this.echoSamples - frames.length);
1021
+ this.reserve = [];
1022
+ this.reserveLength = 0;
1023
+ this.buffer = [];
1024
+ this.bufferLength = 0;
1025
+ return;
1026
+ }
983
1027
  if (this.turnListening) {
984
1028
  this.turnDetector?.push(frames, sampleRate);
985
1029
  }
@@ -989,28 +1033,31 @@ var MicRecorder = class extends EventEmitter4 {
989
1033
  this.reserve = [];
990
1034
  this.reserveLength = 0;
991
1035
  }
992
- if (this.isRecording) {
993
- this.buffer.push(frames);
994
- this.bufferLength += frames.length;
995
- this.flushChunks();
996
- return;
997
- }
998
1036
  this.reserve.push(frames);
999
1037
  this.reserveLength += frames.length;
1000
- const reserveMax = Math.round(this.vad.delay / 1e3 * sampleRate);
1001
- while (this.reserve.length > 1 && this.reserveLength - this.reserve[0].length >= reserveMax) {
1002
- this.reserveLength -= this.reserve[0].length;
1003
- this.reserve.shift();
1038
+ this.reserveLength = trimFrames(
1039
+ this.reserve,
1040
+ this.reserveLength,
1041
+ Math.round(this.vad.delay / 1e3 * sampleRate)
1042
+ );
1043
+ if (!this.isRecording) return;
1044
+ this.buffer.push(frames);
1045
+ this.bufferLength += frames.length;
1046
+ if (this.speakingConfirmed) {
1047
+ this.flushChunks();
1048
+ return;
1004
1049
  }
1050
+ this.bufferLength = trimFrames(
1051
+ this.buffer,
1052
+ this.bufferLength,
1053
+ Math.round((this.vad.delay + CONFIRM_MARGIN) / 1e3 * sampleRate)
1054
+ );
1005
1055
  };
1006
1056
  this.onStartSpeaking = () => {
1007
1057
  this.speakingConfirmed = false;
1008
- this.queuedChunks.length = 0;
1009
1058
  this.isRecording = true;
1010
- this.buffer = this.reserve;
1059
+ this.buffer = [...this.reserve];
1011
1060
  this.bufferLength = this.reserveLength;
1012
- this.reserve = [];
1013
- this.reserveLength = 0;
1014
1061
  if (this.turnDetector && !this.turnListening) {
1015
1062
  this.turnDetector.reset();
1016
1063
  this.turnListening = true;
@@ -1018,7 +1065,6 @@ var MicRecorder = class extends EventEmitter4 {
1018
1065
  this.turnDetector.push(frames, this.sourceSampleRate);
1019
1066
  }
1020
1067
  }
1021
- this.flushChunks();
1022
1068
  };
1023
1069
  this.onConfirmSpeaking = () => {
1024
1070
  this.speakingConfirmed = true;
@@ -1028,10 +1074,7 @@ var MicRecorder = class extends EventEmitter4 {
1028
1074
  this.turnOpen = true;
1029
1075
  this.emit("StartSpeaking");
1030
1076
  }
1031
- for (const chunk of this.queuedChunks) {
1032
- this.emit("Chunk", chunk);
1033
- }
1034
- this.queuedChunks.length = 0;
1077
+ this.flushChunks();
1035
1078
  };
1036
1079
  this.onCancelSpeaking = () => {
1037
1080
  this.stopRecording();
@@ -1098,7 +1141,6 @@ var MicRecorder = class extends EventEmitter4 {
1098
1141
  stopRecording() {
1099
1142
  this.isRecording = false;
1100
1143
  this.speakingConfirmed = false;
1101
- this.queuedChunks.length = 0;
1102
1144
  this.buffer = [];
1103
1145
  this.bufferLength = 0;
1104
1146
  }
@@ -1110,7 +1152,7 @@ var MicRecorder = class extends EventEmitter4 {
1110
1152
  const chunkLength = Math.round(
1111
1153
  CHUNK_INTERVAL / 1e3 * this.sourceSampleRate
1112
1154
  );
1113
- if (this.bufferLength === 0) return;
1155
+ if (!this.speakingConfirmed || this.bufferLength === 0) return;
1114
1156
  if (this.bufferLength < chunkLength && !final) return;
1115
1157
  const merged = concatFloat32(this.buffer);
1116
1158
  let offset = 0;
@@ -1132,10 +1174,6 @@ var MicRecorder = class extends EventEmitter4 {
1132
1174
  const pcm = floatToPcm16(
1133
1175
  resample(samples, this.sourceSampleRate, SAMPLE_RATE)
1134
1176
  );
1135
- if (!this.speakingConfirmed) {
1136
- this.queuedChunks.push(pcm);
1137
- return;
1138
- }
1139
1177
  this.emit("Chunk", pcm);
1140
1178
  }
1141
1179
  resetBuffers() {
@@ -1147,7 +1185,6 @@ var MicRecorder = class extends EventEmitter4 {
1147
1185
  this.reserveLength = 0;
1148
1186
  this.buffer = [];
1149
1187
  this.bufferLength = 0;
1150
- this.queuedChunks.length = 0;
1151
1188
  }
1152
1189
  changeState(state) {
1153
1190
  const hasChanged = Object.keys(state).some(
@@ -1158,6 +1195,13 @@ var MicRecorder = class extends EventEmitter4 {
1158
1195
  this.emit("StateChange", this.state);
1159
1196
  }
1160
1197
  };
1198
+ function trimFrames(frames, length, max) {
1199
+ while (frames.length > 1 && length - frames[0].length >= max) {
1200
+ length -= frames[0].length;
1201
+ frames.shift();
1202
+ }
1203
+ return length;
1204
+ }
1161
1205
 
1162
1206
  // src/audio/Pcm16AudioStream.ts
1163
1207
  import { EventEmitter as EventEmitter5 } from "eventemitter3";
@@ -1318,6 +1362,13 @@ var Pcm16AudioStream = class extends EventEmitter5 {
1318
1362
  // src/audio/types.ts
1319
1363
  import { EventEmitter as EventEmitter6 } from "eventemitter3";
1320
1364
  var MicDriver = class extends EventEmitter6 {
1365
+ /**
1366
+ * Whether the system removes the speaker from what the microphone hears.
1367
+ * False when it is known to be off, undefined when the system does not say.
1368
+ */
1369
+ get echoCancellation() {
1370
+ return void 0;
1371
+ }
1321
1372
  };
1322
1373
  var SpeakerDriver = class extends EventEmitter6 {
1323
1374
  };
@@ -1350,8 +1401,30 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
1350
1401
  MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
1351
1402
  MicdropServerCommands2["EndCall"] = "EndCall";
1352
1403
  MicdropServerCommands2["ToolCall"] = "ToolCall";
1404
+ MicdropServerCommands2["Classification"] = "Classification";
1353
1405
  return MicdropServerCommands2;
1354
1406
  })(MicdropServerCommands || {});
1407
+ function currentTurn(conversation) {
1408
+ let start = conversation.length;
1409
+ while (start > 0) {
1410
+ const item = conversation[start - 1];
1411
+ if (item.role !== "user" || item.metadata?.classification) break;
1412
+ start--;
1413
+ }
1414
+ const messages = conversation.slice(start);
1415
+ return {
1416
+ transcript: messages.map((message) => message.content).join(" "),
1417
+ messages,
1418
+ before: conversation.slice(0, start)
1419
+ };
1420
+ }
1421
+ function getTurnClassification(conversation) {
1422
+ for (let index = conversation.length - 1; index >= 0; index--) {
1423
+ const item = conversation[index];
1424
+ if (item.role === "user") return item.metadata?.classification;
1425
+ }
1426
+ return void 0;
1427
+ }
1355
1428
 
1356
1429
  // src/client/MicdropClientError.ts
1357
1430
  var MicdropClientErrorCode = /* @__PURE__ */ ((MicdropClientErrorCode2) => {
@@ -1417,6 +1490,9 @@ var MicdropClient = class extends EventEmitter7 {
1417
1490
  this._isMuted = false;
1418
1491
  this._isPaused = false;
1419
1492
  this._isReconnecting = false;
1493
+ // From start() until the connection opens, the microphone included: opening
1494
+ // it can take a while, for the permission prompt or the browser itself
1495
+ this._isStartingCall = false;
1420
1496
  this.reconnectAttempt = 0;
1421
1497
  this.start = async (options) => {
1422
1498
  this.error = void 0;
@@ -1428,18 +1504,25 @@ var MicdropClient = class extends EventEmitter7 {
1428
1504
  this._isMuted = false;
1429
1505
  this._isPaused = false;
1430
1506
  this._isReconnecting = false;
1507
+ this._isStartingCall = true;
1431
1508
  this.reconnectAttempt = 0;
1432
1509
  if (this.reconnectTimer) {
1433
1510
  clearTimeout(this.reconnectTimer);
1434
1511
  this.reconnectTimer = void 0;
1435
1512
  }
1513
+ this.notifyStateChange();
1436
1514
  if (!this.micRecorder || options?.vad) {
1437
1515
  await this.startMic({ vad: options?.vad });
1438
1516
  }
1439
1517
  await this.startWS();
1518
+ if (this.isWSStarted && this._isStartingCall) {
1519
+ this._isStartingCall = false;
1520
+ this.notifyStateChange();
1521
+ }
1440
1522
  };
1441
1523
  this.stop = async () => {
1442
1524
  this.partialAssistantMessage = "";
1525
+ this._isStartingCall = false;
1443
1526
  this._isProcessing = false;
1444
1527
  this._isMuted = false;
1445
1528
  this._isPaused = false;
@@ -1466,7 +1549,7 @@ var MicdropClient = class extends EventEmitter7 {
1466
1549
  this.notifyStateChange();
1467
1550
  };
1468
1551
  this.unmute = () => {
1469
- if (!this.isPaused && !(this.options.disableInterruption && Speaker.isPlaying)) {
1552
+ if (!this.isPaused && !(this.isInterruptionDisabled && Speaker.isPlaying)) {
1470
1553
  this.vad?.resume();
1471
1554
  }
1472
1555
  this._isMuted = false;
@@ -1507,6 +1590,9 @@ var MicdropClient = class extends EventEmitter7 {
1507
1590
  this.options.turnDetector,
1508
1591
  this.options.turnMaxWait
1509
1592
  );
1593
+ this.micRecorder.setSpeakerPlaying(
1594
+ this.isInterruptionDisabled && Speaker.isPlaying
1595
+ );
1510
1596
  this.micRecorder.on("StateChange", () => {
1511
1597
  this.notifyStateChange();
1512
1598
  });
@@ -1538,6 +1624,9 @@ var MicdropClient = class extends EventEmitter7 {
1538
1624
  } else {
1539
1625
  storage.removeItem("micdrop.micDevice" /* MicDevice */);
1540
1626
  }
1627
+ if (Mic.echoCancellation === false && !this.options.disableInterruption) {
1628
+ this.log("No echo cancellation, interruptions are disabled");
1629
+ }
1541
1630
  await this.micRecorder.start(Mic);
1542
1631
  await Speaker.start();
1543
1632
  await this.updateDevices();
@@ -1598,6 +1687,7 @@ var MicdropClient = class extends EventEmitter7 {
1598
1687
  this.connectionTimer = void 0;
1599
1688
  }
1600
1689
  this._isReconnecting = false;
1690
+ this._isStartingCall = false;
1601
1691
  this.reconnectAttempt = 0;
1602
1692
  this.error = void 0;
1603
1693
  this.notifyStateChange();
@@ -1658,6 +1748,19 @@ var MicdropClient = class extends EventEmitter7 {
1658
1748
  } catch (error) {
1659
1749
  console.error("[MicdropClient] Error parsing tool call:", data, error);
1660
1750
  }
1751
+ } else if (data.startsWith("Classification" /* Classification */)) {
1752
+ try {
1753
+ const classification = JSON.parse(
1754
+ data.substring("Classification" /* Classification */.length + 1)
1755
+ );
1756
+ this.emit("Classification", classification);
1757
+ } catch (error) {
1758
+ console.error(
1759
+ "[MicdropClient] Error parsing classification:",
1760
+ data,
1761
+ error
1762
+ );
1763
+ }
1661
1764
  }
1662
1765
  };
1663
1766
  this.onWSClose = (event) => {
@@ -1681,20 +1784,22 @@ var MicdropClient = class extends EventEmitter7 {
1681
1784
  };
1682
1785
  this.onSpeakerStartPlaying = () => {
1683
1786
  this.log("Speaker started");
1684
- if (this.options.disableInterruption) {
1787
+ if (this.isInterruptionDisabled) {
1788
+ this.micRecorder?.setSpeakerPlaying(true);
1685
1789
  this.vad?.pause();
1686
1790
  }
1687
1791
  this.notifyStateChange();
1688
1792
  };
1689
1793
  this.onSpeakerStopPlaying = () => {
1690
1794
  this.log("Speaker stopped");
1691
- if (this.options.disableInterruption) {
1795
+ this.micRecorder?.setSpeakerPlaying(false);
1796
+ if (this.isInterruptionDisabled) {
1692
1797
  setTimeout(() => {
1693
1798
  if (!this.isMuted) {
1694
1799
  this.vad?.resume();
1695
1800
  this.notifyStateChange();
1696
1801
  }
1697
- }, 200);
1802
+ }, ECHO_TAIL);
1698
1803
  } else {
1699
1804
  this.notifyStateChange();
1700
1805
  }
@@ -1710,7 +1815,7 @@ var MicdropClient = class extends EventEmitter7 {
1710
1815
  return (this.isWSStarted || this._isReconnecting) && (this.micRecorder?.state.isStarted || false);
1711
1816
  }
1712
1817
  get isStarting() {
1713
- return (this.isWSStarting || this.micRecorder?.state.isStarting || false) && !this._isReconnecting;
1818
+ return (this._isStartingCall || this.isWSStarting || this.micRecorder?.state.isStarting || false) && !this._isReconnecting;
1714
1819
  }
1715
1820
  get isReconnecting() {
1716
1821
  return this._isReconnecting;
@@ -1745,6 +1850,14 @@ var MicdropClient = class extends EventEmitter7 {
1745
1850
  get isAssistantSpeaking() {
1746
1851
  return Speaker.isPlaying;
1747
1852
  }
1853
+ /**
1854
+ * Whether the user is kept from speaking over the assistant. Without echo
1855
+ * cancellation the microphone hears the assistant, which would interrupt
1856
+ * itself and end up in the transcript.
1857
+ */
1858
+ get isInterruptionDisabled() {
1859
+ return !!this.options.disableInterruption || Mic.echoCancellation === false;
1860
+ }
1748
1861
  get micDeviceId() {
1749
1862
  return Mic.deviceId;
1750
1863
  }
@@ -1895,7 +2008,9 @@ if (!globalScope2.micdropClient) {
1895
2008
  }
1896
2009
  var Micdrop = globalScope2.micdropClient;
1897
2010
  export {
2011
+ CONFIRM_MARGIN,
1898
2012
  DEFAULT_TURN_MAX_WAIT,
2013
+ ECHO_TAIL,
1899
2014
  Mic,
1900
2015
  MicController,
1901
2016
  MicDriver,
@@ -1921,10 +2036,12 @@ export {
1921
2036
  VolumeMeter,
1922
2037
  VolumeVAD,
1923
2038
  concatFloat32,
2039
+ currentTurn,
1924
2040
  equalVADConfig,
1925
2041
  floatToPcm16,
1926
2042
  getClientErrorFromWSCloseEvent,
1927
2043
  getClientErrorFromWSCloseEventCode,
2044
+ getTurnClassification,
1928
2045
  getVAD,
1929
2046
  getVolume,
1930
2047
  isRecoverableError,