@micdrop/client 3.2.1 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -20,7 +20,9 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
20
20
  // src/index.ts
21
21
  var index_exports = {};
22
22
  __export(index_exports, {
23
+ CONFIRM_MARGIN: () => CONFIRM_MARGIN,
23
24
  DEFAULT_TURN_MAX_WAIT: () => DEFAULT_TURN_MAX_WAIT,
25
+ ECHO_TAIL: () => ECHO_TAIL,
24
26
  Mic: () => Mic,
25
27
  MicController: () => MicController,
26
28
  MicDriver: () => MicDriver,
@@ -46,10 +48,12 @@ __export(index_exports, {
46
48
  VolumeMeter: () => VolumeMeter,
47
49
  VolumeVAD: () => VolumeVAD,
48
50
  concatFloat32: () => concatFloat32,
51
+ currentTurn: () => currentTurn,
49
52
  equalVADConfig: () => equalVADConfig,
50
53
  floatToPcm16: () => floatToPcm16,
51
54
  getClientErrorFromWSCloseEvent: () => getClientErrorFromWSCloseEvent,
52
55
  getClientErrorFromWSCloseEventCode: () => getClientErrorFromWSCloseEventCode,
56
+ getTurnClassification: () => getTurnClassification,
53
57
  getVAD: () => getVAD,
54
58
  getVolume: () => getVolume,
55
59
  isRecoverableError: () => isRecoverableError,
@@ -287,6 +291,10 @@ var MicController = class extends import_eventemitter3.EventEmitter {
287
291
  get deviceId() {
288
292
  return this.driver?.deviceId;
289
293
  }
294
+ /** Whether echo cancellation is on, undefined when nobody knows */
295
+ get echoCancellation() {
296
+ return this.driver?.echoCancellation;
297
+ }
290
298
  /**
291
299
  * Asks for the microphone permission and starts capturing
292
300
  * @param deviceId - Input device to record from, defaults to the system one
@@ -554,18 +562,19 @@ var MultipleVAD = class extends VAD {
554
562
  constructor(vads) {
555
563
  super();
556
564
  this.name = "MultipleVAD";
565
+ // Every VAD has to hear the speech, but only one has to be sure of it. A
566
+ // short word like "No" is over before the volume VAD gathers the loud
567
+ // reports it needs to confirm, while Silero confirms it right away.
557
568
  this.onStatusChange = () => {
558
569
  const isAllSilence = this.vads.every(
559
570
  (vad) => vad.status === 0 /* Silence */
560
571
  );
561
- const isAllSpeaking = this.vads.every(
562
- (vad) => vad.status === 2 /* Speaking */
563
- );
572
+ const isConfirmed = this.vads.every((vad) => vad.status !== 0 /* Silence */) && this.vads.some((vad) => vad.status === 2 /* Speaking */);
564
573
  switch (this.status) {
565
574
  case 0 /* Silence */:
566
575
  if (isAllSilence) break;
567
576
  this.emit("StartSpeaking");
568
- if (isAllSpeaking) {
577
+ if (isConfirmed) {
569
578
  this.emit("ConfirmSpeaking");
570
579
  }
571
580
  break;
@@ -574,7 +583,7 @@ var MultipleVAD = class extends VAD {
574
583
  this.emit("CancelSpeaking");
575
584
  break;
576
585
  }
577
- if (isAllSpeaking) {
586
+ if (isConfirmed) {
578
587
  this.emit("ConfirmSpeaking");
579
588
  }
580
589
  break;
@@ -586,6 +595,7 @@ var MultipleVAD = class extends VAD {
586
595
  }
587
596
  };
588
597
  this.vads = vads.map((vad) => getVAD(vad));
598
+ this.delay = Math.max(...this.vads.map((vad) => vad.delay));
589
599
  }
590
600
  get isStarted() {
591
601
  return this.vads.some((vad) => vad.isStarted);
@@ -594,18 +604,19 @@ var MultipleVAD = class extends VAD {
594
604
  return this.vads.every((vad) => vad.isPaused);
595
605
  }
596
606
  async start(mic) {
607
+ this.setStatus(0 /* Silence */);
597
608
  for (const vad of this.vads) {
598
- if (vad.isStarted) {
599
- continue;
600
- }
609
+ vad.off("ChangeStatus", this.onStatusChange);
601
610
  vad.on("ChangeStatus", this.onStatusChange);
602
- await vad.start(mic);
611
+ if (!vad.isStarted) {
612
+ await vad.start(mic);
613
+ }
603
614
  }
604
615
  }
605
616
  async stop() {
606
617
  for (const vad of this.vads) {
607
- vad.off("ChangeStatus", this.onStatusChange);
608
618
  await vad.stop();
619
+ vad.off("ChangeStatus", this.onStatusChange);
609
620
  }
610
621
  }
611
622
  async pause() {
@@ -639,10 +650,11 @@ var SileroVAD = class extends VAD {
639
650
  super();
640
651
  this.name = "SileroVAD";
641
652
  this.options = defaultOptions;
642
- // The model decides on the first window it hears speech in, so it is quick.
643
- // The reserve still covers a few windows, since scoring runs asynchronously
644
- // and may lag behind the audio on a busy device.
645
- this.delay = 3 * MS_PER_FRAME;
653
+ // The model scores a window as speech a few windows after the voice began:
654
+ // a soft consonant or a breathy onset stays under the threshold until the
655
+ // vowel comes. Scoring also runs asynchronously and may lag behind the audio
656
+ // on a busy device, so the reserve covers about a third of a second.
657
+ this.delay = 10 * MS_PER_FRAME;
646
658
  this._isPaused = false;
647
659
  this.buffer = [];
648
660
  this.bufferLength = 0;
@@ -811,9 +823,11 @@ var VolumeVAD = class extends VAD {
811
823
  this.options = defaultOptions2;
812
824
  // The microphone reports a level about every 100 ms, and two consecutive loud
813
825
  // ones are needed before a turn opens. Speech can therefore begin up to two
814
- // reports before it is noticed, and a third is kept as a margin: without that
815
- // reserve the first syllable of a sentence never reaches the server.
816
- this.delay = 300;
826
+ // reports before it is noticed. A quiet onset (a soft consonant, a word
827
+ // spoken low) stays under the threshold for a while too, so the reserve adds
828
+ // a margin: without it the first syllable of a sentence never reaches the
829
+ // server.
830
+ this.delay = 500;
817
831
  this.running = false;
818
832
  this._isPaused = false;
819
833
  this.speaking = false;
@@ -956,6 +970,8 @@ function equalVADConfig(a, b) {
956
970
  var SAMPLE_RATE = 16e3;
957
971
  var CHUNK_INTERVAL = 100;
958
972
  var DEFAULT_TURN_MAX_WAIT = 4e3;
973
+ var CONFIRM_MARGIN = 300;
974
+ var ECHO_TAIL = 300;
959
975
  var defaultMicRecorderState = {
960
976
  isStarting: false,
961
977
  isStarted: false,
@@ -974,7 +990,9 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
974
990
  this.reserveLength = 0;
975
991
  this.buffer = [];
976
992
  this.bufferLength = 0;
977
- this.queuedChunks = [];
993
+ // Samples left before the echo of the speaker has died out
994
+ this.echoSamples = 0;
995
+ this.speakerPlaying = false;
978
996
  // The turn is open between the first confirmed word and the moment the
979
997
  // server is told to answer, which can span several pauses
980
998
  this.turnOpen = false;
@@ -1050,7 +1068,30 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1050
1068
  this.start(mic);
1051
1069
  }
1052
1070
  };
1071
+ /**
1072
+ * Leaves out what the microphone hears while the speaker plays, so the
1073
+ * assistant is never sent back to the server as if the user had said it.
1074
+ *
1075
+ * Only for calls the user may not interrupt: an interruption relies on echo
1076
+ * cancellation, and needs every word the user says over the assistant.
1077
+ * @param playing - True while the assistant is heard
1078
+ */
1079
+ this.setSpeakerPlaying = (playing) => {
1080
+ if (playing === this.speakerPlaying) return;
1081
+ this.speakerPlaying = playing;
1082
+ if (!playing) {
1083
+ this.echoSamples = Math.round(ECHO_TAIL / 1e3 * this.sourceSampleRate);
1084
+ }
1085
+ };
1053
1086
  this.onFrames = (frames, sampleRate) => {
1087
+ if (this.speakerPlaying || this.echoSamples > 0) {
1088
+ this.echoSamples = Math.max(0, this.echoSamples - frames.length);
1089
+ this.reserve = [];
1090
+ this.reserveLength = 0;
1091
+ this.buffer = [];
1092
+ this.bufferLength = 0;
1093
+ return;
1094
+ }
1054
1095
  if (this.turnListening) {
1055
1096
  this.turnDetector?.push(frames, sampleRate);
1056
1097
  }
@@ -1060,28 +1101,31 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1060
1101
  this.reserve = [];
1061
1102
  this.reserveLength = 0;
1062
1103
  }
1063
- if (this.isRecording) {
1064
- this.buffer.push(frames);
1065
- this.bufferLength += frames.length;
1066
- this.flushChunks();
1067
- return;
1068
- }
1069
1104
  this.reserve.push(frames);
1070
1105
  this.reserveLength += frames.length;
1071
- const reserveMax = Math.round(this.vad.delay / 1e3 * sampleRate);
1072
- while (this.reserve.length > 1 && this.reserveLength - this.reserve[0].length >= reserveMax) {
1073
- this.reserveLength -= this.reserve[0].length;
1074
- this.reserve.shift();
1106
+ this.reserveLength = trimFrames(
1107
+ this.reserve,
1108
+ this.reserveLength,
1109
+ Math.round(this.vad.delay / 1e3 * sampleRate)
1110
+ );
1111
+ if (!this.isRecording) return;
1112
+ this.buffer.push(frames);
1113
+ this.bufferLength += frames.length;
1114
+ if (this.speakingConfirmed) {
1115
+ this.flushChunks();
1116
+ return;
1075
1117
  }
1118
+ this.bufferLength = trimFrames(
1119
+ this.buffer,
1120
+ this.bufferLength,
1121
+ Math.round((this.vad.delay + CONFIRM_MARGIN) / 1e3 * sampleRate)
1122
+ );
1076
1123
  };
1077
1124
  this.onStartSpeaking = () => {
1078
1125
  this.speakingConfirmed = false;
1079
- this.queuedChunks.length = 0;
1080
1126
  this.isRecording = true;
1081
- this.buffer = this.reserve;
1127
+ this.buffer = [...this.reserve];
1082
1128
  this.bufferLength = this.reserveLength;
1083
- this.reserve = [];
1084
- this.reserveLength = 0;
1085
1129
  if (this.turnDetector && !this.turnListening) {
1086
1130
  this.turnDetector.reset();
1087
1131
  this.turnListening = true;
@@ -1089,7 +1133,6 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1089
1133
  this.turnDetector.push(frames, this.sourceSampleRate);
1090
1134
  }
1091
1135
  }
1092
- this.flushChunks();
1093
1136
  };
1094
1137
  this.onConfirmSpeaking = () => {
1095
1138
  this.speakingConfirmed = true;
@@ -1099,10 +1142,7 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1099
1142
  this.turnOpen = true;
1100
1143
  this.emit("StartSpeaking");
1101
1144
  }
1102
- for (const chunk of this.queuedChunks) {
1103
- this.emit("Chunk", chunk);
1104
- }
1105
- this.queuedChunks.length = 0;
1145
+ this.flushChunks();
1106
1146
  };
1107
1147
  this.onCancelSpeaking = () => {
1108
1148
  this.stopRecording();
@@ -1169,7 +1209,6 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1169
1209
  stopRecording() {
1170
1210
  this.isRecording = false;
1171
1211
  this.speakingConfirmed = false;
1172
- this.queuedChunks.length = 0;
1173
1212
  this.buffer = [];
1174
1213
  this.bufferLength = 0;
1175
1214
  }
@@ -1181,7 +1220,7 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1181
1220
  const chunkLength = Math.round(
1182
1221
  CHUNK_INTERVAL / 1e3 * this.sourceSampleRate
1183
1222
  );
1184
- if (this.bufferLength === 0) return;
1223
+ if (!this.speakingConfirmed || this.bufferLength === 0) return;
1185
1224
  if (this.bufferLength < chunkLength && !final) return;
1186
1225
  const merged = concatFloat32(this.buffer);
1187
1226
  let offset = 0;
@@ -1203,10 +1242,6 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1203
1242
  const pcm = floatToPcm16(
1204
1243
  resample(samples, this.sourceSampleRate, SAMPLE_RATE)
1205
1244
  );
1206
- if (!this.speakingConfirmed) {
1207
- this.queuedChunks.push(pcm);
1208
- return;
1209
- }
1210
1245
  this.emit("Chunk", pcm);
1211
1246
  }
1212
1247
  resetBuffers() {
@@ -1218,7 +1253,6 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1218
1253
  this.reserveLength = 0;
1219
1254
  this.buffer = [];
1220
1255
  this.bufferLength = 0;
1221
- this.queuedChunks.length = 0;
1222
1256
  }
1223
1257
  changeState(state) {
1224
1258
  const hasChanged = Object.keys(state).some(
@@ -1229,6 +1263,13 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
1229
1263
  this.emit("StateChange", this.state);
1230
1264
  }
1231
1265
  };
1266
+ function trimFrames(frames, length, max) {
1267
+ while (frames.length > 1 && length - frames[0].length >= max) {
1268
+ length -= frames[0].length;
1269
+ frames.shift();
1270
+ }
1271
+ return length;
1272
+ }
1232
1273
 
1233
1274
  // src/audio/Pcm16AudioStream.ts
1234
1275
  var import_eventemitter35 = require("eventemitter3");
@@ -1389,6 +1430,13 @@ var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
1389
1430
  // src/audio/types.ts
1390
1431
  var import_eventemitter36 = require("eventemitter3");
1391
1432
  var MicDriver = class extends import_eventemitter36.EventEmitter {
1433
+ /**
1434
+ * Whether the system removes the speaker from what the microphone hears.
1435
+ * False when it is known to be off, undefined when the system does not say.
1436
+ */
1437
+ get echoCancellation() {
1438
+ return void 0;
1439
+ }
1392
1440
  };
1393
1441
  var SpeakerDriver = class extends import_eventemitter36.EventEmitter {
1394
1442
  };
@@ -1421,8 +1469,30 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
1421
1469
  MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
1422
1470
  MicdropServerCommands2["EndCall"] = "EndCall";
1423
1471
  MicdropServerCommands2["ToolCall"] = "ToolCall";
1472
+ MicdropServerCommands2["Classification"] = "Classification";
1424
1473
  return MicdropServerCommands2;
1425
1474
  })(MicdropServerCommands || {});
1475
+ function currentTurn(conversation) {
1476
+ let start = conversation.length;
1477
+ while (start > 0) {
1478
+ const item = conversation[start - 1];
1479
+ if (item.role !== "user" || item.metadata?.classification) break;
1480
+ start--;
1481
+ }
1482
+ const messages = conversation.slice(start);
1483
+ return {
1484
+ transcript: messages.map((message) => message.content).join(" "),
1485
+ messages,
1486
+ before: conversation.slice(0, start)
1487
+ };
1488
+ }
1489
+ function getTurnClassification(conversation) {
1490
+ for (let index = conversation.length - 1; index >= 0; index--) {
1491
+ const item = conversation[index];
1492
+ if (item.role === "user") return item.metadata?.classification;
1493
+ }
1494
+ return void 0;
1495
+ }
1426
1496
 
1427
1497
  // src/client/MicdropClientError.ts
1428
1498
  var MicdropClientErrorCode = /* @__PURE__ */ ((MicdropClientErrorCode2) => {
@@ -1547,7 +1617,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1547
1617
  this.notifyStateChange();
1548
1618
  };
1549
1619
  this.unmute = () => {
1550
- if (!this.isPaused && !(this.options.disableInterruption && Speaker.isPlaying)) {
1620
+ if (!this.isPaused && !(this.isInterruptionDisabled && Speaker.isPlaying)) {
1551
1621
  this.vad?.resume();
1552
1622
  }
1553
1623
  this._isMuted = false;
@@ -1588,6 +1658,9 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1588
1658
  this.options.turnDetector,
1589
1659
  this.options.turnMaxWait
1590
1660
  );
1661
+ this.micRecorder.setSpeakerPlaying(
1662
+ this.isInterruptionDisabled && Speaker.isPlaying
1663
+ );
1591
1664
  this.micRecorder.on("StateChange", () => {
1592
1665
  this.notifyStateChange();
1593
1666
  });
@@ -1619,6 +1692,9 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1619
1692
  } else {
1620
1693
  storage.removeItem("micdrop.micDevice" /* MicDevice */);
1621
1694
  }
1695
+ if (Mic.echoCancellation === false && !this.options.disableInterruption) {
1696
+ this.log("No echo cancellation, interruptions are disabled");
1697
+ }
1622
1698
  await this.micRecorder.start(Mic);
1623
1699
  await Speaker.start();
1624
1700
  await this.updateDevices();
@@ -1740,6 +1816,19 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1740
1816
  } catch (error) {
1741
1817
  console.error("[MicdropClient] Error parsing tool call:", data, error);
1742
1818
  }
1819
+ } else if (data.startsWith("Classification" /* Classification */)) {
1820
+ try {
1821
+ const classification = JSON.parse(
1822
+ data.substring("Classification" /* Classification */.length + 1)
1823
+ );
1824
+ this.emit("Classification", classification);
1825
+ } catch (error) {
1826
+ console.error(
1827
+ "[MicdropClient] Error parsing classification:",
1828
+ data,
1829
+ error
1830
+ );
1831
+ }
1743
1832
  }
1744
1833
  };
1745
1834
  this.onWSClose = (event) => {
@@ -1763,20 +1852,22 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1763
1852
  };
1764
1853
  this.onSpeakerStartPlaying = () => {
1765
1854
  this.log("Speaker started");
1766
- if (this.options.disableInterruption) {
1855
+ if (this.isInterruptionDisabled) {
1856
+ this.micRecorder?.setSpeakerPlaying(true);
1767
1857
  this.vad?.pause();
1768
1858
  }
1769
1859
  this.notifyStateChange();
1770
1860
  };
1771
1861
  this.onSpeakerStopPlaying = () => {
1772
1862
  this.log("Speaker stopped");
1773
- if (this.options.disableInterruption) {
1863
+ this.micRecorder?.setSpeakerPlaying(false);
1864
+ if (this.isInterruptionDisabled) {
1774
1865
  setTimeout(() => {
1775
1866
  if (!this.isMuted) {
1776
1867
  this.vad?.resume();
1777
1868
  this.notifyStateChange();
1778
1869
  }
1779
- }, 200);
1870
+ }, ECHO_TAIL);
1780
1871
  } else {
1781
1872
  this.notifyStateChange();
1782
1873
  }
@@ -1827,6 +1918,14 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
1827
1918
  get isAssistantSpeaking() {
1828
1919
  return Speaker.isPlaying;
1829
1920
  }
1921
+ /**
1922
+ * Whether the user is kept from speaking over the assistant. Without echo
1923
+ * cancellation the microphone hears the assistant, which would interrupt
1924
+ * itself and end up in the transcript.
1925
+ */
1926
+ get isInterruptionDisabled() {
1927
+ return !!this.options.disableInterruption || Mic.echoCancellation === false;
1928
+ }
1830
1929
  get micDeviceId() {
1831
1930
  return Mic.deviceId;
1832
1931
  }
@@ -1978,7 +2077,9 @@ if (!globalScope2.micdropClient) {
1978
2077
  var Micdrop = globalScope2.micdropClient;
1979
2078
  // Annotate the CommonJS export names for ESM import in node:
1980
2079
  0 && (module.exports = {
2080
+ CONFIRM_MARGIN,
1981
2081
  DEFAULT_TURN_MAX_WAIT,
2082
+ ECHO_TAIL,
1982
2083
  Mic,
1983
2084
  MicController,
1984
2085
  MicDriver,
@@ -2004,10 +2105,12 @@ var Micdrop = globalScope2.micdropClient;
2004
2105
  VolumeMeter,
2005
2106
  VolumeVAD,
2006
2107
  concatFloat32,
2108
+ currentTurn,
2007
2109
  equalVADConfig,
2008
2110
  floatToPcm16,
2009
2111
  getClientErrorFromWSCloseEvent,
2010
2112
  getClientErrorFromWSCloseEventCode,
2113
+ getTurnClassification,
2011
2114
  getVAD,
2012
2115
  getVolume,
2013
2116
  isRecoverableError,