@micdrop/client 3.2.1 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -223,6 +223,10 @@ var MicController = class extends EventEmitter {
223
223
  get deviceId() {
224
224
  return this.driver?.deviceId;
225
225
  }
226
+ /** Whether echo cancellation is on, undefined when nobody knows */
227
+ get echoCancellation() {
228
+ return this.driver?.echoCancellation;
229
+ }
226
230
  /**
227
231
  * Asks for the microphone permission and starts capturing
228
232
  * @param deviceId - Input device to record from, defaults to the system one
@@ -490,18 +494,19 @@ var MultipleVAD = class extends VAD {
490
494
  constructor(vads) {
491
495
  super();
492
496
  this.name = "MultipleVAD";
497
+ // Every VAD has to hear the speech, but only one has to be sure of it. A
498
+ // short word like "No" is over before the volume VAD gathers the loud
499
+ // reports it needs to confirm, while Silero confirms it right away.
493
500
  this.onStatusChange = () => {
494
501
  const isAllSilence = this.vads.every(
495
502
  (vad) => vad.status === 0 /* Silence */
496
503
  );
497
- const isAllSpeaking = this.vads.every(
498
- (vad) => vad.status === 2 /* Speaking */
499
- );
504
+ const isConfirmed = this.vads.every((vad) => vad.status !== 0 /* Silence */) && this.vads.some((vad) => vad.status === 2 /* Speaking */);
500
505
  switch (this.status) {
501
506
  case 0 /* Silence */:
502
507
  if (isAllSilence) break;
503
508
  this.emit("StartSpeaking");
504
- if (isAllSpeaking) {
509
+ if (isConfirmed) {
505
510
  this.emit("ConfirmSpeaking");
506
511
  }
507
512
  break;
@@ -510,7 +515,7 @@ var MultipleVAD = class extends VAD {
510
515
  this.emit("CancelSpeaking");
511
516
  break;
512
517
  }
513
- if (isAllSpeaking) {
518
+ if (isConfirmed) {
514
519
  this.emit("ConfirmSpeaking");
515
520
  }
516
521
  break;
@@ -522,6 +527,7 @@ var MultipleVAD = class extends VAD {
522
527
  }
523
528
  };
524
529
  this.vads = vads.map((vad) => getVAD(vad));
530
+ this.delay = Math.max(...this.vads.map((vad) => vad.delay));
525
531
  }
526
532
  get isStarted() {
527
533
  return this.vads.some((vad) => vad.isStarted);
@@ -530,18 +536,19 @@ var MultipleVAD = class extends VAD {
530
536
  return this.vads.every((vad) => vad.isPaused);
531
537
  }
532
538
  async start(mic) {
539
+ this.setStatus(0 /* Silence */);
533
540
  for (const vad of this.vads) {
534
- if (vad.isStarted) {
535
- continue;
536
- }
541
+ vad.off("ChangeStatus", this.onStatusChange);
537
542
  vad.on("ChangeStatus", this.onStatusChange);
538
- await vad.start(mic);
543
+ if (!vad.isStarted) {
544
+ await vad.start(mic);
545
+ }
539
546
  }
540
547
  }
541
548
  async stop() {
542
549
  for (const vad of this.vads) {
543
- vad.off("ChangeStatus", this.onStatusChange);
544
550
  await vad.stop();
551
+ vad.off("ChangeStatus", this.onStatusChange);
545
552
  }
546
553
  }
547
554
  async pause() {
@@ -575,10 +582,11 @@ var SileroVAD = class extends VAD {
575
582
  super();
576
583
  this.name = "SileroVAD";
577
584
  this.options = defaultOptions;
578
- // The model decides on the first window it hears speech in, so it is quick.
579
- // The reserve still covers a few windows, since scoring runs asynchronously
580
- // and may lag behind the audio on a busy device.
581
- this.delay = 3 * MS_PER_FRAME;
585
+ // The model scores a window as speech a few windows after the voice began:
586
+ // a soft consonant or a breathy onset stays under the threshold until the
587
+ // vowel comes. Scoring also runs asynchronously and may lag behind the audio
588
+ // on a busy device, so the reserve covers about a third of a second.
589
+ this.delay = 10 * MS_PER_FRAME;
582
590
  this._isPaused = false;
583
591
  this.buffer = [];
584
592
  this.bufferLength = 0;
@@ -747,9 +755,11 @@ var VolumeVAD = class extends VAD {
747
755
  this.options = defaultOptions2;
748
756
  // The microphone reports a level about every 100 ms, and two consecutive loud
749
757
  // ones are needed before a turn opens. Speech can therefore begin up to two
750
- // reports before it is noticed, and a third is kept as a margin: without that
751
- // reserve the first syllable of a sentence never reaches the server.
752
- this.delay = 300;
758
+ // reports before it is noticed. A quiet onset (a soft consonant, a word
759
+ // spoken low) stays under the threshold for a while too, so the reserve adds
760
+ // a margin: without it the first syllable of a sentence never reaches the
761
+ // server.
762
+ this.delay = 500;
753
763
  this.running = false;
754
764
  this._isPaused = false;
755
765
  this.speaking = false;
@@ -892,6 +902,8 @@ function equalVADConfig(a, b) {
892
902
  var SAMPLE_RATE = 16e3;
893
903
  var CHUNK_INTERVAL = 100;
894
904
  var DEFAULT_TURN_MAX_WAIT = 4e3;
905
+ var CONFIRM_MARGIN = 300;
906
+ var ECHO_TAIL = 300;
895
907
  var defaultMicRecorderState = {
896
908
  isStarting: false,
897
909
  isStarted: false,
@@ -910,7 +922,9 @@ var MicRecorder = class extends EventEmitter4 {
910
922
  this.reserveLength = 0;
911
923
  this.buffer = [];
912
924
  this.bufferLength = 0;
913
- this.queuedChunks = [];
925
+ // Samples left before the echo of the speaker has died out
926
+ this.echoSamples = 0;
927
+ this.speakerPlaying = false;
914
928
  // The turn is open between the first confirmed word and the moment the
915
929
  // server is told to answer, which can span several pauses
916
930
  this.turnOpen = false;
@@ -986,7 +1000,30 @@ var MicRecorder = class extends EventEmitter4 {
986
1000
  this.start(mic);
987
1001
  }
988
1002
  };
1003
+ /**
1004
+ * Leaves out what the microphone hears while the speaker plays, so the
1005
+ * assistant is never sent back to the server as if the user had said it.
1006
+ *
1007
+ * Only for calls the user may not interrupt: an interruption relies on echo
1008
+ * cancellation, and needs every word the user says over the assistant.
1009
+ * @param playing - True while the assistant is heard
1010
+ */
1011
+ this.setSpeakerPlaying = (playing) => {
1012
+ if (playing === this.speakerPlaying) return;
1013
+ this.speakerPlaying = playing;
1014
+ if (!playing) {
1015
+ this.echoSamples = Math.round(ECHO_TAIL / 1e3 * this.sourceSampleRate);
1016
+ }
1017
+ };
989
1018
  this.onFrames = (frames, sampleRate) => {
1019
+ if (this.speakerPlaying || this.echoSamples > 0) {
1020
+ this.echoSamples = Math.max(0, this.echoSamples - frames.length);
1021
+ this.reserve = [];
1022
+ this.reserveLength = 0;
1023
+ this.buffer = [];
1024
+ this.bufferLength = 0;
1025
+ return;
1026
+ }
990
1027
  if (this.turnListening) {
991
1028
  this.turnDetector?.push(frames, sampleRate);
992
1029
  }
@@ -996,28 +1033,31 @@ var MicRecorder = class extends EventEmitter4 {
996
1033
  this.reserve = [];
997
1034
  this.reserveLength = 0;
998
1035
  }
999
- if (this.isRecording) {
1000
- this.buffer.push(frames);
1001
- this.bufferLength += frames.length;
1002
- this.flushChunks();
1003
- return;
1004
- }
1005
1036
  this.reserve.push(frames);
1006
1037
  this.reserveLength += frames.length;
1007
- const reserveMax = Math.round(this.vad.delay / 1e3 * sampleRate);
1008
- while (this.reserve.length > 1 && this.reserveLength - this.reserve[0].length >= reserveMax) {
1009
- this.reserveLength -= this.reserve[0].length;
1010
- this.reserve.shift();
1038
+ this.reserveLength = trimFrames(
1039
+ this.reserve,
1040
+ this.reserveLength,
1041
+ Math.round(this.vad.delay / 1e3 * sampleRate)
1042
+ );
1043
+ if (!this.isRecording) return;
1044
+ this.buffer.push(frames);
1045
+ this.bufferLength += frames.length;
1046
+ if (this.speakingConfirmed) {
1047
+ this.flushChunks();
1048
+ return;
1011
1049
  }
1050
+ this.bufferLength = trimFrames(
1051
+ this.buffer,
1052
+ this.bufferLength,
1053
+ Math.round((this.vad.delay + CONFIRM_MARGIN) / 1e3 * sampleRate)
1054
+ );
1012
1055
  };
1013
1056
  this.onStartSpeaking = () => {
1014
1057
  this.speakingConfirmed = false;
1015
- this.queuedChunks.length = 0;
1016
1058
  this.isRecording = true;
1017
- this.buffer = this.reserve;
1059
+ this.buffer = [...this.reserve];
1018
1060
  this.bufferLength = this.reserveLength;
1019
- this.reserve = [];
1020
- this.reserveLength = 0;
1021
1061
  if (this.turnDetector && !this.turnListening) {
1022
1062
  this.turnDetector.reset();
1023
1063
  this.turnListening = true;
@@ -1025,7 +1065,6 @@ var MicRecorder = class extends EventEmitter4 {
1025
1065
  this.turnDetector.push(frames, this.sourceSampleRate);
1026
1066
  }
1027
1067
  }
1028
- this.flushChunks();
1029
1068
  };
1030
1069
  this.onConfirmSpeaking = () => {
1031
1070
  this.speakingConfirmed = true;
@@ -1035,10 +1074,7 @@ var MicRecorder = class extends EventEmitter4 {
1035
1074
  this.turnOpen = true;
1036
1075
  this.emit("StartSpeaking");
1037
1076
  }
1038
- for (const chunk of this.queuedChunks) {
1039
- this.emit("Chunk", chunk);
1040
- }
1041
- this.queuedChunks.length = 0;
1077
+ this.flushChunks();
1042
1078
  };
1043
1079
  this.onCancelSpeaking = () => {
1044
1080
  this.stopRecording();
@@ -1105,7 +1141,6 @@ var MicRecorder = class extends EventEmitter4 {
1105
1141
  stopRecording() {
1106
1142
  this.isRecording = false;
1107
1143
  this.speakingConfirmed = false;
1108
- this.queuedChunks.length = 0;
1109
1144
  this.buffer = [];
1110
1145
  this.bufferLength = 0;
1111
1146
  }
@@ -1117,7 +1152,7 @@ var MicRecorder = class extends EventEmitter4 {
1117
1152
  const chunkLength = Math.round(
1118
1153
  CHUNK_INTERVAL / 1e3 * this.sourceSampleRate
1119
1154
  );
1120
- if (this.bufferLength === 0) return;
1155
+ if (!this.speakingConfirmed || this.bufferLength === 0) return;
1121
1156
  if (this.bufferLength < chunkLength && !final) return;
1122
1157
  const merged = concatFloat32(this.buffer);
1123
1158
  let offset = 0;
@@ -1139,10 +1174,6 @@ var MicRecorder = class extends EventEmitter4 {
1139
1174
  const pcm = floatToPcm16(
1140
1175
  resample(samples, this.sourceSampleRate, SAMPLE_RATE)
1141
1176
  );
1142
- if (!this.speakingConfirmed) {
1143
- this.queuedChunks.push(pcm);
1144
- return;
1145
- }
1146
1177
  this.emit("Chunk", pcm);
1147
1178
  }
1148
1179
  resetBuffers() {
@@ -1154,7 +1185,6 @@ var MicRecorder = class extends EventEmitter4 {
1154
1185
  this.reserveLength = 0;
1155
1186
  this.buffer = [];
1156
1187
  this.bufferLength = 0;
1157
- this.queuedChunks.length = 0;
1158
1188
  }
1159
1189
  changeState(state) {
1160
1190
  const hasChanged = Object.keys(state).some(
@@ -1165,6 +1195,13 @@ var MicRecorder = class extends EventEmitter4 {
1165
1195
  this.emit("StateChange", this.state);
1166
1196
  }
1167
1197
  };
1198
+ function trimFrames(frames, length, max) {
1199
+ while (frames.length > 1 && length - frames[0].length >= max) {
1200
+ length -= frames[0].length;
1201
+ frames.shift();
1202
+ }
1203
+ return length;
1204
+ }
1168
1205
 
1169
1206
  // src/audio/Pcm16AudioStream.ts
1170
1207
  import { EventEmitter as EventEmitter5 } from "eventemitter3";
@@ -1184,6 +1221,8 @@ var Pcm16AudioStream = class extends EventEmitter5 {
1184
1221
  this.prebufferDuration = 0;
1185
1222
  this.volumeTimers = [];
1186
1223
  this.meter = new VolumeMeter();
1224
+ // Wall time before which no utterance starts, see warmUp()
1225
+ this.holdUntil = 0;
1187
1226
  }
1188
1227
  /**
1189
1228
  * Queues samples for playback
@@ -1199,6 +1238,34 @@ var Pcm16AudioStream = class extends EventEmitter5 {
1199
1238
  );
1200
1239
  this.handleSamples(samples);
1201
1240
  }
1241
+ /**
1242
+ * Plays silence for a while, and holds the next utterance until it is over.
1243
+ *
1244
+ * Firefox mangles what it plays while a Bluetooth headset switches to its
1245
+ * call profile, which the headset does as soon as its microphone opens. The
1246
+ * first answer then comes out all at once. Silence played through that switch
1247
+ * goes unheard.
1248
+ * @param duration - How long to play silence, in milliseconds
1249
+ */
1250
+ warmUp(duration) {
1251
+ this.holdUntil = Date.now() + duration;
1252
+ const samples = new Float32Array(
1253
+ Math.round(duration / 1e3 * this.sink.sampleRate)
1254
+ );
1255
+ this.warmUpAudio?.stop();
1256
+ this.warmUpAudio = this.sink.schedule(
1257
+ samples,
1258
+ this.sink.currentTime,
1259
+ () => this.warmUpAudio = void 0
1260
+ );
1261
+ }
1262
+ /** Stops playing, for a stream no longer used */
1263
+ destroy() {
1264
+ this.stopAudio();
1265
+ this.warmUpAudio?.stop();
1266
+ this.warmUpAudio = void 0;
1267
+ this.removeAllListeners();
1268
+ }
1202
1269
  /** Drops everything queued and stops playing immediately */
1203
1270
  stopAudio() {
1204
1271
  for (const source of this.sources) {
@@ -1238,6 +1305,14 @@ var Pcm16AudioStream = class extends EventEmitter5 {
1238
1305
  clearTimeout(this.quietTimer);
1239
1306
  this.quietTimer = void 0;
1240
1307
  }
1308
+ const hold = this.holdUntil - Date.now();
1309
+ if (hold > 0) {
1310
+ this.holdTimer ?? (this.holdTimer = setTimeout(() => {
1311
+ this.holdTimer = void 0;
1312
+ this.flushPrebuffer();
1313
+ }, hold));
1314
+ return;
1315
+ }
1241
1316
  this.prebuffering = false;
1242
1317
  const buffers = this.prebuffer;
1243
1318
  this.prebuffer = [];
@@ -1253,6 +1328,10 @@ var Pcm16AudioStream = class extends EventEmitter5 {
1253
1328
  clearTimeout(this.endTimer);
1254
1329
  this.endTimer = void 0;
1255
1330
  }
1331
+ if (this.holdTimer !== void 0) {
1332
+ clearTimeout(this.holdTimer);
1333
+ this.holdTimer = void 0;
1334
+ }
1256
1335
  this.prebuffering = true;
1257
1336
  this.prebuffer = [];
1258
1337
  this.prebufferDuration = 0;
@@ -1325,6 +1404,13 @@ var Pcm16AudioStream = class extends EventEmitter5 {
1325
1404
  // src/audio/types.ts
1326
1405
  import { EventEmitter as EventEmitter6 } from "eventemitter3";
1327
1406
  var MicDriver = class extends EventEmitter6 {
1407
+ /**
1408
+ * Whether the system removes the speaker from what the microphone hears.
1409
+ * False when it is known to be off, undefined when the system does not say.
1410
+ */
1411
+ get echoCancellation() {
1412
+ return void 0;
1413
+ }
1328
1414
  };
1329
1415
  var SpeakerDriver = class extends EventEmitter6 {
1330
1416
  };
@@ -1357,8 +1443,30 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
1357
1443
  MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
1358
1444
  MicdropServerCommands2["EndCall"] = "EndCall";
1359
1445
  MicdropServerCommands2["ToolCall"] = "ToolCall";
1446
+ MicdropServerCommands2["Classification"] = "Classification";
1360
1447
  return MicdropServerCommands2;
1361
1448
  })(MicdropServerCommands || {});
1449
+ function currentTurn(conversation) {
1450
+ let start = conversation.length;
1451
+ while (start > 0) {
1452
+ const item = conversation[start - 1];
1453
+ if (item.role !== "user" || item.metadata?.classification) break;
1454
+ start--;
1455
+ }
1456
+ const messages = conversation.slice(start);
1457
+ return {
1458
+ transcript: messages.map((message) => message.content).join(" "),
1459
+ messages,
1460
+ before: conversation.slice(0, start)
1461
+ };
1462
+ }
1463
+ function getTurnClassification(conversation) {
1464
+ for (let index = conversation.length - 1; index >= 0; index--) {
1465
+ const item = conversation[index];
1466
+ if (item.role === "user") return item.metadata?.classification;
1467
+ }
1468
+ return void 0;
1469
+ }
1362
1470
 
1363
1471
  // src/client/MicdropClientError.ts
1364
1472
  var MicdropClientErrorCode = /* @__PURE__ */ ((MicdropClientErrorCode2) => {
@@ -1483,7 +1591,7 @@ var MicdropClient = class extends EventEmitter7 {
1483
1591
  this.notifyStateChange();
1484
1592
  };
1485
1593
  this.unmute = () => {
1486
- if (!this.isPaused && !(this.options.disableInterruption && Speaker.isPlaying)) {
1594
+ if (!this.isPaused && !(this.isInterruptionDisabled && Speaker.isPlaying)) {
1487
1595
  this.vad?.resume();
1488
1596
  }
1489
1597
  this._isMuted = false;
@@ -1524,6 +1632,9 @@ var MicdropClient = class extends EventEmitter7 {
1524
1632
  this.options.turnDetector,
1525
1633
  this.options.turnMaxWait
1526
1634
  );
1635
+ this.micRecorder.setSpeakerPlaying(
1636
+ this.isInterruptionDisabled && Speaker.isPlaying
1637
+ );
1527
1638
  this.micRecorder.on("StateChange", () => {
1528
1639
  this.notifyStateChange();
1529
1640
  });
@@ -1555,6 +1666,9 @@ var MicdropClient = class extends EventEmitter7 {
1555
1666
  } else {
1556
1667
  storage.removeItem("micdrop.micDevice" /* MicDevice */);
1557
1668
  }
1669
+ if (Mic.echoCancellation === false && !this.options.disableInterruption) {
1670
+ this.log("No echo cancellation, interruptions are disabled");
1671
+ }
1558
1672
  await this.micRecorder.start(Mic);
1559
1673
  await Speaker.start();
1560
1674
  await this.updateDevices();
@@ -1676,6 +1790,19 @@ var MicdropClient = class extends EventEmitter7 {
1676
1790
  } catch (error) {
1677
1791
  console.error("[MicdropClient] Error parsing tool call:", data, error);
1678
1792
  }
1793
+ } else if (data.startsWith("Classification" /* Classification */)) {
1794
+ try {
1795
+ const classification = JSON.parse(
1796
+ data.substring("Classification" /* Classification */.length + 1)
1797
+ );
1798
+ this.emit("Classification", classification);
1799
+ } catch (error) {
1800
+ console.error(
1801
+ "[MicdropClient] Error parsing classification:",
1802
+ data,
1803
+ error
1804
+ );
1805
+ }
1679
1806
  }
1680
1807
  };
1681
1808
  this.onWSClose = (event) => {
@@ -1699,20 +1826,22 @@ var MicdropClient = class extends EventEmitter7 {
1699
1826
  };
1700
1827
  this.onSpeakerStartPlaying = () => {
1701
1828
  this.log("Speaker started");
1702
- if (this.options.disableInterruption) {
1829
+ if (this.isInterruptionDisabled) {
1830
+ this.micRecorder?.setSpeakerPlaying(true);
1703
1831
  this.vad?.pause();
1704
1832
  }
1705
1833
  this.notifyStateChange();
1706
1834
  };
1707
1835
  this.onSpeakerStopPlaying = () => {
1708
1836
  this.log("Speaker stopped");
1709
- if (this.options.disableInterruption) {
1837
+ this.micRecorder?.setSpeakerPlaying(false);
1838
+ if (this.isInterruptionDisabled) {
1710
1839
  setTimeout(() => {
1711
1840
  if (!this.isMuted) {
1712
1841
  this.vad?.resume();
1713
1842
  this.notifyStateChange();
1714
1843
  }
1715
- }, 200);
1844
+ }, ECHO_TAIL);
1716
1845
  } else {
1717
1846
  this.notifyStateChange();
1718
1847
  }
@@ -1763,6 +1892,14 @@ var MicdropClient = class extends EventEmitter7 {
1763
1892
  get isAssistantSpeaking() {
1764
1893
  return Speaker.isPlaying;
1765
1894
  }
1895
+ /**
1896
+ * Whether the user is kept from speaking over the assistant. Without echo
1897
+ * cancellation the microphone hears the assistant, which would interrupt
1898
+ * itself and end up in the transcript.
1899
+ */
1900
+ get isInterruptionDisabled() {
1901
+ return !!this.options.disableInterruption || Mic.echoCancellation === false;
1902
+ }
1766
1903
  get micDeviceId() {
1767
1904
  return Mic.deviceId;
1768
1905
  }
@@ -1913,7 +2050,9 @@ if (!globalScope2.micdropClient) {
1913
2050
  }
1914
2051
  var Micdrop = globalScope2.micdropClient;
1915
2052
  export {
2053
+ CONFIRM_MARGIN,
1916
2054
  DEFAULT_TURN_MAX_WAIT,
2055
+ ECHO_TAIL,
1917
2056
  Mic,
1918
2057
  MicController,
1919
2058
  MicDriver,
@@ -1939,10 +2078,12 @@ export {
1939
2078
  VolumeMeter,
1940
2079
  VolumeVAD,
1941
2080
  concatFloat32,
2081
+ currentTurn,
1942
2082
  equalVADConfig,
1943
2083
  floatToPcm16,
1944
2084
  getClientErrorFromWSCloseEvent,
1945
2085
  getClientErrorFromWSCloseEventCode,
2086
+ getTurnClassification,
1946
2087
  getVAD,
1947
2088
  getVolume,
1948
2089
  isRecoverableError,