@micdrop/client 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +152 -49
- package/dist/index.d.ts +152 -49
- package/dist/index.js +142 -6
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +141 -6
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -563,8 +563,8 @@ var MS_PER_FRAME = SILERO_FRAME_SAMPLES / SILERO_SAMPLE_RATE * 1e3;
|
|
|
563
563
|
var defaultOptions = {
|
|
564
564
|
positiveSpeechThreshold: 0.18,
|
|
565
565
|
negativeSpeechThreshold: 0.11,
|
|
566
|
-
minSpeechFrames:
|
|
567
|
-
redemptionFrames:
|
|
566
|
+
minSpeechFrames: 4,
|
|
567
|
+
redemptionFrames: 8
|
|
568
568
|
};
|
|
569
569
|
var loadModel;
|
|
570
570
|
function setSileroModelLoader(loader) {
|
|
@@ -884,15 +884,18 @@ function equalVADConfig(a, b) {
|
|
|
884
884
|
// src/audio/MicRecorder.ts
|
|
885
885
|
var SAMPLE_RATE = 16e3;
|
|
886
886
|
var CHUNK_INTERVAL = 100;
|
|
887
|
+
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
887
888
|
var defaultMicRecorderState = {
|
|
888
889
|
isStarting: false,
|
|
889
890
|
isStarted: false,
|
|
890
891
|
isSpeaking: false
|
|
891
892
|
};
|
|
892
893
|
var MicRecorder = class extends EventEmitter4 {
|
|
893
|
-
constructor(vadConfig) {
|
|
894
|
+
constructor(vadConfig, turnDetector, turnMaxWait = DEFAULT_TURN_MAX_WAIT) {
|
|
894
895
|
super();
|
|
895
896
|
this.vadConfig = vadConfig;
|
|
897
|
+
this.turnDetector = turnDetector;
|
|
898
|
+
this.turnMaxWait = turnMaxWait;
|
|
896
899
|
this.isRecording = false;
|
|
897
900
|
this.speakingConfirmed = false;
|
|
898
901
|
this.sourceSampleRate = SAMPLE_RATE;
|
|
@@ -901,6 +904,12 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
901
904
|
this.buffer = [];
|
|
902
905
|
this.bufferLength = 0;
|
|
903
906
|
this.queuedChunks = [];
|
|
907
|
+
// The turn is open between the first confirmed word and the moment the
|
|
908
|
+
// server is told to answer, which can span several pauses
|
|
909
|
+
this.turnOpen = false;
|
|
910
|
+
this.turnListening = false;
|
|
911
|
+
// Bumped whenever the turn moves on, so a late answer is dropped
|
|
912
|
+
this.turnRound = 0;
|
|
904
913
|
this.start = async (mic) => {
|
|
905
914
|
if (this.state.isStarted) {
|
|
906
915
|
throw new Error("MicRecorder is already started");
|
|
@@ -933,6 +942,7 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
933
942
|
isStarted: false,
|
|
934
943
|
isSpeaking: false
|
|
935
944
|
});
|
|
945
|
+
this.cancelPendingClose();
|
|
936
946
|
try {
|
|
937
947
|
this.mic?.off("Frames", this.onFrames);
|
|
938
948
|
this.mic = void 0;
|
|
@@ -946,6 +956,17 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
946
956
|
}
|
|
947
957
|
this.resetBuffers();
|
|
948
958
|
};
|
|
959
|
+
/**
|
|
960
|
+
* Changes what decides when a turn is over
|
|
961
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
962
|
+
*/
|
|
963
|
+
this.changeTurnDetector = (turnDetector) => {
|
|
964
|
+
if (turnDetector === this.turnDetector) return;
|
|
965
|
+
if (this.turnOpen) {
|
|
966
|
+
this.endTurn();
|
|
967
|
+
}
|
|
968
|
+
this.turnDetector = turnDetector;
|
|
969
|
+
};
|
|
949
970
|
this.changeVad = (vadConfig) => {
|
|
950
971
|
if (equalVADConfig(vadConfig, this.vadConfig)) return;
|
|
951
972
|
const mic = this.mic;
|
|
@@ -959,6 +980,9 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
959
980
|
}
|
|
960
981
|
};
|
|
961
982
|
this.onFrames = (frames, sampleRate) => {
|
|
983
|
+
if (this.turnListening) {
|
|
984
|
+
this.turnDetector?.push(frames, sampleRate);
|
|
985
|
+
}
|
|
962
986
|
if (sampleRate !== this.sourceSampleRate) {
|
|
963
987
|
this.flushChunks();
|
|
964
988
|
this.sourceSampleRate = sampleRate;
|
|
@@ -987,12 +1011,23 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
987
1011
|
this.bufferLength = this.reserveLength;
|
|
988
1012
|
this.reserve = [];
|
|
989
1013
|
this.reserveLength = 0;
|
|
1014
|
+
if (this.turnDetector && !this.turnListening) {
|
|
1015
|
+
this.turnDetector.reset();
|
|
1016
|
+
this.turnListening = true;
|
|
1017
|
+
for (const frames of this.buffer) {
|
|
1018
|
+
this.turnDetector.push(frames, this.sourceSampleRate);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
990
1021
|
this.flushChunks();
|
|
991
1022
|
};
|
|
992
1023
|
this.onConfirmSpeaking = () => {
|
|
993
1024
|
this.speakingConfirmed = true;
|
|
994
1025
|
this.changeState({ isSpeaking: true });
|
|
995
|
-
this.
|
|
1026
|
+
this.cancelPendingClose();
|
|
1027
|
+
if (!this.turnOpen) {
|
|
1028
|
+
this.turnOpen = true;
|
|
1029
|
+
this.emit("StartSpeaking");
|
|
1030
|
+
}
|
|
996
1031
|
for (const chunk of this.queuedChunks) {
|
|
997
1032
|
this.emit("Chunk", chunk);
|
|
998
1033
|
}
|
|
@@ -1000,6 +1035,9 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1000
1035
|
};
|
|
1001
1036
|
this.onCancelSpeaking = () => {
|
|
1002
1037
|
this.stopRecording();
|
|
1038
|
+
if (!this.turnOpen) {
|
|
1039
|
+
this.turnListening = false;
|
|
1040
|
+
}
|
|
1003
1041
|
};
|
|
1004
1042
|
this.onStopSpeaking = () => {
|
|
1005
1043
|
const wasConfirmed = this.speakingConfirmed;
|
|
@@ -1008,11 +1046,55 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1008
1046
|
this.flushChunks(true);
|
|
1009
1047
|
}
|
|
1010
1048
|
this.stopRecording();
|
|
1011
|
-
this.
|
|
1049
|
+
this.closeTurn();
|
|
1012
1050
|
};
|
|
1013
1051
|
this.state = defaultMicRecorderState;
|
|
1014
1052
|
this.vad = getVAD(vadConfig);
|
|
1015
1053
|
}
|
|
1054
|
+
/**
|
|
1055
|
+
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
1056
|
+
*
|
|
1057
|
+
* A microphone that is no longer listening ends it straight away: a muted or
|
|
1058
|
+
* paused call has nothing more to say, whatever the sentence sounded like.
|
|
1059
|
+
*/
|
|
1060
|
+
async closeTurn() {
|
|
1061
|
+
if (!this.turnDetector || !this.turnOpen || !this.vad.isStarted || this.vad.isPaused) {
|
|
1062
|
+
this.endTurn();
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
1065
|
+
const round = this.turnRound;
|
|
1066
|
+
try {
|
|
1067
|
+
const { complete } = await this.turnDetector.predict();
|
|
1068
|
+
if (round !== this.turnRound) return;
|
|
1069
|
+
if (complete) {
|
|
1070
|
+
this.endTurn();
|
|
1071
|
+
return;
|
|
1072
|
+
}
|
|
1073
|
+
} catch (error) {
|
|
1074
|
+
console.error("[Micdrop] Turn detection failed", error);
|
|
1075
|
+
this.endTurn();
|
|
1076
|
+
return;
|
|
1077
|
+
}
|
|
1078
|
+
this.cancelPendingClose();
|
|
1079
|
+
this.turnTimer = setTimeout(() => {
|
|
1080
|
+
this.turnTimer = void 0;
|
|
1081
|
+
this.endTurn();
|
|
1082
|
+
}, this.turnMaxWait);
|
|
1083
|
+
}
|
|
1084
|
+
endTurn() {
|
|
1085
|
+
this.cancelPendingClose();
|
|
1086
|
+
this.turnOpen = false;
|
|
1087
|
+
this.turnListening = false;
|
|
1088
|
+
this.emit("StopSpeaking");
|
|
1089
|
+
}
|
|
1090
|
+
/** Drops the pending close, both the timer and the answer still owed */
|
|
1091
|
+
cancelPendingClose() {
|
|
1092
|
+
this.turnRound++;
|
|
1093
|
+
if (this.turnTimer) {
|
|
1094
|
+
clearTimeout(this.turnTimer);
|
|
1095
|
+
this.turnTimer = void 0;
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1016
1098
|
stopRecording() {
|
|
1017
1099
|
this.isRecording = false;
|
|
1018
1100
|
this.speakingConfirmed = false;
|
|
@@ -1059,6 +1141,8 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1059
1141
|
resetBuffers() {
|
|
1060
1142
|
this.isRecording = false;
|
|
1061
1143
|
this.speakingConfirmed = false;
|
|
1144
|
+
this.turnOpen = false;
|
|
1145
|
+
this.turnListening = false;
|
|
1062
1146
|
this.reserve = [];
|
|
1063
1147
|
this.reserveLength = 0;
|
|
1064
1148
|
this.buffer = [];
|
|
@@ -1261,6 +1345,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
|
1261
1345
|
})(MicdropClientCommands || {});
|
|
1262
1346
|
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
1263
1347
|
MicdropServerCommands2["Message"] = "Message";
|
|
1348
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
1264
1349
|
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
1265
1350
|
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
1266
1351
|
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
@@ -1323,6 +1408,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1323
1408
|
super();
|
|
1324
1409
|
this.options = options;
|
|
1325
1410
|
this.conversation = [];
|
|
1411
|
+
this.partialAssistantMessage = "";
|
|
1326
1412
|
this.speakerDevices = [];
|
|
1327
1413
|
this.micDevices = [];
|
|
1328
1414
|
this.startTime = 0;
|
|
@@ -1337,6 +1423,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1337
1423
|
this.options = { ...this.options, ...options };
|
|
1338
1424
|
this.startTime = Date.now();
|
|
1339
1425
|
this.conversation = [];
|
|
1426
|
+
this.partialAssistantMessage = "";
|
|
1340
1427
|
this._isProcessing = true;
|
|
1341
1428
|
this._isMuted = false;
|
|
1342
1429
|
this._isPaused = false;
|
|
@@ -1352,6 +1439,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1352
1439
|
await this.startWS();
|
|
1353
1440
|
};
|
|
1354
1441
|
this.stop = async () => {
|
|
1442
|
+
this.partialAssistantMessage = "";
|
|
1355
1443
|
this._isProcessing = false;
|
|
1356
1444
|
this._isMuted = false;
|
|
1357
1445
|
this._isPaused = false;
|
|
@@ -1414,7 +1502,11 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1414
1502
|
this.micRecorder.changeVad(options.vad);
|
|
1415
1503
|
}
|
|
1416
1504
|
} else {
|
|
1417
|
-
this.micRecorder = new MicRecorder(
|
|
1505
|
+
this.micRecorder = new MicRecorder(
|
|
1506
|
+
this.options.vad,
|
|
1507
|
+
this.options.turnDetector,
|
|
1508
|
+
this.options.turnMaxWait
|
|
1509
|
+
);
|
|
1418
1510
|
this.micRecorder.on("StateChange", () => {
|
|
1419
1511
|
this.notifyStateChange();
|
|
1420
1512
|
});
|
|
@@ -1425,6 +1517,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1425
1517
|
this.micRecorder.on("StartSpeaking", () => {
|
|
1426
1518
|
this.log("User start speaking");
|
|
1427
1519
|
this.ws?.send("StartSpeaking" /* StartSpeaking */);
|
|
1520
|
+
this.partialAssistantMessage = "";
|
|
1428
1521
|
this._isProcessing = false;
|
|
1429
1522
|
this.notifyStateChange();
|
|
1430
1523
|
Speaker.stopAudio();
|
|
@@ -1460,6 +1553,28 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1460
1553
|
throw error;
|
|
1461
1554
|
}
|
|
1462
1555
|
};
|
|
1556
|
+
/**
|
|
1557
|
+
* Changes what decides when a turn is over, during a call or before one
|
|
1558
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
1559
|
+
*/
|
|
1560
|
+
this.setTurnDetector = (turnDetector) => {
|
|
1561
|
+
this.options.turnDetector = turnDetector;
|
|
1562
|
+
this.micRecorder?.changeTurnDetector(turnDetector);
|
|
1563
|
+
};
|
|
1564
|
+
/**
|
|
1565
|
+
* Says how long a held turn waits for the rest of the sentence.
|
|
1566
|
+
*
|
|
1567
|
+
* The way out of a wrong verdict: a detector can hear an unfinished sentence
|
|
1568
|
+
* where there is none, and the speaker who never comes back still gets an
|
|
1569
|
+
* answer once this runs out.
|
|
1570
|
+
* @param milliseconds - How long to wait past the detector asking to
|
|
1571
|
+
*/
|
|
1572
|
+
this.setTurnMaxWait = (milliseconds) => {
|
|
1573
|
+
this.options.turnMaxWait = milliseconds;
|
|
1574
|
+
if (this.micRecorder) {
|
|
1575
|
+
this.micRecorder.turnMaxWait = milliseconds;
|
|
1576
|
+
}
|
|
1577
|
+
};
|
|
1463
1578
|
this.changeMicDevice = async (deviceId) => {
|
|
1464
1579
|
await this.startMic({ deviceId });
|
|
1465
1580
|
};
|
|
@@ -1517,9 +1632,12 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1517
1632
|
} catch (error) {
|
|
1518
1633
|
console.error("[MicdropClient] Error parsing message:", data, error);
|
|
1519
1634
|
}
|
|
1635
|
+
} else if (data.startsWith("PartialAssistantMessage" /* PartialAssistantMessage */)) {
|
|
1636
|
+
this.setPartialAssistantMessage(data);
|
|
1520
1637
|
} else if (data === "EndCall" /* EndCall */) {
|
|
1521
1638
|
this.emit("EndCall");
|
|
1522
1639
|
} else if (data === "SkipAnswer" /* SkipAnswer */) {
|
|
1640
|
+
this.partialAssistantMessage = "";
|
|
1523
1641
|
this._isProcessing = false;
|
|
1524
1642
|
this.notifyStateChange();
|
|
1525
1643
|
} else if (data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
|
|
@@ -1647,6 +1765,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1647
1765
|
isMicStarted: this.isMicStarted,
|
|
1648
1766
|
isMicMuted: this.isMicMuted,
|
|
1649
1767
|
conversation: this.conversation,
|
|
1768
|
+
partialAssistantMessage: this.partialAssistantMessage,
|
|
1650
1769
|
error: this.error,
|
|
1651
1770
|
micDeviceId: this.micDeviceId,
|
|
1652
1771
|
speakerDeviceId: this.speakerDeviceId,
|
|
@@ -1734,8 +1853,23 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1734
1853
|
}, config.delayMs);
|
|
1735
1854
|
}
|
|
1736
1855
|
addMessage(message) {
|
|
1856
|
+
if (message.role === "assistant") this.partialAssistantMessage = "";
|
|
1737
1857
|
this.conversation = [...this.conversation, message];
|
|
1738
1858
|
this.notifyStateChange();
|
|
1859
|
+
this.emit("Message", message);
|
|
1860
|
+
}
|
|
1861
|
+
setPartialAssistantMessage(data) {
|
|
1862
|
+
const command = "PartialAssistantMessage" /* PartialAssistantMessage */;
|
|
1863
|
+
let content;
|
|
1864
|
+
try {
|
|
1865
|
+
content = JSON.parse(data.substring(command.length + 1));
|
|
1866
|
+
} catch (error) {
|
|
1867
|
+
console.error("[MicdropClient] Error parsing partial answer:", data);
|
|
1868
|
+
return;
|
|
1869
|
+
}
|
|
1870
|
+
this.partialAssistantMessage = content;
|
|
1871
|
+
this.emit("PartialAssistantMessage", content);
|
|
1872
|
+
this.notifyStateChange();
|
|
1739
1873
|
}
|
|
1740
1874
|
setError(error) {
|
|
1741
1875
|
console.error("[MicdropClient] Error:", error);
|
|
@@ -1761,6 +1895,7 @@ if (!globalScope2.micdropClient) {
|
|
|
1761
1895
|
}
|
|
1762
1896
|
var Micdrop = globalScope2.micdropClient;
|
|
1763
1897
|
export {
|
|
1898
|
+
DEFAULT_TURN_MAX_WAIT,
|
|
1764
1899
|
Mic,
|
|
1765
1900
|
MicController,
|
|
1766
1901
|
MicDriver,
|