@micdrop/client 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +152 -49
- package/dist/index.d.ts +152 -49
- package/dist/index.js +142 -6
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +141 -6
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -20,6 +20,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
20
20
|
// src/index.ts
|
|
21
21
|
var index_exports = {};
|
|
22
22
|
__export(index_exports, {
|
|
23
|
+
DEFAULT_TURN_MAX_WAIT: () => DEFAULT_TURN_MAX_WAIT,
|
|
23
24
|
Mic: () => Mic,
|
|
24
25
|
MicController: () => MicController,
|
|
25
26
|
MicDriver: () => MicDriver,
|
|
@@ -626,8 +627,8 @@ var MS_PER_FRAME = SILERO_FRAME_SAMPLES / SILERO_SAMPLE_RATE * 1e3;
|
|
|
626
627
|
var defaultOptions = {
|
|
627
628
|
positiveSpeechThreshold: 0.18,
|
|
628
629
|
negativeSpeechThreshold: 0.11,
|
|
629
|
-
minSpeechFrames:
|
|
630
|
-
redemptionFrames:
|
|
630
|
+
minSpeechFrames: 4,
|
|
631
|
+
redemptionFrames: 8
|
|
631
632
|
};
|
|
632
633
|
var loadModel;
|
|
633
634
|
function setSileroModelLoader(loader) {
|
|
@@ -947,15 +948,18 @@ function equalVADConfig(a, b) {
|
|
|
947
948
|
// src/audio/MicRecorder.ts
|
|
948
949
|
var SAMPLE_RATE = 16e3;
|
|
949
950
|
var CHUNK_INTERVAL = 100;
|
|
951
|
+
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
950
952
|
var defaultMicRecorderState = {
|
|
951
953
|
isStarting: false,
|
|
952
954
|
isStarted: false,
|
|
953
955
|
isSpeaking: false
|
|
954
956
|
};
|
|
955
957
|
var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
956
|
-
constructor(vadConfig) {
|
|
958
|
+
constructor(vadConfig, turnDetector, turnMaxWait = DEFAULT_TURN_MAX_WAIT) {
|
|
957
959
|
super();
|
|
958
960
|
this.vadConfig = vadConfig;
|
|
961
|
+
this.turnDetector = turnDetector;
|
|
962
|
+
this.turnMaxWait = turnMaxWait;
|
|
959
963
|
this.isRecording = false;
|
|
960
964
|
this.speakingConfirmed = false;
|
|
961
965
|
this.sourceSampleRate = SAMPLE_RATE;
|
|
@@ -964,6 +968,12 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
964
968
|
this.buffer = [];
|
|
965
969
|
this.bufferLength = 0;
|
|
966
970
|
this.queuedChunks = [];
|
|
971
|
+
// The turn is open between the first confirmed word and the moment the
|
|
972
|
+
// server is told to answer, which can span several pauses
|
|
973
|
+
this.turnOpen = false;
|
|
974
|
+
this.turnListening = false;
|
|
975
|
+
// Bumped whenever the turn moves on, so a late answer is dropped
|
|
976
|
+
this.turnRound = 0;
|
|
967
977
|
this.start = async (mic) => {
|
|
968
978
|
if (this.state.isStarted) {
|
|
969
979
|
throw new Error("MicRecorder is already started");
|
|
@@ -996,6 +1006,7 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
996
1006
|
isStarted: false,
|
|
997
1007
|
isSpeaking: false
|
|
998
1008
|
});
|
|
1009
|
+
this.cancelPendingClose();
|
|
999
1010
|
try {
|
|
1000
1011
|
this.mic?.off("Frames", this.onFrames);
|
|
1001
1012
|
this.mic = void 0;
|
|
@@ -1009,6 +1020,17 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1009
1020
|
}
|
|
1010
1021
|
this.resetBuffers();
|
|
1011
1022
|
};
|
|
1023
|
+
/**
|
|
1024
|
+
* Changes what decides when a turn is over
|
|
1025
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
1026
|
+
*/
|
|
1027
|
+
this.changeTurnDetector = (turnDetector) => {
|
|
1028
|
+
if (turnDetector === this.turnDetector) return;
|
|
1029
|
+
if (this.turnOpen) {
|
|
1030
|
+
this.endTurn();
|
|
1031
|
+
}
|
|
1032
|
+
this.turnDetector = turnDetector;
|
|
1033
|
+
};
|
|
1012
1034
|
this.changeVad = (vadConfig) => {
|
|
1013
1035
|
if (equalVADConfig(vadConfig, this.vadConfig)) return;
|
|
1014
1036
|
const mic = this.mic;
|
|
@@ -1022,6 +1044,9 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1022
1044
|
}
|
|
1023
1045
|
};
|
|
1024
1046
|
this.onFrames = (frames, sampleRate) => {
|
|
1047
|
+
if (this.turnListening) {
|
|
1048
|
+
this.turnDetector?.push(frames, sampleRate);
|
|
1049
|
+
}
|
|
1025
1050
|
if (sampleRate !== this.sourceSampleRate) {
|
|
1026
1051
|
this.flushChunks();
|
|
1027
1052
|
this.sourceSampleRate = sampleRate;
|
|
@@ -1050,12 +1075,23 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1050
1075
|
this.bufferLength = this.reserveLength;
|
|
1051
1076
|
this.reserve = [];
|
|
1052
1077
|
this.reserveLength = 0;
|
|
1078
|
+
if (this.turnDetector && !this.turnListening) {
|
|
1079
|
+
this.turnDetector.reset();
|
|
1080
|
+
this.turnListening = true;
|
|
1081
|
+
for (const frames of this.buffer) {
|
|
1082
|
+
this.turnDetector.push(frames, this.sourceSampleRate);
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1053
1085
|
this.flushChunks();
|
|
1054
1086
|
};
|
|
1055
1087
|
this.onConfirmSpeaking = () => {
|
|
1056
1088
|
this.speakingConfirmed = true;
|
|
1057
1089
|
this.changeState({ isSpeaking: true });
|
|
1058
|
-
this.
|
|
1090
|
+
this.cancelPendingClose();
|
|
1091
|
+
if (!this.turnOpen) {
|
|
1092
|
+
this.turnOpen = true;
|
|
1093
|
+
this.emit("StartSpeaking");
|
|
1094
|
+
}
|
|
1059
1095
|
for (const chunk of this.queuedChunks) {
|
|
1060
1096
|
this.emit("Chunk", chunk);
|
|
1061
1097
|
}
|
|
@@ -1063,6 +1099,9 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1063
1099
|
};
|
|
1064
1100
|
this.onCancelSpeaking = () => {
|
|
1065
1101
|
this.stopRecording();
|
|
1102
|
+
if (!this.turnOpen) {
|
|
1103
|
+
this.turnListening = false;
|
|
1104
|
+
}
|
|
1066
1105
|
};
|
|
1067
1106
|
this.onStopSpeaking = () => {
|
|
1068
1107
|
const wasConfirmed = this.speakingConfirmed;
|
|
@@ -1071,11 +1110,55 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1071
1110
|
this.flushChunks(true);
|
|
1072
1111
|
}
|
|
1073
1112
|
this.stopRecording();
|
|
1074
|
-
this.
|
|
1113
|
+
this.closeTurn();
|
|
1075
1114
|
};
|
|
1076
1115
|
this.state = defaultMicRecorderState;
|
|
1077
1116
|
this.vad = getVAD(vadConfig);
|
|
1078
1117
|
}
|
|
1118
|
+
/**
|
|
1119
|
+
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
1120
|
+
*
|
|
1121
|
+
* A microphone that is no longer listening ends it straight away: a muted or
|
|
1122
|
+
* paused call has nothing more to say, whatever the sentence sounded like.
|
|
1123
|
+
*/
|
|
1124
|
+
async closeTurn() {
|
|
1125
|
+
if (!this.turnDetector || !this.turnOpen || !this.vad.isStarted || this.vad.isPaused) {
|
|
1126
|
+
this.endTurn();
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
const round = this.turnRound;
|
|
1130
|
+
try {
|
|
1131
|
+
const { complete } = await this.turnDetector.predict();
|
|
1132
|
+
if (round !== this.turnRound) return;
|
|
1133
|
+
if (complete) {
|
|
1134
|
+
this.endTurn();
|
|
1135
|
+
return;
|
|
1136
|
+
}
|
|
1137
|
+
} catch (error) {
|
|
1138
|
+
console.error("[Micdrop] Turn detection failed", error);
|
|
1139
|
+
this.endTurn();
|
|
1140
|
+
return;
|
|
1141
|
+
}
|
|
1142
|
+
this.cancelPendingClose();
|
|
1143
|
+
this.turnTimer = setTimeout(() => {
|
|
1144
|
+
this.turnTimer = void 0;
|
|
1145
|
+
this.endTurn();
|
|
1146
|
+
}, this.turnMaxWait);
|
|
1147
|
+
}
|
|
1148
|
+
endTurn() {
|
|
1149
|
+
this.cancelPendingClose();
|
|
1150
|
+
this.turnOpen = false;
|
|
1151
|
+
this.turnListening = false;
|
|
1152
|
+
this.emit("StopSpeaking");
|
|
1153
|
+
}
|
|
1154
|
+
/** Drops the pending close, both the timer and the answer still owed */
|
|
1155
|
+
cancelPendingClose() {
|
|
1156
|
+
this.turnRound++;
|
|
1157
|
+
if (this.turnTimer) {
|
|
1158
|
+
clearTimeout(this.turnTimer);
|
|
1159
|
+
this.turnTimer = void 0;
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1079
1162
|
stopRecording() {
|
|
1080
1163
|
this.isRecording = false;
|
|
1081
1164
|
this.speakingConfirmed = false;
|
|
@@ -1122,6 +1205,8 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1122
1205
|
resetBuffers() {
|
|
1123
1206
|
this.isRecording = false;
|
|
1124
1207
|
this.speakingConfirmed = false;
|
|
1208
|
+
this.turnOpen = false;
|
|
1209
|
+
this.turnListening = false;
|
|
1125
1210
|
this.reserve = [];
|
|
1126
1211
|
this.reserveLength = 0;
|
|
1127
1212
|
this.buffer = [];
|
|
@@ -1324,6 +1409,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
|
1324
1409
|
})(MicdropClientCommands || {});
|
|
1325
1410
|
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
1326
1411
|
MicdropServerCommands2["Message"] = "Message";
|
|
1412
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
1327
1413
|
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
1328
1414
|
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
1329
1415
|
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
@@ -1386,6 +1472,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1386
1472
|
super();
|
|
1387
1473
|
this.options = options;
|
|
1388
1474
|
this.conversation = [];
|
|
1475
|
+
this.partialAssistantMessage = "";
|
|
1389
1476
|
this.speakerDevices = [];
|
|
1390
1477
|
this.micDevices = [];
|
|
1391
1478
|
this.startTime = 0;
|
|
@@ -1400,6 +1487,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1400
1487
|
this.options = { ...this.options, ...options };
|
|
1401
1488
|
this.startTime = Date.now();
|
|
1402
1489
|
this.conversation = [];
|
|
1490
|
+
this.partialAssistantMessage = "";
|
|
1403
1491
|
this._isProcessing = true;
|
|
1404
1492
|
this._isMuted = false;
|
|
1405
1493
|
this._isPaused = false;
|
|
@@ -1415,6 +1503,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1415
1503
|
await this.startWS();
|
|
1416
1504
|
};
|
|
1417
1505
|
this.stop = async () => {
|
|
1506
|
+
this.partialAssistantMessage = "";
|
|
1418
1507
|
this._isProcessing = false;
|
|
1419
1508
|
this._isMuted = false;
|
|
1420
1509
|
this._isPaused = false;
|
|
@@ -1477,7 +1566,11 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1477
1566
|
this.micRecorder.changeVad(options.vad);
|
|
1478
1567
|
}
|
|
1479
1568
|
} else {
|
|
1480
|
-
this.micRecorder = new MicRecorder(
|
|
1569
|
+
this.micRecorder = new MicRecorder(
|
|
1570
|
+
this.options.vad,
|
|
1571
|
+
this.options.turnDetector,
|
|
1572
|
+
this.options.turnMaxWait
|
|
1573
|
+
);
|
|
1481
1574
|
this.micRecorder.on("StateChange", () => {
|
|
1482
1575
|
this.notifyStateChange();
|
|
1483
1576
|
});
|
|
@@ -1488,6 +1581,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1488
1581
|
this.micRecorder.on("StartSpeaking", () => {
|
|
1489
1582
|
this.log("User start speaking");
|
|
1490
1583
|
this.ws?.send("StartSpeaking" /* StartSpeaking */);
|
|
1584
|
+
this.partialAssistantMessage = "";
|
|
1491
1585
|
this._isProcessing = false;
|
|
1492
1586
|
this.notifyStateChange();
|
|
1493
1587
|
Speaker.stopAudio();
|
|
@@ -1523,6 +1617,28 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1523
1617
|
throw error;
|
|
1524
1618
|
}
|
|
1525
1619
|
};
|
|
1620
|
+
/**
|
|
1621
|
+
* Changes what decides when a turn is over, during a call or before one
|
|
1622
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
1623
|
+
*/
|
|
1624
|
+
this.setTurnDetector = (turnDetector) => {
|
|
1625
|
+
this.options.turnDetector = turnDetector;
|
|
1626
|
+
this.micRecorder?.changeTurnDetector(turnDetector);
|
|
1627
|
+
};
|
|
1628
|
+
/**
|
|
1629
|
+
* Says how long a held turn waits for the rest of the sentence.
|
|
1630
|
+
*
|
|
1631
|
+
* The way out of a wrong verdict: a detector can hear an unfinished sentence
|
|
1632
|
+
* where there is none, and the speaker who never comes back still gets an
|
|
1633
|
+
* answer once this runs out.
|
|
1634
|
+
* @param milliseconds - How long to wait past the detector asking to
|
|
1635
|
+
*/
|
|
1636
|
+
this.setTurnMaxWait = (milliseconds) => {
|
|
1637
|
+
this.options.turnMaxWait = milliseconds;
|
|
1638
|
+
if (this.micRecorder) {
|
|
1639
|
+
this.micRecorder.turnMaxWait = milliseconds;
|
|
1640
|
+
}
|
|
1641
|
+
};
|
|
1526
1642
|
this.changeMicDevice = async (deviceId) => {
|
|
1527
1643
|
await this.startMic({ deviceId });
|
|
1528
1644
|
};
|
|
@@ -1580,9 +1696,12 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1580
1696
|
} catch (error) {
|
|
1581
1697
|
console.error("[MicdropClient] Error parsing message:", data, error);
|
|
1582
1698
|
}
|
|
1699
|
+
} else if (data.startsWith("PartialAssistantMessage" /* PartialAssistantMessage */)) {
|
|
1700
|
+
this.setPartialAssistantMessage(data);
|
|
1583
1701
|
} else if (data === "EndCall" /* EndCall */) {
|
|
1584
1702
|
this.emit("EndCall");
|
|
1585
1703
|
} else if (data === "SkipAnswer" /* SkipAnswer */) {
|
|
1704
|
+
this.partialAssistantMessage = "";
|
|
1586
1705
|
this._isProcessing = false;
|
|
1587
1706
|
this.notifyStateChange();
|
|
1588
1707
|
} else if (data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
|
|
@@ -1710,6 +1829,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1710
1829
|
isMicStarted: this.isMicStarted,
|
|
1711
1830
|
isMicMuted: this.isMicMuted,
|
|
1712
1831
|
conversation: this.conversation,
|
|
1832
|
+
partialAssistantMessage: this.partialAssistantMessage,
|
|
1713
1833
|
error: this.error,
|
|
1714
1834
|
micDeviceId: this.micDeviceId,
|
|
1715
1835
|
speakerDeviceId: this.speakerDeviceId,
|
|
@@ -1797,8 +1917,23 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1797
1917
|
}, config.delayMs);
|
|
1798
1918
|
}
|
|
1799
1919
|
addMessage(message) {
|
|
1920
|
+
if (message.role === "assistant") this.partialAssistantMessage = "";
|
|
1800
1921
|
this.conversation = [...this.conversation, message];
|
|
1801
1922
|
this.notifyStateChange();
|
|
1923
|
+
this.emit("Message", message);
|
|
1924
|
+
}
|
|
1925
|
+
setPartialAssistantMessage(data) {
|
|
1926
|
+
const command = "PartialAssistantMessage" /* PartialAssistantMessage */;
|
|
1927
|
+
let content;
|
|
1928
|
+
try {
|
|
1929
|
+
content = JSON.parse(data.substring(command.length + 1));
|
|
1930
|
+
} catch (error) {
|
|
1931
|
+
console.error("[MicdropClient] Error parsing partial answer:", data);
|
|
1932
|
+
return;
|
|
1933
|
+
}
|
|
1934
|
+
this.partialAssistantMessage = content;
|
|
1935
|
+
this.emit("PartialAssistantMessage", content);
|
|
1936
|
+
this.notifyStateChange();
|
|
1802
1937
|
}
|
|
1803
1938
|
setError(error) {
|
|
1804
1939
|
console.error("[MicdropClient] Error:", error);
|
|
@@ -1825,6 +1960,7 @@ if (!globalScope2.micdropClient) {
|
|
|
1825
1960
|
var Micdrop = globalScope2.micdropClient;
|
|
1826
1961
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1827
1962
|
0 && (module.exports = {
|
|
1963
|
+
DEFAULT_TURN_MAX_WAIT,
|
|
1828
1964
|
Mic,
|
|
1829
1965
|
MicController,
|
|
1830
1966
|
MicDriver,
|