@micdrop/client 3.4.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/index.d.mts +50 -2
- package/dist/index.d.ts +50 -2
- package/dist/index.js +142 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +142 -2
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -429,6 +429,17 @@ var SpeakerController = class extends EventEmitter2 {
|
|
|
429
429
|
if (!this.driver) return [];
|
|
430
430
|
return this.driver.getDevices();
|
|
431
431
|
}
|
|
432
|
+
/**
|
|
433
|
+
* Plays by another route, for when the microphone hears the current one
|
|
434
|
+
* @returns False if the platform has no other route to offer
|
|
435
|
+
*/
|
|
436
|
+
async useEchoFallback() {
|
|
437
|
+
return await this.driver?.useEchoFallback() ?? false;
|
|
438
|
+
}
|
|
439
|
+
/** Goes back to the usual route */
|
|
440
|
+
resetEchoFallback() {
|
|
441
|
+
this.driver?.resetEchoFallback();
|
|
442
|
+
}
|
|
432
443
|
};
|
|
433
444
|
|
|
434
445
|
// src/audio/MicRecorder.ts
|
|
@@ -931,6 +942,8 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
931
942
|
this.turnListening = false;
|
|
932
943
|
// Bumped whenever the turn moves on, so a late answer is dropped
|
|
933
944
|
this.turnRound = 0;
|
|
945
|
+
// The VAD is being restarted, what it reports on the way is not speech
|
|
946
|
+
this.isDroppingSpeech = false;
|
|
934
947
|
this.start = async (mic) => {
|
|
935
948
|
if (this.state.isStarted) {
|
|
936
949
|
throw new Error("MicRecorder is already started");
|
|
@@ -1054,6 +1067,7 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1054
1067
|
);
|
|
1055
1068
|
};
|
|
1056
1069
|
this.onStartSpeaking = () => {
|
|
1070
|
+
if (this.isDroppingSpeech) return;
|
|
1057
1071
|
this.speakingConfirmed = false;
|
|
1058
1072
|
this.isRecording = true;
|
|
1059
1073
|
this.buffer = [...this.reserve];
|
|
@@ -1067,6 +1081,11 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1067
1081
|
}
|
|
1068
1082
|
};
|
|
1069
1083
|
this.onConfirmSpeaking = () => {
|
|
1084
|
+
if (this.isDroppingSpeech) return;
|
|
1085
|
+
if (!this.turnOpen && this.isEcho?.()) {
|
|
1086
|
+
this.dropSpeech();
|
|
1087
|
+
return;
|
|
1088
|
+
}
|
|
1070
1089
|
this.speakingConfirmed = true;
|
|
1071
1090
|
this.changeState({ isSpeaking: true });
|
|
1072
1091
|
this.cancelPendingClose();
|
|
@@ -1077,12 +1096,14 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1077
1096
|
this.flushChunks();
|
|
1078
1097
|
};
|
|
1079
1098
|
this.onCancelSpeaking = () => {
|
|
1099
|
+
if (this.isDroppingSpeech) return;
|
|
1080
1100
|
this.stopRecording();
|
|
1081
1101
|
if (!this.turnOpen) {
|
|
1082
1102
|
this.turnListening = false;
|
|
1083
1103
|
}
|
|
1084
1104
|
};
|
|
1085
1105
|
this.onStopSpeaking = () => {
|
|
1106
|
+
if (this.isDroppingSpeech) return;
|
|
1086
1107
|
const wasConfirmed = this.speakingConfirmed;
|
|
1087
1108
|
this.changeState({ isSpeaking: false });
|
|
1088
1109
|
if (wasConfirmed) {
|
|
@@ -1094,6 +1115,26 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1094
1115
|
this.state = defaultMicRecorderState;
|
|
1095
1116
|
this.vad = getVAD(vadConfig);
|
|
1096
1117
|
}
|
|
1118
|
+
/**
|
|
1119
|
+
* Forgets the speech heard so far.
|
|
1120
|
+
*
|
|
1121
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
1122
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
1123
|
+
*/
|
|
1124
|
+
async dropSpeech() {
|
|
1125
|
+
this.isDroppingSpeech = true;
|
|
1126
|
+
this.stopRecording();
|
|
1127
|
+
this.turnListening = false;
|
|
1128
|
+
const vad = this.vad;
|
|
1129
|
+
try {
|
|
1130
|
+
await vad.pause();
|
|
1131
|
+
if (!this.speakerPlaying) await vad.resume();
|
|
1132
|
+
} catch (error) {
|
|
1133
|
+
console.error("[Micdrop] Restarting the VAD failed", error);
|
|
1134
|
+
} finally {
|
|
1135
|
+
this.isDroppingSpeech = false;
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1097
1138
|
/**
|
|
1098
1139
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
1099
1140
|
*
|
|
@@ -1413,6 +1454,16 @@ var MicDriver = class extends EventEmitter6 {
|
|
|
1413
1454
|
}
|
|
1414
1455
|
};
|
|
1415
1456
|
var SpeakerDriver = class extends EventEmitter6 {
|
|
1457
|
+
/**
|
|
1458
|
+
* Plays by another route, for when the microphone hears the current one
|
|
1459
|
+
* @returns False if the platform has no other route to offer
|
|
1460
|
+
*/
|
|
1461
|
+
async useEchoFallback() {
|
|
1462
|
+
return false;
|
|
1463
|
+
}
|
|
1464
|
+
/** Goes back to the usual route */
|
|
1465
|
+
resetEchoFallback() {
|
|
1466
|
+
}
|
|
1416
1467
|
};
|
|
1417
1468
|
|
|
1418
1469
|
// src/audio/index.ts
|
|
@@ -1512,6 +1563,51 @@ function isRecoverableError(error) {
|
|
|
1512
1563
|
return error.code === "Connection" /* Connection */ || error.code === "InternalServer" /* InternalServer */ || error.code === "Unknown" /* Unknown */;
|
|
1513
1564
|
}
|
|
1514
1565
|
|
|
1566
|
+
// src/client/EchoGuard.ts
|
|
1567
|
+
var ECHO_WATCH = 2e3;
|
|
1568
|
+
var EchoGuard = class {
|
|
1569
|
+
constructor() {
|
|
1570
|
+
this.stage = "normal";
|
|
1571
|
+
// The next sound opens a watch, the first of a call or after a new route
|
|
1572
|
+
this.isDue = true;
|
|
1573
|
+
}
|
|
1574
|
+
/** @param now - The current time, in milliseconds */
|
|
1575
|
+
speakerStarted(now) {
|
|
1576
|
+
if (!this.isDue) return;
|
|
1577
|
+
this.isDue = false;
|
|
1578
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1579
|
+
}
|
|
1580
|
+
/** @param now - The current time, in milliseconds */
|
|
1581
|
+
speakerStopped(now) {
|
|
1582
|
+
if (this.watchEnd === void 0) return;
|
|
1583
|
+
if (this.stage === "fallback" && now < this.watchEnd) {
|
|
1584
|
+
this.isDue = true;
|
|
1585
|
+
}
|
|
1586
|
+
this.watchEnd = void 0;
|
|
1587
|
+
}
|
|
1588
|
+
/**
|
|
1589
|
+
* Moves to the next stage if the speech heard can only be echo
|
|
1590
|
+
* @param now - The current time, in milliseconds
|
|
1591
|
+
* @returns True if the speech is to be ignored
|
|
1592
|
+
*/
|
|
1593
|
+
speechHeard(now) {
|
|
1594
|
+
if (this.watchEnd === void 0 || now >= this.watchEnd) return false;
|
|
1595
|
+
if (this.stage === "normal") {
|
|
1596
|
+
this.stage = "fallback";
|
|
1597
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1598
|
+
} else {
|
|
1599
|
+
this.stage = "disabled";
|
|
1600
|
+
this.watchEnd = void 0;
|
|
1601
|
+
}
|
|
1602
|
+
return true;
|
|
1603
|
+
}
|
|
1604
|
+
reset() {
|
|
1605
|
+
this.stage = "normal";
|
|
1606
|
+
this.isDue = true;
|
|
1607
|
+
this.watchEnd = void 0;
|
|
1608
|
+
}
|
|
1609
|
+
};
|
|
1610
|
+
|
|
1515
1611
|
// src/client/MicdropClient.ts
|
|
1516
1612
|
var DEFAULT_RECONNECT_OPTIONS = {
|
|
1517
1613
|
maxAttempts: Infinity,
|
|
@@ -1536,6 +1632,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1536
1632
|
// it can take a while, for the permission prompt or the browser itself
|
|
1537
1633
|
this._isStartingCall = false;
|
|
1538
1634
|
this.reconnectAttempt = 0;
|
|
1635
|
+
this.echoGuard = new EchoGuard();
|
|
1539
1636
|
this.start = async (options) => {
|
|
1540
1637
|
this.error = void 0;
|
|
1541
1638
|
this.options = { ...this.options, ...options };
|
|
@@ -1579,6 +1676,8 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1579
1676
|
console.error("[MicdropClient] Error stopping WebSocket", error);
|
|
1580
1677
|
}
|
|
1581
1678
|
Speaker.stopAudio();
|
|
1679
|
+
this.echoGuard.reset();
|
|
1680
|
+
Speaker.resetEchoFallback();
|
|
1582
1681
|
try {
|
|
1583
1682
|
await this.stopMic();
|
|
1584
1683
|
} catch (error) {
|
|
@@ -1635,6 +1734,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1635
1734
|
this.micRecorder.setSpeakerPlaying(
|
|
1636
1735
|
this.isInterruptionDisabled && Speaker.isPlaying
|
|
1637
1736
|
);
|
|
1737
|
+
this.micRecorder.isEcho = this.isEchoOfAssistant;
|
|
1638
1738
|
this.micRecorder.on("StateChange", () => {
|
|
1639
1739
|
this.notifyStateChange();
|
|
1640
1740
|
});
|
|
@@ -1826,6 +1926,9 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1826
1926
|
};
|
|
1827
1927
|
this.onSpeakerStartPlaying = () => {
|
|
1828
1928
|
this.log("Speaker started");
|
|
1929
|
+
if (this.isWSStarted) {
|
|
1930
|
+
this.echoGuard.speakerStarted(Date.now());
|
|
1931
|
+
}
|
|
1829
1932
|
if (this.isInterruptionDisabled) {
|
|
1830
1933
|
this.micRecorder?.setSpeakerPlaying(true);
|
|
1831
1934
|
this.vad?.pause();
|
|
@@ -1834,6 +1937,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1834
1937
|
};
|
|
1835
1938
|
this.onSpeakerStopPlaying = () => {
|
|
1836
1939
|
this.log("Speaker stopped");
|
|
1940
|
+
this.echoGuard.speakerStopped(Date.now());
|
|
1837
1941
|
this.micRecorder?.setSpeakerPlaying(false);
|
|
1838
1942
|
if (this.isInterruptionDisabled) {
|
|
1839
1943
|
setTimeout(() => {
|
|
@@ -1846,6 +1950,28 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1846
1950
|
this.notifyStateChange();
|
|
1847
1951
|
}
|
|
1848
1952
|
};
|
|
1953
|
+
/**
|
|
1954
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
1955
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
1956
|
+
* neither the speaker nor the server hear of it.
|
|
1957
|
+
*/
|
|
1958
|
+
this.isEchoOfAssistant = () => {
|
|
1959
|
+
if (this.isInterruptionDisabled || !Speaker.isPlaying) return false;
|
|
1960
|
+
if (!this.echoGuard.speechHeard(Date.now())) return false;
|
|
1961
|
+
this.log("Echo of the assistant, moving to", this.echoGuard.stage);
|
|
1962
|
+
if (this.echoGuard.stage === "fallback") {
|
|
1963
|
+
Speaker.useEchoFallback().then((moved) => {
|
|
1964
|
+
if (!moved && this.echoGuard.stage === "fallback") {
|
|
1965
|
+
this.echoGuard.stage = "disabled";
|
|
1966
|
+
this.onInterruptionsGivenUp();
|
|
1967
|
+
}
|
|
1968
|
+
});
|
|
1969
|
+
} else {
|
|
1970
|
+
this.onInterruptionsGivenUp();
|
|
1971
|
+
}
|
|
1972
|
+
this.notifyStateChange();
|
|
1973
|
+
return true;
|
|
1974
|
+
};
|
|
1849
1975
|
Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
|
|
1850
1976
|
Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
|
|
1851
1977
|
Mic.on("DeviceChange", this.updateDevices);
|
|
@@ -1895,10 +2021,15 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1895
2021
|
/**
|
|
1896
2022
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
1897
2023
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
1898
|
-
* itself and end up in the transcript.
|
|
2024
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
2025
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
1899
2026
|
*/
|
|
1900
2027
|
get isInterruptionDisabled() {
|
|
1901
|
-
return !!this.options.disableInterruption || Mic.echoCancellation === false;
|
|
2028
|
+
return !!this.options.disableInterruption || Mic.echoCancellation === false || this.echoGuard.stage === "disabled";
|
|
2029
|
+
}
|
|
2030
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
2031
|
+
get echoStage() {
|
|
2032
|
+
return this.echoGuard.stage;
|
|
1902
2033
|
}
|
|
1903
2034
|
get micDeviceId() {
|
|
1904
2035
|
return Mic.deviceId;
|
|
@@ -2007,6 +2138,15 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
2007
2138
|
}
|
|
2008
2139
|
}, config.delayMs);
|
|
2009
2140
|
}
|
|
2141
|
+
// The same as when the speaker starts on a call without interruptions
|
|
2142
|
+
onInterruptionsGivenUp() {
|
|
2143
|
+
this.log("Interruptions disabled until the end of the call");
|
|
2144
|
+
if (Speaker.isPlaying) {
|
|
2145
|
+
this.micRecorder?.setSpeakerPlaying(true);
|
|
2146
|
+
this.vad?.pause();
|
|
2147
|
+
}
|
|
2148
|
+
this.notifyStateChange();
|
|
2149
|
+
}
|
|
2010
2150
|
addMessage(message) {
|
|
2011
2151
|
if (message.role === "assistant") this.partialAssistantMessage = "";
|
|
2012
2152
|
this.conversation = [...this.conversation, message];
|