@micdrop/client 3.3.1 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/index.d.mts +65 -2
- package/dist/index.d.ts +65 -2
- package/dist/index.js +184 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +184 -2
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -429,6 +429,17 @@ var SpeakerController = class extends EventEmitter2 {
|
|
|
429
429
|
if (!this.driver) return [];
|
|
430
430
|
return this.driver.getDevices();
|
|
431
431
|
}
|
|
432
|
+
/**
|
|
433
|
+
* Plays by another route, for when the microphone hears the current one
|
|
434
|
+
* @returns False if the platform has no other route to offer
|
|
435
|
+
*/
|
|
436
|
+
async useEchoFallback() {
|
|
437
|
+
return await this.driver?.useEchoFallback() ?? false;
|
|
438
|
+
}
|
|
439
|
+
/** Goes back to the usual route */
|
|
440
|
+
resetEchoFallback() {
|
|
441
|
+
this.driver?.resetEchoFallback();
|
|
442
|
+
}
|
|
432
443
|
};
|
|
433
444
|
|
|
434
445
|
// src/audio/MicRecorder.ts
|
|
@@ -931,6 +942,8 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
931
942
|
this.turnListening = false;
|
|
932
943
|
// Bumped whenever the turn moves on, so a late answer is dropped
|
|
933
944
|
this.turnRound = 0;
|
|
945
|
+
// The VAD is being restarted, what it reports on the way is not speech
|
|
946
|
+
this.isDroppingSpeech = false;
|
|
934
947
|
this.start = async (mic) => {
|
|
935
948
|
if (this.state.isStarted) {
|
|
936
949
|
throw new Error("MicRecorder is already started");
|
|
@@ -1054,6 +1067,7 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1054
1067
|
);
|
|
1055
1068
|
};
|
|
1056
1069
|
this.onStartSpeaking = () => {
|
|
1070
|
+
if (this.isDroppingSpeech) return;
|
|
1057
1071
|
this.speakingConfirmed = false;
|
|
1058
1072
|
this.isRecording = true;
|
|
1059
1073
|
this.buffer = [...this.reserve];
|
|
@@ -1067,6 +1081,11 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1067
1081
|
}
|
|
1068
1082
|
};
|
|
1069
1083
|
this.onConfirmSpeaking = () => {
|
|
1084
|
+
if (this.isDroppingSpeech) return;
|
|
1085
|
+
if (!this.turnOpen && this.isEcho?.()) {
|
|
1086
|
+
this.dropSpeech();
|
|
1087
|
+
return;
|
|
1088
|
+
}
|
|
1070
1089
|
this.speakingConfirmed = true;
|
|
1071
1090
|
this.changeState({ isSpeaking: true });
|
|
1072
1091
|
this.cancelPendingClose();
|
|
@@ -1077,12 +1096,14 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1077
1096
|
this.flushChunks();
|
|
1078
1097
|
};
|
|
1079
1098
|
this.onCancelSpeaking = () => {
|
|
1099
|
+
if (this.isDroppingSpeech) return;
|
|
1080
1100
|
this.stopRecording();
|
|
1081
1101
|
if (!this.turnOpen) {
|
|
1082
1102
|
this.turnListening = false;
|
|
1083
1103
|
}
|
|
1084
1104
|
};
|
|
1085
1105
|
this.onStopSpeaking = () => {
|
|
1106
|
+
if (this.isDroppingSpeech) return;
|
|
1086
1107
|
const wasConfirmed = this.speakingConfirmed;
|
|
1087
1108
|
this.changeState({ isSpeaking: false });
|
|
1088
1109
|
if (wasConfirmed) {
|
|
@@ -1094,6 +1115,26 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1094
1115
|
this.state = defaultMicRecorderState;
|
|
1095
1116
|
this.vad = getVAD(vadConfig);
|
|
1096
1117
|
}
|
|
1118
|
+
/**
|
|
1119
|
+
* Forgets the speech heard so far.
|
|
1120
|
+
*
|
|
1121
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
1122
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
1123
|
+
*/
|
|
1124
|
+
async dropSpeech() {
|
|
1125
|
+
this.isDroppingSpeech = true;
|
|
1126
|
+
this.stopRecording();
|
|
1127
|
+
this.turnListening = false;
|
|
1128
|
+
const vad = this.vad;
|
|
1129
|
+
try {
|
|
1130
|
+
await vad.pause();
|
|
1131
|
+
if (!this.speakerPlaying) await vad.resume();
|
|
1132
|
+
} catch (error) {
|
|
1133
|
+
console.error("[Micdrop] Restarting the VAD failed", error);
|
|
1134
|
+
} finally {
|
|
1135
|
+
this.isDroppingSpeech = false;
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1097
1138
|
/**
|
|
1098
1139
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
1099
1140
|
*
|
|
@@ -1221,6 +1262,8 @@ var Pcm16AudioStream = class extends EventEmitter5 {
|
|
|
1221
1262
|
this.prebufferDuration = 0;
|
|
1222
1263
|
this.volumeTimers = [];
|
|
1223
1264
|
this.meter = new VolumeMeter();
|
|
1265
|
+
// Wall time before which no utterance starts, see warmUp()
|
|
1266
|
+
this.holdUntil = 0;
|
|
1224
1267
|
}
|
|
1225
1268
|
/**
|
|
1226
1269
|
* Queues samples for playback
|
|
@@ -1236,6 +1279,34 @@ var Pcm16AudioStream = class extends EventEmitter5 {
|
|
|
1236
1279
|
);
|
|
1237
1280
|
this.handleSamples(samples);
|
|
1238
1281
|
}
|
|
1282
|
+
/**
|
|
1283
|
+
* Plays silence for a while, and holds the next utterance until it is over.
|
|
1284
|
+
*
|
|
1285
|
+
* Firefox mangles what it plays while a Bluetooth headset switches to its
|
|
1286
|
+
* call profile, which the headset does as soon as its microphone opens. The
|
|
1287
|
+
* first answer then comes out all at once. Silence played through that switch
|
|
1288
|
+
* goes unheard.
|
|
1289
|
+
* @param duration - How long to play silence, in milliseconds
|
|
1290
|
+
*/
|
|
1291
|
+
warmUp(duration) {
|
|
1292
|
+
this.holdUntil = Date.now() + duration;
|
|
1293
|
+
const samples = new Float32Array(
|
|
1294
|
+
Math.round(duration / 1e3 * this.sink.sampleRate)
|
|
1295
|
+
);
|
|
1296
|
+
this.warmUpAudio?.stop();
|
|
1297
|
+
this.warmUpAudio = this.sink.schedule(
|
|
1298
|
+
samples,
|
|
1299
|
+
this.sink.currentTime,
|
|
1300
|
+
() => this.warmUpAudio = void 0
|
|
1301
|
+
);
|
|
1302
|
+
}
|
|
1303
|
+
/** Stops playing, for a stream no longer used */
|
|
1304
|
+
destroy() {
|
|
1305
|
+
this.stopAudio();
|
|
1306
|
+
this.warmUpAudio?.stop();
|
|
1307
|
+
this.warmUpAudio = void 0;
|
|
1308
|
+
this.removeAllListeners();
|
|
1309
|
+
}
|
|
1239
1310
|
/** Drops everything queued and stops playing immediately */
|
|
1240
1311
|
stopAudio() {
|
|
1241
1312
|
for (const source of this.sources) {
|
|
@@ -1275,6 +1346,14 @@ var Pcm16AudioStream = class extends EventEmitter5 {
|
|
|
1275
1346
|
clearTimeout(this.quietTimer);
|
|
1276
1347
|
this.quietTimer = void 0;
|
|
1277
1348
|
}
|
|
1349
|
+
const hold = this.holdUntil - Date.now();
|
|
1350
|
+
if (hold > 0) {
|
|
1351
|
+
this.holdTimer ?? (this.holdTimer = setTimeout(() => {
|
|
1352
|
+
this.holdTimer = void 0;
|
|
1353
|
+
this.flushPrebuffer();
|
|
1354
|
+
}, hold));
|
|
1355
|
+
return;
|
|
1356
|
+
}
|
|
1278
1357
|
this.prebuffering = false;
|
|
1279
1358
|
const buffers = this.prebuffer;
|
|
1280
1359
|
this.prebuffer = [];
|
|
@@ -1290,6 +1369,10 @@ var Pcm16AudioStream = class extends EventEmitter5 {
|
|
|
1290
1369
|
clearTimeout(this.endTimer);
|
|
1291
1370
|
this.endTimer = void 0;
|
|
1292
1371
|
}
|
|
1372
|
+
if (this.holdTimer !== void 0) {
|
|
1373
|
+
clearTimeout(this.holdTimer);
|
|
1374
|
+
this.holdTimer = void 0;
|
|
1375
|
+
}
|
|
1293
1376
|
this.prebuffering = true;
|
|
1294
1377
|
this.prebuffer = [];
|
|
1295
1378
|
this.prebufferDuration = 0;
|
|
@@ -1371,6 +1454,16 @@ var MicDriver = class extends EventEmitter6 {
|
|
|
1371
1454
|
}
|
|
1372
1455
|
};
|
|
1373
1456
|
var SpeakerDriver = class extends EventEmitter6 {
|
|
1457
|
+
/**
|
|
1458
|
+
* Plays by another route, for when the microphone hears the current one
|
|
1459
|
+
* @returns False if the platform has no other route to offer
|
|
1460
|
+
*/
|
|
1461
|
+
async useEchoFallback() {
|
|
1462
|
+
return false;
|
|
1463
|
+
}
|
|
1464
|
+
/** Goes back to the usual route */
|
|
1465
|
+
resetEchoFallback() {
|
|
1466
|
+
}
|
|
1374
1467
|
};
|
|
1375
1468
|
|
|
1376
1469
|
// src/audio/index.ts
|
|
@@ -1470,6 +1563,51 @@ function isRecoverableError(error) {
|
|
|
1470
1563
|
return error.code === "Connection" /* Connection */ || error.code === "InternalServer" /* InternalServer */ || error.code === "Unknown" /* Unknown */;
|
|
1471
1564
|
}
|
|
1472
1565
|
|
|
1566
|
+
// src/client/EchoGuard.ts
|
|
1567
|
+
var ECHO_WATCH = 2e3;
|
|
1568
|
+
var EchoGuard = class {
|
|
1569
|
+
constructor() {
|
|
1570
|
+
this.stage = "normal";
|
|
1571
|
+
// The next sound opens a watch, the first of a call or after a new route
|
|
1572
|
+
this.isDue = true;
|
|
1573
|
+
}
|
|
1574
|
+
/** @param now - The current time, in milliseconds */
|
|
1575
|
+
speakerStarted(now) {
|
|
1576
|
+
if (!this.isDue) return;
|
|
1577
|
+
this.isDue = false;
|
|
1578
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1579
|
+
}
|
|
1580
|
+
/** @param now - The current time, in milliseconds */
|
|
1581
|
+
speakerStopped(now) {
|
|
1582
|
+
if (this.watchEnd === void 0) return;
|
|
1583
|
+
if (this.stage === "fallback" && now < this.watchEnd) {
|
|
1584
|
+
this.isDue = true;
|
|
1585
|
+
}
|
|
1586
|
+
this.watchEnd = void 0;
|
|
1587
|
+
}
|
|
1588
|
+
/**
|
|
1589
|
+
* Moves to the next stage if the speech heard can only be echo
|
|
1590
|
+
* @param now - The current time, in milliseconds
|
|
1591
|
+
* @returns True if the speech is to be ignored
|
|
1592
|
+
*/
|
|
1593
|
+
speechHeard(now) {
|
|
1594
|
+
if (this.watchEnd === void 0 || now >= this.watchEnd) return false;
|
|
1595
|
+
if (this.stage === "normal") {
|
|
1596
|
+
this.stage = "fallback";
|
|
1597
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1598
|
+
} else {
|
|
1599
|
+
this.stage = "disabled";
|
|
1600
|
+
this.watchEnd = void 0;
|
|
1601
|
+
}
|
|
1602
|
+
return true;
|
|
1603
|
+
}
|
|
1604
|
+
reset() {
|
|
1605
|
+
this.stage = "normal";
|
|
1606
|
+
this.isDue = true;
|
|
1607
|
+
this.watchEnd = void 0;
|
|
1608
|
+
}
|
|
1609
|
+
};
|
|
1610
|
+
|
|
1473
1611
|
// src/client/MicdropClient.ts
|
|
1474
1612
|
var DEFAULT_RECONNECT_OPTIONS = {
|
|
1475
1613
|
maxAttempts: Infinity,
|
|
@@ -1494,6 +1632,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1494
1632
|
// it can take a while, for the permission prompt or the browser itself
|
|
1495
1633
|
this._isStartingCall = false;
|
|
1496
1634
|
this.reconnectAttempt = 0;
|
|
1635
|
+
this.echoGuard = new EchoGuard();
|
|
1497
1636
|
this.start = async (options) => {
|
|
1498
1637
|
this.error = void 0;
|
|
1499
1638
|
this.options = { ...this.options, ...options };
|
|
@@ -1537,6 +1676,8 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1537
1676
|
console.error("[MicdropClient] Error stopping WebSocket", error);
|
|
1538
1677
|
}
|
|
1539
1678
|
Speaker.stopAudio();
|
|
1679
|
+
this.echoGuard.reset();
|
|
1680
|
+
Speaker.resetEchoFallback();
|
|
1540
1681
|
try {
|
|
1541
1682
|
await this.stopMic();
|
|
1542
1683
|
} catch (error) {
|
|
@@ -1593,6 +1734,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1593
1734
|
this.micRecorder.setSpeakerPlaying(
|
|
1594
1735
|
this.isInterruptionDisabled && Speaker.isPlaying
|
|
1595
1736
|
);
|
|
1737
|
+
this.micRecorder.isEcho = this.isEchoOfAssistant;
|
|
1596
1738
|
this.micRecorder.on("StateChange", () => {
|
|
1597
1739
|
this.notifyStateChange();
|
|
1598
1740
|
});
|
|
@@ -1784,6 +1926,9 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1784
1926
|
};
|
|
1785
1927
|
this.onSpeakerStartPlaying = () => {
|
|
1786
1928
|
this.log("Speaker started");
|
|
1929
|
+
if (this.isWSStarted) {
|
|
1930
|
+
this.echoGuard.speakerStarted(Date.now());
|
|
1931
|
+
}
|
|
1787
1932
|
if (this.isInterruptionDisabled) {
|
|
1788
1933
|
this.micRecorder?.setSpeakerPlaying(true);
|
|
1789
1934
|
this.vad?.pause();
|
|
@@ -1792,6 +1937,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1792
1937
|
};
|
|
1793
1938
|
this.onSpeakerStopPlaying = () => {
|
|
1794
1939
|
this.log("Speaker stopped");
|
|
1940
|
+
this.echoGuard.speakerStopped(Date.now());
|
|
1795
1941
|
this.micRecorder?.setSpeakerPlaying(false);
|
|
1796
1942
|
if (this.isInterruptionDisabled) {
|
|
1797
1943
|
setTimeout(() => {
|
|
@@ -1804,6 +1950,28 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1804
1950
|
this.notifyStateChange();
|
|
1805
1951
|
}
|
|
1806
1952
|
};
|
|
1953
|
+
/**
|
|
1954
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
1955
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
1956
|
+
* neither the speaker nor the server hear of it.
|
|
1957
|
+
*/
|
|
1958
|
+
this.isEchoOfAssistant = () => {
|
|
1959
|
+
if (this.isInterruptionDisabled || !Speaker.isPlaying) return false;
|
|
1960
|
+
if (!this.echoGuard.speechHeard(Date.now())) return false;
|
|
1961
|
+
this.log("Echo of the assistant, moving to", this.echoGuard.stage);
|
|
1962
|
+
if (this.echoGuard.stage === "fallback") {
|
|
1963
|
+
Speaker.useEchoFallback().then((moved) => {
|
|
1964
|
+
if (!moved && this.echoGuard.stage === "fallback") {
|
|
1965
|
+
this.echoGuard.stage = "disabled";
|
|
1966
|
+
this.onInterruptionsGivenUp();
|
|
1967
|
+
}
|
|
1968
|
+
});
|
|
1969
|
+
} else {
|
|
1970
|
+
this.onInterruptionsGivenUp();
|
|
1971
|
+
}
|
|
1972
|
+
this.notifyStateChange();
|
|
1973
|
+
return true;
|
|
1974
|
+
};
|
|
1807
1975
|
Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
|
|
1808
1976
|
Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
|
|
1809
1977
|
Mic.on("DeviceChange", this.updateDevices);
|
|
@@ -1853,10 +2021,15 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1853
2021
|
/**
|
|
1854
2022
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
1855
2023
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
1856
|
-
* itself and end up in the transcript.
|
|
2024
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
2025
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
1857
2026
|
*/
|
|
1858
2027
|
get isInterruptionDisabled() {
|
|
1859
|
-
return !!this.options.disableInterruption || Mic.echoCancellation === false;
|
|
2028
|
+
return !!this.options.disableInterruption || Mic.echoCancellation === false || this.echoGuard.stage === "disabled";
|
|
2029
|
+
}
|
|
2030
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
2031
|
+
get echoStage() {
|
|
2032
|
+
return this.echoGuard.stage;
|
|
1860
2033
|
}
|
|
1861
2034
|
get micDeviceId() {
|
|
1862
2035
|
return Mic.deviceId;
|
|
@@ -1965,6 +2138,15 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1965
2138
|
}
|
|
1966
2139
|
}, config.delayMs);
|
|
1967
2140
|
}
|
|
2141
|
+
// The same as when the speaker starts on a call without interruptions
|
|
2142
|
+
onInterruptionsGivenUp() {
|
|
2143
|
+
this.log("Interruptions disabled until the end of the call");
|
|
2144
|
+
if (Speaker.isPlaying) {
|
|
2145
|
+
this.micRecorder?.setSpeakerPlaying(true);
|
|
2146
|
+
this.vad?.pause();
|
|
2147
|
+
}
|
|
2148
|
+
this.notifyStateChange();
|
|
2149
|
+
}
|
|
1968
2150
|
addMessage(message) {
|
|
1969
2151
|
if (message.role === "assistant") this.partialAssistantMessage = "";
|
|
1970
2152
|
this.conversation = [...this.conversation, message];
|