@micdrop/client 3.4.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/index.d.mts +158 -6
- package/dist/index.d.ts +158 -6
- package/dist/index.js +398 -78
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +396 -78
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -429,6 +429,17 @@ var SpeakerController = class extends EventEmitter2 {
|
|
|
429
429
|
if (!this.driver) return [];
|
|
430
430
|
return this.driver.getDevices();
|
|
431
431
|
}
|
|
432
|
+
/**
|
|
433
|
+
* Plays by another route, for when the microphone hears the current one
|
|
434
|
+
* @returns False if the platform has no other route to offer
|
|
435
|
+
*/
|
|
436
|
+
async useEchoFallback() {
|
|
437
|
+
return await this.driver?.useEchoFallback() ?? false;
|
|
438
|
+
}
|
|
439
|
+
/** Goes back to the usual route */
|
|
440
|
+
resetEchoFallback() {
|
|
441
|
+
this.driver?.resetEchoFallback();
|
|
442
|
+
}
|
|
432
443
|
};
|
|
433
444
|
|
|
434
445
|
// src/audio/MicRecorder.ts
|
|
@@ -598,6 +609,9 @@ var SileroVAD = class extends VAD {
|
|
|
598
609
|
// Bumped whenever the state is reset, so a window scored before that is
|
|
599
610
|
// dropped rather than reopening a turn
|
|
600
611
|
this.round = 0;
|
|
612
|
+
// Bumped by every start and stop, so a start overtaken while its model
|
|
613
|
+
// loaded lets go of it
|
|
614
|
+
this.startId = 0;
|
|
601
615
|
this.onFrames = (frames, sampleRate) => {
|
|
602
616
|
if (this._isPaused || !this.model) return;
|
|
603
617
|
if (sampleRate !== this.bufferRate) {
|
|
@@ -633,19 +647,33 @@ var SileroVAD = class extends VAD {
|
|
|
633
647
|
"No Silero model. Add `import '@micdrop/web/silero'` in a browser or `import '@micdrop/react-native/silero'` on a phone, or call setSileroModelLoader()."
|
|
634
648
|
);
|
|
635
649
|
}
|
|
636
|
-
this.
|
|
650
|
+
this.mic = mic;
|
|
651
|
+
const startId = ++this.startId;
|
|
652
|
+
let model;
|
|
653
|
+
try {
|
|
654
|
+
model = await loadModel();
|
|
655
|
+
} catch (error) {
|
|
656
|
+
if (startId === this.startId) this.mic = void 0;
|
|
657
|
+
throw error;
|
|
658
|
+
}
|
|
659
|
+
if (startId !== this.startId) {
|
|
660
|
+
await model.release();
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
this.model = model;
|
|
637
664
|
this.resetState();
|
|
638
665
|
this.setStatus(0 /* Silence */);
|
|
639
|
-
this.mic = mic;
|
|
640
666
|
mic.on("Frames", this.onFrames);
|
|
641
667
|
}
|
|
642
668
|
async stop() {
|
|
669
|
+
this.startId++;
|
|
643
670
|
this.mic?.off("Frames", this.onFrames);
|
|
644
671
|
this.mic = void 0;
|
|
645
672
|
this.closeOpenTurn();
|
|
646
673
|
this.resetState();
|
|
647
|
-
|
|
674
|
+
const model = this.model;
|
|
648
675
|
this.model = void 0;
|
|
676
|
+
await model?.release();
|
|
649
677
|
}
|
|
650
678
|
async pause() {
|
|
651
679
|
if (this._isPaused) return;
|
|
@@ -931,6 +959,8 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
931
959
|
this.turnListening = false;
|
|
932
960
|
// Bumped whenever the turn moves on, so a late answer is dropped
|
|
933
961
|
this.turnRound = 0;
|
|
962
|
+
// The VAD is being restarted, what it reports on the way is not speech
|
|
963
|
+
this.isDroppingSpeech = false;
|
|
934
964
|
this.start = async (mic) => {
|
|
935
965
|
if (this.state.isStarted) {
|
|
936
966
|
throw new Error("MicRecorder is already started");
|
|
@@ -963,6 +993,9 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
963
993
|
isStarted: false,
|
|
964
994
|
isSpeaking: false
|
|
965
995
|
});
|
|
996
|
+
if (this.turnOpen) {
|
|
997
|
+
this.endTurn();
|
|
998
|
+
}
|
|
966
999
|
this.cancelPendingClose();
|
|
967
1000
|
try {
|
|
968
1001
|
this.mic?.off("Frames", this.onFrames);
|
|
@@ -1054,6 +1087,7 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1054
1087
|
);
|
|
1055
1088
|
};
|
|
1056
1089
|
this.onStartSpeaking = () => {
|
|
1090
|
+
if (this.isDroppingSpeech) return;
|
|
1057
1091
|
this.speakingConfirmed = false;
|
|
1058
1092
|
this.isRecording = true;
|
|
1059
1093
|
this.buffer = [...this.reserve];
|
|
@@ -1067,6 +1101,11 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1067
1101
|
}
|
|
1068
1102
|
};
|
|
1069
1103
|
this.onConfirmSpeaking = () => {
|
|
1104
|
+
if (this.isDroppingSpeech) return;
|
|
1105
|
+
if (!this.turnOpen && this.isEcho?.()) {
|
|
1106
|
+
this.dropSpeech();
|
|
1107
|
+
return;
|
|
1108
|
+
}
|
|
1070
1109
|
this.speakingConfirmed = true;
|
|
1071
1110
|
this.changeState({ isSpeaking: true });
|
|
1072
1111
|
this.cancelPendingClose();
|
|
@@ -1077,12 +1116,14 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1077
1116
|
this.flushChunks();
|
|
1078
1117
|
};
|
|
1079
1118
|
this.onCancelSpeaking = () => {
|
|
1119
|
+
if (this.isDroppingSpeech) return;
|
|
1080
1120
|
this.stopRecording();
|
|
1081
1121
|
if (!this.turnOpen) {
|
|
1082
1122
|
this.turnListening = false;
|
|
1083
1123
|
}
|
|
1084
1124
|
};
|
|
1085
1125
|
this.onStopSpeaking = () => {
|
|
1126
|
+
if (this.isDroppingSpeech) return;
|
|
1086
1127
|
const wasConfirmed = this.speakingConfirmed;
|
|
1087
1128
|
this.changeState({ isSpeaking: false });
|
|
1088
1129
|
if (wasConfirmed) {
|
|
@@ -1094,6 +1135,26 @@ var MicRecorder = class extends EventEmitter4 {
|
|
|
1094
1135
|
this.state = defaultMicRecorderState;
|
|
1095
1136
|
this.vad = getVAD(vadConfig);
|
|
1096
1137
|
}
|
|
1138
|
+
/**
|
|
1139
|
+
* Forgets the speech heard so far.
|
|
1140
|
+
*
|
|
1141
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
1142
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
1143
|
+
*/
|
|
1144
|
+
async dropSpeech() {
|
|
1145
|
+
this.isDroppingSpeech = true;
|
|
1146
|
+
this.stopRecording();
|
|
1147
|
+
this.turnListening = false;
|
|
1148
|
+
const vad = this.vad;
|
|
1149
|
+
try {
|
|
1150
|
+
await vad.pause();
|
|
1151
|
+
if (!this.speakerPlaying) await vad.resume();
|
|
1152
|
+
} catch (error) {
|
|
1153
|
+
console.error("[Micdrop] Restarting the VAD failed", error);
|
|
1154
|
+
} finally {
|
|
1155
|
+
this.isDroppingSpeech = false;
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1097
1158
|
/**
|
|
1098
1159
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
1099
1160
|
*
|
|
@@ -1413,8 +1474,104 @@ var MicDriver = class extends EventEmitter6 {
|
|
|
1413
1474
|
}
|
|
1414
1475
|
};
|
|
1415
1476
|
var SpeakerDriver = class extends EventEmitter6 {
|
|
1477
|
+
/**
|
|
1478
|
+
* Plays by another route, for when the microphone hears the current one
|
|
1479
|
+
* @returns False if the platform has no other route to offer
|
|
1480
|
+
*/
|
|
1481
|
+
async useEchoFallback() {
|
|
1482
|
+
return false;
|
|
1483
|
+
}
|
|
1484
|
+
/** Goes back to the usual route */
|
|
1485
|
+
resetEchoFallback() {
|
|
1486
|
+
}
|
|
1416
1487
|
};
|
|
1417
1488
|
|
|
1489
|
+
// src/audio/vad/lazySession.ts
|
|
1490
|
+
var LazySession = class {
|
|
1491
|
+
/**
|
|
1492
|
+
* @param name - What the session runs, to name it in errors
|
|
1493
|
+
* @param create - Creates the session, called once unless it fails
|
|
1494
|
+
*/
|
|
1495
|
+
constructor(name, create) {
|
|
1496
|
+
this.name = name;
|
|
1497
|
+
this.create = create;
|
|
1498
|
+
this.state = { status: "unloaded" };
|
|
1499
|
+
this.runs = /* @__PURE__ */ new Set();
|
|
1500
|
+
}
|
|
1501
|
+
get status() {
|
|
1502
|
+
return this.state.status;
|
|
1503
|
+
}
|
|
1504
|
+
/** The session, loaded on the first call */
|
|
1505
|
+
get() {
|
|
1506
|
+
const state = this.state;
|
|
1507
|
+
if (state.status === "loaded") return Promise.resolve(state.session);
|
|
1508
|
+
if (state.status === "loading") return state.promise;
|
|
1509
|
+
if (state.status === "released") return Promise.reject(this.releasedError());
|
|
1510
|
+
const promise = this.create().then(
|
|
1511
|
+
async (session) => {
|
|
1512
|
+
if (this.isLoading(promise)) {
|
|
1513
|
+
this.state = { status: "loaded", session };
|
|
1514
|
+
return session;
|
|
1515
|
+
}
|
|
1516
|
+
await session.release();
|
|
1517
|
+
throw this.releasedError();
|
|
1518
|
+
},
|
|
1519
|
+
(error) => {
|
|
1520
|
+
if (this.isLoading(promise)) this.state = { status: "unloaded" };
|
|
1521
|
+
throw error;
|
|
1522
|
+
}
|
|
1523
|
+
);
|
|
1524
|
+
this.state = { status: "loading", promise };
|
|
1525
|
+
return promise;
|
|
1526
|
+
}
|
|
1527
|
+
/**
|
|
1528
|
+
* Runs something on the session, which a release waits for
|
|
1529
|
+
* @param run - Uses the session, and only until its promise settles
|
|
1530
|
+
*/
|
|
1531
|
+
async use(run) {
|
|
1532
|
+
const session = await this.get();
|
|
1533
|
+
if (this.state.status !== "loaded" || this.state.session !== session) {
|
|
1534
|
+
throw this.releasedError();
|
|
1535
|
+
}
|
|
1536
|
+
const pending = run(session);
|
|
1537
|
+
this.runs.add(pending);
|
|
1538
|
+
try {
|
|
1539
|
+
return await pending;
|
|
1540
|
+
} finally {
|
|
1541
|
+
this.runs.delete(pending);
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
/** Releases the session, for good */
|
|
1545
|
+
async release() {
|
|
1546
|
+
const state = this.state;
|
|
1547
|
+
this.state = { status: "released" };
|
|
1548
|
+
if (state.status !== "loaded") return;
|
|
1549
|
+
await Promise.allSettled(this.runs);
|
|
1550
|
+
await state.session.release();
|
|
1551
|
+
}
|
|
1552
|
+
isLoading(promise) {
|
|
1553
|
+
return this.state.status === "loading" && this.state.promise === promise;
|
|
1554
|
+
}
|
|
1555
|
+
releasedError() {
|
|
1556
|
+
return new Error(`The ${this.name} model was released`);
|
|
1557
|
+
}
|
|
1558
|
+
};
|
|
1559
|
+
function createDownloadCache(download) {
|
|
1560
|
+
const downloads = /* @__PURE__ */ new Map();
|
|
1561
|
+
return (url) => {
|
|
1562
|
+
let pending = downloads.get(url);
|
|
1563
|
+
if (!pending) {
|
|
1564
|
+
const started = download(url);
|
|
1565
|
+
downloads.set(url, started);
|
|
1566
|
+
started.catch(() => {
|
|
1567
|
+
if (downloads.get(url) === started) downloads.delete(url);
|
|
1568
|
+
});
|
|
1569
|
+
pending = started;
|
|
1570
|
+
}
|
|
1571
|
+
return pending;
|
|
1572
|
+
};
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1418
1575
|
// src/audio/index.ts
|
|
1419
1576
|
var globalScope = globalThis;
|
|
1420
1577
|
if (!globalScope.micdropMic) {
|
|
@@ -1512,6 +1669,51 @@ function isRecoverableError(error) {
|
|
|
1512
1669
|
return error.code === "Connection" /* Connection */ || error.code === "InternalServer" /* InternalServer */ || error.code === "Unknown" /* Unknown */;
|
|
1513
1670
|
}
|
|
1514
1671
|
|
|
1672
|
+
// src/client/EchoGuard.ts
|
|
1673
|
+
var ECHO_WATCH = 2e3;
|
|
1674
|
+
var EchoGuard = class {
|
|
1675
|
+
constructor() {
|
|
1676
|
+
this.stage = "normal";
|
|
1677
|
+
// The next sound opens a watch, the first of a call or after a new route
|
|
1678
|
+
this.isDue = true;
|
|
1679
|
+
}
|
|
1680
|
+
/** @param now - The current time, in milliseconds */
|
|
1681
|
+
speakerStarted(now) {
|
|
1682
|
+
if (!this.isDue) return;
|
|
1683
|
+
this.isDue = false;
|
|
1684
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1685
|
+
}
|
|
1686
|
+
/** @param now - The current time, in milliseconds */
|
|
1687
|
+
speakerStopped(now) {
|
|
1688
|
+
if (this.watchEnd === void 0) return;
|
|
1689
|
+
if (this.stage === "fallback" && now < this.watchEnd) {
|
|
1690
|
+
this.isDue = true;
|
|
1691
|
+
}
|
|
1692
|
+
this.watchEnd = void 0;
|
|
1693
|
+
}
|
|
1694
|
+
/**
|
|
1695
|
+
* Moves to the next stage if the speech heard can only be echo
|
|
1696
|
+
* @param now - The current time, in milliseconds
|
|
1697
|
+
* @returns True if the speech is to be ignored
|
|
1698
|
+
*/
|
|
1699
|
+
speechHeard(now) {
|
|
1700
|
+
if (this.watchEnd === void 0 || now >= this.watchEnd) return false;
|
|
1701
|
+
if (this.stage === "normal") {
|
|
1702
|
+
this.stage = "fallback";
|
|
1703
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1704
|
+
} else {
|
|
1705
|
+
this.stage = "disabled";
|
|
1706
|
+
this.watchEnd = void 0;
|
|
1707
|
+
}
|
|
1708
|
+
return true;
|
|
1709
|
+
}
|
|
1710
|
+
reset() {
|
|
1711
|
+
this.stage = "normal";
|
|
1712
|
+
this.isDue = true;
|
|
1713
|
+
this.watchEnd = void 0;
|
|
1714
|
+
}
|
|
1715
|
+
};
|
|
1716
|
+
|
|
1515
1717
|
// src/client/MicdropClient.ts
|
|
1516
1718
|
var DEFAULT_RECONNECT_OPTIONS = {
|
|
1517
1719
|
maxAttempts: Infinity,
|
|
@@ -1531,12 +1733,13 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1531
1733
|
this._isProcessing = false;
|
|
1532
1734
|
this._isMuted = false;
|
|
1533
1735
|
this._isPaused = false;
|
|
1534
|
-
|
|
1535
|
-
//
|
|
1536
|
-
|
|
1537
|
-
this._isStartingCall = false;
|
|
1736
|
+
// The call as start() and the connection leave it. The public status also
|
|
1737
|
+
// weighs the microphone, which a device change restarts during a call.
|
|
1738
|
+
this.callStatus = "stopped";
|
|
1538
1739
|
this.reconnectAttempt = 0;
|
|
1740
|
+
this.echoGuard = new EchoGuard();
|
|
1539
1741
|
this.start = async (options) => {
|
|
1742
|
+
this.cancelPendingStop();
|
|
1540
1743
|
this.error = void 0;
|
|
1541
1744
|
this.options = { ...this.options, ...options };
|
|
1542
1745
|
this.startTime = Date.now();
|
|
@@ -1545,46 +1748,79 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1545
1748
|
this._isProcessing = true;
|
|
1546
1749
|
this._isMuted = false;
|
|
1547
1750
|
this._isPaused = false;
|
|
1548
|
-
this.
|
|
1549
|
-
this._isStartingCall = true;
|
|
1751
|
+
this.callStatus = "starting";
|
|
1550
1752
|
this.reconnectAttempt = 0;
|
|
1551
|
-
|
|
1552
|
-
clearTimeout(this.reconnectTimer);
|
|
1553
|
-
this.reconnectTimer = void 0;
|
|
1554
|
-
}
|
|
1753
|
+
this.clearReconnectTimer();
|
|
1555
1754
|
this.notifyStateChange();
|
|
1556
|
-
|
|
1557
|
-
|
|
1755
|
+
try {
|
|
1756
|
+
if (!this.micRecorder || options?.vad) {
|
|
1757
|
+
await this.startMic({ vad: options?.vad });
|
|
1758
|
+
}
|
|
1759
|
+
if (this.callStatus !== "starting") return;
|
|
1760
|
+
this.openWS();
|
|
1761
|
+
} catch (error) {
|
|
1762
|
+
if (this.callStatus !== "stopped") {
|
|
1763
|
+
this.setError(toClientError(error));
|
|
1764
|
+
await this.stop();
|
|
1765
|
+
}
|
|
1766
|
+
throw error;
|
|
1558
1767
|
}
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
this._isStartingCall = false;
|
|
1768
|
+
if (this.isWSStarted) {
|
|
1769
|
+
this.callStatus = "started";
|
|
1562
1770
|
this.notifyStateChange();
|
|
1563
1771
|
}
|
|
1564
1772
|
};
|
|
1565
1773
|
this.stop = async () => {
|
|
1774
|
+
this.cancelPendingStop();
|
|
1566
1775
|
this.partialAssistantMessage = "";
|
|
1567
|
-
this.
|
|
1776
|
+
this.callStatus = "stopped";
|
|
1568
1777
|
this._isProcessing = false;
|
|
1569
1778
|
this._isMuted = false;
|
|
1570
1779
|
this._isPaused = false;
|
|
1571
|
-
this.
|
|
1572
|
-
if (this.reconnectTimer) {
|
|
1573
|
-
clearTimeout(this.reconnectTimer);
|
|
1574
|
-
this.reconnectTimer = void 0;
|
|
1575
|
-
}
|
|
1780
|
+
this.clearReconnectTimer();
|
|
1576
1781
|
try {
|
|
1577
1782
|
this.stopWS();
|
|
1578
1783
|
} catch (error) {
|
|
1579
1784
|
console.error("[MicdropClient] Error stopping WebSocket", error);
|
|
1580
1785
|
}
|
|
1581
1786
|
Speaker.stopAudio();
|
|
1787
|
+
this.echoGuard.reset();
|
|
1788
|
+
Speaker.resetEchoFallback();
|
|
1582
1789
|
try {
|
|
1583
1790
|
await this.stopMic();
|
|
1584
1791
|
} catch (error) {
|
|
1585
1792
|
console.error("[MicdropClient] Error stopping microphone", error);
|
|
1586
1793
|
}
|
|
1587
1794
|
};
|
|
1795
|
+
/**
|
|
1796
|
+
* Stops the call once the assistant finished its answer, the one it is
|
|
1797
|
+
* still preparing included, such as the goodbye that follows an `EndCall`.
|
|
1798
|
+
* Stops right away when no answer is under way.
|
|
1799
|
+
*
|
|
1800
|
+
* A `start()` or a `stop()` meanwhile cancels the wait, so it never acts on
|
|
1801
|
+
* the next call.
|
|
1802
|
+
* @returns Resolves once the call is stopped, or the wait cancelled
|
|
1803
|
+
*/
|
|
1804
|
+
this.stopAfterAnswer = async () => {
|
|
1805
|
+
if (!this.isAnswering) return this.stop();
|
|
1806
|
+
this.cancelPendingStop();
|
|
1807
|
+
return new Promise((resolve) => {
|
|
1808
|
+
const detach = () => {
|
|
1809
|
+
this.off("StateChange", onStateChange);
|
|
1810
|
+
this.pendingStop = void 0;
|
|
1811
|
+
};
|
|
1812
|
+
const onStateChange = () => {
|
|
1813
|
+
if (this.isAnswering) return;
|
|
1814
|
+
detach();
|
|
1815
|
+
this.stop().then(resolve);
|
|
1816
|
+
};
|
|
1817
|
+
this.on("StateChange", onStateChange);
|
|
1818
|
+
this.pendingStop = () => {
|
|
1819
|
+
detach();
|
|
1820
|
+
resolve();
|
|
1821
|
+
};
|
|
1822
|
+
});
|
|
1823
|
+
};
|
|
1588
1824
|
this.mute = () => {
|
|
1589
1825
|
this.vad?.pause();
|
|
1590
1826
|
this._isMuted = true;
|
|
@@ -1635,6 +1871,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1635
1871
|
this.micRecorder.setSpeakerPlaying(
|
|
1636
1872
|
this.isInterruptionDisabled && Speaker.isPlaying
|
|
1637
1873
|
);
|
|
1874
|
+
this.micRecorder.isEcho = this.isEchoOfAssistant;
|
|
1638
1875
|
this.micRecorder.on("StateChange", () => {
|
|
1639
1876
|
this.notifyStateChange();
|
|
1640
1877
|
});
|
|
@@ -1660,7 +1897,9 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1660
1897
|
});
|
|
1661
1898
|
}
|
|
1662
1899
|
const deviceId = options.deviceId ?? storage.getItem("micdrop.micDevice" /* MicDevice */) ?? void 0;
|
|
1900
|
+
const recorder = this.micRecorder;
|
|
1663
1901
|
await Mic.start(deviceId);
|
|
1902
|
+
if (this.micRecorder !== recorder || !Mic.isStarted) return;
|
|
1664
1903
|
if (Mic.deviceId) {
|
|
1665
1904
|
storage.setItem("micdrop.micDevice" /* MicDevice */, Mic.deviceId);
|
|
1666
1905
|
} else {
|
|
@@ -1728,8 +1967,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1728
1967
|
clearTimeout(this.connectionTimer);
|
|
1729
1968
|
this.connectionTimer = void 0;
|
|
1730
1969
|
}
|
|
1731
|
-
this.
|
|
1732
|
-
this._isStartingCall = false;
|
|
1970
|
+
this.callStatus = "started";
|
|
1733
1971
|
this.reconnectAttempt = 0;
|
|
1734
1972
|
this.error = void 0;
|
|
1735
1973
|
this.notifyStateChange();
|
|
@@ -1814,7 +2052,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1814
2052
|
const config = { ...DEFAULT_RECONNECT_OPTIONS, ...this.options.reconnect };
|
|
1815
2053
|
const canReconnect = error && isRecoverableError(error) && config.maxAttempts > 0;
|
|
1816
2054
|
if (canReconnect) {
|
|
1817
|
-
this.
|
|
2055
|
+
this.callStatus = "reconnecting";
|
|
1818
2056
|
this.stopWS();
|
|
1819
2057
|
this.scheduleReconnect(config);
|
|
1820
2058
|
} else {
|
|
@@ -1826,6 +2064,9 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1826
2064
|
};
|
|
1827
2065
|
this.onSpeakerStartPlaying = () => {
|
|
1828
2066
|
this.log("Speaker started");
|
|
2067
|
+
if (this.isWSStarted) {
|
|
2068
|
+
this.echoGuard.speakerStarted(Date.now());
|
|
2069
|
+
}
|
|
1829
2070
|
if (this.isInterruptionDisabled) {
|
|
1830
2071
|
this.micRecorder?.setSpeakerPlaying(true);
|
|
1831
2072
|
this.vad?.pause();
|
|
@@ -1834,6 +2075,7 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1834
2075
|
};
|
|
1835
2076
|
this.onSpeakerStopPlaying = () => {
|
|
1836
2077
|
this.log("Speaker stopped");
|
|
2078
|
+
this.echoGuard.speakerStopped(Date.now());
|
|
1837
2079
|
this.micRecorder?.setSpeakerPlaying(false);
|
|
1838
2080
|
if (this.isInterruptionDisabled) {
|
|
1839
2081
|
setTimeout(() => {
|
|
@@ -1846,6 +2088,28 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1846
2088
|
this.notifyStateChange();
|
|
1847
2089
|
}
|
|
1848
2090
|
};
|
|
2091
|
+
/**
|
|
2092
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
2093
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
2094
|
+
* neither the speaker nor the server hear of it.
|
|
2095
|
+
*/
|
|
2096
|
+
this.isEchoOfAssistant = () => {
|
|
2097
|
+
if (this.isInterruptionDisabled || !Speaker.isPlaying) return false;
|
|
2098
|
+
if (!this.echoGuard.speechHeard(Date.now())) return false;
|
|
2099
|
+
this.log("Echo of the assistant, moving to", this.echoGuard.stage);
|
|
2100
|
+
if (this.echoGuard.stage === "fallback") {
|
|
2101
|
+
Speaker.useEchoFallback().then((moved) => {
|
|
2102
|
+
if (!moved && this.echoGuard.stage === "fallback") {
|
|
2103
|
+
this.echoGuard.stage = "disabled";
|
|
2104
|
+
this.onInterruptionsGivenUp();
|
|
2105
|
+
}
|
|
2106
|
+
});
|
|
2107
|
+
} else {
|
|
2108
|
+
this.onInterruptionsGivenUp();
|
|
2109
|
+
}
|
|
2110
|
+
this.notifyStateChange();
|
|
2111
|
+
return true;
|
|
2112
|
+
};
|
|
1849
2113
|
Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
|
|
1850
2114
|
Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
|
|
1851
2115
|
Mic.on("DeviceChange", this.updateDevices);
|
|
@@ -1853,14 +2117,30 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1853
2117
|
get vad() {
|
|
1854
2118
|
return this.micRecorder?.vad;
|
|
1855
2119
|
}
|
|
2120
|
+
get status() {
|
|
2121
|
+
const mic = this.micRecorder?.state;
|
|
2122
|
+
if (this.callStatus === "reconnecting") return "reconnecting";
|
|
2123
|
+
if (this.callStatus === "started" && mic?.isStarted) return "started";
|
|
2124
|
+
if (this.callStatus !== "stopped" || mic?.isStarting) return "starting";
|
|
2125
|
+
return "stopped";
|
|
2126
|
+
}
|
|
2127
|
+
get activity() {
|
|
2128
|
+
if (this.isUserSpeaking) return "userSpeaking";
|
|
2129
|
+
if (this.isAssistantSpeaking) return "assistantSpeaking";
|
|
2130
|
+
if (this.isProcessing) return "processing";
|
|
2131
|
+
if (this.isListening) return "listening";
|
|
2132
|
+
return "idle";
|
|
2133
|
+
}
|
|
2134
|
+
/** The call is on, reconnecting included */
|
|
1856
2135
|
get isStarted() {
|
|
1857
|
-
|
|
2136
|
+
const status = this.status;
|
|
2137
|
+
return status === "started" || status === "reconnecting";
|
|
1858
2138
|
}
|
|
1859
2139
|
get isStarting() {
|
|
1860
|
-
return
|
|
2140
|
+
return this.status === "starting";
|
|
1861
2141
|
}
|
|
1862
2142
|
get isReconnecting() {
|
|
1863
|
-
return this.
|
|
2143
|
+
return this.status === "reconnecting";
|
|
1864
2144
|
}
|
|
1865
2145
|
get isMuted() {
|
|
1866
2146
|
return this._isMuted;
|
|
@@ -1895,10 +2175,15 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1895
2175
|
/**
|
|
1896
2176
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
1897
2177
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
1898
|
-
* itself and end up in the transcript.
|
|
2178
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
2179
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
1899
2180
|
*/
|
|
1900
2181
|
get isInterruptionDisabled() {
|
|
1901
|
-
return !!this.options.disableInterruption || Mic.echoCancellation === false;
|
|
2182
|
+
return !!this.options.disableInterruption || Mic.echoCancellation === false || this.echoGuard.stage === "disabled";
|
|
2183
|
+
}
|
|
2184
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
2185
|
+
get echoStage() {
|
|
2186
|
+
return this.echoGuard.stage;
|
|
1902
2187
|
}
|
|
1903
2188
|
get micDeviceId() {
|
|
1904
2189
|
return Mic.deviceId;
|
|
@@ -1908,6 +2193,8 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1908
2193
|
}
|
|
1909
2194
|
get state() {
|
|
1910
2195
|
return {
|
|
2196
|
+
status: this.status,
|
|
2197
|
+
activity: this.activity,
|
|
1911
2198
|
isStarting: this.isStarting,
|
|
1912
2199
|
isStarted: this.isStarted,
|
|
1913
2200
|
isReconnecting: this.isReconnecting,
|
|
@@ -1928,62 +2215,69 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1928
2215
|
speakerDevices: this.speakerDevices
|
|
1929
2216
|
};
|
|
1930
2217
|
}
|
|
2218
|
+
/** The assistant prepares or speaks an answer */
|
|
2219
|
+
get isAnswering() {
|
|
2220
|
+
const { activity } = this;
|
|
2221
|
+
return activity === "processing" || activity === "assistantSpeaking";
|
|
2222
|
+
}
|
|
2223
|
+
cancelPendingStop() {
|
|
2224
|
+
this.pendingStop?.();
|
|
2225
|
+
}
|
|
1931
2226
|
async stopMic() {
|
|
1932
2227
|
this.micRecorder?.stop();
|
|
1933
2228
|
this.micRecorder = void 0;
|
|
1934
2229
|
await Mic.stop();
|
|
1935
2230
|
this.notifyStateChange();
|
|
1936
2231
|
}
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
this.ws.binaryType = "arraybuffer";
|
|
1951
|
-
this.notifyStateChange();
|
|
1952
|
-
this.ws.onopen = this.onWSOpen;
|
|
1953
|
-
this.ws.onmessage = this.onWSMessage;
|
|
1954
|
-
this.ws.onclose = this.onWSClose;
|
|
1955
|
-
this.ws.onerror = this.onWSError;
|
|
1956
|
-
this.connectionTimer = setTimeout(() => {
|
|
1957
|
-
this.connectionTimer = void 0;
|
|
1958
|
-
if (this.ws?.readyState === 0) {
|
|
1959
|
-
this.log("WebSocket connection timeout");
|
|
1960
|
-
this.ws.close();
|
|
1961
|
-
}
|
|
1962
|
-
}, this.options.reconnect?.connectionTimeout ?? DEFAULT_RECONNECT_OPTIONS.connectionTimeout);
|
|
1963
|
-
} catch (error) {
|
|
1964
|
-
this.setError(
|
|
1965
|
-
error instanceof MicdropClientError ? error : new MicdropClientError(
|
|
1966
|
-
"Connection" /* Connection */,
|
|
1967
|
-
error?.message
|
|
1968
|
-
)
|
|
1969
|
-
);
|
|
1970
|
-
await this.stop();
|
|
1971
|
-
throw error;
|
|
2232
|
+
/**
|
|
2233
|
+
* Opens the connection, unless one is already open or opening.
|
|
2234
|
+
*
|
|
2235
|
+
* The socket answers through handlers bound to it, so one left behind by a
|
|
2236
|
+
* stop or a reconnection never acts on the call that followed.
|
|
2237
|
+
*/
|
|
2238
|
+
openWS() {
|
|
2239
|
+
if (this.ws) {
|
|
2240
|
+
this.log("WebSocket is already started");
|
|
2241
|
+
return;
|
|
2242
|
+
}
|
|
2243
|
+
if (!this.isMicStarted) {
|
|
2244
|
+
throw new MicdropClientError("Mic" /* Mic */);
|
|
1972
2245
|
}
|
|
2246
|
+
if (!this.options.url) {
|
|
2247
|
+
throw new MicdropClientError("MissingUrl" /* MissingUrl */);
|
|
2248
|
+
}
|
|
2249
|
+
const ws = new WebSocket(this.options.url);
|
|
2250
|
+
ws.binaryType = "arraybuffer";
|
|
2251
|
+
this.ws = ws;
|
|
2252
|
+
this.notifyStateChange();
|
|
2253
|
+
const isCurrent = () => this.ws === ws;
|
|
2254
|
+
ws.onopen = () => isCurrent() && this.onWSOpen();
|
|
2255
|
+
ws.onmessage = (event) => isCurrent() && this.onWSMessage(event);
|
|
2256
|
+
ws.onclose = (event) => isCurrent() && this.onWSClose(event);
|
|
2257
|
+
ws.onerror = () => isCurrent() && this.onWSError();
|
|
2258
|
+
this.connectionTimer = setTimeout(() => {
|
|
2259
|
+
this.connectionTimer = void 0;
|
|
2260
|
+
if (isCurrent() && ws.readyState === 0) {
|
|
2261
|
+
this.log("WebSocket connection timeout");
|
|
2262
|
+
ws.close();
|
|
2263
|
+
}
|
|
2264
|
+
}, this.options.reconnect?.connectionTimeout ?? DEFAULT_RECONNECT_OPTIONS.connectionTimeout);
|
|
1973
2265
|
}
|
|
1974
2266
|
stopWS() {
|
|
1975
2267
|
if (this.connectionTimer) {
|
|
1976
2268
|
clearTimeout(this.connectionTimer);
|
|
1977
2269
|
this.connectionTimer = void 0;
|
|
1978
2270
|
}
|
|
1979
|
-
|
|
1980
|
-
if (
|
|
1981
|
-
this.ws.close();
|
|
1982
|
-
}
|
|
2271
|
+
const ws = this.ws;
|
|
2272
|
+
if (!ws) return;
|
|
1983
2273
|
this.ws = void 0;
|
|
2274
|
+
if (ws.readyState === 1 || ws.readyState === 0) {
|
|
2275
|
+
ws.close();
|
|
2276
|
+
}
|
|
1984
2277
|
this.notifyStateChange();
|
|
1985
2278
|
}
|
|
1986
2279
|
scheduleReconnect(config) {
|
|
2280
|
+
if (this.callStatus !== "reconnecting") return;
|
|
1987
2281
|
if (this.reconnectAttempt >= config.maxAttempts) {
|
|
1988
2282
|
this.stop();
|
|
1989
2283
|
this.setError(new MicdropClientError("Connection" /* Connection */));
|
|
@@ -1996,17 +2290,32 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
1996
2290
|
if (!this.isPaused && !this.isMuted) {
|
|
1997
2291
|
this.vad?.pause?.();
|
|
1998
2292
|
}
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2293
|
+
this.clearReconnectTimer();
|
|
2294
|
+
this.reconnectTimer = setTimeout(() => {
|
|
2295
|
+
this.reconnectTimer = void 0;
|
|
2296
|
+
if (this.callStatus !== "reconnecting") return;
|
|
2003
2297
|
try {
|
|
2004
|
-
|
|
2005
|
-
} catch {
|
|
2006
|
-
this.
|
|
2298
|
+
this.openWS();
|
|
2299
|
+
} catch (error) {
|
|
2300
|
+
this.setError(toClientError(error));
|
|
2301
|
+
this.stop();
|
|
2007
2302
|
}
|
|
2008
2303
|
}, config.delayMs);
|
|
2009
2304
|
}
|
|
2305
|
+
clearReconnectTimer() {
|
|
2306
|
+
if (!this.reconnectTimer) return;
|
|
2307
|
+
clearTimeout(this.reconnectTimer);
|
|
2308
|
+
this.reconnectTimer = void 0;
|
|
2309
|
+
}
|
|
2310
|
+
// The same as when the speaker starts on a call without interruptions
|
|
2311
|
+
onInterruptionsGivenUp() {
|
|
2312
|
+
this.log("Interruptions disabled until the end of the call");
|
|
2313
|
+
if (Speaker.isPlaying) {
|
|
2314
|
+
this.micRecorder?.setSpeakerPlaying(true);
|
|
2315
|
+
this.vad?.pause();
|
|
2316
|
+
}
|
|
2317
|
+
this.notifyStateChange();
|
|
2318
|
+
}
|
|
2010
2319
|
addMessage(message) {
|
|
2011
2320
|
if (message.role === "assistant") this.partialAssistantMessage = "";
|
|
2012
2321
|
this.conversation = [...this.conversation, message];
|
|
@@ -2042,6 +2351,13 @@ var MicdropClient = class extends EventEmitter7 {
|
|
|
2042
2351
|
console.log(`[MicdropClient ${Date.now() - this.startTime}]`, ...message);
|
|
2043
2352
|
}
|
|
2044
2353
|
};
|
|
2354
|
+
function toClientError(error) {
|
|
2355
|
+
if (error instanceof MicdropClientError) return error;
|
|
2356
|
+
return new MicdropClientError(
|
|
2357
|
+
"Connection" /* Connection */,
|
|
2358
|
+
error?.message
|
|
2359
|
+
);
|
|
2360
|
+
}
|
|
2045
2361
|
|
|
2046
2362
|
// src/Micdrop.ts
|
|
2047
2363
|
var globalScope2 = globalThis;
|
|
@@ -2053,6 +2369,7 @@ export {
|
|
|
2053
2369
|
CONFIRM_MARGIN,
|
|
2054
2370
|
DEFAULT_TURN_MAX_WAIT,
|
|
2055
2371
|
ECHO_TAIL,
|
|
2372
|
+
LazySession,
|
|
2056
2373
|
Mic,
|
|
2057
2374
|
MicController,
|
|
2058
2375
|
MicDriver,
|
|
@@ -2078,6 +2395,7 @@ export {
|
|
|
2078
2395
|
VolumeMeter,
|
|
2079
2396
|
VolumeVAD,
|
|
2080
2397
|
concatFloat32,
|
|
2398
|
+
createDownloadCache,
|
|
2081
2399
|
currentTurn,
|
|
2082
2400
|
equalVADConfig,
|
|
2083
2401
|
floatToPcm16,
|