@micdrop/client 3.4.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/index.d.mts +158 -6
- package/dist/index.d.ts +158 -6
- package/dist/index.js +398 -78
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +396 -78
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -23,6 +23,7 @@ __export(index_exports, {
|
|
|
23
23
|
CONFIRM_MARGIN: () => CONFIRM_MARGIN,
|
|
24
24
|
DEFAULT_TURN_MAX_WAIT: () => DEFAULT_TURN_MAX_WAIT,
|
|
25
25
|
ECHO_TAIL: () => ECHO_TAIL,
|
|
26
|
+
LazySession: () => LazySession,
|
|
26
27
|
Mic: () => Mic,
|
|
27
28
|
MicController: () => MicController,
|
|
28
29
|
MicDriver: () => MicDriver,
|
|
@@ -48,6 +49,7 @@ __export(index_exports, {
|
|
|
48
49
|
VolumeMeter: () => VolumeMeter,
|
|
49
50
|
VolumeVAD: () => VolumeVAD,
|
|
50
51
|
concatFloat32: () => concatFloat32,
|
|
52
|
+
createDownloadCache: () => createDownloadCache,
|
|
51
53
|
currentTurn: () => currentTurn,
|
|
52
54
|
equalVADConfig: () => equalVADConfig,
|
|
53
55
|
floatToPcm16: () => floatToPcm16,
|
|
@@ -497,6 +499,17 @@ var SpeakerController = class extends import_eventemitter32.EventEmitter {
|
|
|
497
499
|
if (!this.driver) return [];
|
|
498
500
|
return this.driver.getDevices();
|
|
499
501
|
}
|
|
502
|
+
/**
|
|
503
|
+
* Plays by another route, for when the microphone hears the current one
|
|
504
|
+
* @returns False if the platform has no other route to offer
|
|
505
|
+
*/
|
|
506
|
+
async useEchoFallback() {
|
|
507
|
+
return await this.driver?.useEchoFallback() ?? false;
|
|
508
|
+
}
|
|
509
|
+
/** Goes back to the usual route */
|
|
510
|
+
resetEchoFallback() {
|
|
511
|
+
this.driver?.resetEchoFallback();
|
|
512
|
+
}
|
|
500
513
|
};
|
|
501
514
|
|
|
502
515
|
// src/audio/MicRecorder.ts
|
|
@@ -666,6 +679,9 @@ var SileroVAD = class extends VAD {
|
|
|
666
679
|
// Bumped whenever the state is reset, so a window scored before that is
|
|
667
680
|
// dropped rather than reopening a turn
|
|
668
681
|
this.round = 0;
|
|
682
|
+
// Bumped by every start and stop, so a start overtaken while its model
|
|
683
|
+
// loaded lets go of it
|
|
684
|
+
this.startId = 0;
|
|
669
685
|
this.onFrames = (frames, sampleRate) => {
|
|
670
686
|
if (this._isPaused || !this.model) return;
|
|
671
687
|
if (sampleRate !== this.bufferRate) {
|
|
@@ -701,19 +717,33 @@ var SileroVAD = class extends VAD {
|
|
|
701
717
|
"No Silero model. Add `import '@micdrop/web/silero'` in a browser or `import '@micdrop/react-native/silero'` on a phone, or call setSileroModelLoader()."
|
|
702
718
|
);
|
|
703
719
|
}
|
|
704
|
-
this.
|
|
720
|
+
this.mic = mic;
|
|
721
|
+
const startId = ++this.startId;
|
|
722
|
+
let model;
|
|
723
|
+
try {
|
|
724
|
+
model = await loadModel();
|
|
725
|
+
} catch (error) {
|
|
726
|
+
if (startId === this.startId) this.mic = void 0;
|
|
727
|
+
throw error;
|
|
728
|
+
}
|
|
729
|
+
if (startId !== this.startId) {
|
|
730
|
+
await model.release();
|
|
731
|
+
return;
|
|
732
|
+
}
|
|
733
|
+
this.model = model;
|
|
705
734
|
this.resetState();
|
|
706
735
|
this.setStatus(0 /* Silence */);
|
|
707
|
-
this.mic = mic;
|
|
708
736
|
mic.on("Frames", this.onFrames);
|
|
709
737
|
}
|
|
710
738
|
async stop() {
|
|
739
|
+
this.startId++;
|
|
711
740
|
this.mic?.off("Frames", this.onFrames);
|
|
712
741
|
this.mic = void 0;
|
|
713
742
|
this.closeOpenTurn();
|
|
714
743
|
this.resetState();
|
|
715
|
-
|
|
744
|
+
const model = this.model;
|
|
716
745
|
this.model = void 0;
|
|
746
|
+
await model?.release();
|
|
717
747
|
}
|
|
718
748
|
async pause() {
|
|
719
749
|
if (this._isPaused) return;
|
|
@@ -999,6 +1029,8 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
999
1029
|
this.turnListening = false;
|
|
1000
1030
|
// Bumped whenever the turn moves on, so a late answer is dropped
|
|
1001
1031
|
this.turnRound = 0;
|
|
1032
|
+
// The VAD is being restarted, what it reports on the way is not speech
|
|
1033
|
+
this.isDroppingSpeech = false;
|
|
1002
1034
|
this.start = async (mic) => {
|
|
1003
1035
|
if (this.state.isStarted) {
|
|
1004
1036
|
throw new Error("MicRecorder is already started");
|
|
@@ -1031,6 +1063,9 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1031
1063
|
isStarted: false,
|
|
1032
1064
|
isSpeaking: false
|
|
1033
1065
|
});
|
|
1066
|
+
if (this.turnOpen) {
|
|
1067
|
+
this.endTurn();
|
|
1068
|
+
}
|
|
1034
1069
|
this.cancelPendingClose();
|
|
1035
1070
|
try {
|
|
1036
1071
|
this.mic?.off("Frames", this.onFrames);
|
|
@@ -1122,6 +1157,7 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1122
1157
|
);
|
|
1123
1158
|
};
|
|
1124
1159
|
this.onStartSpeaking = () => {
|
|
1160
|
+
if (this.isDroppingSpeech) return;
|
|
1125
1161
|
this.speakingConfirmed = false;
|
|
1126
1162
|
this.isRecording = true;
|
|
1127
1163
|
this.buffer = [...this.reserve];
|
|
@@ -1135,6 +1171,11 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1135
1171
|
}
|
|
1136
1172
|
};
|
|
1137
1173
|
this.onConfirmSpeaking = () => {
|
|
1174
|
+
if (this.isDroppingSpeech) return;
|
|
1175
|
+
if (!this.turnOpen && this.isEcho?.()) {
|
|
1176
|
+
this.dropSpeech();
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1138
1179
|
this.speakingConfirmed = true;
|
|
1139
1180
|
this.changeState({ isSpeaking: true });
|
|
1140
1181
|
this.cancelPendingClose();
|
|
@@ -1145,12 +1186,14 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1145
1186
|
this.flushChunks();
|
|
1146
1187
|
};
|
|
1147
1188
|
this.onCancelSpeaking = () => {
|
|
1189
|
+
if (this.isDroppingSpeech) return;
|
|
1148
1190
|
this.stopRecording();
|
|
1149
1191
|
if (!this.turnOpen) {
|
|
1150
1192
|
this.turnListening = false;
|
|
1151
1193
|
}
|
|
1152
1194
|
};
|
|
1153
1195
|
this.onStopSpeaking = () => {
|
|
1196
|
+
if (this.isDroppingSpeech) return;
|
|
1154
1197
|
const wasConfirmed = this.speakingConfirmed;
|
|
1155
1198
|
this.changeState({ isSpeaking: false });
|
|
1156
1199
|
if (wasConfirmed) {
|
|
@@ -1162,6 +1205,26 @@ var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1162
1205
|
this.state = defaultMicRecorderState;
|
|
1163
1206
|
this.vad = getVAD(vadConfig);
|
|
1164
1207
|
}
|
|
1208
|
+
/**
|
|
1209
|
+
* Forgets the speech heard so far.
|
|
1210
|
+
*
|
|
1211
|
+
* The VAD starts over rather than staying on it: an echo that carries on
|
|
1212
|
+
* would keep it speaking past the moment it could be told from a person.
|
|
1213
|
+
*/
|
|
1214
|
+
async dropSpeech() {
|
|
1215
|
+
this.isDroppingSpeech = true;
|
|
1216
|
+
this.stopRecording();
|
|
1217
|
+
this.turnListening = false;
|
|
1218
|
+
const vad = this.vad;
|
|
1219
|
+
try {
|
|
1220
|
+
await vad.pause();
|
|
1221
|
+
if (!this.speakerPlaying) await vad.resume();
|
|
1222
|
+
} catch (error) {
|
|
1223
|
+
console.error("[Micdrop] Restarting the VAD failed", error);
|
|
1224
|
+
} finally {
|
|
1225
|
+
this.isDroppingSpeech = false;
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1165
1228
|
/**
|
|
1166
1229
|
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
1167
1230
|
*
|
|
@@ -1481,8 +1544,104 @@ var MicDriver = class extends import_eventemitter36.EventEmitter {
|
|
|
1481
1544
|
}
|
|
1482
1545
|
};
|
|
1483
1546
|
var SpeakerDriver = class extends import_eventemitter36.EventEmitter {
|
|
1547
|
+
/**
|
|
1548
|
+
* Plays by another route, for when the microphone hears the current one
|
|
1549
|
+
* @returns False if the platform has no other route to offer
|
|
1550
|
+
*/
|
|
1551
|
+
async useEchoFallback() {
|
|
1552
|
+
return false;
|
|
1553
|
+
}
|
|
1554
|
+
/** Goes back to the usual route */
|
|
1555
|
+
resetEchoFallback() {
|
|
1556
|
+
}
|
|
1484
1557
|
};
|
|
1485
1558
|
|
|
1559
|
+
// src/audio/vad/lazySession.ts
|
|
1560
|
+
var LazySession = class {
|
|
1561
|
+
/**
|
|
1562
|
+
* @param name - What the session runs, to name it in errors
|
|
1563
|
+
* @param create - Creates the session, called once unless it fails
|
|
1564
|
+
*/
|
|
1565
|
+
constructor(name, create) {
|
|
1566
|
+
this.name = name;
|
|
1567
|
+
this.create = create;
|
|
1568
|
+
this.state = { status: "unloaded" };
|
|
1569
|
+
this.runs = /* @__PURE__ */ new Set();
|
|
1570
|
+
}
|
|
1571
|
+
get status() {
|
|
1572
|
+
return this.state.status;
|
|
1573
|
+
}
|
|
1574
|
+
/** The session, loaded on the first call */
|
|
1575
|
+
get() {
|
|
1576
|
+
const state = this.state;
|
|
1577
|
+
if (state.status === "loaded") return Promise.resolve(state.session);
|
|
1578
|
+
if (state.status === "loading") return state.promise;
|
|
1579
|
+
if (state.status === "released") return Promise.reject(this.releasedError());
|
|
1580
|
+
const promise = this.create().then(
|
|
1581
|
+
async (session) => {
|
|
1582
|
+
if (this.isLoading(promise)) {
|
|
1583
|
+
this.state = { status: "loaded", session };
|
|
1584
|
+
return session;
|
|
1585
|
+
}
|
|
1586
|
+
await session.release();
|
|
1587
|
+
throw this.releasedError();
|
|
1588
|
+
},
|
|
1589
|
+
(error) => {
|
|
1590
|
+
if (this.isLoading(promise)) this.state = { status: "unloaded" };
|
|
1591
|
+
throw error;
|
|
1592
|
+
}
|
|
1593
|
+
);
|
|
1594
|
+
this.state = { status: "loading", promise };
|
|
1595
|
+
return promise;
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* Runs something on the session, which a release waits for
|
|
1599
|
+
* @param run - Uses the session, and only until its promise settles
|
|
1600
|
+
*/
|
|
1601
|
+
async use(run) {
|
|
1602
|
+
const session = await this.get();
|
|
1603
|
+
if (this.state.status !== "loaded" || this.state.session !== session) {
|
|
1604
|
+
throw this.releasedError();
|
|
1605
|
+
}
|
|
1606
|
+
const pending = run(session);
|
|
1607
|
+
this.runs.add(pending);
|
|
1608
|
+
try {
|
|
1609
|
+
return await pending;
|
|
1610
|
+
} finally {
|
|
1611
|
+
this.runs.delete(pending);
|
|
1612
|
+
}
|
|
1613
|
+
}
|
|
1614
|
+
/** Releases the session, for good */
|
|
1615
|
+
async release() {
|
|
1616
|
+
const state = this.state;
|
|
1617
|
+
this.state = { status: "released" };
|
|
1618
|
+
if (state.status !== "loaded") return;
|
|
1619
|
+
await Promise.allSettled(this.runs);
|
|
1620
|
+
await state.session.release();
|
|
1621
|
+
}
|
|
1622
|
+
isLoading(promise) {
|
|
1623
|
+
return this.state.status === "loading" && this.state.promise === promise;
|
|
1624
|
+
}
|
|
1625
|
+
releasedError() {
|
|
1626
|
+
return new Error(`The ${this.name} model was released`);
|
|
1627
|
+
}
|
|
1628
|
+
};
|
|
1629
|
+
function createDownloadCache(download) {
|
|
1630
|
+
const downloads = /* @__PURE__ */ new Map();
|
|
1631
|
+
return (url) => {
|
|
1632
|
+
let pending = downloads.get(url);
|
|
1633
|
+
if (!pending) {
|
|
1634
|
+
const started = download(url);
|
|
1635
|
+
downloads.set(url, started);
|
|
1636
|
+
started.catch(() => {
|
|
1637
|
+
if (downloads.get(url) === started) downloads.delete(url);
|
|
1638
|
+
});
|
|
1639
|
+
pending = started;
|
|
1640
|
+
}
|
|
1641
|
+
return pending;
|
|
1642
|
+
};
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1486
1645
|
// src/audio/index.ts
|
|
1487
1646
|
var globalScope = globalThis;
|
|
1488
1647
|
if (!globalScope.micdropMic) {
|
|
@@ -1580,6 +1739,51 @@ function isRecoverableError(error) {
|
|
|
1580
1739
|
return error.code === "Connection" /* Connection */ || error.code === "InternalServer" /* InternalServer */ || error.code === "Unknown" /* Unknown */;
|
|
1581
1740
|
}
|
|
1582
1741
|
|
|
1742
|
+
// src/client/EchoGuard.ts
|
|
1743
|
+
var ECHO_WATCH = 2e3;
|
|
1744
|
+
var EchoGuard = class {
|
|
1745
|
+
constructor() {
|
|
1746
|
+
this.stage = "normal";
|
|
1747
|
+
// The next sound opens a watch, the first of a call or after a new route
|
|
1748
|
+
this.isDue = true;
|
|
1749
|
+
}
|
|
1750
|
+
/** @param now - The current time, in milliseconds */
|
|
1751
|
+
speakerStarted(now) {
|
|
1752
|
+
if (!this.isDue) return;
|
|
1753
|
+
this.isDue = false;
|
|
1754
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1755
|
+
}
|
|
1756
|
+
/** @param now - The current time, in milliseconds */
|
|
1757
|
+
speakerStopped(now) {
|
|
1758
|
+
if (this.watchEnd === void 0) return;
|
|
1759
|
+
if (this.stage === "fallback" && now < this.watchEnd) {
|
|
1760
|
+
this.isDue = true;
|
|
1761
|
+
}
|
|
1762
|
+
this.watchEnd = void 0;
|
|
1763
|
+
}
|
|
1764
|
+
/**
|
|
1765
|
+
* Moves to the next stage if the speech heard can only be echo
|
|
1766
|
+
* @param now - The current time, in milliseconds
|
|
1767
|
+
* @returns True if the speech is to be ignored
|
|
1768
|
+
*/
|
|
1769
|
+
speechHeard(now) {
|
|
1770
|
+
if (this.watchEnd === void 0 || now >= this.watchEnd) return false;
|
|
1771
|
+
if (this.stage === "normal") {
|
|
1772
|
+
this.stage = "fallback";
|
|
1773
|
+
this.watchEnd = now + ECHO_WATCH;
|
|
1774
|
+
} else {
|
|
1775
|
+
this.stage = "disabled";
|
|
1776
|
+
this.watchEnd = void 0;
|
|
1777
|
+
}
|
|
1778
|
+
return true;
|
|
1779
|
+
}
|
|
1780
|
+
reset() {
|
|
1781
|
+
this.stage = "normal";
|
|
1782
|
+
this.isDue = true;
|
|
1783
|
+
this.watchEnd = void 0;
|
|
1784
|
+
}
|
|
1785
|
+
};
|
|
1786
|
+
|
|
1583
1787
|
// src/client/MicdropClient.ts
|
|
1584
1788
|
var DEFAULT_RECONNECT_OPTIONS = {
|
|
1585
1789
|
maxAttempts: Infinity,
|
|
@@ -1599,12 +1803,13 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1599
1803
|
this._isProcessing = false;
|
|
1600
1804
|
this._isMuted = false;
|
|
1601
1805
|
this._isPaused = false;
|
|
1602
|
-
|
|
1603
|
-
//
|
|
1604
|
-
|
|
1605
|
-
this._isStartingCall = false;
|
|
1806
|
+
// The call as start() and the connection leave it. The public status also
|
|
1807
|
+
// weighs the microphone, which a device change restarts during a call.
|
|
1808
|
+
this.callStatus = "stopped";
|
|
1606
1809
|
this.reconnectAttempt = 0;
|
|
1810
|
+
this.echoGuard = new EchoGuard();
|
|
1607
1811
|
this.start = async (options) => {
|
|
1812
|
+
this.cancelPendingStop();
|
|
1608
1813
|
this.error = void 0;
|
|
1609
1814
|
this.options = { ...this.options, ...options };
|
|
1610
1815
|
this.startTime = Date.now();
|
|
@@ -1613,46 +1818,79 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1613
1818
|
this._isProcessing = true;
|
|
1614
1819
|
this._isMuted = false;
|
|
1615
1820
|
this._isPaused = false;
|
|
1616
|
-
this.
|
|
1617
|
-
this._isStartingCall = true;
|
|
1821
|
+
this.callStatus = "starting";
|
|
1618
1822
|
this.reconnectAttempt = 0;
|
|
1619
|
-
|
|
1620
|
-
clearTimeout(this.reconnectTimer);
|
|
1621
|
-
this.reconnectTimer = void 0;
|
|
1622
|
-
}
|
|
1823
|
+
this.clearReconnectTimer();
|
|
1623
1824
|
this.notifyStateChange();
|
|
1624
|
-
|
|
1625
|
-
|
|
1825
|
+
try {
|
|
1826
|
+
if (!this.micRecorder || options?.vad) {
|
|
1827
|
+
await this.startMic({ vad: options?.vad });
|
|
1828
|
+
}
|
|
1829
|
+
if (this.callStatus !== "starting") return;
|
|
1830
|
+
this.openWS();
|
|
1831
|
+
} catch (error) {
|
|
1832
|
+
if (this.callStatus !== "stopped") {
|
|
1833
|
+
this.setError(toClientError(error));
|
|
1834
|
+
await this.stop();
|
|
1835
|
+
}
|
|
1836
|
+
throw error;
|
|
1626
1837
|
}
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
this._isStartingCall = false;
|
|
1838
|
+
if (this.isWSStarted) {
|
|
1839
|
+
this.callStatus = "started";
|
|
1630
1840
|
this.notifyStateChange();
|
|
1631
1841
|
}
|
|
1632
1842
|
};
|
|
1633
1843
|
this.stop = async () => {
|
|
1844
|
+
this.cancelPendingStop();
|
|
1634
1845
|
this.partialAssistantMessage = "";
|
|
1635
|
-
this.
|
|
1846
|
+
this.callStatus = "stopped";
|
|
1636
1847
|
this._isProcessing = false;
|
|
1637
1848
|
this._isMuted = false;
|
|
1638
1849
|
this._isPaused = false;
|
|
1639
|
-
this.
|
|
1640
|
-
if (this.reconnectTimer) {
|
|
1641
|
-
clearTimeout(this.reconnectTimer);
|
|
1642
|
-
this.reconnectTimer = void 0;
|
|
1643
|
-
}
|
|
1850
|
+
this.clearReconnectTimer();
|
|
1644
1851
|
try {
|
|
1645
1852
|
this.stopWS();
|
|
1646
1853
|
} catch (error) {
|
|
1647
1854
|
console.error("[MicdropClient] Error stopping WebSocket", error);
|
|
1648
1855
|
}
|
|
1649
1856
|
Speaker.stopAudio();
|
|
1857
|
+
this.echoGuard.reset();
|
|
1858
|
+
Speaker.resetEchoFallback();
|
|
1650
1859
|
try {
|
|
1651
1860
|
await this.stopMic();
|
|
1652
1861
|
} catch (error) {
|
|
1653
1862
|
console.error("[MicdropClient] Error stopping microphone", error);
|
|
1654
1863
|
}
|
|
1655
1864
|
};
|
|
1865
|
+
/**
|
|
1866
|
+
* Stops the call once the assistant finished its answer, the one it is
|
|
1867
|
+
* still preparing included, such as the goodbye that follows an `EndCall`.
|
|
1868
|
+
* Stops right away when no answer is under way.
|
|
1869
|
+
*
|
|
1870
|
+
* A `start()` or a `stop()` meanwhile cancels the wait, so it never acts on
|
|
1871
|
+
* the next call.
|
|
1872
|
+
* @returns Resolves once the call is stopped, or the wait cancelled
|
|
1873
|
+
*/
|
|
1874
|
+
this.stopAfterAnswer = async () => {
|
|
1875
|
+
if (!this.isAnswering) return this.stop();
|
|
1876
|
+
this.cancelPendingStop();
|
|
1877
|
+
return new Promise((resolve) => {
|
|
1878
|
+
const detach = () => {
|
|
1879
|
+
this.off("StateChange", onStateChange);
|
|
1880
|
+
this.pendingStop = void 0;
|
|
1881
|
+
};
|
|
1882
|
+
const onStateChange = () => {
|
|
1883
|
+
if (this.isAnswering) return;
|
|
1884
|
+
detach();
|
|
1885
|
+
this.stop().then(resolve);
|
|
1886
|
+
};
|
|
1887
|
+
this.on("StateChange", onStateChange);
|
|
1888
|
+
this.pendingStop = () => {
|
|
1889
|
+
detach();
|
|
1890
|
+
resolve();
|
|
1891
|
+
};
|
|
1892
|
+
});
|
|
1893
|
+
};
|
|
1656
1894
|
this.mute = () => {
|
|
1657
1895
|
this.vad?.pause();
|
|
1658
1896
|
this._isMuted = true;
|
|
@@ -1703,6 +1941,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1703
1941
|
this.micRecorder.setSpeakerPlaying(
|
|
1704
1942
|
this.isInterruptionDisabled && Speaker.isPlaying
|
|
1705
1943
|
);
|
|
1944
|
+
this.micRecorder.isEcho = this.isEchoOfAssistant;
|
|
1706
1945
|
this.micRecorder.on("StateChange", () => {
|
|
1707
1946
|
this.notifyStateChange();
|
|
1708
1947
|
});
|
|
@@ -1728,7 +1967,9 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1728
1967
|
});
|
|
1729
1968
|
}
|
|
1730
1969
|
const deviceId = options.deviceId ?? storage.getItem("micdrop.micDevice" /* MicDevice */) ?? void 0;
|
|
1970
|
+
const recorder = this.micRecorder;
|
|
1731
1971
|
await Mic.start(deviceId);
|
|
1972
|
+
if (this.micRecorder !== recorder || !Mic.isStarted) return;
|
|
1732
1973
|
if (Mic.deviceId) {
|
|
1733
1974
|
storage.setItem("micdrop.micDevice" /* MicDevice */, Mic.deviceId);
|
|
1734
1975
|
} else {
|
|
@@ -1796,8 +2037,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1796
2037
|
clearTimeout(this.connectionTimer);
|
|
1797
2038
|
this.connectionTimer = void 0;
|
|
1798
2039
|
}
|
|
1799
|
-
this.
|
|
1800
|
-
this._isStartingCall = false;
|
|
2040
|
+
this.callStatus = "started";
|
|
1801
2041
|
this.reconnectAttempt = 0;
|
|
1802
2042
|
this.error = void 0;
|
|
1803
2043
|
this.notifyStateChange();
|
|
@@ -1882,7 +2122,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1882
2122
|
const config = { ...DEFAULT_RECONNECT_OPTIONS, ...this.options.reconnect };
|
|
1883
2123
|
const canReconnect = error && isRecoverableError(error) && config.maxAttempts > 0;
|
|
1884
2124
|
if (canReconnect) {
|
|
1885
|
-
this.
|
|
2125
|
+
this.callStatus = "reconnecting";
|
|
1886
2126
|
this.stopWS();
|
|
1887
2127
|
this.scheduleReconnect(config);
|
|
1888
2128
|
} else {
|
|
@@ -1894,6 +2134,9 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1894
2134
|
};
|
|
1895
2135
|
this.onSpeakerStartPlaying = () => {
|
|
1896
2136
|
this.log("Speaker started");
|
|
2137
|
+
if (this.isWSStarted) {
|
|
2138
|
+
this.echoGuard.speakerStarted(Date.now());
|
|
2139
|
+
}
|
|
1897
2140
|
if (this.isInterruptionDisabled) {
|
|
1898
2141
|
this.micRecorder?.setSpeakerPlaying(true);
|
|
1899
2142
|
this.vad?.pause();
|
|
@@ -1902,6 +2145,7 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1902
2145
|
};
|
|
1903
2146
|
this.onSpeakerStopPlaying = () => {
|
|
1904
2147
|
this.log("Speaker stopped");
|
|
2148
|
+
this.echoGuard.speakerStopped(Date.now());
|
|
1905
2149
|
this.micRecorder?.setSpeakerPlaying(false);
|
|
1906
2150
|
if (this.isInterruptionDisabled) {
|
|
1907
2151
|
setTimeout(() => {
|
|
@@ -1914,6 +2158,28 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1914
2158
|
this.notifyStateChange();
|
|
1915
2159
|
}
|
|
1916
2160
|
};
|
|
2161
|
+
/**
|
|
2162
|
+
* Treats speech heard just as the assistant starts as its echo, and steps up
|
|
2163
|
+
* what keeps it out of the microphone. The interruption never happens, so
|
|
2164
|
+
* neither the speaker nor the server hear of it.
|
|
2165
|
+
*/
|
|
2166
|
+
this.isEchoOfAssistant = () => {
|
|
2167
|
+
if (this.isInterruptionDisabled || !Speaker.isPlaying) return false;
|
|
2168
|
+
if (!this.echoGuard.speechHeard(Date.now())) return false;
|
|
2169
|
+
this.log("Echo of the assistant, moving to", this.echoGuard.stage);
|
|
2170
|
+
if (this.echoGuard.stage === "fallback") {
|
|
2171
|
+
Speaker.useEchoFallback().then((moved) => {
|
|
2172
|
+
if (!moved && this.echoGuard.stage === "fallback") {
|
|
2173
|
+
this.echoGuard.stage = "disabled";
|
|
2174
|
+
this.onInterruptionsGivenUp();
|
|
2175
|
+
}
|
|
2176
|
+
});
|
|
2177
|
+
} else {
|
|
2178
|
+
this.onInterruptionsGivenUp();
|
|
2179
|
+
}
|
|
2180
|
+
this.notifyStateChange();
|
|
2181
|
+
return true;
|
|
2182
|
+
};
|
|
1917
2183
|
Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
|
|
1918
2184
|
Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
|
|
1919
2185
|
Mic.on("DeviceChange", this.updateDevices);
|
|
@@ -1921,14 +2187,30 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1921
2187
|
get vad() {
|
|
1922
2188
|
return this.micRecorder?.vad;
|
|
1923
2189
|
}
|
|
2190
|
+
get status() {
|
|
2191
|
+
const mic = this.micRecorder?.state;
|
|
2192
|
+
if (this.callStatus === "reconnecting") return "reconnecting";
|
|
2193
|
+
if (this.callStatus === "started" && mic?.isStarted) return "started";
|
|
2194
|
+
if (this.callStatus !== "stopped" || mic?.isStarting) return "starting";
|
|
2195
|
+
return "stopped";
|
|
2196
|
+
}
|
|
2197
|
+
get activity() {
|
|
2198
|
+
if (this.isUserSpeaking) return "userSpeaking";
|
|
2199
|
+
if (this.isAssistantSpeaking) return "assistantSpeaking";
|
|
2200
|
+
if (this.isProcessing) return "processing";
|
|
2201
|
+
if (this.isListening) return "listening";
|
|
2202
|
+
return "idle";
|
|
2203
|
+
}
|
|
2204
|
+
/** The call is on, reconnecting included */
|
|
1924
2205
|
get isStarted() {
|
|
1925
|
-
|
|
2206
|
+
const status = this.status;
|
|
2207
|
+
return status === "started" || status === "reconnecting";
|
|
1926
2208
|
}
|
|
1927
2209
|
get isStarting() {
|
|
1928
|
-
return
|
|
2210
|
+
return this.status === "starting";
|
|
1929
2211
|
}
|
|
1930
2212
|
get isReconnecting() {
|
|
1931
|
-
return this.
|
|
2213
|
+
return this.status === "reconnecting";
|
|
1932
2214
|
}
|
|
1933
2215
|
get isMuted() {
|
|
1934
2216
|
return this._isMuted;
|
|
@@ -1963,10 +2245,15 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1963
2245
|
/**
|
|
1964
2246
|
* Whether the user is kept from speaking over the assistant. Without echo
|
|
1965
2247
|
* cancellation the microphone hears the assistant, which would interrupt
|
|
1966
|
-
* itself and end up in the transcript.
|
|
2248
|
+
* itself and end up in the transcript. A call also gets here on its own when
|
|
2249
|
+
* the assistant keeps cutting itself off, see {@link EchoGuard}.
|
|
1967
2250
|
*/
|
|
1968
2251
|
get isInterruptionDisabled() {
|
|
1969
|
-
return !!this.options.disableInterruption || Mic.echoCancellation === false;
|
|
2252
|
+
return !!this.options.disableInterruption || Mic.echoCancellation === false || this.echoGuard.stage === "disabled";
|
|
2253
|
+
}
|
|
2254
|
+
/** How far this call has gone to keep the assistant out of the microphone */
|
|
2255
|
+
get echoStage() {
|
|
2256
|
+
return this.echoGuard.stage;
|
|
1970
2257
|
}
|
|
1971
2258
|
get micDeviceId() {
|
|
1972
2259
|
return Mic.deviceId;
|
|
@@ -1976,6 +2263,8 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1976
2263
|
}
|
|
1977
2264
|
get state() {
|
|
1978
2265
|
return {
|
|
2266
|
+
status: this.status,
|
|
2267
|
+
activity: this.activity,
|
|
1979
2268
|
isStarting: this.isStarting,
|
|
1980
2269
|
isStarted: this.isStarted,
|
|
1981
2270
|
isReconnecting: this.isReconnecting,
|
|
@@ -1996,62 +2285,69 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
1996
2285
|
speakerDevices: this.speakerDevices
|
|
1997
2286
|
};
|
|
1998
2287
|
}
|
|
2288
|
+
/** The assistant prepares or speaks an answer */
|
|
2289
|
+
get isAnswering() {
|
|
2290
|
+
const { activity } = this;
|
|
2291
|
+
return activity === "processing" || activity === "assistantSpeaking";
|
|
2292
|
+
}
|
|
2293
|
+
cancelPendingStop() {
|
|
2294
|
+
this.pendingStop?.();
|
|
2295
|
+
}
|
|
1999
2296
|
async stopMic() {
|
|
2000
2297
|
this.micRecorder?.stop();
|
|
2001
2298
|
this.micRecorder = void 0;
|
|
2002
2299
|
await Mic.stop();
|
|
2003
2300
|
this.notifyStateChange();
|
|
2004
2301
|
}
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
this.ws.binaryType = "arraybuffer";
|
|
2019
|
-
this.notifyStateChange();
|
|
2020
|
-
this.ws.onopen = this.onWSOpen;
|
|
2021
|
-
this.ws.onmessage = this.onWSMessage;
|
|
2022
|
-
this.ws.onclose = this.onWSClose;
|
|
2023
|
-
this.ws.onerror = this.onWSError;
|
|
2024
|
-
this.connectionTimer = setTimeout(() => {
|
|
2025
|
-
this.connectionTimer = void 0;
|
|
2026
|
-
if (this.ws?.readyState === 0) {
|
|
2027
|
-
this.log("WebSocket connection timeout");
|
|
2028
|
-
this.ws.close();
|
|
2029
|
-
}
|
|
2030
|
-
}, this.options.reconnect?.connectionTimeout ?? DEFAULT_RECONNECT_OPTIONS.connectionTimeout);
|
|
2031
|
-
} catch (error) {
|
|
2032
|
-
this.setError(
|
|
2033
|
-
error instanceof MicdropClientError ? error : new MicdropClientError(
|
|
2034
|
-
"Connection" /* Connection */,
|
|
2035
|
-
error?.message
|
|
2036
|
-
)
|
|
2037
|
-
);
|
|
2038
|
-
await this.stop();
|
|
2039
|
-
throw error;
|
|
2302
|
+
/**
|
|
2303
|
+
* Opens the connection, unless one is already open or opening.
|
|
2304
|
+
*
|
|
2305
|
+
* The socket answers through handlers bound to it, so one left behind by a
|
|
2306
|
+
* stop or a reconnection never acts on the call that followed.
|
|
2307
|
+
*/
|
|
2308
|
+
openWS() {
|
|
2309
|
+
if (this.ws) {
|
|
2310
|
+
this.log("WebSocket is already started");
|
|
2311
|
+
return;
|
|
2312
|
+
}
|
|
2313
|
+
if (!this.isMicStarted) {
|
|
2314
|
+
throw new MicdropClientError("Mic" /* Mic */);
|
|
2040
2315
|
}
|
|
2316
|
+
if (!this.options.url) {
|
|
2317
|
+
throw new MicdropClientError("MissingUrl" /* MissingUrl */);
|
|
2318
|
+
}
|
|
2319
|
+
const ws = new WebSocket(this.options.url);
|
|
2320
|
+
ws.binaryType = "arraybuffer";
|
|
2321
|
+
this.ws = ws;
|
|
2322
|
+
this.notifyStateChange();
|
|
2323
|
+
const isCurrent = () => this.ws === ws;
|
|
2324
|
+
ws.onopen = () => isCurrent() && this.onWSOpen();
|
|
2325
|
+
ws.onmessage = (event) => isCurrent() && this.onWSMessage(event);
|
|
2326
|
+
ws.onclose = (event) => isCurrent() && this.onWSClose(event);
|
|
2327
|
+
ws.onerror = () => isCurrent() && this.onWSError();
|
|
2328
|
+
this.connectionTimer = setTimeout(() => {
|
|
2329
|
+
this.connectionTimer = void 0;
|
|
2330
|
+
if (isCurrent() && ws.readyState === 0) {
|
|
2331
|
+
this.log("WebSocket connection timeout");
|
|
2332
|
+
ws.close();
|
|
2333
|
+
}
|
|
2334
|
+
}, this.options.reconnect?.connectionTimeout ?? DEFAULT_RECONNECT_OPTIONS.connectionTimeout);
|
|
2041
2335
|
}
|
|
2042
2336
|
stopWS() {
|
|
2043
2337
|
if (this.connectionTimer) {
|
|
2044
2338
|
clearTimeout(this.connectionTimer);
|
|
2045
2339
|
this.connectionTimer = void 0;
|
|
2046
2340
|
}
|
|
2047
|
-
|
|
2048
|
-
if (
|
|
2049
|
-
this.ws.close();
|
|
2050
|
-
}
|
|
2341
|
+
const ws = this.ws;
|
|
2342
|
+
if (!ws) return;
|
|
2051
2343
|
this.ws = void 0;
|
|
2344
|
+
if (ws.readyState === 1 || ws.readyState === 0) {
|
|
2345
|
+
ws.close();
|
|
2346
|
+
}
|
|
2052
2347
|
this.notifyStateChange();
|
|
2053
2348
|
}
|
|
2054
2349
|
scheduleReconnect(config) {
|
|
2350
|
+
if (this.callStatus !== "reconnecting") return;
|
|
2055
2351
|
if (this.reconnectAttempt >= config.maxAttempts) {
|
|
2056
2352
|
this.stop();
|
|
2057
2353
|
this.setError(new MicdropClientError("Connection" /* Connection */));
|
|
@@ -2064,17 +2360,32 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
2064
2360
|
if (!this.isPaused && !this.isMuted) {
|
|
2065
2361
|
this.vad?.pause?.();
|
|
2066
2362
|
}
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2363
|
+
this.clearReconnectTimer();
|
|
2364
|
+
this.reconnectTimer = setTimeout(() => {
|
|
2365
|
+
this.reconnectTimer = void 0;
|
|
2366
|
+
if (this.callStatus !== "reconnecting") return;
|
|
2071
2367
|
try {
|
|
2072
|
-
|
|
2073
|
-
} catch {
|
|
2074
|
-
this.
|
|
2368
|
+
this.openWS();
|
|
2369
|
+
} catch (error) {
|
|
2370
|
+
this.setError(toClientError(error));
|
|
2371
|
+
this.stop();
|
|
2075
2372
|
}
|
|
2076
2373
|
}, config.delayMs);
|
|
2077
2374
|
}
|
|
2375
|
+
clearReconnectTimer() {
|
|
2376
|
+
if (!this.reconnectTimer) return;
|
|
2377
|
+
clearTimeout(this.reconnectTimer);
|
|
2378
|
+
this.reconnectTimer = void 0;
|
|
2379
|
+
}
|
|
2380
|
+
// The same as when the speaker starts on a call without interruptions
|
|
2381
|
+
onInterruptionsGivenUp() {
|
|
2382
|
+
this.log("Interruptions disabled until the end of the call");
|
|
2383
|
+
if (Speaker.isPlaying) {
|
|
2384
|
+
this.micRecorder?.setSpeakerPlaying(true);
|
|
2385
|
+
this.vad?.pause();
|
|
2386
|
+
}
|
|
2387
|
+
this.notifyStateChange();
|
|
2388
|
+
}
|
|
2078
2389
|
addMessage(message) {
|
|
2079
2390
|
if (message.role === "assistant") this.partialAssistantMessage = "";
|
|
2080
2391
|
this.conversation = [...this.conversation, message];
|
|
@@ -2110,6 +2421,13 @@ var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
|
2110
2421
|
console.log(`[MicdropClient ${Date.now() - this.startTime}]`, ...message);
|
|
2111
2422
|
}
|
|
2112
2423
|
};
|
|
2424
|
+
function toClientError(error) {
|
|
2425
|
+
if (error instanceof MicdropClientError) return error;
|
|
2426
|
+
return new MicdropClientError(
|
|
2427
|
+
"Connection" /* Connection */,
|
|
2428
|
+
error?.message
|
|
2429
|
+
);
|
|
2430
|
+
}
|
|
2113
2431
|
|
|
2114
2432
|
// src/Micdrop.ts
|
|
2115
2433
|
var globalScope2 = globalThis;
|
|
@@ -2122,6 +2440,7 @@ var Micdrop = globalScope2.micdropClient;
|
|
|
2122
2440
|
CONFIRM_MARGIN,
|
|
2123
2441
|
DEFAULT_TURN_MAX_WAIT,
|
|
2124
2442
|
ECHO_TAIL,
|
|
2443
|
+
LazySession,
|
|
2125
2444
|
Mic,
|
|
2126
2445
|
MicController,
|
|
2127
2446
|
MicDriver,
|
|
@@ -2147,6 +2466,7 @@ var Micdrop = globalScope2.micdropClient;
|
|
|
2147
2466
|
VolumeMeter,
|
|
2148
2467
|
VolumeVAD,
|
|
2149
2468
|
concatFloat32,
|
|
2469
|
+
createDownloadCache,
|
|
2150
2470
|
currentTurn,
|
|
2151
2471
|
equalVADConfig,
|
|
2152
2472
|
floatToPcm16,
|