@micdrop/server 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/index.d.mts +191 -5
- package/dist/index.d.ts +191 -5
- package/dist/index.js +380 -25
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +378 -25
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -51,9 +51,11 @@ __export(index_exports, {
|
|
|
51
51
|
MockSTT: () => MockSTT,
|
|
52
52
|
MockTTS: () => MockTTS,
|
|
53
53
|
Pcm16Resampler: () => Pcm16Resampler,
|
|
54
|
+
Realtime: () => Realtime,
|
|
54
55
|
STT: () => STT,
|
|
55
56
|
SentenceSplitter: () => SentenceSplitter,
|
|
56
57
|
SentenceTTS: () => SentenceTTS,
|
|
58
|
+
SharedInstances: () => SharedInstances,
|
|
57
59
|
TTS: () => TTS,
|
|
58
60
|
float32ToPcm16: () => float32ToPcm16,
|
|
59
61
|
handleError: () => handleError,
|
|
@@ -89,7 +91,9 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
89
91
|
const answerCount = ++this.answerCount;
|
|
90
92
|
const stream = new import_stream.PassThrough();
|
|
91
93
|
this.answering = true;
|
|
92
|
-
Promise.resolve().then(
|
|
94
|
+
Promise.resolve().then(
|
|
95
|
+
() => this.options.onBeforeAnswer?.bind(this)(stream)
|
|
96
|
+
).then((skip) => {
|
|
93
97
|
if (skip) return;
|
|
94
98
|
return this.generateAnswer(stream);
|
|
95
99
|
}).finally(() => {
|
|
@@ -132,16 +136,16 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
132
136
|
metadata
|
|
133
137
|
};
|
|
134
138
|
this.conversation.push(message);
|
|
135
|
-
this.
|
|
139
|
+
this.emitAgentEvent("Message", message);
|
|
136
140
|
}
|
|
137
141
|
addToolMessage(message) {
|
|
138
142
|
this.log("Adding tool message:", message);
|
|
139
143
|
this.conversation.push(message);
|
|
140
|
-
this.
|
|
144
|
+
this.emitAgentEvent("Message", message);
|
|
141
145
|
}
|
|
142
146
|
endCall() {
|
|
143
147
|
this.log("Ending call");
|
|
144
|
-
this.
|
|
148
|
+
this.emitAgentEvent("EndCall");
|
|
145
149
|
}
|
|
146
150
|
cancelLastUserMessage() {
|
|
147
151
|
this.log("Cancelling last user message");
|
|
@@ -151,11 +155,11 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
151
155
|
if (lastMessageIndex !== -1) {
|
|
152
156
|
this.conversation.splice(lastMessageIndex, 1);
|
|
153
157
|
}
|
|
154
|
-
this.
|
|
158
|
+
this.emitAgentEvent("CancelLastUserMessage");
|
|
155
159
|
}
|
|
156
160
|
skipAnswer() {
|
|
157
161
|
this.log("Skipping answer");
|
|
158
|
-
this.
|
|
162
|
+
this.emitAgentEvent("SkipAnswer");
|
|
159
163
|
}
|
|
160
164
|
getDefaultTools() {
|
|
161
165
|
const tools = [];
|
|
@@ -201,7 +205,7 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
201
205
|
output: JSON.stringify(output ?? null)
|
|
202
206
|
});
|
|
203
207
|
if (tool.emitOutput) {
|
|
204
|
-
this.
|
|
208
|
+
this.emitAgentEvent("ToolCall", {
|
|
205
209
|
name: toolCall.toolName,
|
|
206
210
|
parameters,
|
|
207
211
|
output
|
|
@@ -260,6 +264,16 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
260
264
|
}
|
|
261
265
|
return { message, metadata };
|
|
262
266
|
}
|
|
267
|
+
/**
|
|
268
|
+
* Emits one of the events every agent has.
|
|
269
|
+
*
|
|
270
|
+
* A subclass can add events of its own, and TypeScript then no longer knows
|
|
271
|
+
* the shared ones are among them.
|
|
272
|
+
*/
|
|
273
|
+
emitAgentEvent(event, ...args) {
|
|
274
|
+
const emitter = this;
|
|
275
|
+
emitter.emit(event, ...args);
|
|
276
|
+
}
|
|
263
277
|
log(...message) {
|
|
264
278
|
this.logger?.log(...message);
|
|
265
279
|
}
|
|
@@ -533,8 +547,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
533
547
|
this.log("Connection closed");
|
|
534
548
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
535
549
|
const conversation = this.conversation;
|
|
536
|
-
this.
|
|
537
|
-
this.config.stt
|
|
550
|
+
this.agent?.destroy();
|
|
551
|
+
this.config.stt?.destroy();
|
|
538
552
|
this.config.tts?.destroy();
|
|
539
553
|
this.emit("End", {
|
|
540
554
|
conversation,
|
|
@@ -577,7 +591,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
577
591
|
this.answerUserTurn();
|
|
578
592
|
}
|
|
579
593
|
};
|
|
580
|
-
this.
|
|
594
|
+
this.onAssistantAudio = (audio) => {
|
|
581
595
|
if (!this.socket) return;
|
|
582
596
|
this.log(`Send audio chunk (${audio.byteLength} bytes)`);
|
|
583
597
|
this.socket.send(audio);
|
|
@@ -589,9 +603,23 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
589
603
|
this.socket = socket;
|
|
590
604
|
this.config = config;
|
|
591
605
|
this.log(`Call started`);
|
|
592
|
-
config
|
|
593
|
-
|
|
594
|
-
|
|
606
|
+
const { realtime } = config;
|
|
607
|
+
if (realtime) {
|
|
608
|
+
realtime.on("Audio", this.onAssistantAudio);
|
|
609
|
+
if (config.partialMessages) {
|
|
610
|
+
realtime.on(
|
|
611
|
+
"PartialMessage",
|
|
612
|
+
(content) => this.sendPartialAnswer(content)
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
} else {
|
|
616
|
+
if (!config.stt) {
|
|
617
|
+
throw new Error("MicdropServer needs an stt, or a realtime model");
|
|
618
|
+
}
|
|
619
|
+
config.stt.on("Transcript", this.onTranscriptSTT);
|
|
620
|
+
config.tts?.on("Audio", this.onAssistantAudio);
|
|
621
|
+
}
|
|
622
|
+
const agent = this.agent;
|
|
595
623
|
if (agent) {
|
|
596
624
|
agent.on("Message", this.onMessageAgent);
|
|
597
625
|
agent.on(
|
|
@@ -619,7 +647,12 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
619
647
|
}
|
|
620
648
|
/** Everything said so far, whether an agent or the server keeps it */
|
|
621
649
|
get conversation() {
|
|
622
|
-
return this.
|
|
650
|
+
return this.agent?.conversation ?? this.ownConversation;
|
|
651
|
+
}
|
|
652
|
+
/** What holds the conversation and answers, a realtime model included */
|
|
653
|
+
get agent() {
|
|
654
|
+
if (!this.config) return void 0;
|
|
655
|
+
return this.config.realtime ?? this.config.agent;
|
|
623
656
|
}
|
|
624
657
|
log(...message) {
|
|
625
658
|
this.logger?.log(...message);
|
|
@@ -645,7 +678,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
645
678
|
}
|
|
646
679
|
cancel() {
|
|
647
680
|
this.config?.tts?.cancel();
|
|
648
|
-
this.
|
|
681
|
+
this.agent?.cancel();
|
|
649
682
|
this.operationQueue = [];
|
|
650
683
|
}
|
|
651
684
|
onUserAudio(chunk) {
|
|
@@ -660,6 +693,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
660
693
|
this.currentUserStream?.end();
|
|
661
694
|
this.currentUserStream = void 0;
|
|
662
695
|
this.cancel();
|
|
696
|
+
this.config?.realtime?.discardTurn();
|
|
663
697
|
}
|
|
664
698
|
onStartSpeaking() {
|
|
665
699
|
if (!this.config) return;
|
|
@@ -669,8 +703,13 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
669
703
|
this.config.turnDetector?.reset();
|
|
670
704
|
this.turnComplete = void 0;
|
|
671
705
|
this.releaseHeldTurn();
|
|
672
|
-
this.config.
|
|
673
|
-
|
|
706
|
+
if (this.config.realtime) {
|
|
707
|
+
this.cancel();
|
|
708
|
+
this.config.realtime.startTurn(this.currentUserStream);
|
|
709
|
+
} else {
|
|
710
|
+
this.config.stt?.transcribe(this.currentUserStream);
|
|
711
|
+
this.cancel();
|
|
712
|
+
}
|
|
674
713
|
}
|
|
675
714
|
onStopSpeaking() {
|
|
676
715
|
const hasNoUserSpeech = !this.currentUserStream || this.userSpeechChunks === 0;
|
|
@@ -682,6 +721,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
682
721
|
return;
|
|
683
722
|
}
|
|
684
723
|
this.turnComplete = this.predictTurnComplete();
|
|
724
|
+
if (this.config?.realtime) {
|
|
725
|
+
this.answerUserTurn();
|
|
726
|
+
return;
|
|
727
|
+
}
|
|
685
728
|
const conversation = this.conversation;
|
|
686
729
|
const lastMessage = conversation[conversation.length - 1];
|
|
687
730
|
if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
|
|
@@ -714,7 +757,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
714
757
|
}
|
|
715
758
|
this.releaseHeldTurn();
|
|
716
759
|
this.cancel();
|
|
717
|
-
this.
|
|
760
|
+
this.answerTurn();
|
|
718
761
|
}
|
|
719
762
|
/**
|
|
720
763
|
* Answers anyway if the rest of the sentence never comes.
|
|
@@ -728,9 +771,18 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
728
771
|
this.heldTurnTimer = void 0;
|
|
729
772
|
this.log("Nothing more came, answering");
|
|
730
773
|
this.cancel();
|
|
731
|
-
this.
|
|
774
|
+
this.answerTurn();
|
|
732
775
|
}, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
|
|
733
776
|
}
|
|
777
|
+
/** Answers what the user just said */
|
|
778
|
+
answerTurn() {
|
|
779
|
+
const realtime = this.config?.realtime;
|
|
780
|
+
if (realtime) {
|
|
781
|
+
realtime.endTurn();
|
|
782
|
+
} else {
|
|
783
|
+
this.answer();
|
|
784
|
+
}
|
|
785
|
+
}
|
|
734
786
|
releaseHeldTurn() {
|
|
735
787
|
if (!this.heldTurnTimer) return;
|
|
736
788
|
clearTimeout(this.heldTurnTimer);
|
|
@@ -738,16 +790,16 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
738
790
|
}
|
|
739
791
|
/** Adds a message to the conversation and sends it to the client */
|
|
740
792
|
addUserMessage(text, metadata) {
|
|
741
|
-
if (this.
|
|
742
|
-
this.
|
|
793
|
+
if (this.agent) {
|
|
794
|
+
this.agent.addUserMessage(text, metadata);
|
|
743
795
|
} else {
|
|
744
796
|
this.addOwnMessage("user", text, metadata);
|
|
745
797
|
}
|
|
746
798
|
}
|
|
747
799
|
/** Adds a message to the conversation and sends it to the client */
|
|
748
800
|
addAssistantMessage(text, metadata) {
|
|
749
|
-
if (this.
|
|
750
|
-
this.
|
|
801
|
+
if (this.agent) {
|
|
802
|
+
this.agent.addAssistantMessage(text, metadata);
|
|
751
803
|
} else {
|
|
752
804
|
this.addOwnMessage("assistant", text, metadata);
|
|
753
805
|
}
|
|
@@ -782,9 +834,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
782
834
|
sendFirstMessage() {
|
|
783
835
|
if (!this.config) return;
|
|
784
836
|
if (this.config.firstMessage) {
|
|
785
|
-
this.
|
|
837
|
+
this.agent?.addAssistantMessage(this.config.firstMessage);
|
|
786
838
|
this.speak(this.config.firstMessage);
|
|
787
|
-
} else if (this.config.generateFirstMessage && this.
|
|
839
|
+
} else if (this.config.generateFirstMessage && this.agent) {
|
|
788
840
|
this.answer();
|
|
789
841
|
} else {
|
|
790
842
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -797,6 +849,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
797
849
|
}
|
|
798
850
|
async _answer() {
|
|
799
851
|
if (!this.config) return;
|
|
852
|
+
if (this.config.realtime) {
|
|
853
|
+
this.config.realtime.respond();
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
800
856
|
if (!this.config.agent) {
|
|
801
857
|
this.log("No agent, skipping answer");
|
|
802
858
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -844,6 +900,11 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
844
900
|
}
|
|
845
901
|
async _speak(message) {
|
|
846
902
|
if (!this.socket || !this.config) return;
|
|
903
|
+
if (this.config.realtime) {
|
|
904
|
+
const text = typeof message === "string" ? message : await readText(message);
|
|
905
|
+
if (text.trim() !== "") this.config.realtime.say(text);
|
|
906
|
+
return;
|
|
907
|
+
}
|
|
847
908
|
let textStream;
|
|
848
909
|
if (typeof message === "string") {
|
|
849
910
|
if (!this.config.agent) {
|
|
@@ -882,6 +943,14 @@ function hasContent(stream) {
|
|
|
882
943
|
stream.on("end", onEnd);
|
|
883
944
|
});
|
|
884
945
|
}
|
|
946
|
+
function readText(stream) {
|
|
947
|
+
return new Promise((resolve) => {
|
|
948
|
+
let text = "";
|
|
949
|
+
stream.on("data", (chunk) => text += chunk.toString());
|
|
950
|
+
stream.once("end", () => resolve(text));
|
|
951
|
+
stream.once("error", () => resolve(text));
|
|
952
|
+
});
|
|
953
|
+
}
|
|
885
954
|
function drain(stream) {
|
|
886
955
|
return new Promise((resolve) => {
|
|
887
956
|
stream.on("data", () => {
|
|
@@ -892,6 +961,252 @@ function drain(stream) {
|
|
|
892
961
|
});
|
|
893
962
|
}
|
|
894
963
|
|
|
964
|
+
// src/realtime/Realtime.ts
|
|
965
|
+
var USER_TRANSCRIPT_TIMEOUT = 3e3;
|
|
966
|
+
function newAnswer(record) {
|
|
967
|
+
return { transcript: "", hasAudio: false, record, resumed: false };
|
|
968
|
+
}
|
|
969
|
+
var Realtime = class extends Agent {
|
|
970
|
+
constructor() {
|
|
971
|
+
super(...arguments);
|
|
972
|
+
// A turn opens with its first audio, so a blip without any sends nothing
|
|
973
|
+
this.turn = "none";
|
|
974
|
+
// Output of an interrupted answer can still be on its way
|
|
975
|
+
this.discarding = false;
|
|
976
|
+
// Answers asked for while another one was being given
|
|
977
|
+
this.queue = [];
|
|
978
|
+
this.answerWaiters = [];
|
|
979
|
+
this.userTranscript = "";
|
|
980
|
+
this.awaitingUserTranscript = false;
|
|
981
|
+
this.afterUserTranscript = [];
|
|
982
|
+
}
|
|
983
|
+
/**
|
|
984
|
+
* The user started speaking.
|
|
985
|
+
*
|
|
986
|
+
* The assistant stops, and the audio read from the stream goes to the model.
|
|
987
|
+
* A turn left open, because the sentence sounded unfinished, carries on.
|
|
988
|
+
*/
|
|
989
|
+
startTurn(audio) {
|
|
990
|
+
this.cancel();
|
|
991
|
+
if (this.turn === "none") this.turn = "pending";
|
|
992
|
+
audio.on("data", (chunk) => this.sendAudio(chunk));
|
|
993
|
+
}
|
|
994
|
+
/** Closes the turn of the user, and has the model answer it */
|
|
995
|
+
endTurn() {
|
|
996
|
+
if (this.turn !== "open") return;
|
|
997
|
+
this.turn = "none";
|
|
998
|
+
this.awaitUserTranscript();
|
|
999
|
+
this.beginAnswer(true);
|
|
1000
|
+
this.closeTurn();
|
|
1001
|
+
}
|
|
1002
|
+
/** Drops the turn of the user, without an answer */
|
|
1003
|
+
discardTurn() {
|
|
1004
|
+
this.cancel();
|
|
1005
|
+
if (this.turn === "open") {
|
|
1006
|
+
this.clearTurn();
|
|
1007
|
+
this.discarding = true;
|
|
1008
|
+
}
|
|
1009
|
+
this.turn = "none";
|
|
1010
|
+
}
|
|
1011
|
+
/** Has the model speak without waiting for the user, to open a call */
|
|
1012
|
+
respond() {
|
|
1013
|
+
this.whenIdle(() => {
|
|
1014
|
+
this.beginAnswer(true);
|
|
1015
|
+
this.generate();
|
|
1016
|
+
});
|
|
1017
|
+
}
|
|
1018
|
+
/**
|
|
1019
|
+
* Has the model say this text, word for word.
|
|
1020
|
+
*
|
|
1021
|
+
* The caller records the text in the conversation, as with a text to speech,
|
|
1022
|
+
* so the transcript of what the model says is not recorded again.
|
|
1023
|
+
*/
|
|
1024
|
+
say(text) {
|
|
1025
|
+
this.whenIdle(() => {
|
|
1026
|
+
this.beginAnswer(false);
|
|
1027
|
+
this.speakText(text);
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
cancel() {
|
|
1031
|
+
this.queue = [];
|
|
1032
|
+
const answer = this.currentAnswer;
|
|
1033
|
+
if (!answer) return;
|
|
1034
|
+
this.log("Cancel");
|
|
1035
|
+
this.currentAnswer = void 0;
|
|
1036
|
+
this.discarding = true;
|
|
1037
|
+
this.cancelAnswer();
|
|
1038
|
+
this.recordAnswer(answer);
|
|
1039
|
+
this.resolveAnswerWaiters();
|
|
1040
|
+
}
|
|
1041
|
+
/**
|
|
1042
|
+
* Adds a message the model did not hear, and tells the model about it.
|
|
1043
|
+
*
|
|
1044
|
+
* The transcripts of the call are recorded apart, so they are not sent back.
|
|
1045
|
+
*/
|
|
1046
|
+
addMessage(role, text, metadata) {
|
|
1047
|
+
super.addMessage(role, text, metadata);
|
|
1048
|
+
if (text.trim() !== "") this.injectMessage(role, text);
|
|
1049
|
+
}
|
|
1050
|
+
addTool(tool) {
|
|
1051
|
+
super.addTool(tool);
|
|
1052
|
+
this.updateTools();
|
|
1053
|
+
}
|
|
1054
|
+
removeTool(name) {
|
|
1055
|
+
super.removeTool(name);
|
|
1056
|
+
this.updateTools();
|
|
1057
|
+
}
|
|
1058
|
+
/**
|
|
1059
|
+
* Lists the events that have listeners.
|
|
1060
|
+
*
|
|
1061
|
+
* Typed as the events of any agent, so a realtime model goes wherever an
|
|
1062
|
+
* agent is expected. The two extra events do not change what that code reads.
|
|
1063
|
+
*/
|
|
1064
|
+
eventNames() {
|
|
1065
|
+
return super.eventNames();
|
|
1066
|
+
}
|
|
1067
|
+
destroy() {
|
|
1068
|
+
super.destroy();
|
|
1069
|
+
this.clearUserTranscriptTimer();
|
|
1070
|
+
this.afterUserTranscript = [];
|
|
1071
|
+
this.resolveAnswerWaiters();
|
|
1072
|
+
}
|
|
1073
|
+
generateAnswer(stream) {
|
|
1074
|
+
let written = 0;
|
|
1075
|
+
const onPartial = (transcript) => {
|
|
1076
|
+
stream.write(transcript.slice(written));
|
|
1077
|
+
written = transcript.length;
|
|
1078
|
+
};
|
|
1079
|
+
this.on("PartialMessage", onPartial);
|
|
1080
|
+
this.respond();
|
|
1081
|
+
return new Promise((resolve) => {
|
|
1082
|
+
this.answerWaiters.push(() => {
|
|
1083
|
+
this.off("PartialMessage", onPartial);
|
|
1084
|
+
resolve();
|
|
1085
|
+
});
|
|
1086
|
+
});
|
|
1087
|
+
}
|
|
1088
|
+
/** Nobody is speaking, nor about to, so the connection can be renewed */
|
|
1089
|
+
get isIdle() {
|
|
1090
|
+
return this.turn === "none" && !this.currentAnswer;
|
|
1091
|
+
}
|
|
1092
|
+
/** Voice of the answer, PCM16 at 16 kHz */
|
|
1093
|
+
emitAudio(chunk) {
|
|
1094
|
+
const answer = this.activeAnswer();
|
|
1095
|
+
if (!answer) return;
|
|
1096
|
+
answer.hasAudio = true;
|
|
1097
|
+
this.emit("Audio", chunk);
|
|
1098
|
+
}
|
|
1099
|
+
/** Transcript of the answer, as it is spoken */
|
|
1100
|
+
addAnswerTranscript(text) {
|
|
1101
|
+
const answer = this.activeAnswer();
|
|
1102
|
+
if (!answer) return;
|
|
1103
|
+
if (answer.resumed) {
|
|
1104
|
+
answer.resumed = false;
|
|
1105
|
+
if (answer.transcript !== "" && !/\s$/.test(answer.transcript)) {
|
|
1106
|
+
text = ` ${text.trimStart()}`;
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
answer.transcript += text;
|
|
1110
|
+
if (answer.record) this.emit("PartialMessage", answer.transcript);
|
|
1111
|
+
}
|
|
1112
|
+
/**
|
|
1113
|
+
* The model finished its answer, or finished dropping an interrupted one.
|
|
1114
|
+
*
|
|
1115
|
+
* An answer without audio tells the client to stop waiting for one.
|
|
1116
|
+
*/
|
|
1117
|
+
endAnswer() {
|
|
1118
|
+
if (this.discarding) {
|
|
1119
|
+
this.discarding = false;
|
|
1120
|
+
return;
|
|
1121
|
+
}
|
|
1122
|
+
const answer = this.currentAnswer;
|
|
1123
|
+
if (!answer) return;
|
|
1124
|
+
this.currentAnswer = void 0;
|
|
1125
|
+
if (!answer.hasAudio) this.emitAgentEvent("SkipAnswer");
|
|
1126
|
+
this.recordAnswer(answer);
|
|
1127
|
+
this.resolveAnswerWaiters();
|
|
1128
|
+
this.queue.shift()?.();
|
|
1129
|
+
}
|
|
1130
|
+
/** Runs a tool the model called, and records it in the conversation */
|
|
1131
|
+
async runTool(toolCall) {
|
|
1132
|
+
const result = await this.executeTool(toolCall);
|
|
1133
|
+
if (this.currentAnswer) this.currentAnswer.resumed = true;
|
|
1134
|
+
return result;
|
|
1135
|
+
}
|
|
1136
|
+
/** Transcript of the turn of the user, possibly in several pieces */
|
|
1137
|
+
addUserTranscript(text) {
|
|
1138
|
+
this.userTranscript += text;
|
|
1139
|
+
}
|
|
1140
|
+
/** The transcript of the turn is complete, it goes in the conversation */
|
|
1141
|
+
commitUserTranscript() {
|
|
1142
|
+
this.clearUserTranscriptTimer();
|
|
1143
|
+
this.awaitingUserTranscript = false;
|
|
1144
|
+
const text = this.userTranscript.trim();
|
|
1145
|
+
this.userTranscript = "";
|
|
1146
|
+
if (text !== "") super.addMessage("user", text);
|
|
1147
|
+
const pending = this.afterUserTranscript;
|
|
1148
|
+
this.afterUserTranscript = [];
|
|
1149
|
+
pending.forEach((record) => record());
|
|
1150
|
+
}
|
|
1151
|
+
sendAudio(chunk) {
|
|
1152
|
+
if (this.turn === "none") return;
|
|
1153
|
+
if (this.turn === "pending") {
|
|
1154
|
+
this.turn = "open";
|
|
1155
|
+
this.openTurn();
|
|
1156
|
+
}
|
|
1157
|
+
this.appendAudio(chunk);
|
|
1158
|
+
}
|
|
1159
|
+
beginAnswer(record) {
|
|
1160
|
+
this.discarding = false;
|
|
1161
|
+
this.currentAnswer = newAnswer(record);
|
|
1162
|
+
}
|
|
1163
|
+
/**
|
|
1164
|
+
* The answer output belongs to, if it is still wanted.
|
|
1165
|
+
*
|
|
1166
|
+
* The model can speak without being asked, to follow up on a tool call for
|
|
1167
|
+
* instance, which opens an answer of its own.
|
|
1168
|
+
*/
|
|
1169
|
+
activeAnswer() {
|
|
1170
|
+
if (this.discarding) return void 0;
|
|
1171
|
+
if (!this.currentAnswer) this.currentAnswer = newAnswer(true);
|
|
1172
|
+
return this.currentAnswer;
|
|
1173
|
+
}
|
|
1174
|
+
whenIdle(start) {
|
|
1175
|
+
if (this.currentAnswer) {
|
|
1176
|
+
this.queue.push(start);
|
|
1177
|
+
} else {
|
|
1178
|
+
start();
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
recordAnswer(answer) {
|
|
1182
|
+
if (!answer.record) return;
|
|
1183
|
+
const record = () => super.addMessage("assistant", answer.transcript.trim());
|
|
1184
|
+
if (this.awaitingUserTranscript) {
|
|
1185
|
+
this.afterUserTranscript.push(record);
|
|
1186
|
+
} else {
|
|
1187
|
+
record();
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
awaitUserTranscript() {
|
|
1191
|
+
this.awaitingUserTranscript = true;
|
|
1192
|
+
this.clearUserTranscriptTimer();
|
|
1193
|
+
this.userTranscriptTimer = setTimeout(() => {
|
|
1194
|
+
this.log("No transcript of the turn, recording the answer anyway");
|
|
1195
|
+
this.commitUserTranscript();
|
|
1196
|
+
}, USER_TRANSCRIPT_TIMEOUT);
|
|
1197
|
+
}
|
|
1198
|
+
clearUserTranscriptTimer() {
|
|
1199
|
+
if (!this.userTranscriptTimer) return;
|
|
1200
|
+
clearTimeout(this.userTranscriptTimer);
|
|
1201
|
+
this.userTranscriptTimer = void 0;
|
|
1202
|
+
}
|
|
1203
|
+
resolveAnswerWaiters() {
|
|
1204
|
+
const waiters = this.answerWaiters;
|
|
1205
|
+
this.answerWaiters = [];
|
|
1206
|
+
waiters.forEach((resolve) => resolve());
|
|
1207
|
+
}
|
|
1208
|
+
};
|
|
1209
|
+
|
|
895
1210
|
// src/recorder/MicdropRecorder.ts
|
|
896
1211
|
var import_eventemitter33 = require("eventemitter3");
|
|
897
1212
|
var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
@@ -1012,6 +1327,44 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
1012
1327
|
}
|
|
1013
1328
|
};
|
|
1014
1329
|
|
|
1330
|
+
// src/SharedInstances.ts
|
|
1331
|
+
var SharedInstances = class {
|
|
1332
|
+
constructor() {
|
|
1333
|
+
this.entries = /* @__PURE__ */ new Map();
|
|
1334
|
+
}
|
|
1335
|
+
/**
|
|
1336
|
+
* Returns the instance for this key, calling `create` the first time.
|
|
1337
|
+
*
|
|
1338
|
+
* The key can be any JSON-serializable value, usually an array of the
|
|
1339
|
+
* options that define the instance. Concurrent callers get the same promise
|
|
1340
|
+
* instead of loading the model twice.
|
|
1341
|
+
*/
|
|
1342
|
+
load(key, create) {
|
|
1343
|
+
const id = typeof key === "string" ? key : JSON.stringify(key);
|
|
1344
|
+
const existing = this.entries.get(id);
|
|
1345
|
+
if (existing) return existing;
|
|
1346
|
+
let loading;
|
|
1347
|
+
try {
|
|
1348
|
+
loading = create();
|
|
1349
|
+
} catch (error) {
|
|
1350
|
+
return Promise.reject(error);
|
|
1351
|
+
}
|
|
1352
|
+
loading.catch(() => {
|
|
1353
|
+
if (this.entries.get(id) === loading) this.entries.delete(id);
|
|
1354
|
+
});
|
|
1355
|
+
this.entries.set(id, loading);
|
|
1356
|
+
return loading;
|
|
1357
|
+
}
|
|
1358
|
+
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
1359
|
+
has(key) {
|
|
1360
|
+
return this.entries.has(typeof key === "string" ? key : JSON.stringify(key));
|
|
1361
|
+
}
|
|
1362
|
+
/** Drops every instance, so the next call loads again. */
|
|
1363
|
+
clear() {
|
|
1364
|
+
this.entries.clear();
|
|
1365
|
+
}
|
|
1366
|
+
};
|
|
1367
|
+
|
|
1015
1368
|
// src/stt/STT.ts
|
|
1016
1369
|
var import_eventemitter34 = require("eventemitter3");
|
|
1017
1370
|
var STT = class extends import_eventemitter34.EventEmitter {
|
|
@@ -1362,9 +1715,11 @@ async function waitForParams(socket, validate) {
|
|
|
1362
1715
|
MockSTT,
|
|
1363
1716
|
MockTTS,
|
|
1364
1717
|
Pcm16Resampler,
|
|
1718
|
+
Realtime,
|
|
1365
1719
|
STT,
|
|
1366
1720
|
SentenceSplitter,
|
|
1367
1721
|
SentenceTTS,
|
|
1722
|
+
SharedInstances,
|
|
1368
1723
|
TTS,
|
|
1369
1724
|
float32ToPcm16,
|
|
1370
1725
|
handleError,
|