@micdrop/server 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/index.d.mts +152 -5
- package/dist/index.d.ts +152 -5
- package/dist/index.js +340 -25
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +339 -25
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -51,6 +51,7 @@ __export(index_exports, {
|
|
|
51
51
|
MockSTT: () => MockSTT,
|
|
52
52
|
MockTTS: () => MockTTS,
|
|
53
53
|
Pcm16Resampler: () => Pcm16Resampler,
|
|
54
|
+
Realtime: () => Realtime,
|
|
54
55
|
STT: () => STT,
|
|
55
56
|
SentenceSplitter: () => SentenceSplitter,
|
|
56
57
|
SentenceTTS: () => SentenceTTS,
|
|
@@ -90,7 +91,9 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
90
91
|
const answerCount = ++this.answerCount;
|
|
91
92
|
const stream = new import_stream.PassThrough();
|
|
92
93
|
this.answering = true;
|
|
93
|
-
Promise.resolve().then(
|
|
94
|
+
Promise.resolve().then(
|
|
95
|
+
() => this.options.onBeforeAnswer?.bind(this)(stream)
|
|
96
|
+
).then((skip) => {
|
|
94
97
|
if (skip) return;
|
|
95
98
|
return this.generateAnswer(stream);
|
|
96
99
|
}).finally(() => {
|
|
@@ -133,16 +136,16 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
133
136
|
metadata
|
|
134
137
|
};
|
|
135
138
|
this.conversation.push(message);
|
|
136
|
-
this.
|
|
139
|
+
this.emitAgentEvent("Message", message);
|
|
137
140
|
}
|
|
138
141
|
addToolMessage(message) {
|
|
139
142
|
this.log("Adding tool message:", message);
|
|
140
143
|
this.conversation.push(message);
|
|
141
|
-
this.
|
|
144
|
+
this.emitAgentEvent("Message", message);
|
|
142
145
|
}
|
|
143
146
|
endCall() {
|
|
144
147
|
this.log("Ending call");
|
|
145
|
-
this.
|
|
148
|
+
this.emitAgentEvent("EndCall");
|
|
146
149
|
}
|
|
147
150
|
cancelLastUserMessage() {
|
|
148
151
|
this.log("Cancelling last user message");
|
|
@@ -152,11 +155,11 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
152
155
|
if (lastMessageIndex !== -1) {
|
|
153
156
|
this.conversation.splice(lastMessageIndex, 1);
|
|
154
157
|
}
|
|
155
|
-
this.
|
|
158
|
+
this.emitAgentEvent("CancelLastUserMessage");
|
|
156
159
|
}
|
|
157
160
|
skipAnswer() {
|
|
158
161
|
this.log("Skipping answer");
|
|
159
|
-
this.
|
|
162
|
+
this.emitAgentEvent("SkipAnswer");
|
|
160
163
|
}
|
|
161
164
|
getDefaultTools() {
|
|
162
165
|
const tools = [];
|
|
@@ -202,7 +205,7 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
202
205
|
output: JSON.stringify(output ?? null)
|
|
203
206
|
});
|
|
204
207
|
if (tool.emitOutput) {
|
|
205
|
-
this.
|
|
208
|
+
this.emitAgentEvent("ToolCall", {
|
|
206
209
|
name: toolCall.toolName,
|
|
207
210
|
parameters,
|
|
208
211
|
output
|
|
@@ -261,6 +264,16 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
261
264
|
}
|
|
262
265
|
return { message, metadata };
|
|
263
266
|
}
|
|
267
|
+
/**
|
|
268
|
+
* Emits one of the events every agent has.
|
|
269
|
+
*
|
|
270
|
+
* A subclass can add events of its own, and TypeScript then no longer knows
|
|
271
|
+
* the shared ones are among them.
|
|
272
|
+
*/
|
|
273
|
+
emitAgentEvent(event, ...args) {
|
|
274
|
+
const emitter = this;
|
|
275
|
+
emitter.emit(event, ...args);
|
|
276
|
+
}
|
|
264
277
|
log(...message) {
|
|
265
278
|
this.logger?.log(...message);
|
|
266
279
|
}
|
|
@@ -534,8 +547,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
534
547
|
this.log("Connection closed");
|
|
535
548
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
536
549
|
const conversation = this.conversation;
|
|
537
|
-
this.
|
|
538
|
-
this.config.stt
|
|
550
|
+
this.agent?.destroy();
|
|
551
|
+
this.config.stt?.destroy();
|
|
539
552
|
this.config.tts?.destroy();
|
|
540
553
|
this.emit("End", {
|
|
541
554
|
conversation,
|
|
@@ -578,7 +591,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
578
591
|
this.answerUserTurn();
|
|
579
592
|
}
|
|
580
593
|
};
|
|
581
|
-
this.
|
|
594
|
+
this.onAssistantAudio = (audio) => {
|
|
582
595
|
if (!this.socket) return;
|
|
583
596
|
this.log(`Send audio chunk (${audio.byteLength} bytes)`);
|
|
584
597
|
this.socket.send(audio);
|
|
@@ -590,9 +603,23 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
590
603
|
this.socket = socket;
|
|
591
604
|
this.config = config;
|
|
592
605
|
this.log(`Call started`);
|
|
593
|
-
config
|
|
594
|
-
|
|
595
|
-
|
|
606
|
+
const { realtime } = config;
|
|
607
|
+
if (realtime) {
|
|
608
|
+
realtime.on("Audio", this.onAssistantAudio);
|
|
609
|
+
if (config.partialMessages) {
|
|
610
|
+
realtime.on(
|
|
611
|
+
"PartialMessage",
|
|
612
|
+
(content) => this.sendPartialAnswer(content)
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
} else {
|
|
616
|
+
if (!config.stt) {
|
|
617
|
+
throw new Error("MicdropServer needs an stt, or a realtime model");
|
|
618
|
+
}
|
|
619
|
+
config.stt.on("Transcript", this.onTranscriptSTT);
|
|
620
|
+
config.tts?.on("Audio", this.onAssistantAudio);
|
|
621
|
+
}
|
|
622
|
+
const agent = this.agent;
|
|
596
623
|
if (agent) {
|
|
597
624
|
agent.on("Message", this.onMessageAgent);
|
|
598
625
|
agent.on(
|
|
@@ -620,7 +647,12 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
620
647
|
}
|
|
621
648
|
/** Everything said so far, whether an agent or the server keeps it */
|
|
622
649
|
get conversation() {
|
|
623
|
-
return this.
|
|
650
|
+
return this.agent?.conversation ?? this.ownConversation;
|
|
651
|
+
}
|
|
652
|
+
/** What holds the conversation and answers, a realtime model included */
|
|
653
|
+
get agent() {
|
|
654
|
+
if (!this.config) return void 0;
|
|
655
|
+
return this.config.realtime ?? this.config.agent;
|
|
624
656
|
}
|
|
625
657
|
log(...message) {
|
|
626
658
|
this.logger?.log(...message);
|
|
@@ -646,7 +678,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
646
678
|
}
|
|
647
679
|
cancel() {
|
|
648
680
|
this.config?.tts?.cancel();
|
|
649
|
-
this.
|
|
681
|
+
this.agent?.cancel();
|
|
650
682
|
this.operationQueue = [];
|
|
651
683
|
}
|
|
652
684
|
onUserAudio(chunk) {
|
|
@@ -661,6 +693,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
661
693
|
this.currentUserStream?.end();
|
|
662
694
|
this.currentUserStream = void 0;
|
|
663
695
|
this.cancel();
|
|
696
|
+
this.config?.realtime?.discardTurn();
|
|
664
697
|
}
|
|
665
698
|
onStartSpeaking() {
|
|
666
699
|
if (!this.config) return;
|
|
@@ -670,8 +703,13 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
670
703
|
this.config.turnDetector?.reset();
|
|
671
704
|
this.turnComplete = void 0;
|
|
672
705
|
this.releaseHeldTurn();
|
|
673
|
-
this.config.
|
|
674
|
-
|
|
706
|
+
if (this.config.realtime) {
|
|
707
|
+
this.cancel();
|
|
708
|
+
this.config.realtime.startTurn(this.currentUserStream);
|
|
709
|
+
} else {
|
|
710
|
+
this.config.stt?.transcribe(this.currentUserStream);
|
|
711
|
+
this.cancel();
|
|
712
|
+
}
|
|
675
713
|
}
|
|
676
714
|
onStopSpeaking() {
|
|
677
715
|
const hasNoUserSpeech = !this.currentUserStream || this.userSpeechChunks === 0;
|
|
@@ -683,6 +721,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
683
721
|
return;
|
|
684
722
|
}
|
|
685
723
|
this.turnComplete = this.predictTurnComplete();
|
|
724
|
+
if (this.config?.realtime) {
|
|
725
|
+
this.answerUserTurn();
|
|
726
|
+
return;
|
|
727
|
+
}
|
|
686
728
|
const conversation = this.conversation;
|
|
687
729
|
const lastMessage = conversation[conversation.length - 1];
|
|
688
730
|
if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
|
|
@@ -715,7 +757,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
715
757
|
}
|
|
716
758
|
this.releaseHeldTurn();
|
|
717
759
|
this.cancel();
|
|
718
|
-
this.
|
|
760
|
+
this.answerTurn();
|
|
719
761
|
}
|
|
720
762
|
/**
|
|
721
763
|
* Answers anyway if the rest of the sentence never comes.
|
|
@@ -729,9 +771,18 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
729
771
|
this.heldTurnTimer = void 0;
|
|
730
772
|
this.log("Nothing more came, answering");
|
|
731
773
|
this.cancel();
|
|
732
|
-
this.
|
|
774
|
+
this.answerTurn();
|
|
733
775
|
}, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
|
|
734
776
|
}
|
|
777
|
+
/** Answers what the user just said */
|
|
778
|
+
answerTurn() {
|
|
779
|
+
const realtime = this.config?.realtime;
|
|
780
|
+
if (realtime) {
|
|
781
|
+
realtime.endTurn();
|
|
782
|
+
} else {
|
|
783
|
+
this.answer();
|
|
784
|
+
}
|
|
785
|
+
}
|
|
735
786
|
releaseHeldTurn() {
|
|
736
787
|
if (!this.heldTurnTimer) return;
|
|
737
788
|
clearTimeout(this.heldTurnTimer);
|
|
@@ -739,16 +790,16 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
739
790
|
}
|
|
740
791
|
/** Adds a message to the conversation and sends it to the client */
|
|
741
792
|
addUserMessage(text, metadata) {
|
|
742
|
-
if (this.
|
|
743
|
-
this.
|
|
793
|
+
if (this.agent) {
|
|
794
|
+
this.agent.addUserMessage(text, metadata);
|
|
744
795
|
} else {
|
|
745
796
|
this.addOwnMessage("user", text, metadata);
|
|
746
797
|
}
|
|
747
798
|
}
|
|
748
799
|
/** Adds a message to the conversation and sends it to the client */
|
|
749
800
|
addAssistantMessage(text, metadata) {
|
|
750
|
-
if (this.
|
|
751
|
-
this.
|
|
801
|
+
if (this.agent) {
|
|
802
|
+
this.agent.addAssistantMessage(text, metadata);
|
|
752
803
|
} else {
|
|
753
804
|
this.addOwnMessage("assistant", text, metadata);
|
|
754
805
|
}
|
|
@@ -783,9 +834,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
783
834
|
sendFirstMessage() {
|
|
784
835
|
if (!this.config) return;
|
|
785
836
|
if (this.config.firstMessage) {
|
|
786
|
-
this.
|
|
837
|
+
this.agent?.addAssistantMessage(this.config.firstMessage);
|
|
787
838
|
this.speak(this.config.firstMessage);
|
|
788
|
-
} else if (this.config.generateFirstMessage && this.
|
|
839
|
+
} else if (this.config.generateFirstMessage && this.agent) {
|
|
789
840
|
this.answer();
|
|
790
841
|
} else {
|
|
791
842
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -798,6 +849,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
798
849
|
}
|
|
799
850
|
async _answer() {
|
|
800
851
|
if (!this.config) return;
|
|
852
|
+
if (this.config.realtime) {
|
|
853
|
+
this.config.realtime.respond();
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
801
856
|
if (!this.config.agent) {
|
|
802
857
|
this.log("No agent, skipping answer");
|
|
803
858
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -845,6 +900,11 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
845
900
|
}
|
|
846
901
|
async _speak(message) {
|
|
847
902
|
if (!this.socket || !this.config) return;
|
|
903
|
+
if (this.config.realtime) {
|
|
904
|
+
const text = typeof message === "string" ? message : await readText(message);
|
|
905
|
+
if (text.trim() !== "") this.config.realtime.say(text);
|
|
906
|
+
return;
|
|
907
|
+
}
|
|
848
908
|
let textStream;
|
|
849
909
|
if (typeof message === "string") {
|
|
850
910
|
if (!this.config.agent) {
|
|
@@ -883,6 +943,14 @@ function hasContent(stream) {
|
|
|
883
943
|
stream.on("end", onEnd);
|
|
884
944
|
});
|
|
885
945
|
}
|
|
946
|
+
function readText(stream) {
|
|
947
|
+
return new Promise((resolve) => {
|
|
948
|
+
let text = "";
|
|
949
|
+
stream.on("data", (chunk) => text += chunk.toString());
|
|
950
|
+
stream.once("end", () => resolve(text));
|
|
951
|
+
stream.once("error", () => resolve(text));
|
|
952
|
+
});
|
|
953
|
+
}
|
|
886
954
|
function drain(stream) {
|
|
887
955
|
return new Promise((resolve) => {
|
|
888
956
|
stream.on("data", () => {
|
|
@@ -893,6 +961,252 @@ function drain(stream) {
|
|
|
893
961
|
});
|
|
894
962
|
}
|
|
895
963
|
|
|
964
|
+
// src/realtime/Realtime.ts
|
|
965
|
+
var USER_TRANSCRIPT_TIMEOUT = 3e3;
|
|
966
|
+
function newAnswer(record) {
|
|
967
|
+
return { transcript: "", hasAudio: false, record, resumed: false };
|
|
968
|
+
}
|
|
969
|
+
var Realtime = class extends Agent {
|
|
970
|
+
constructor() {
|
|
971
|
+
super(...arguments);
|
|
972
|
+
// A turn opens with its first audio, so a blip without any sends nothing
|
|
973
|
+
this.turn = "none";
|
|
974
|
+
// Output of an interrupted answer can still be on its way
|
|
975
|
+
this.discarding = false;
|
|
976
|
+
// Answers asked for while another one was being given
|
|
977
|
+
this.queue = [];
|
|
978
|
+
this.answerWaiters = [];
|
|
979
|
+
this.userTranscript = "";
|
|
980
|
+
this.awaitingUserTranscript = false;
|
|
981
|
+
this.afterUserTranscript = [];
|
|
982
|
+
}
|
|
983
|
+
/**
|
|
984
|
+
* The user started speaking.
|
|
985
|
+
*
|
|
986
|
+
* The assistant stops, and the audio read from the stream goes to the model.
|
|
987
|
+
* A turn left open, because the sentence sounded unfinished, carries on.
|
|
988
|
+
*/
|
|
989
|
+
startTurn(audio) {
|
|
990
|
+
this.cancel();
|
|
991
|
+
if (this.turn === "none") this.turn = "pending";
|
|
992
|
+
audio.on("data", (chunk) => this.sendAudio(chunk));
|
|
993
|
+
}
|
|
994
|
+
/** Closes the turn of the user, and has the model answer it */
|
|
995
|
+
endTurn() {
|
|
996
|
+
if (this.turn !== "open") return;
|
|
997
|
+
this.turn = "none";
|
|
998
|
+
this.awaitUserTranscript();
|
|
999
|
+
this.beginAnswer(true);
|
|
1000
|
+
this.closeTurn();
|
|
1001
|
+
}
|
|
1002
|
+
/** Drops the turn of the user, without an answer */
|
|
1003
|
+
discardTurn() {
|
|
1004
|
+
this.cancel();
|
|
1005
|
+
if (this.turn === "open") {
|
|
1006
|
+
this.clearTurn();
|
|
1007
|
+
this.discarding = true;
|
|
1008
|
+
}
|
|
1009
|
+
this.turn = "none";
|
|
1010
|
+
}
|
|
1011
|
+
/** Has the model speak without waiting for the user, to open a call */
|
|
1012
|
+
respond() {
|
|
1013
|
+
this.whenIdle(() => {
|
|
1014
|
+
this.beginAnswer(true);
|
|
1015
|
+
this.generate();
|
|
1016
|
+
});
|
|
1017
|
+
}
|
|
1018
|
+
/**
|
|
1019
|
+
* Has the model say this text, word for word.
|
|
1020
|
+
*
|
|
1021
|
+
* The caller records the text in the conversation, as with a text to speech,
|
|
1022
|
+
* so the transcript of what the model says is not recorded again.
|
|
1023
|
+
*/
|
|
1024
|
+
say(text) {
|
|
1025
|
+
this.whenIdle(() => {
|
|
1026
|
+
this.beginAnswer(false);
|
|
1027
|
+
this.speakText(text);
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
cancel() {
|
|
1031
|
+
this.queue = [];
|
|
1032
|
+
const answer = this.currentAnswer;
|
|
1033
|
+
if (!answer) return;
|
|
1034
|
+
this.log("Cancel");
|
|
1035
|
+
this.currentAnswer = void 0;
|
|
1036
|
+
this.discarding = true;
|
|
1037
|
+
this.cancelAnswer();
|
|
1038
|
+
this.recordAnswer(answer);
|
|
1039
|
+
this.resolveAnswerWaiters();
|
|
1040
|
+
}
|
|
1041
|
+
/**
|
|
1042
|
+
* Adds a message the model did not hear, and tells the model about it.
|
|
1043
|
+
*
|
|
1044
|
+
* The transcripts of the call are recorded apart, so they are not sent back.
|
|
1045
|
+
*/
|
|
1046
|
+
addMessage(role, text, metadata) {
|
|
1047
|
+
super.addMessage(role, text, metadata);
|
|
1048
|
+
if (text.trim() !== "") this.injectMessage(role, text);
|
|
1049
|
+
}
|
|
1050
|
+
addTool(tool) {
|
|
1051
|
+
super.addTool(tool);
|
|
1052
|
+
this.updateTools();
|
|
1053
|
+
}
|
|
1054
|
+
removeTool(name) {
|
|
1055
|
+
super.removeTool(name);
|
|
1056
|
+
this.updateTools();
|
|
1057
|
+
}
|
|
1058
|
+
/**
|
|
1059
|
+
* Lists the events that have listeners.
|
|
1060
|
+
*
|
|
1061
|
+
* Typed as the events of any agent, so a realtime model goes wherever an
|
|
1062
|
+
* agent is expected. The two extra events do not change what that code reads.
|
|
1063
|
+
*/
|
|
1064
|
+
eventNames() {
|
|
1065
|
+
return super.eventNames();
|
|
1066
|
+
}
|
|
1067
|
+
destroy() {
|
|
1068
|
+
super.destroy();
|
|
1069
|
+
this.clearUserTranscriptTimer();
|
|
1070
|
+
this.afterUserTranscript = [];
|
|
1071
|
+
this.resolveAnswerWaiters();
|
|
1072
|
+
}
|
|
1073
|
+
generateAnswer(stream) {
|
|
1074
|
+
let written = 0;
|
|
1075
|
+
const onPartial = (transcript) => {
|
|
1076
|
+
stream.write(transcript.slice(written));
|
|
1077
|
+
written = transcript.length;
|
|
1078
|
+
};
|
|
1079
|
+
this.on("PartialMessage", onPartial);
|
|
1080
|
+
this.respond();
|
|
1081
|
+
return new Promise((resolve) => {
|
|
1082
|
+
this.answerWaiters.push(() => {
|
|
1083
|
+
this.off("PartialMessage", onPartial);
|
|
1084
|
+
resolve();
|
|
1085
|
+
});
|
|
1086
|
+
});
|
|
1087
|
+
}
|
|
1088
|
+
/** Nobody is speaking, nor about to, so the connection can be renewed */
|
|
1089
|
+
get isIdle() {
|
|
1090
|
+
return this.turn === "none" && !this.currentAnswer;
|
|
1091
|
+
}
|
|
1092
|
+
/** Voice of the answer, PCM16 at 16 kHz */
|
|
1093
|
+
emitAudio(chunk) {
|
|
1094
|
+
const answer = this.activeAnswer();
|
|
1095
|
+
if (!answer) return;
|
|
1096
|
+
answer.hasAudio = true;
|
|
1097
|
+
this.emit("Audio", chunk);
|
|
1098
|
+
}
|
|
1099
|
+
/** Transcript of the answer, as it is spoken */
|
|
1100
|
+
addAnswerTranscript(text) {
|
|
1101
|
+
const answer = this.activeAnswer();
|
|
1102
|
+
if (!answer) return;
|
|
1103
|
+
if (answer.resumed) {
|
|
1104
|
+
answer.resumed = false;
|
|
1105
|
+
if (answer.transcript !== "" && !/\s$/.test(answer.transcript)) {
|
|
1106
|
+
text = ` ${text.trimStart()}`;
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
answer.transcript += text;
|
|
1110
|
+
if (answer.record) this.emit("PartialMessage", answer.transcript);
|
|
1111
|
+
}
|
|
1112
|
+
/**
|
|
1113
|
+
* The model finished its answer, or finished dropping an interrupted one.
|
|
1114
|
+
*
|
|
1115
|
+
* An answer without audio tells the client to stop waiting for one.
|
|
1116
|
+
*/
|
|
1117
|
+
endAnswer() {
|
|
1118
|
+
if (this.discarding) {
|
|
1119
|
+
this.discarding = false;
|
|
1120
|
+
return;
|
|
1121
|
+
}
|
|
1122
|
+
const answer = this.currentAnswer;
|
|
1123
|
+
if (!answer) return;
|
|
1124
|
+
this.currentAnswer = void 0;
|
|
1125
|
+
if (!answer.hasAudio) this.emitAgentEvent("SkipAnswer");
|
|
1126
|
+
this.recordAnswer(answer);
|
|
1127
|
+
this.resolveAnswerWaiters();
|
|
1128
|
+
this.queue.shift()?.();
|
|
1129
|
+
}
|
|
1130
|
+
/** Runs a tool the model called, and records it in the conversation */
|
|
1131
|
+
async runTool(toolCall) {
|
|
1132
|
+
const result = await this.executeTool(toolCall);
|
|
1133
|
+
if (this.currentAnswer) this.currentAnswer.resumed = true;
|
|
1134
|
+
return result;
|
|
1135
|
+
}
|
|
1136
|
+
/** Transcript of the turn of the user, possibly in several pieces */
|
|
1137
|
+
addUserTranscript(text) {
|
|
1138
|
+
this.userTranscript += text;
|
|
1139
|
+
}
|
|
1140
|
+
/** The transcript of the turn is complete, it goes in the conversation */
|
|
1141
|
+
commitUserTranscript() {
|
|
1142
|
+
this.clearUserTranscriptTimer();
|
|
1143
|
+
this.awaitingUserTranscript = false;
|
|
1144
|
+
const text = this.userTranscript.trim();
|
|
1145
|
+
this.userTranscript = "";
|
|
1146
|
+
if (text !== "") super.addMessage("user", text);
|
|
1147
|
+
const pending = this.afterUserTranscript;
|
|
1148
|
+
this.afterUserTranscript = [];
|
|
1149
|
+
pending.forEach((record) => record());
|
|
1150
|
+
}
|
|
1151
|
+
sendAudio(chunk) {
|
|
1152
|
+
if (this.turn === "none") return;
|
|
1153
|
+
if (this.turn === "pending") {
|
|
1154
|
+
this.turn = "open";
|
|
1155
|
+
this.openTurn();
|
|
1156
|
+
}
|
|
1157
|
+
this.appendAudio(chunk);
|
|
1158
|
+
}
|
|
1159
|
+
beginAnswer(record) {
|
|
1160
|
+
this.discarding = false;
|
|
1161
|
+
this.currentAnswer = newAnswer(record);
|
|
1162
|
+
}
|
|
1163
|
+
/**
|
|
1164
|
+
* The answer output belongs to, if it is still wanted.
|
|
1165
|
+
*
|
|
1166
|
+
* The model can speak without being asked, to follow up on a tool call for
|
|
1167
|
+
* instance, which opens an answer of its own.
|
|
1168
|
+
*/
|
|
1169
|
+
activeAnswer() {
|
|
1170
|
+
if (this.discarding) return void 0;
|
|
1171
|
+
if (!this.currentAnswer) this.currentAnswer = newAnswer(true);
|
|
1172
|
+
return this.currentAnswer;
|
|
1173
|
+
}
|
|
1174
|
+
whenIdle(start) {
|
|
1175
|
+
if (this.currentAnswer) {
|
|
1176
|
+
this.queue.push(start);
|
|
1177
|
+
} else {
|
|
1178
|
+
start();
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
recordAnswer(answer) {
|
|
1182
|
+
if (!answer.record) return;
|
|
1183
|
+
const record = () => super.addMessage("assistant", answer.transcript.trim());
|
|
1184
|
+
if (this.awaitingUserTranscript) {
|
|
1185
|
+
this.afterUserTranscript.push(record);
|
|
1186
|
+
} else {
|
|
1187
|
+
record();
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
awaitUserTranscript() {
|
|
1191
|
+
this.awaitingUserTranscript = true;
|
|
1192
|
+
this.clearUserTranscriptTimer();
|
|
1193
|
+
this.userTranscriptTimer = setTimeout(() => {
|
|
1194
|
+
this.log("No transcript of the turn, recording the answer anyway");
|
|
1195
|
+
this.commitUserTranscript();
|
|
1196
|
+
}, USER_TRANSCRIPT_TIMEOUT);
|
|
1197
|
+
}
|
|
1198
|
+
clearUserTranscriptTimer() {
|
|
1199
|
+
if (!this.userTranscriptTimer) return;
|
|
1200
|
+
clearTimeout(this.userTranscriptTimer);
|
|
1201
|
+
this.userTranscriptTimer = void 0;
|
|
1202
|
+
}
|
|
1203
|
+
resolveAnswerWaiters() {
|
|
1204
|
+
const waiters = this.answerWaiters;
|
|
1205
|
+
this.answerWaiters = [];
|
|
1206
|
+
waiters.forEach((resolve) => resolve());
|
|
1207
|
+
}
|
|
1208
|
+
};
|
|
1209
|
+
|
|
896
1210
|
// src/recorder/MicdropRecorder.ts
|
|
897
1211
|
var import_eventemitter33 = require("eventemitter3");
|
|
898
1212
|
var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
@@ -1401,6 +1715,7 @@ async function waitForParams(socket, validate) {
|
|
|
1401
1715
|
MockSTT,
|
|
1402
1716
|
MockTTS,
|
|
1403
1717
|
Pcm16Resampler,
|
|
1718
|
+
Realtime,
|
|
1404
1719
|
STT,
|
|
1405
1720
|
SentenceSplitter,
|
|
1406
1721
|
SentenceTTS,
|