@micdrop/server 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/index.d.mts +152 -5
- package/dist/index.d.ts +152 -5
- package/dist/index.js +340 -25
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +339 -25
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -25,7 +25,9 @@ var Agent = class extends EventEmitter {
|
|
|
25
25
|
const answerCount = ++this.answerCount;
|
|
26
26
|
const stream = new PassThrough();
|
|
27
27
|
this.answering = true;
|
|
28
|
-
Promise.resolve().then(
|
|
28
|
+
Promise.resolve().then(
|
|
29
|
+
() => this.options.onBeforeAnswer?.bind(this)(stream)
|
|
30
|
+
).then((skip) => {
|
|
29
31
|
if (skip) return;
|
|
30
32
|
return this.generateAnswer(stream);
|
|
31
33
|
}).finally(() => {
|
|
@@ -68,16 +70,16 @@ var Agent = class extends EventEmitter {
|
|
|
68
70
|
metadata
|
|
69
71
|
};
|
|
70
72
|
this.conversation.push(message);
|
|
71
|
-
this.
|
|
73
|
+
this.emitAgentEvent("Message", message);
|
|
72
74
|
}
|
|
73
75
|
addToolMessage(message) {
|
|
74
76
|
this.log("Adding tool message:", message);
|
|
75
77
|
this.conversation.push(message);
|
|
76
|
-
this.
|
|
78
|
+
this.emitAgentEvent("Message", message);
|
|
77
79
|
}
|
|
78
80
|
endCall() {
|
|
79
81
|
this.log("Ending call");
|
|
80
|
-
this.
|
|
82
|
+
this.emitAgentEvent("EndCall");
|
|
81
83
|
}
|
|
82
84
|
cancelLastUserMessage() {
|
|
83
85
|
this.log("Cancelling last user message");
|
|
@@ -87,11 +89,11 @@ var Agent = class extends EventEmitter {
|
|
|
87
89
|
if (lastMessageIndex !== -1) {
|
|
88
90
|
this.conversation.splice(lastMessageIndex, 1);
|
|
89
91
|
}
|
|
90
|
-
this.
|
|
92
|
+
this.emitAgentEvent("CancelLastUserMessage");
|
|
91
93
|
}
|
|
92
94
|
skipAnswer() {
|
|
93
95
|
this.log("Skipping answer");
|
|
94
|
-
this.
|
|
96
|
+
this.emitAgentEvent("SkipAnswer");
|
|
95
97
|
}
|
|
96
98
|
getDefaultTools() {
|
|
97
99
|
const tools = [];
|
|
@@ -137,7 +139,7 @@ var Agent = class extends EventEmitter {
|
|
|
137
139
|
output: JSON.stringify(output ?? null)
|
|
138
140
|
});
|
|
139
141
|
if (tool.emitOutput) {
|
|
140
|
-
this.
|
|
142
|
+
this.emitAgentEvent("ToolCall", {
|
|
141
143
|
name: toolCall.toolName,
|
|
142
144
|
parameters,
|
|
143
145
|
output
|
|
@@ -196,6 +198,16 @@ var Agent = class extends EventEmitter {
|
|
|
196
198
|
}
|
|
197
199
|
return { message, metadata };
|
|
198
200
|
}
|
|
201
|
+
/**
|
|
202
|
+
* Emits one of the events every agent has.
|
|
203
|
+
*
|
|
204
|
+
* A subclass can add events of its own, and TypeScript then no longer knows
|
|
205
|
+
* the shared ones are among them.
|
|
206
|
+
*/
|
|
207
|
+
emitAgentEvent(event, ...args) {
|
|
208
|
+
const emitter = this;
|
|
209
|
+
emitter.emit(event, ...args);
|
|
210
|
+
}
|
|
199
211
|
log(...message) {
|
|
200
212
|
this.logger?.log(...message);
|
|
201
213
|
}
|
|
@@ -469,8 +481,8 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
469
481
|
this.log("Connection closed");
|
|
470
482
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
471
483
|
const conversation = this.conversation;
|
|
472
|
-
this.
|
|
473
|
-
this.config.stt
|
|
484
|
+
this.agent?.destroy();
|
|
485
|
+
this.config.stt?.destroy();
|
|
474
486
|
this.config.tts?.destroy();
|
|
475
487
|
this.emit("End", {
|
|
476
488
|
conversation,
|
|
@@ -513,7 +525,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
513
525
|
this.answerUserTurn();
|
|
514
526
|
}
|
|
515
527
|
};
|
|
516
|
-
this.
|
|
528
|
+
this.onAssistantAudio = (audio) => {
|
|
517
529
|
if (!this.socket) return;
|
|
518
530
|
this.log(`Send audio chunk (${audio.byteLength} bytes)`);
|
|
519
531
|
this.socket.send(audio);
|
|
@@ -525,9 +537,23 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
525
537
|
this.socket = socket;
|
|
526
538
|
this.config = config;
|
|
527
539
|
this.log(`Call started`);
|
|
528
|
-
config
|
|
529
|
-
|
|
530
|
-
|
|
540
|
+
const { realtime } = config;
|
|
541
|
+
if (realtime) {
|
|
542
|
+
realtime.on("Audio", this.onAssistantAudio);
|
|
543
|
+
if (config.partialMessages) {
|
|
544
|
+
realtime.on(
|
|
545
|
+
"PartialMessage",
|
|
546
|
+
(content) => this.sendPartialAnswer(content)
|
|
547
|
+
);
|
|
548
|
+
}
|
|
549
|
+
} else {
|
|
550
|
+
if (!config.stt) {
|
|
551
|
+
throw new Error("MicdropServer needs an stt, or a realtime model");
|
|
552
|
+
}
|
|
553
|
+
config.stt.on("Transcript", this.onTranscriptSTT);
|
|
554
|
+
config.tts?.on("Audio", this.onAssistantAudio);
|
|
555
|
+
}
|
|
556
|
+
const agent = this.agent;
|
|
531
557
|
if (agent) {
|
|
532
558
|
agent.on("Message", this.onMessageAgent);
|
|
533
559
|
agent.on(
|
|
@@ -555,7 +581,12 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
555
581
|
}
|
|
556
582
|
/** Everything said so far, whether an agent or the server keeps it */
|
|
557
583
|
get conversation() {
|
|
558
|
-
return this.
|
|
584
|
+
return this.agent?.conversation ?? this.ownConversation;
|
|
585
|
+
}
|
|
586
|
+
/** What holds the conversation and answers, a realtime model included */
|
|
587
|
+
get agent() {
|
|
588
|
+
if (!this.config) return void 0;
|
|
589
|
+
return this.config.realtime ?? this.config.agent;
|
|
559
590
|
}
|
|
560
591
|
log(...message) {
|
|
561
592
|
this.logger?.log(...message);
|
|
@@ -581,7 +612,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
581
612
|
}
|
|
582
613
|
cancel() {
|
|
583
614
|
this.config?.tts?.cancel();
|
|
584
|
-
this.
|
|
615
|
+
this.agent?.cancel();
|
|
585
616
|
this.operationQueue = [];
|
|
586
617
|
}
|
|
587
618
|
onUserAudio(chunk) {
|
|
@@ -596,6 +627,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
596
627
|
this.currentUserStream?.end();
|
|
597
628
|
this.currentUserStream = void 0;
|
|
598
629
|
this.cancel();
|
|
630
|
+
this.config?.realtime?.discardTurn();
|
|
599
631
|
}
|
|
600
632
|
onStartSpeaking() {
|
|
601
633
|
if (!this.config) return;
|
|
@@ -605,8 +637,13 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
605
637
|
this.config.turnDetector?.reset();
|
|
606
638
|
this.turnComplete = void 0;
|
|
607
639
|
this.releaseHeldTurn();
|
|
608
|
-
this.config.
|
|
609
|
-
|
|
640
|
+
if (this.config.realtime) {
|
|
641
|
+
this.cancel();
|
|
642
|
+
this.config.realtime.startTurn(this.currentUserStream);
|
|
643
|
+
} else {
|
|
644
|
+
this.config.stt?.transcribe(this.currentUserStream);
|
|
645
|
+
this.cancel();
|
|
646
|
+
}
|
|
610
647
|
}
|
|
611
648
|
onStopSpeaking() {
|
|
612
649
|
const hasNoUserSpeech = !this.currentUserStream || this.userSpeechChunks === 0;
|
|
@@ -618,6 +655,10 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
618
655
|
return;
|
|
619
656
|
}
|
|
620
657
|
this.turnComplete = this.predictTurnComplete();
|
|
658
|
+
if (this.config?.realtime) {
|
|
659
|
+
this.answerUserTurn();
|
|
660
|
+
return;
|
|
661
|
+
}
|
|
621
662
|
const conversation = this.conversation;
|
|
622
663
|
const lastMessage = conversation[conversation.length - 1];
|
|
623
664
|
if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
|
|
@@ -650,7 +691,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
650
691
|
}
|
|
651
692
|
this.releaseHeldTurn();
|
|
652
693
|
this.cancel();
|
|
653
|
-
this.
|
|
694
|
+
this.answerTurn();
|
|
654
695
|
}
|
|
655
696
|
/**
|
|
656
697
|
* Answers anyway if the rest of the sentence never comes.
|
|
@@ -664,9 +705,18 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
664
705
|
this.heldTurnTimer = void 0;
|
|
665
706
|
this.log("Nothing more came, answering");
|
|
666
707
|
this.cancel();
|
|
667
|
-
this.
|
|
708
|
+
this.answerTurn();
|
|
668
709
|
}, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
|
|
669
710
|
}
|
|
711
|
+
/** Answers what the user just said */
|
|
712
|
+
answerTurn() {
|
|
713
|
+
const realtime = this.config?.realtime;
|
|
714
|
+
if (realtime) {
|
|
715
|
+
realtime.endTurn();
|
|
716
|
+
} else {
|
|
717
|
+
this.answer();
|
|
718
|
+
}
|
|
719
|
+
}
|
|
670
720
|
releaseHeldTurn() {
|
|
671
721
|
if (!this.heldTurnTimer) return;
|
|
672
722
|
clearTimeout(this.heldTurnTimer);
|
|
@@ -674,16 +724,16 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
674
724
|
}
|
|
675
725
|
/** Adds a message to the conversation and sends it to the client */
|
|
676
726
|
addUserMessage(text, metadata) {
|
|
677
|
-
if (this.
|
|
678
|
-
this.
|
|
727
|
+
if (this.agent) {
|
|
728
|
+
this.agent.addUserMessage(text, metadata);
|
|
679
729
|
} else {
|
|
680
730
|
this.addOwnMessage("user", text, metadata);
|
|
681
731
|
}
|
|
682
732
|
}
|
|
683
733
|
/** Adds a message to the conversation and sends it to the client */
|
|
684
734
|
addAssistantMessage(text, metadata) {
|
|
685
|
-
if (this.
|
|
686
|
-
this.
|
|
735
|
+
if (this.agent) {
|
|
736
|
+
this.agent.addAssistantMessage(text, metadata);
|
|
687
737
|
} else {
|
|
688
738
|
this.addOwnMessage("assistant", text, metadata);
|
|
689
739
|
}
|
|
@@ -718,9 +768,9 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
718
768
|
sendFirstMessage() {
|
|
719
769
|
if (!this.config) return;
|
|
720
770
|
if (this.config.firstMessage) {
|
|
721
|
-
this.
|
|
771
|
+
this.agent?.addAssistantMessage(this.config.firstMessage);
|
|
722
772
|
this.speak(this.config.firstMessage);
|
|
723
|
-
} else if (this.config.generateFirstMessage && this.
|
|
773
|
+
} else if (this.config.generateFirstMessage && this.agent) {
|
|
724
774
|
this.answer();
|
|
725
775
|
} else {
|
|
726
776
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -733,6 +783,10 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
733
783
|
}
|
|
734
784
|
async _answer() {
|
|
735
785
|
if (!this.config) return;
|
|
786
|
+
if (this.config.realtime) {
|
|
787
|
+
this.config.realtime.respond();
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
736
790
|
if (!this.config.agent) {
|
|
737
791
|
this.log("No agent, skipping answer");
|
|
738
792
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -780,6 +834,11 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
780
834
|
}
|
|
781
835
|
async _speak(message) {
|
|
782
836
|
if (!this.socket || !this.config) return;
|
|
837
|
+
if (this.config.realtime) {
|
|
838
|
+
const text = typeof message === "string" ? message : await readText(message);
|
|
839
|
+
if (text.trim() !== "") this.config.realtime.say(text);
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
783
842
|
let textStream;
|
|
784
843
|
if (typeof message === "string") {
|
|
785
844
|
if (!this.config.agent) {
|
|
@@ -818,6 +877,14 @@ function hasContent(stream) {
|
|
|
818
877
|
stream.on("end", onEnd);
|
|
819
878
|
});
|
|
820
879
|
}
|
|
880
|
+
function readText(stream) {
|
|
881
|
+
return new Promise((resolve) => {
|
|
882
|
+
let text = "";
|
|
883
|
+
stream.on("data", (chunk) => text += chunk.toString());
|
|
884
|
+
stream.once("end", () => resolve(text));
|
|
885
|
+
stream.once("error", () => resolve(text));
|
|
886
|
+
});
|
|
887
|
+
}
|
|
821
888
|
function drain(stream) {
|
|
822
889
|
return new Promise((resolve) => {
|
|
823
890
|
stream.on("data", () => {
|
|
@@ -828,6 +895,252 @@ function drain(stream) {
|
|
|
828
895
|
});
|
|
829
896
|
}
|
|
830
897
|
|
|
898
|
+
// src/realtime/Realtime.ts
|
|
899
|
+
var USER_TRANSCRIPT_TIMEOUT = 3e3;
|
|
900
|
+
function newAnswer(record) {
|
|
901
|
+
return { transcript: "", hasAudio: false, record, resumed: false };
|
|
902
|
+
}
|
|
903
|
+
var Realtime = class extends Agent {
|
|
904
|
+
constructor() {
|
|
905
|
+
super(...arguments);
|
|
906
|
+
// A turn opens with its first audio, so a blip without any sends nothing
|
|
907
|
+
this.turn = "none";
|
|
908
|
+
// Output of an interrupted answer can still be on its way
|
|
909
|
+
this.discarding = false;
|
|
910
|
+
// Answers asked for while another one was being given
|
|
911
|
+
this.queue = [];
|
|
912
|
+
this.answerWaiters = [];
|
|
913
|
+
this.userTranscript = "";
|
|
914
|
+
this.awaitingUserTranscript = false;
|
|
915
|
+
this.afterUserTranscript = [];
|
|
916
|
+
}
|
|
917
|
+
/**
|
|
918
|
+
* The user started speaking.
|
|
919
|
+
*
|
|
920
|
+
* The assistant stops, and the audio read from the stream goes to the model.
|
|
921
|
+
* A turn left open, because the sentence sounded unfinished, carries on.
|
|
922
|
+
*/
|
|
923
|
+
startTurn(audio) {
|
|
924
|
+
this.cancel();
|
|
925
|
+
if (this.turn === "none") this.turn = "pending";
|
|
926
|
+
audio.on("data", (chunk) => this.sendAudio(chunk));
|
|
927
|
+
}
|
|
928
|
+
/** Closes the turn of the user, and has the model answer it */
|
|
929
|
+
endTurn() {
|
|
930
|
+
if (this.turn !== "open") return;
|
|
931
|
+
this.turn = "none";
|
|
932
|
+
this.awaitUserTranscript();
|
|
933
|
+
this.beginAnswer(true);
|
|
934
|
+
this.closeTurn();
|
|
935
|
+
}
|
|
936
|
+
/** Drops the turn of the user, without an answer */
|
|
937
|
+
discardTurn() {
|
|
938
|
+
this.cancel();
|
|
939
|
+
if (this.turn === "open") {
|
|
940
|
+
this.clearTurn();
|
|
941
|
+
this.discarding = true;
|
|
942
|
+
}
|
|
943
|
+
this.turn = "none";
|
|
944
|
+
}
|
|
945
|
+
/** Has the model speak without waiting for the user, to open a call */
|
|
946
|
+
respond() {
|
|
947
|
+
this.whenIdle(() => {
|
|
948
|
+
this.beginAnswer(true);
|
|
949
|
+
this.generate();
|
|
950
|
+
});
|
|
951
|
+
}
|
|
952
|
+
/**
|
|
953
|
+
* Has the model say this text, word for word.
|
|
954
|
+
*
|
|
955
|
+
* The caller records the text in the conversation, as with a text to speech,
|
|
956
|
+
* so the transcript of what the model says is not recorded again.
|
|
957
|
+
*/
|
|
958
|
+
say(text) {
|
|
959
|
+
this.whenIdle(() => {
|
|
960
|
+
this.beginAnswer(false);
|
|
961
|
+
this.speakText(text);
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
cancel() {
|
|
965
|
+
this.queue = [];
|
|
966
|
+
const answer = this.currentAnswer;
|
|
967
|
+
if (!answer) return;
|
|
968
|
+
this.log("Cancel");
|
|
969
|
+
this.currentAnswer = void 0;
|
|
970
|
+
this.discarding = true;
|
|
971
|
+
this.cancelAnswer();
|
|
972
|
+
this.recordAnswer(answer);
|
|
973
|
+
this.resolveAnswerWaiters();
|
|
974
|
+
}
|
|
975
|
+
/**
|
|
976
|
+
* Adds a message the model did not hear, and tells the model about it.
|
|
977
|
+
*
|
|
978
|
+
* The transcripts of the call are recorded apart, so they are not sent back.
|
|
979
|
+
*/
|
|
980
|
+
addMessage(role, text, metadata) {
|
|
981
|
+
super.addMessage(role, text, metadata);
|
|
982
|
+
if (text.trim() !== "") this.injectMessage(role, text);
|
|
983
|
+
}
|
|
984
|
+
addTool(tool) {
|
|
985
|
+
super.addTool(tool);
|
|
986
|
+
this.updateTools();
|
|
987
|
+
}
|
|
988
|
+
removeTool(name) {
|
|
989
|
+
super.removeTool(name);
|
|
990
|
+
this.updateTools();
|
|
991
|
+
}
|
|
992
|
+
/**
|
|
993
|
+
* Lists the events that have listeners.
|
|
994
|
+
*
|
|
995
|
+
* Typed as the events of any agent, so a realtime model goes wherever an
|
|
996
|
+
* agent is expected. The two extra events do not change what that code reads.
|
|
997
|
+
*/
|
|
998
|
+
eventNames() {
|
|
999
|
+
return super.eventNames();
|
|
1000
|
+
}
|
|
1001
|
+
destroy() {
|
|
1002
|
+
super.destroy();
|
|
1003
|
+
this.clearUserTranscriptTimer();
|
|
1004
|
+
this.afterUserTranscript = [];
|
|
1005
|
+
this.resolveAnswerWaiters();
|
|
1006
|
+
}
|
|
1007
|
+
generateAnswer(stream) {
|
|
1008
|
+
let written = 0;
|
|
1009
|
+
const onPartial = (transcript) => {
|
|
1010
|
+
stream.write(transcript.slice(written));
|
|
1011
|
+
written = transcript.length;
|
|
1012
|
+
};
|
|
1013
|
+
this.on("PartialMessage", onPartial);
|
|
1014
|
+
this.respond();
|
|
1015
|
+
return new Promise((resolve) => {
|
|
1016
|
+
this.answerWaiters.push(() => {
|
|
1017
|
+
this.off("PartialMessage", onPartial);
|
|
1018
|
+
resolve();
|
|
1019
|
+
});
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
/** Nobody is speaking, nor about to, so the connection can be renewed */
|
|
1023
|
+
get isIdle() {
|
|
1024
|
+
return this.turn === "none" && !this.currentAnswer;
|
|
1025
|
+
}
|
|
1026
|
+
/** Voice of the answer, PCM16 at 16 kHz */
|
|
1027
|
+
emitAudio(chunk) {
|
|
1028
|
+
const answer = this.activeAnswer();
|
|
1029
|
+
if (!answer) return;
|
|
1030
|
+
answer.hasAudio = true;
|
|
1031
|
+
this.emit("Audio", chunk);
|
|
1032
|
+
}
|
|
1033
|
+
/** Transcript of the answer, as it is spoken */
|
|
1034
|
+
addAnswerTranscript(text) {
|
|
1035
|
+
const answer = this.activeAnswer();
|
|
1036
|
+
if (!answer) return;
|
|
1037
|
+
if (answer.resumed) {
|
|
1038
|
+
answer.resumed = false;
|
|
1039
|
+
if (answer.transcript !== "" && !/\s$/.test(answer.transcript)) {
|
|
1040
|
+
text = ` ${text.trimStart()}`;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
answer.transcript += text;
|
|
1044
|
+
if (answer.record) this.emit("PartialMessage", answer.transcript);
|
|
1045
|
+
}
|
|
1046
|
+
/**
|
|
1047
|
+
* The model finished its answer, or finished dropping an interrupted one.
|
|
1048
|
+
*
|
|
1049
|
+
* An answer without audio tells the client to stop waiting for one.
|
|
1050
|
+
*/
|
|
1051
|
+
endAnswer() {
|
|
1052
|
+
if (this.discarding) {
|
|
1053
|
+
this.discarding = false;
|
|
1054
|
+
return;
|
|
1055
|
+
}
|
|
1056
|
+
const answer = this.currentAnswer;
|
|
1057
|
+
if (!answer) return;
|
|
1058
|
+
this.currentAnswer = void 0;
|
|
1059
|
+
if (!answer.hasAudio) this.emitAgentEvent("SkipAnswer");
|
|
1060
|
+
this.recordAnswer(answer);
|
|
1061
|
+
this.resolveAnswerWaiters();
|
|
1062
|
+
this.queue.shift()?.();
|
|
1063
|
+
}
|
|
1064
|
+
/** Runs a tool the model called, and records it in the conversation */
|
|
1065
|
+
async runTool(toolCall) {
|
|
1066
|
+
const result = await this.executeTool(toolCall);
|
|
1067
|
+
if (this.currentAnswer) this.currentAnswer.resumed = true;
|
|
1068
|
+
return result;
|
|
1069
|
+
}
|
|
1070
|
+
/** Transcript of the turn of the user, possibly in several pieces */
|
|
1071
|
+
addUserTranscript(text) {
|
|
1072
|
+
this.userTranscript += text;
|
|
1073
|
+
}
|
|
1074
|
+
/** The transcript of the turn is complete, it goes in the conversation */
|
|
1075
|
+
commitUserTranscript() {
|
|
1076
|
+
this.clearUserTranscriptTimer();
|
|
1077
|
+
this.awaitingUserTranscript = false;
|
|
1078
|
+
const text = this.userTranscript.trim();
|
|
1079
|
+
this.userTranscript = "";
|
|
1080
|
+
if (text !== "") super.addMessage("user", text);
|
|
1081
|
+
const pending = this.afterUserTranscript;
|
|
1082
|
+
this.afterUserTranscript = [];
|
|
1083
|
+
pending.forEach((record) => record());
|
|
1084
|
+
}
|
|
1085
|
+
sendAudio(chunk) {
|
|
1086
|
+
if (this.turn === "none") return;
|
|
1087
|
+
if (this.turn === "pending") {
|
|
1088
|
+
this.turn = "open";
|
|
1089
|
+
this.openTurn();
|
|
1090
|
+
}
|
|
1091
|
+
this.appendAudio(chunk);
|
|
1092
|
+
}
|
|
1093
|
+
beginAnswer(record) {
|
|
1094
|
+
this.discarding = false;
|
|
1095
|
+
this.currentAnswer = newAnswer(record);
|
|
1096
|
+
}
|
|
1097
|
+
/**
|
|
1098
|
+
* The answer output belongs to, if it is still wanted.
|
|
1099
|
+
*
|
|
1100
|
+
* The model can speak without being asked, to follow up on a tool call for
|
|
1101
|
+
* instance, which opens an answer of its own.
|
|
1102
|
+
*/
|
|
1103
|
+
activeAnswer() {
|
|
1104
|
+
if (this.discarding) return void 0;
|
|
1105
|
+
if (!this.currentAnswer) this.currentAnswer = newAnswer(true);
|
|
1106
|
+
return this.currentAnswer;
|
|
1107
|
+
}
|
|
1108
|
+
whenIdle(start) {
|
|
1109
|
+
if (this.currentAnswer) {
|
|
1110
|
+
this.queue.push(start);
|
|
1111
|
+
} else {
|
|
1112
|
+
start();
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
recordAnswer(answer) {
|
|
1116
|
+
if (!answer.record) return;
|
|
1117
|
+
const record = () => super.addMessage("assistant", answer.transcript.trim());
|
|
1118
|
+
if (this.awaitingUserTranscript) {
|
|
1119
|
+
this.afterUserTranscript.push(record);
|
|
1120
|
+
} else {
|
|
1121
|
+
record();
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
awaitUserTranscript() {
|
|
1125
|
+
this.awaitingUserTranscript = true;
|
|
1126
|
+
this.clearUserTranscriptTimer();
|
|
1127
|
+
this.userTranscriptTimer = setTimeout(() => {
|
|
1128
|
+
this.log("No transcript of the turn, recording the answer anyway");
|
|
1129
|
+
this.commitUserTranscript();
|
|
1130
|
+
}, USER_TRANSCRIPT_TIMEOUT);
|
|
1131
|
+
}
|
|
1132
|
+
clearUserTranscriptTimer() {
|
|
1133
|
+
if (!this.userTranscriptTimer) return;
|
|
1134
|
+
clearTimeout(this.userTranscriptTimer);
|
|
1135
|
+
this.userTranscriptTimer = void 0;
|
|
1136
|
+
}
|
|
1137
|
+
resolveAnswerWaiters() {
|
|
1138
|
+
const waiters = this.answerWaiters;
|
|
1139
|
+
this.answerWaiters = [];
|
|
1140
|
+
waiters.forEach((resolve) => resolve());
|
|
1141
|
+
}
|
|
1142
|
+
};
|
|
1143
|
+
|
|
831
1144
|
// src/recorder/MicdropRecorder.ts
|
|
832
1145
|
import { EventEmitter as EventEmitter3 } from "eventemitter3";
|
|
833
1146
|
var MicdropRecorder = class extends EventEmitter3 {
|
|
@@ -1335,6 +1648,7 @@ export {
|
|
|
1335
1648
|
MockSTT,
|
|
1336
1649
|
MockTTS,
|
|
1337
1650
|
Pcm16Resampler,
|
|
1651
|
+
Realtime,
|
|
1338
1652
|
STT,
|
|
1339
1653
|
SentenceSplitter,
|
|
1340
1654
|
SentenceTTS,
|