@micdrop/server 2.7.0 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -51,6 +51,7 @@ __export(index_exports, {
51
51
  MockSTT: () => MockSTT,
52
52
  MockTTS: () => MockTTS,
53
53
  Pcm16Resampler: () => Pcm16Resampler,
54
+ Realtime: () => Realtime,
54
55
  STT: () => STT,
55
56
  SentenceSplitter: () => SentenceSplitter,
56
57
  SentenceTTS: () => SentenceTTS,
@@ -90,7 +91,9 @@ var Agent = class extends import_eventemitter3.EventEmitter {
90
91
  const answerCount = ++this.answerCount;
91
92
  const stream = new import_stream.PassThrough();
92
93
  this.answering = true;
93
- Promise.resolve().then(() => this.options.onBeforeAnswer?.bind(this)(stream)).then((skip) => {
94
+ Promise.resolve().then(
95
+ () => this.options.onBeforeAnswer?.bind(this)(stream)
96
+ ).then((skip) => {
94
97
  if (skip) return;
95
98
  return this.generateAnswer(stream);
96
99
  }).finally(() => {
@@ -133,16 +136,16 @@ var Agent = class extends import_eventemitter3.EventEmitter {
133
136
  metadata
134
137
  };
135
138
  this.conversation.push(message);
136
- this.emit("Message", message);
139
+ this.emitAgentEvent("Message", message);
137
140
  }
138
141
  addToolMessage(message) {
139
142
  this.log("Adding tool message:", message);
140
143
  this.conversation.push(message);
141
- this.emit("Message", message);
144
+ this.emitAgentEvent("Message", message);
142
145
  }
143
146
  endCall() {
144
147
  this.log("Ending call");
145
- this.emit("EndCall");
148
+ this.emitAgentEvent("EndCall");
146
149
  }
147
150
  cancelLastUserMessage() {
148
151
  this.log("Cancelling last user message");
@@ -152,11 +155,11 @@ var Agent = class extends import_eventemitter3.EventEmitter {
152
155
  if (lastMessageIndex !== -1) {
153
156
  this.conversation.splice(lastMessageIndex, 1);
154
157
  }
155
- this.emit("CancelLastUserMessage");
158
+ this.emitAgentEvent("CancelLastUserMessage");
156
159
  }
157
160
  skipAnswer() {
158
161
  this.log("Skipping answer");
159
- this.emit("SkipAnswer");
162
+ this.emitAgentEvent("SkipAnswer");
160
163
  }
161
164
  getDefaultTools() {
162
165
  const tools = [];
@@ -202,7 +205,7 @@ var Agent = class extends import_eventemitter3.EventEmitter {
202
205
  output: JSON.stringify(output ?? null)
203
206
  });
204
207
  if (tool.emitOutput) {
205
- this.emit("ToolCall", {
208
+ this.emitAgentEvent("ToolCall", {
206
209
  name: toolCall.toolName,
207
210
  parameters,
208
211
  output
@@ -261,6 +264,16 @@ var Agent = class extends import_eventemitter3.EventEmitter {
261
264
  }
262
265
  return { message, metadata };
263
266
  }
267
+ /**
268
+ * Emits one of the events every agent has.
269
+ *
270
+ * A subclass can add events of its own, and TypeScript then no longer knows
271
+ * the shared ones are among them.
272
+ */
273
+ emitAgentEvent(event, ...args) {
274
+ const emitter = this;
275
+ emitter.emit(event, ...args);
276
+ }
264
277
  log(...message) {
265
278
  this.logger?.log(...message);
266
279
  }
@@ -534,8 +547,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
534
547
  this.log("Connection closed");
535
548
  const duration = Math.round((Date.now() - this.startTime) / 1e3);
536
549
  const conversation = this.conversation;
537
- this.config.agent?.destroy();
538
- this.config.stt.destroy();
550
+ this.agent?.destroy();
551
+ this.config.stt?.destroy();
539
552
  this.config.tts?.destroy();
540
553
  this.emit("End", {
541
554
  conversation,
@@ -578,7 +591,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
578
591
  this.answerUserTurn();
579
592
  }
580
593
  };
581
- this.onAudioTTS = (audio) => {
594
+ this.onAssistantAudio = (audio) => {
582
595
  if (!this.socket) return;
583
596
  this.log(`Send audio chunk (${audio.byteLength} bytes)`);
584
597
  this.socket.send(audio);
@@ -590,9 +603,23 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
590
603
  this.socket = socket;
591
604
  this.config = config;
592
605
  this.log(`Call started`);
593
- config.stt.on("Transcript", this.onTranscriptSTT);
594
- config.tts?.on("Audio", this.onAudioTTS);
595
- const agent = config.agent;
606
+ const { realtime } = config;
607
+ if (realtime) {
608
+ realtime.on("Audio", this.onAssistantAudio);
609
+ if (config.partialMessages) {
610
+ realtime.on(
611
+ "PartialMessage",
612
+ (content) => this.sendPartialAnswer(content)
613
+ );
614
+ }
615
+ } else {
616
+ if (!config.stt) {
617
+ throw new Error("MicdropServer needs an stt, or a realtime model");
618
+ }
619
+ config.stt.on("Transcript", this.onTranscriptSTT);
620
+ config.tts?.on("Audio", this.onAssistantAudio);
621
+ }
622
+ const agent = this.agent;
596
623
  if (agent) {
597
624
  agent.on("Message", this.onMessageAgent);
598
625
  agent.on(
@@ -620,7 +647,12 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
620
647
  }
621
648
  /** Everything said so far, whether an agent or the server keeps it */
622
649
  get conversation() {
623
- return this.config?.agent?.conversation ?? this.ownConversation;
650
+ return this.agent?.conversation ?? this.ownConversation;
651
+ }
652
+ /** What holds the conversation and answers, a realtime model included */
653
+ get agent() {
654
+ if (!this.config) return void 0;
655
+ return this.config.realtime ?? this.config.agent;
624
656
  }
625
657
  log(...message) {
626
658
  this.logger?.log(...message);
@@ -646,7 +678,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
646
678
  }
647
679
  cancel() {
648
680
  this.config?.tts?.cancel();
649
- this.config?.agent?.cancel();
681
+ this.agent?.cancel();
650
682
  this.operationQueue = [];
651
683
  }
652
684
  onUserAudio(chunk) {
@@ -661,6 +693,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
661
693
  this.currentUserStream?.end();
662
694
  this.currentUserStream = void 0;
663
695
  this.cancel();
696
+ this.config?.realtime?.discardTurn();
664
697
  }
665
698
  onStartSpeaking() {
666
699
  if (!this.config) return;
@@ -670,8 +703,13 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
670
703
  this.config.turnDetector?.reset();
671
704
  this.turnComplete = void 0;
672
705
  this.releaseHeldTurn();
673
- this.config.stt.transcribe(this.currentUserStream);
674
- this.cancel();
706
+ if (this.config.realtime) {
707
+ this.cancel();
708
+ this.config.realtime.startTurn(this.currentUserStream);
709
+ } else {
710
+ this.config.stt?.transcribe(this.currentUserStream);
711
+ this.cancel();
712
+ }
675
713
  }
676
714
  onStopSpeaking() {
677
715
  const hasNoUserSpeech = !this.currentUserStream || this.userSpeechChunks === 0;
@@ -683,6 +721,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
683
721
  return;
684
722
  }
685
723
  this.turnComplete = this.predictTurnComplete();
724
+ if (this.config?.realtime) {
725
+ this.answerUserTurn();
726
+ return;
727
+ }
686
728
  const conversation = this.conversation;
687
729
  const lastMessage = conversation[conversation.length - 1];
688
730
  if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
@@ -715,7 +757,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
715
757
  }
716
758
  this.releaseHeldTurn();
717
759
  this.cancel();
718
- this.answer();
760
+ this.answerTurn();
719
761
  }
720
762
  /**
721
763
  * Answers anyway if the rest of the sentence never comes.
@@ -729,9 +771,18 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
729
771
  this.heldTurnTimer = void 0;
730
772
  this.log("Nothing more came, answering");
731
773
  this.cancel();
732
- this.answer();
774
+ this.answerTurn();
733
775
  }, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
734
776
  }
777
+ /** Answers what the user just said */
778
+ answerTurn() {
779
+ const realtime = this.config?.realtime;
780
+ if (realtime) {
781
+ realtime.endTurn();
782
+ } else {
783
+ this.answer();
784
+ }
785
+ }
735
786
  releaseHeldTurn() {
736
787
  if (!this.heldTurnTimer) return;
737
788
  clearTimeout(this.heldTurnTimer);
@@ -739,16 +790,16 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
739
790
  }
740
791
  /** Adds a message to the conversation and sends it to the client */
741
792
  addUserMessage(text, metadata) {
742
- if (this.config?.agent) {
743
- this.config.agent.addUserMessage(text, metadata);
793
+ if (this.agent) {
794
+ this.agent.addUserMessage(text, metadata);
744
795
  } else {
745
796
  this.addOwnMessage("user", text, metadata);
746
797
  }
747
798
  }
748
799
  /** Adds a message to the conversation and sends it to the client */
749
800
  addAssistantMessage(text, metadata) {
750
- if (this.config?.agent) {
751
- this.config.agent.addAssistantMessage(text, metadata);
801
+ if (this.agent) {
802
+ this.agent.addAssistantMessage(text, metadata);
752
803
  } else {
753
804
  this.addOwnMessage("assistant", text, metadata);
754
805
  }
@@ -783,9 +834,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
783
834
  sendFirstMessage() {
784
835
  if (!this.config) return;
785
836
  if (this.config.firstMessage) {
786
- this.config.agent?.addAssistantMessage(this.config.firstMessage);
837
+ this.agent?.addAssistantMessage(this.config.firstMessage);
787
838
  this.speak(this.config.firstMessage);
788
- } else if (this.config.generateFirstMessage && this.config.agent) {
839
+ } else if (this.config.generateFirstMessage && this.agent) {
789
840
  this.answer();
790
841
  } else {
791
842
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -798,6 +849,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
798
849
  }
799
850
  async _answer() {
800
851
  if (!this.config) return;
852
+ if (this.config.realtime) {
853
+ this.config.realtime.respond();
854
+ return;
855
+ }
801
856
  if (!this.config.agent) {
802
857
  this.log("No agent, skipping answer");
803
858
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -845,6 +900,11 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
845
900
  }
846
901
  async _speak(message) {
847
902
  if (!this.socket || !this.config) return;
903
+ if (this.config.realtime) {
904
+ const text = typeof message === "string" ? message : await readText(message);
905
+ if (text.trim() !== "") this.config.realtime.say(text);
906
+ return;
907
+ }
848
908
  let textStream;
849
909
  if (typeof message === "string") {
850
910
  if (!this.config.agent) {
@@ -883,6 +943,14 @@ function hasContent(stream) {
883
943
  stream.on("end", onEnd);
884
944
  });
885
945
  }
946
+ function readText(stream) {
947
+ return new Promise((resolve) => {
948
+ let text = "";
949
+ stream.on("data", (chunk) => text += chunk.toString());
950
+ stream.once("end", () => resolve(text));
951
+ stream.once("error", () => resolve(text));
952
+ });
953
+ }
886
954
  function drain(stream) {
887
955
  return new Promise((resolve) => {
888
956
  stream.on("data", () => {
@@ -893,6 +961,252 @@ function drain(stream) {
893
961
  });
894
962
  }
895
963
 
964
+ // src/realtime/Realtime.ts
965
+ var USER_TRANSCRIPT_TIMEOUT = 3e3;
966
+ function newAnswer(record) {
967
+ return { transcript: "", hasAudio: false, record, resumed: false };
968
+ }
969
+ var Realtime = class extends Agent {
970
+ constructor() {
971
+ super(...arguments);
972
+ // A turn opens with its first audio, so a blip without any sends nothing
973
+ this.turn = "none";
974
+ // Output of an interrupted answer can still be on its way
975
+ this.discarding = false;
976
+ // Answers asked for while another one was being given
977
+ this.queue = [];
978
+ this.answerWaiters = [];
979
+ this.userTranscript = "";
980
+ this.awaitingUserTranscript = false;
981
+ this.afterUserTranscript = [];
982
+ }
983
+ /**
984
+ * The user started speaking.
985
+ *
986
+ * The assistant stops, and the audio read from the stream goes to the model.
987
+ * A turn left open, because the sentence sounded unfinished, carries on.
988
+ */
989
+ startTurn(audio) {
990
+ this.cancel();
991
+ if (this.turn === "none") this.turn = "pending";
992
+ audio.on("data", (chunk) => this.sendAudio(chunk));
993
+ }
994
+ /** Closes the turn of the user, and has the model answer it */
995
+ endTurn() {
996
+ if (this.turn !== "open") return;
997
+ this.turn = "none";
998
+ this.awaitUserTranscript();
999
+ this.beginAnswer(true);
1000
+ this.closeTurn();
1001
+ }
1002
+ /** Drops the turn of the user, without an answer */
1003
+ discardTurn() {
1004
+ this.cancel();
1005
+ if (this.turn === "open") {
1006
+ this.clearTurn();
1007
+ this.discarding = true;
1008
+ }
1009
+ this.turn = "none";
1010
+ }
1011
+ /** Has the model speak without waiting for the user, to open a call */
1012
+ respond() {
1013
+ this.whenIdle(() => {
1014
+ this.beginAnswer(true);
1015
+ this.generate();
1016
+ });
1017
+ }
1018
+ /**
1019
+ * Has the model say this text, word for word.
1020
+ *
1021
+ * The caller records the text in the conversation, as with a text to speech,
1022
+ * so the transcript of what the model says is not recorded again.
1023
+ */
1024
+ say(text) {
1025
+ this.whenIdle(() => {
1026
+ this.beginAnswer(false);
1027
+ this.speakText(text);
1028
+ });
1029
+ }
1030
+ cancel() {
1031
+ this.queue = [];
1032
+ const answer = this.currentAnswer;
1033
+ if (!answer) return;
1034
+ this.log("Cancel");
1035
+ this.currentAnswer = void 0;
1036
+ this.discarding = true;
1037
+ this.cancelAnswer();
1038
+ this.recordAnswer(answer);
1039
+ this.resolveAnswerWaiters();
1040
+ }
1041
+ /**
1042
+ * Adds a message the model did not hear, and tells the model about it.
1043
+ *
1044
+ * The transcripts of the call are recorded apart, so they are not sent back.
1045
+ */
1046
+ addMessage(role, text, metadata) {
1047
+ super.addMessage(role, text, metadata);
1048
+ if (text.trim() !== "") this.injectMessage(role, text);
1049
+ }
1050
+ addTool(tool) {
1051
+ super.addTool(tool);
1052
+ this.updateTools();
1053
+ }
1054
+ removeTool(name) {
1055
+ super.removeTool(name);
1056
+ this.updateTools();
1057
+ }
1058
+ /**
1059
+ * Lists the events that have listeners.
1060
+ *
1061
+ * Typed as the events of any agent, so a realtime model goes wherever an
1062
+ * agent is expected. The two extra events do not change what that code reads.
1063
+ */
1064
+ eventNames() {
1065
+ return super.eventNames();
1066
+ }
1067
+ destroy() {
1068
+ super.destroy();
1069
+ this.clearUserTranscriptTimer();
1070
+ this.afterUserTranscript = [];
1071
+ this.resolveAnswerWaiters();
1072
+ }
1073
+ generateAnswer(stream) {
1074
+ let written = 0;
1075
+ const onPartial = (transcript) => {
1076
+ stream.write(transcript.slice(written));
1077
+ written = transcript.length;
1078
+ };
1079
+ this.on("PartialMessage", onPartial);
1080
+ this.respond();
1081
+ return new Promise((resolve) => {
1082
+ this.answerWaiters.push(() => {
1083
+ this.off("PartialMessage", onPartial);
1084
+ resolve();
1085
+ });
1086
+ });
1087
+ }
1088
+ /** Nobody is speaking, nor about to, so the connection can be renewed */
1089
+ get isIdle() {
1090
+ return this.turn === "none" && !this.currentAnswer;
1091
+ }
1092
+ /** Voice of the answer, PCM16 at 16 kHz */
1093
+ emitAudio(chunk) {
1094
+ const answer = this.activeAnswer();
1095
+ if (!answer) return;
1096
+ answer.hasAudio = true;
1097
+ this.emit("Audio", chunk);
1098
+ }
1099
+ /** Transcript of the answer, as it is spoken */
1100
+ addAnswerTranscript(text) {
1101
+ const answer = this.activeAnswer();
1102
+ if (!answer) return;
1103
+ if (answer.resumed) {
1104
+ answer.resumed = false;
1105
+ if (answer.transcript !== "" && !/\s$/.test(answer.transcript)) {
1106
+ text = ` ${text.trimStart()}`;
1107
+ }
1108
+ }
1109
+ answer.transcript += text;
1110
+ if (answer.record) this.emit("PartialMessage", answer.transcript);
1111
+ }
1112
+ /**
1113
+ * The model finished its answer, or finished dropping an interrupted one.
1114
+ *
1115
+ * An answer without audio tells the client to stop waiting for one.
1116
+ */
1117
+ endAnswer() {
1118
+ if (this.discarding) {
1119
+ this.discarding = false;
1120
+ return;
1121
+ }
1122
+ const answer = this.currentAnswer;
1123
+ if (!answer) return;
1124
+ this.currentAnswer = void 0;
1125
+ if (!answer.hasAudio) this.emitAgentEvent("SkipAnswer");
1126
+ this.recordAnswer(answer);
1127
+ this.resolveAnswerWaiters();
1128
+ this.queue.shift()?.();
1129
+ }
1130
+ /** Runs a tool the model called, and records it in the conversation */
1131
+ async runTool(toolCall) {
1132
+ const result = await this.executeTool(toolCall);
1133
+ if (this.currentAnswer) this.currentAnswer.resumed = true;
1134
+ return result;
1135
+ }
1136
+ /** Transcript of the turn of the user, possibly in several pieces */
1137
+ addUserTranscript(text) {
1138
+ this.userTranscript += text;
1139
+ }
1140
+ /** The transcript of the turn is complete, it goes in the conversation */
1141
+ commitUserTranscript() {
1142
+ this.clearUserTranscriptTimer();
1143
+ this.awaitingUserTranscript = false;
1144
+ const text = this.userTranscript.trim();
1145
+ this.userTranscript = "";
1146
+ if (text !== "") super.addMessage("user", text);
1147
+ const pending = this.afterUserTranscript;
1148
+ this.afterUserTranscript = [];
1149
+ pending.forEach((record) => record());
1150
+ }
1151
+ sendAudio(chunk) {
1152
+ if (this.turn === "none") return;
1153
+ if (this.turn === "pending") {
1154
+ this.turn = "open";
1155
+ this.openTurn();
1156
+ }
1157
+ this.appendAudio(chunk);
1158
+ }
1159
+ beginAnswer(record) {
1160
+ this.discarding = false;
1161
+ this.currentAnswer = newAnswer(record);
1162
+ }
1163
+ /**
1164
+ * The answer output belongs to, if it is still wanted.
1165
+ *
1166
+ * The model can speak without being asked, to follow up on a tool call for
1167
+ * instance, which opens an answer of its own.
1168
+ */
1169
+ activeAnswer() {
1170
+ if (this.discarding) return void 0;
1171
+ if (!this.currentAnswer) this.currentAnswer = newAnswer(true);
1172
+ return this.currentAnswer;
1173
+ }
1174
+ whenIdle(start) {
1175
+ if (this.currentAnswer) {
1176
+ this.queue.push(start);
1177
+ } else {
1178
+ start();
1179
+ }
1180
+ }
1181
+ recordAnswer(answer) {
1182
+ if (!answer.record) return;
1183
+ const record = () => super.addMessage("assistant", answer.transcript.trim());
1184
+ if (this.awaitingUserTranscript) {
1185
+ this.afterUserTranscript.push(record);
1186
+ } else {
1187
+ record();
1188
+ }
1189
+ }
1190
+ awaitUserTranscript() {
1191
+ this.awaitingUserTranscript = true;
1192
+ this.clearUserTranscriptTimer();
1193
+ this.userTranscriptTimer = setTimeout(() => {
1194
+ this.log("No transcript of the turn, recording the answer anyway");
1195
+ this.commitUserTranscript();
1196
+ }, USER_TRANSCRIPT_TIMEOUT);
1197
+ }
1198
+ clearUserTranscriptTimer() {
1199
+ if (!this.userTranscriptTimer) return;
1200
+ clearTimeout(this.userTranscriptTimer);
1201
+ this.userTranscriptTimer = void 0;
1202
+ }
1203
+ resolveAnswerWaiters() {
1204
+ const waiters = this.answerWaiters;
1205
+ this.answerWaiters = [];
1206
+ waiters.forEach((resolve) => resolve());
1207
+ }
1208
+ };
1209
+
896
1210
  // src/recorder/MicdropRecorder.ts
897
1211
  var import_eventemitter33 = require("eventemitter3");
898
1212
  var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
@@ -1401,6 +1715,7 @@ async function waitForParams(socket, validate) {
1401
1715
  MockSTT,
1402
1716
  MockTTS,
1403
1717
  Pcm16Resampler,
1718
+ Realtime,
1404
1719
  STT,
1405
1720
  SentenceSplitter,
1406
1721
  SentenceTTS,