@micdrop/server 2.6.0 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -25,7 +25,9 @@ var Agent = class extends EventEmitter {
25
25
  const answerCount = ++this.answerCount;
26
26
  const stream = new PassThrough();
27
27
  this.answering = true;
28
- Promise.resolve().then(() => this.options.onBeforeAnswer?.bind(this)(stream)).then((skip) => {
28
+ Promise.resolve().then(
29
+ () => this.options.onBeforeAnswer?.bind(this)(stream)
30
+ ).then((skip) => {
29
31
  if (skip) return;
30
32
  return this.generateAnswer(stream);
31
33
  }).finally(() => {
@@ -68,16 +70,16 @@ var Agent = class extends EventEmitter {
68
70
  metadata
69
71
  };
70
72
  this.conversation.push(message);
71
- this.emit("Message", message);
73
+ this.emitAgentEvent("Message", message);
72
74
  }
73
75
  addToolMessage(message) {
74
76
  this.log("Adding tool message:", message);
75
77
  this.conversation.push(message);
76
- this.emit("Message", message);
78
+ this.emitAgentEvent("Message", message);
77
79
  }
78
80
  endCall() {
79
81
  this.log("Ending call");
80
- this.emit("EndCall");
82
+ this.emitAgentEvent("EndCall");
81
83
  }
82
84
  cancelLastUserMessage() {
83
85
  this.log("Cancelling last user message");
@@ -87,11 +89,11 @@ var Agent = class extends EventEmitter {
87
89
  if (lastMessageIndex !== -1) {
88
90
  this.conversation.splice(lastMessageIndex, 1);
89
91
  }
90
- this.emit("CancelLastUserMessage");
92
+ this.emitAgentEvent("CancelLastUserMessage");
91
93
  }
92
94
  skipAnswer() {
93
95
  this.log("Skipping answer");
94
- this.emit("SkipAnswer");
96
+ this.emitAgentEvent("SkipAnswer");
95
97
  }
96
98
  getDefaultTools() {
97
99
  const tools = [];
@@ -137,7 +139,7 @@ var Agent = class extends EventEmitter {
137
139
  output: JSON.stringify(output ?? null)
138
140
  });
139
141
  if (tool.emitOutput) {
140
- this.emit("ToolCall", {
142
+ this.emitAgentEvent("ToolCall", {
141
143
  name: toolCall.toolName,
142
144
  parameters,
143
145
  output
@@ -196,6 +198,16 @@ var Agent = class extends EventEmitter {
196
198
  }
197
199
  return { message, metadata };
198
200
  }
201
+ /**
202
+ * Emits one of the events every agent has.
203
+ *
204
+ * A subclass can add events of its own, and TypeScript then no longer knows
205
+ * the shared ones are among them.
206
+ */
207
+ emitAgentEvent(event, ...args) {
208
+ const emitter = this;
209
+ emitter.emit(event, ...args);
210
+ }
199
211
  log(...message) {
200
212
  this.logger?.log(...message);
201
213
  }
@@ -469,8 +481,8 @@ var MicdropServer = class extends EventEmitter2 {
469
481
  this.log("Connection closed");
470
482
  const duration = Math.round((Date.now() - this.startTime) / 1e3);
471
483
  const conversation = this.conversation;
472
- this.config.agent?.destroy();
473
- this.config.stt.destroy();
484
+ this.agent?.destroy();
485
+ this.config.stt?.destroy();
474
486
  this.config.tts?.destroy();
475
487
  this.emit("End", {
476
488
  conversation,
@@ -513,7 +525,7 @@ var MicdropServer = class extends EventEmitter2 {
513
525
  this.answerUserTurn();
514
526
  }
515
527
  };
516
- this.onAudioTTS = (audio) => {
528
+ this.onAssistantAudio = (audio) => {
517
529
  if (!this.socket) return;
518
530
  this.log(`Send audio chunk (${audio.byteLength} bytes)`);
519
531
  this.socket.send(audio);
@@ -525,9 +537,23 @@ var MicdropServer = class extends EventEmitter2 {
525
537
  this.socket = socket;
526
538
  this.config = config;
527
539
  this.log(`Call started`);
528
- config.stt.on("Transcript", this.onTranscriptSTT);
529
- config.tts?.on("Audio", this.onAudioTTS);
530
- const agent = config.agent;
540
+ const { realtime } = config;
541
+ if (realtime) {
542
+ realtime.on("Audio", this.onAssistantAudio);
543
+ if (config.partialMessages) {
544
+ realtime.on(
545
+ "PartialMessage",
546
+ (content) => this.sendPartialAnswer(content)
547
+ );
548
+ }
549
+ } else {
550
+ if (!config.stt) {
551
+ throw new Error("MicdropServer needs an stt, or a realtime model");
552
+ }
553
+ config.stt.on("Transcript", this.onTranscriptSTT);
554
+ config.tts?.on("Audio", this.onAssistantAudio);
555
+ }
556
+ const agent = this.agent;
531
557
  if (agent) {
532
558
  agent.on("Message", this.onMessageAgent);
533
559
  agent.on(
@@ -555,7 +581,12 @@ var MicdropServer = class extends EventEmitter2 {
555
581
  }
556
582
  /** Everything said so far, whether an agent or the server keeps it */
557
583
  get conversation() {
558
- return this.config?.agent?.conversation ?? this.ownConversation;
584
+ return this.agent?.conversation ?? this.ownConversation;
585
+ }
586
+ /** What holds the conversation and answers, a realtime model included */
587
+ get agent() {
588
+ if (!this.config) return void 0;
589
+ return this.config.realtime ?? this.config.agent;
559
590
  }
560
591
  log(...message) {
561
592
  this.logger?.log(...message);
@@ -581,7 +612,7 @@ var MicdropServer = class extends EventEmitter2 {
581
612
  }
582
613
  cancel() {
583
614
  this.config?.tts?.cancel();
584
- this.config?.agent?.cancel();
615
+ this.agent?.cancel();
585
616
  this.operationQueue = [];
586
617
  }
587
618
  onUserAudio(chunk) {
@@ -596,6 +627,7 @@ var MicdropServer = class extends EventEmitter2 {
596
627
  this.currentUserStream?.end();
597
628
  this.currentUserStream = void 0;
598
629
  this.cancel();
630
+ this.config?.realtime?.discardTurn();
599
631
  }
600
632
  onStartSpeaking() {
601
633
  if (!this.config) return;
@@ -605,8 +637,13 @@ var MicdropServer = class extends EventEmitter2 {
605
637
  this.config.turnDetector?.reset();
606
638
  this.turnComplete = void 0;
607
639
  this.releaseHeldTurn();
608
- this.config.stt.transcribe(this.currentUserStream);
609
- this.cancel();
640
+ if (this.config.realtime) {
641
+ this.cancel();
642
+ this.config.realtime.startTurn(this.currentUserStream);
643
+ } else {
644
+ this.config.stt?.transcribe(this.currentUserStream);
645
+ this.cancel();
646
+ }
610
647
  }
611
648
  onStopSpeaking() {
612
649
  const hasNoUserSpeech = !this.currentUserStream || this.userSpeechChunks === 0;
@@ -618,6 +655,10 @@ var MicdropServer = class extends EventEmitter2 {
618
655
  return;
619
656
  }
620
657
  this.turnComplete = this.predictTurnComplete();
658
+ if (this.config?.realtime) {
659
+ this.answerUserTurn();
660
+ return;
661
+ }
621
662
  const conversation = this.conversation;
622
663
  const lastMessage = conversation[conversation.length - 1];
623
664
  if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
@@ -650,7 +691,7 @@ var MicdropServer = class extends EventEmitter2 {
650
691
  }
651
692
  this.releaseHeldTurn();
652
693
  this.cancel();
653
- this.answer();
694
+ this.answerTurn();
654
695
  }
655
696
  /**
656
697
  * Answers anyway if the rest of the sentence never comes.
@@ -664,9 +705,18 @@ var MicdropServer = class extends EventEmitter2 {
664
705
  this.heldTurnTimer = void 0;
665
706
  this.log("Nothing more came, answering");
666
707
  this.cancel();
667
- this.answer();
708
+ this.answerTurn();
668
709
  }, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
669
710
  }
711
+ /** Answers what the user just said */
712
+ answerTurn() {
713
+ const realtime = this.config?.realtime;
714
+ if (realtime) {
715
+ realtime.endTurn();
716
+ } else {
717
+ this.answer();
718
+ }
719
+ }
670
720
  releaseHeldTurn() {
671
721
  if (!this.heldTurnTimer) return;
672
722
  clearTimeout(this.heldTurnTimer);
@@ -674,16 +724,16 @@ var MicdropServer = class extends EventEmitter2 {
674
724
  }
675
725
  /** Adds a message to the conversation and sends it to the client */
676
726
  addUserMessage(text, metadata) {
677
- if (this.config?.agent) {
678
- this.config.agent.addUserMessage(text, metadata);
727
+ if (this.agent) {
728
+ this.agent.addUserMessage(text, metadata);
679
729
  } else {
680
730
  this.addOwnMessage("user", text, metadata);
681
731
  }
682
732
  }
683
733
  /** Adds a message to the conversation and sends it to the client */
684
734
  addAssistantMessage(text, metadata) {
685
- if (this.config?.agent) {
686
- this.config.agent.addAssistantMessage(text, metadata);
735
+ if (this.agent) {
736
+ this.agent.addAssistantMessage(text, metadata);
687
737
  } else {
688
738
  this.addOwnMessage("assistant", text, metadata);
689
739
  }
@@ -718,9 +768,9 @@ var MicdropServer = class extends EventEmitter2 {
718
768
  sendFirstMessage() {
719
769
  if (!this.config) return;
720
770
  if (this.config.firstMessage) {
721
- this.config.agent?.addAssistantMessage(this.config.firstMessage);
771
+ this.agent?.addAssistantMessage(this.config.firstMessage);
722
772
  this.speak(this.config.firstMessage);
723
- } else if (this.config.generateFirstMessage && this.config.agent) {
773
+ } else if (this.config.generateFirstMessage && this.agent) {
724
774
  this.answer();
725
775
  } else {
726
776
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -733,6 +783,10 @@ var MicdropServer = class extends EventEmitter2 {
733
783
  }
734
784
  async _answer() {
735
785
  if (!this.config) return;
786
+ if (this.config.realtime) {
787
+ this.config.realtime.respond();
788
+ return;
789
+ }
736
790
  if (!this.config.agent) {
737
791
  this.log("No agent, skipping answer");
738
792
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -780,6 +834,11 @@ var MicdropServer = class extends EventEmitter2 {
780
834
  }
781
835
  async _speak(message) {
782
836
  if (!this.socket || !this.config) return;
837
+ if (this.config.realtime) {
838
+ const text = typeof message === "string" ? message : await readText(message);
839
+ if (text.trim() !== "") this.config.realtime.say(text);
840
+ return;
841
+ }
783
842
  let textStream;
784
843
  if (typeof message === "string") {
785
844
  if (!this.config.agent) {
@@ -818,6 +877,14 @@ function hasContent(stream) {
818
877
  stream.on("end", onEnd);
819
878
  });
820
879
  }
880
+ function readText(stream) {
881
+ return new Promise((resolve) => {
882
+ let text = "";
883
+ stream.on("data", (chunk) => text += chunk.toString());
884
+ stream.once("end", () => resolve(text));
885
+ stream.once("error", () => resolve(text));
886
+ });
887
+ }
821
888
  function drain(stream) {
822
889
  return new Promise((resolve) => {
823
890
  stream.on("data", () => {
@@ -828,6 +895,252 @@ function drain(stream) {
828
895
  });
829
896
  }
830
897
 
898
+ // src/realtime/Realtime.ts
899
+ var USER_TRANSCRIPT_TIMEOUT = 3e3;
900
+ function newAnswer(record) {
901
+ return { transcript: "", hasAudio: false, record, resumed: false };
902
+ }
903
+ var Realtime = class extends Agent {
904
+ constructor() {
905
+ super(...arguments);
906
+ // A turn opens with its first audio, so a blip without any sends nothing
907
+ this.turn = "none";
908
+ // Output of an interrupted answer can still be on its way
909
+ this.discarding = false;
910
+ // Answers asked for while another one was being given
911
+ this.queue = [];
912
+ this.answerWaiters = [];
913
+ this.userTranscript = "";
914
+ this.awaitingUserTranscript = false;
915
+ this.afterUserTranscript = [];
916
+ }
917
+ /**
918
+ * The user started speaking.
919
+ *
920
+ * The assistant stops, and the audio read from the stream goes to the model.
921
+ * A turn left open, because the sentence sounded unfinished, carries on.
922
+ */
923
+ startTurn(audio) {
924
+ this.cancel();
925
+ if (this.turn === "none") this.turn = "pending";
926
+ audio.on("data", (chunk) => this.sendAudio(chunk));
927
+ }
928
+ /** Closes the turn of the user, and has the model answer it */
929
+ endTurn() {
930
+ if (this.turn !== "open") return;
931
+ this.turn = "none";
932
+ this.awaitUserTranscript();
933
+ this.beginAnswer(true);
934
+ this.closeTurn();
935
+ }
936
+ /** Drops the turn of the user, without an answer */
937
+ discardTurn() {
938
+ this.cancel();
939
+ if (this.turn === "open") {
940
+ this.clearTurn();
941
+ this.discarding = true;
942
+ }
943
+ this.turn = "none";
944
+ }
945
+ /** Has the model speak without waiting for the user, to open a call */
946
+ respond() {
947
+ this.whenIdle(() => {
948
+ this.beginAnswer(true);
949
+ this.generate();
950
+ });
951
+ }
952
+ /**
953
+ * Has the model say this text, word for word.
954
+ *
955
+ * The caller records the text in the conversation, as with a text to speech,
956
+ * so the transcript of what the model says is not recorded again.
957
+ */
958
+ say(text) {
959
+ this.whenIdle(() => {
960
+ this.beginAnswer(false);
961
+ this.speakText(text);
962
+ });
963
+ }
964
+ cancel() {
965
+ this.queue = [];
966
+ const answer = this.currentAnswer;
967
+ if (!answer) return;
968
+ this.log("Cancel");
969
+ this.currentAnswer = void 0;
970
+ this.discarding = true;
971
+ this.cancelAnswer();
972
+ this.recordAnswer(answer);
973
+ this.resolveAnswerWaiters();
974
+ }
975
+ /**
976
+ * Adds a message the model did not hear, and tells the model about it.
977
+ *
978
+ * The transcripts of the call are recorded apart, so they are not sent back.
979
+ */
980
+ addMessage(role, text, metadata) {
981
+ super.addMessage(role, text, metadata);
982
+ if (text.trim() !== "") this.injectMessage(role, text);
983
+ }
984
+ addTool(tool) {
985
+ super.addTool(tool);
986
+ this.updateTools();
987
+ }
988
+ removeTool(name) {
989
+ super.removeTool(name);
990
+ this.updateTools();
991
+ }
992
+ /**
993
+ * Lists the events that have listeners.
994
+ *
995
+ * Typed as the events of any agent, so a realtime model goes wherever an
996
+ * agent is expected. The two extra events do not change what that code reads.
997
+ */
998
+ eventNames() {
999
+ return super.eventNames();
1000
+ }
1001
+ destroy() {
1002
+ super.destroy();
1003
+ this.clearUserTranscriptTimer();
1004
+ this.afterUserTranscript = [];
1005
+ this.resolveAnswerWaiters();
1006
+ }
1007
+ generateAnswer(stream) {
1008
+ let written = 0;
1009
+ const onPartial = (transcript) => {
1010
+ stream.write(transcript.slice(written));
1011
+ written = transcript.length;
1012
+ };
1013
+ this.on("PartialMessage", onPartial);
1014
+ this.respond();
1015
+ return new Promise((resolve) => {
1016
+ this.answerWaiters.push(() => {
1017
+ this.off("PartialMessage", onPartial);
1018
+ resolve();
1019
+ });
1020
+ });
1021
+ }
1022
+ /** Nobody is speaking, nor about to, so the connection can be renewed */
1023
+ get isIdle() {
1024
+ return this.turn === "none" && !this.currentAnswer;
1025
+ }
1026
+ /** Voice of the answer, PCM16 at 16 kHz */
1027
+ emitAudio(chunk) {
1028
+ const answer = this.activeAnswer();
1029
+ if (!answer) return;
1030
+ answer.hasAudio = true;
1031
+ this.emit("Audio", chunk);
1032
+ }
1033
+ /** Transcript of the answer, as it is spoken */
1034
+ addAnswerTranscript(text) {
1035
+ const answer = this.activeAnswer();
1036
+ if (!answer) return;
1037
+ if (answer.resumed) {
1038
+ answer.resumed = false;
1039
+ if (answer.transcript !== "" && !/\s$/.test(answer.transcript)) {
1040
+ text = ` ${text.trimStart()}`;
1041
+ }
1042
+ }
1043
+ answer.transcript += text;
1044
+ if (answer.record) this.emit("PartialMessage", answer.transcript);
1045
+ }
1046
+ /**
1047
+ * The model finished its answer, or finished dropping an interrupted one.
1048
+ *
1049
+ * An answer without audio tells the client to stop waiting for one.
1050
+ */
1051
+ endAnswer() {
1052
+ if (this.discarding) {
1053
+ this.discarding = false;
1054
+ return;
1055
+ }
1056
+ const answer = this.currentAnswer;
1057
+ if (!answer) return;
1058
+ this.currentAnswer = void 0;
1059
+ if (!answer.hasAudio) this.emitAgentEvent("SkipAnswer");
1060
+ this.recordAnswer(answer);
1061
+ this.resolveAnswerWaiters();
1062
+ this.queue.shift()?.();
1063
+ }
1064
+ /** Runs a tool the model called, and records it in the conversation */
1065
+ async runTool(toolCall) {
1066
+ const result = await this.executeTool(toolCall);
1067
+ if (this.currentAnswer) this.currentAnswer.resumed = true;
1068
+ return result;
1069
+ }
1070
+ /** Transcript of the turn of the user, possibly in several pieces */
1071
+ addUserTranscript(text) {
1072
+ this.userTranscript += text;
1073
+ }
1074
+ /** The transcript of the turn is complete, it goes in the conversation */
1075
+ commitUserTranscript() {
1076
+ this.clearUserTranscriptTimer();
1077
+ this.awaitingUserTranscript = false;
1078
+ const text = this.userTranscript.trim();
1079
+ this.userTranscript = "";
1080
+ if (text !== "") super.addMessage("user", text);
1081
+ const pending = this.afterUserTranscript;
1082
+ this.afterUserTranscript = [];
1083
+ pending.forEach((record) => record());
1084
+ }
1085
+ sendAudio(chunk) {
1086
+ if (this.turn === "none") return;
1087
+ if (this.turn === "pending") {
1088
+ this.turn = "open";
1089
+ this.openTurn();
1090
+ }
1091
+ this.appendAudio(chunk);
1092
+ }
1093
+ beginAnswer(record) {
1094
+ this.discarding = false;
1095
+ this.currentAnswer = newAnswer(record);
1096
+ }
1097
+ /**
1098
+ * The answer output belongs to, if it is still wanted.
1099
+ *
1100
+ * The model can speak without being asked, to follow up on a tool call for
1101
+ * instance, which opens an answer of its own.
1102
+ */
1103
+ activeAnswer() {
1104
+ if (this.discarding) return void 0;
1105
+ if (!this.currentAnswer) this.currentAnswer = newAnswer(true);
1106
+ return this.currentAnswer;
1107
+ }
1108
+ whenIdle(start) {
1109
+ if (this.currentAnswer) {
1110
+ this.queue.push(start);
1111
+ } else {
1112
+ start();
1113
+ }
1114
+ }
1115
+ recordAnswer(answer) {
1116
+ if (!answer.record) return;
1117
+ const record = () => super.addMessage("assistant", answer.transcript.trim());
1118
+ if (this.awaitingUserTranscript) {
1119
+ this.afterUserTranscript.push(record);
1120
+ } else {
1121
+ record();
1122
+ }
1123
+ }
1124
+ awaitUserTranscript() {
1125
+ this.awaitingUserTranscript = true;
1126
+ this.clearUserTranscriptTimer();
1127
+ this.userTranscriptTimer = setTimeout(() => {
1128
+ this.log("No transcript of the turn, recording the answer anyway");
1129
+ this.commitUserTranscript();
1130
+ }, USER_TRANSCRIPT_TIMEOUT);
1131
+ }
1132
+ clearUserTranscriptTimer() {
1133
+ if (!this.userTranscriptTimer) return;
1134
+ clearTimeout(this.userTranscriptTimer);
1135
+ this.userTranscriptTimer = void 0;
1136
+ }
1137
+ resolveAnswerWaiters() {
1138
+ const waiters = this.answerWaiters;
1139
+ this.answerWaiters = [];
1140
+ waiters.forEach((resolve) => resolve());
1141
+ }
1142
+ };
1143
+
831
1144
  // src/recorder/MicdropRecorder.ts
832
1145
  import { EventEmitter as EventEmitter3 } from "eventemitter3";
833
1146
  var MicdropRecorder = class extends EventEmitter3 {
@@ -948,6 +1261,44 @@ var MicdropRecorder = class extends EventEmitter3 {
948
1261
  }
949
1262
  };
950
1263
 
1264
+ // src/SharedInstances.ts
1265
+ var SharedInstances = class {
1266
+ constructor() {
1267
+ this.entries = /* @__PURE__ */ new Map();
1268
+ }
1269
+ /**
1270
+ * Returns the instance for this key, calling `create` the first time.
1271
+ *
1272
+ * The key can be any JSON-serializable value, usually an array of the
1273
+ * options that define the instance. Concurrent callers get the same promise
1274
+ * instead of loading the model twice.
1275
+ */
1276
+ load(key, create) {
1277
+ const id = typeof key === "string" ? key : JSON.stringify(key);
1278
+ const existing = this.entries.get(id);
1279
+ if (existing) return existing;
1280
+ let loading;
1281
+ try {
1282
+ loading = create();
1283
+ } catch (error) {
1284
+ return Promise.reject(error);
1285
+ }
1286
+ loading.catch(() => {
1287
+ if (this.entries.get(id) === loading) this.entries.delete(id);
1288
+ });
1289
+ this.entries.set(id, loading);
1290
+ return loading;
1291
+ }
1292
+ /** Whether an instance is loaded, or being loaded, for this key. */
1293
+ has(key) {
1294
+ return this.entries.has(typeof key === "string" ? key : JSON.stringify(key));
1295
+ }
1296
+ /** Drops every instance, so the next call loads again. */
1297
+ clear() {
1298
+ this.entries.clear();
1299
+ }
1300
+ };
1301
+
951
1302
  // src/stt/STT.ts
952
1303
  import { EventEmitter as EventEmitter4 } from "eventemitter3";
953
1304
  var STT = class extends EventEmitter4 {
@@ -1297,9 +1648,11 @@ export {
1297
1648
  MockSTT,
1298
1649
  MockTTS,
1299
1650
  Pcm16Resampler,
1651
+ Realtime,
1300
1652
  STT,
1301
1653
  SentenceSplitter,
1302
1654
  SentenceTTS,
1655
+ SharedInstances,
1303
1656
  TTS,
1304
1657
  float32ToPcm16,
1305
1658
  handleError,