@micdrop/server 2.7.0 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -37,6 +37,7 @@ __export(index_exports, {
37
37
  AUTO_SEMANTIC_TURN_PROMPT: () => AUTO_SEMANTIC_TURN_PROMPT,
38
38
  AUTO_SEMANTIC_TURN_TOOL_NAME: () => AUTO_SEMANTIC_TURN_TOOL_NAME,
39
39
  Agent: () => Agent,
40
+ Classifier: () => Classifier,
40
41
  FallbackAgent: () => FallbackAgent,
41
42
  FallbackSTT: () => FallbackSTT,
42
43
  FallbackTTS: () => FallbackTTS,
@@ -51,14 +52,18 @@ __export(index_exports, {
51
52
  MockSTT: () => MockSTT,
52
53
  MockTTS: () => MockTTS,
53
54
  Pcm16Resampler: () => Pcm16Resampler,
55
+ Realtime: () => Realtime,
54
56
  STT: () => STT,
55
57
  SentenceSplitter: () => SentenceSplitter,
56
58
  SentenceTTS: () => SentenceTTS,
57
59
  SharedInstances: () => SharedInstances,
58
60
  TTS: () => TTS,
61
+ currentTurn: () => currentTurn,
59
62
  float32ToPcm16: () => float32ToPcm16,
63
+ getTurnClassification: () => getTurnClassification,
60
64
  handleError: () => handleError,
61
65
  pcm16ToFloat32: () => pcm16ToFloat32,
66
+ turnInput: () => turnInput,
62
67
  waitForParams: () => waitForParams
63
68
  });
64
69
  module.exports = __toCommonJS(index_exports);
@@ -90,8 +95,15 @@ var Agent = class extends import_eventemitter3.EventEmitter {
90
95
  const answerCount = ++this.answerCount;
91
96
  const stream = new import_stream.PassThrough();
92
97
  this.answering = true;
93
- Promise.resolve().then(() => this.options.onBeforeAnswer?.bind(this)(stream)).then((skip) => {
94
- if (skip) return;
98
+ Promise.resolve().then(
99
+ () => this.options.onBeforeAnswer?.bind(this)(stream)
100
+ ).then((result) => {
101
+ if (typeof result === "string") {
102
+ stream.write(result);
103
+ this.addAssistantMessage(result);
104
+ return;
105
+ }
106
+ if (result) return;
95
107
  return this.generateAnswer(stream);
96
108
  }).finally(() => {
97
109
  if (stream.writable) {
@@ -133,16 +145,16 @@ var Agent = class extends import_eventemitter3.EventEmitter {
133
145
  metadata
134
146
  };
135
147
  this.conversation.push(message);
136
- this.emit("Message", message);
148
+ this.emitAgentEvent("Message", message);
137
149
  }
138
150
  addToolMessage(message) {
139
151
  this.log("Adding tool message:", message);
140
152
  this.conversation.push(message);
141
- this.emit("Message", message);
153
+ this.emitAgentEvent("Message", message);
142
154
  }
143
155
  endCall() {
144
156
  this.log("Ending call");
145
- this.emit("EndCall");
157
+ this.emitAgentEvent("EndCall");
146
158
  }
147
159
  cancelLastUserMessage() {
148
160
  this.log("Cancelling last user message");
@@ -152,11 +164,11 @@ var Agent = class extends import_eventemitter3.EventEmitter {
152
164
  if (lastMessageIndex !== -1) {
153
165
  this.conversation.splice(lastMessageIndex, 1);
154
166
  }
155
- this.emit("CancelLastUserMessage");
167
+ this.emitAgentEvent("CancelLastUserMessage");
156
168
  }
157
169
  skipAnswer() {
158
170
  this.log("Skipping answer");
159
- this.emit("SkipAnswer");
171
+ this.emitAgentEvent("SkipAnswer");
160
172
  }
161
173
  getDefaultTools() {
162
174
  const tools = [];
@@ -202,7 +214,7 @@ var Agent = class extends import_eventemitter3.EventEmitter {
202
214
  output: JSON.stringify(output ?? null)
203
215
  });
204
216
  if (tool.emitOutput) {
205
- this.emit("ToolCall", {
217
+ this.emitAgentEvent("ToolCall", {
206
218
  name: toolCall.toolName,
207
219
  parameters,
208
220
  output
@@ -261,6 +273,16 @@ var Agent = class extends import_eventemitter3.EventEmitter {
261
273
  }
262
274
  return { message, metadata };
263
275
  }
276
+ /**
277
+ * Emits one of the events every agent has.
278
+ *
279
+ * A subclass can add events of its own, and TypeScript then no longer knows
280
+ * the shared ones are among them.
281
+ */
282
+ emitAgentEvent(event, ...args) {
283
+ const emitter = this;
284
+ emitter.emit(event, ...args);
285
+ }
264
286
  log(...message) {
265
287
  this.logger?.log(...message);
266
288
  }
@@ -405,6 +427,144 @@ var MockAgent = class extends Agent {
405
427
  }
406
428
  };
407
429
 
430
+ // src/classifier/Classifier.ts
431
+ var import_eventemitter32 = require("eventemitter3");
432
+ var Classifier = class extends import_eventemitter32.EventEmitter {
433
+ constructor() {
434
+ super(...arguments);
435
+ this.controllers = /* @__PURE__ */ new Set();
436
+ }
437
+ /**
438
+ * Classifies an input, and emits the result as `Classification`
439
+ * @param input - What to classify: a text, or JSON with named fields
440
+ * @returns The classification, or undefined when cancelled or failed
441
+ */
442
+ classify(input) {
443
+ const controller = new AbortController();
444
+ this.controllers.add(controller);
445
+ const pending = this.run(input, controller.signal).finally(() => {
446
+ this.controllers.delete(controller);
447
+ if (this._pending === pending) this._pending = void 0;
448
+ });
449
+ this._pending = pending;
450
+ return pending;
451
+ }
452
+ /** The last classification started, until it is done */
453
+ get pending() {
454
+ return this._pending;
455
+ }
456
+ /** Drops every classification in progress, which then emit nothing */
457
+ cancel() {
458
+ this.controllers.forEach((controller) => controller.abort());
459
+ this.controllers.clear();
460
+ this._pending = void 0;
461
+ }
462
+ log(...message) {
463
+ this.logger?.log(...message);
464
+ }
465
+ destroy() {
466
+ this.log("Destroyed");
467
+ this.cancel();
468
+ this.removeAllListeners();
469
+ }
470
+ async run(input, signal) {
471
+ const start = Date.now();
472
+ try {
473
+ const result = await this.evaluate(input, signal);
474
+ if (signal.aborted) return;
475
+ const classification = {
476
+ input,
477
+ result,
478
+ duration: Date.now() - start
479
+ };
480
+ this.log(
481
+ `Classified in ${classification.duration} ms:`,
482
+ JSON.stringify(input)
483
+ );
484
+ this.lastClassification = classification;
485
+ this.emit("Classification", classification);
486
+ return classification;
487
+ } catch (error) {
488
+ if (!signal.aborted) this.log("Classification failed:", error);
489
+ return;
490
+ }
491
+ }
492
+ };
493
+
494
+ // src/types.ts
495
+ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
496
+ MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
497
+ MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
498
+ MicdropClientCommands2["Mute"] = "Mute";
499
+ return MicdropClientCommands2;
500
+ })(MicdropClientCommands || {});
501
+ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
502
+ MicdropServerCommands2["Message"] = "Message";
503
+ MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
504
+ MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
505
+ MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
506
+ MicdropServerCommands2["EndCall"] = "EndCall";
507
+ MicdropServerCommands2["ToolCall"] = "ToolCall";
508
+ MicdropServerCommands2["Classification"] = "Classification";
509
+ return MicdropServerCommands2;
510
+ })(MicdropServerCommands || {});
511
+ function currentTurn(conversation) {
512
+ let start = conversation.length;
513
+ while (start > 0) {
514
+ const item = conversation[start - 1];
515
+ if (item.role !== "user" || item.metadata?.classification) break;
516
+ start--;
517
+ }
518
+ const messages = conversation.slice(start);
519
+ return {
520
+ transcript: messages.map((message) => message.content).join(" "),
521
+ messages,
522
+ before: conversation.slice(0, start)
523
+ };
524
+ }
525
+ function getTurnClassification(conversation) {
526
+ for (let index = conversation.length - 1; index >= 0; index--) {
527
+ const item = conversation[index];
528
+ if (item.role === "user") return item.metadata?.classification;
529
+ }
530
+ return void 0;
531
+ }
532
+
533
+ // src/classifier/turnInput.ts
534
+ function turnInput(conversation, history = 1) {
535
+ const turn = currentTurn(conversation);
536
+ const input = { history: [], turn: turn.transcript };
537
+ let rest = turn.before;
538
+ for (let n = 0; n < history && rest.length; n++) {
539
+ let end = rest.length;
540
+ const answers = [];
541
+ while (end > 0 && rest[end - 1].role !== "user") {
542
+ const item = rest[end - 1];
543
+ if (item.role === "assistant" && item.content.trim()) {
544
+ answers.unshift(item.content);
545
+ }
546
+ end--;
547
+ }
548
+ let start = end;
549
+ while (start > 0) {
550
+ const item = rest[start - 1];
551
+ if (item.role !== "user") break;
552
+ if (start < end && item.metadata?.classification)
553
+ break;
554
+ start--;
555
+ }
556
+ if (answers.length) {
557
+ input.history.unshift({ role: "assistant", text: answers.join(" ") });
558
+ }
559
+ if (start < end) {
560
+ const text = rest.slice(start, end).map((item) => "content" in item ? item.content : "").join(" ");
561
+ input.history.unshift({ role: "user", text });
562
+ }
563
+ rest = rest.slice(0, start);
564
+ }
565
+ return { input, message: turn.messages[turn.messages.length - 1] };
566
+ }
567
+
408
568
  // src/audio/Pcm16Resampler.ts
409
569
  var Pcm16Resampler = class {
410
570
  // Fractional position into the first sample of the buffer
@@ -492,30 +652,12 @@ function handleError(socket, error) {
492
652
  }
493
653
 
494
654
  // src/MicdropServer.ts
495
- var import_eventemitter32 = require("eventemitter3");
655
+ var import_eventemitter33 = require("eventemitter3");
496
656
  var import_stream2 = require("stream");
497
-
498
- // src/types.ts
499
- var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
500
- MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
501
- MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
502
- MicdropClientCommands2["Mute"] = "Mute";
503
- return MicdropClientCommands2;
504
- })(MicdropClientCommands || {});
505
- var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
506
- MicdropServerCommands2["Message"] = "Message";
507
- MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
508
- MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
509
- MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
510
- MicdropServerCommands2["EndCall"] = "EndCall";
511
- MicdropServerCommands2["ToolCall"] = "ToolCall";
512
- return MicdropServerCommands2;
513
- })(MicdropServerCommands || {});
514
-
515
- // src/MicdropServer.ts
516
657
  var USER_SAMPLE_RATE = 16e3;
517
658
  var DEFAULT_TURN_MAX_WAIT = 4e3;
518
- var MicdropServer = class extends import_eventemitter32.EventEmitter {
659
+ var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
660
+ var MicdropServer = class extends import_eventemitter33.EventEmitter {
519
661
  constructor(socket, config) {
520
662
  super();
521
663
  this.socket = null;
@@ -534,9 +676,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
534
676
  this.log("Connection closed");
535
677
  const duration = Math.round((Date.now() - this.startTime) / 1e3);
536
678
  const conversation = this.conversation;
537
- this.config.agent?.destroy();
538
- this.config.stt.destroy();
679
+ this.agent?.destroy();
680
+ this.config.stt?.destroy();
539
681
  this.config.tts?.destroy();
682
+ this.config.classifier?.destroy();
540
683
  this.emit("End", {
541
684
  conversation,
542
685
  duration
@@ -578,7 +721,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
578
721
  this.answerUserTurn();
579
722
  }
580
723
  };
581
- this.onAudioTTS = (audio) => {
724
+ this.onAssistantAudio = (audio) => {
582
725
  if (!this.socket) return;
583
726
  this.log(`Send audio chunk (${audio.byteLength} bytes)`);
584
727
  this.socket.send(audio);
@@ -590,9 +733,23 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
590
733
  this.socket = socket;
591
734
  this.config = config;
592
735
  this.log(`Call started`);
593
- config.stt.on("Transcript", this.onTranscriptSTT);
594
- config.tts?.on("Audio", this.onAudioTTS);
595
- const agent = config.agent;
736
+ const { realtime } = config;
737
+ if (realtime) {
738
+ realtime.on("Audio", this.onAssistantAudio);
739
+ if (config.partialMessages) {
740
+ realtime.on(
741
+ "PartialMessage",
742
+ (content) => this.sendPartialAnswer(content)
743
+ );
744
+ }
745
+ } else {
746
+ if (!config.stt) {
747
+ throw new Error("MicdropServer needs an stt, or a realtime model");
748
+ }
749
+ config.stt.on("Transcript", this.onTranscriptSTT);
750
+ config.tts?.on("Audio", this.onAssistantAudio);
751
+ }
752
+ const agent = this.agent;
596
753
  if (agent) {
597
754
  agent.on("Message", this.onMessageAgent);
598
755
  agent.on(
@@ -614,13 +771,26 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
614
771
  )
615
772
  );
616
773
  }
774
+ if (config.classifierOptions?.sendToClient) {
775
+ config.classifier?.on(
776
+ "Classification",
777
+ (classification) => this.socket?.send(
778
+ `${"Classification" /* Classification */} ${JSON.stringify(classification)}`
779
+ )
780
+ );
781
+ }
617
782
  queueMicrotask(() => this.sendFirstMessage());
618
783
  socket.on("close", this.onClose);
619
784
  socket.on("message", this.onMessage);
620
785
  }
621
786
  /** Everything said so far, whether an agent or the server keeps it */
622
787
  get conversation() {
623
- return this.config?.agent?.conversation ?? this.ownConversation;
788
+ return this.agent?.conversation ?? this.ownConversation;
789
+ }
790
+ /** What holds the conversation and answers, a realtime model included */
791
+ get agent() {
792
+ if (!this.config) return void 0;
793
+ return this.config.realtime ?? this.config.agent;
624
794
  }
625
795
  log(...message) {
626
796
  this.logger?.log(...message);
@@ -645,8 +815,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
645
815
  this.processQueue();
646
816
  }
647
817
  cancel() {
818
+ this.config?.classifier?.cancel();
648
819
  this.config?.tts?.cancel();
649
- this.config?.agent?.cancel();
820
+ this.agent?.cancel();
650
821
  this.operationQueue = [];
651
822
  }
652
823
  onUserAudio(chunk) {
@@ -661,6 +832,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
661
832
  this.currentUserStream?.end();
662
833
  this.currentUserStream = void 0;
663
834
  this.cancel();
835
+ this.config?.realtime?.discardTurn();
664
836
  }
665
837
  onStartSpeaking() {
666
838
  if (!this.config) return;
@@ -670,8 +842,13 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
670
842
  this.config.turnDetector?.reset();
671
843
  this.turnComplete = void 0;
672
844
  this.releaseHeldTurn();
673
- this.config.stt.transcribe(this.currentUserStream);
674
- this.cancel();
845
+ if (this.config.realtime) {
846
+ this.cancel();
847
+ this.config.realtime.startTurn(this.currentUserStream);
848
+ } else {
849
+ this.config.stt?.transcribe(this.currentUserStream);
850
+ this.cancel();
851
+ }
675
852
  }
676
853
  onStopSpeaking() {
677
854
  const hasNoUserSpeech = !this.currentUserStream || this.userSpeechChunks === 0;
@@ -683,6 +860,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
683
860
  return;
684
861
  }
685
862
  this.turnComplete = this.predictTurnComplete();
863
+ if (this.config?.realtime) {
864
+ this.answerUserTurn();
865
+ return;
866
+ }
686
867
  const conversation = this.conversation;
687
868
  const lastMessage = conversation[conversation.length - 1];
688
869
  if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
@@ -715,7 +896,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
715
896
  }
716
897
  this.releaseHeldTurn();
717
898
  this.cancel();
718
- this.answer();
899
+ this.answerTurn();
719
900
  }
720
901
  /**
721
902
  * Answers anyway if the rest of the sentence never comes.
@@ -729,9 +910,19 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
729
910
  this.heldTurnTimer = void 0;
730
911
  this.log("Nothing more came, answering");
731
912
  this.cancel();
732
- this.answer();
913
+ this.answerTurn();
733
914
  }, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
734
915
  }
916
+ /** Answers what the user just said */
917
+ answerTurn() {
918
+ const realtime = this.config?.realtime;
919
+ if (realtime) {
920
+ realtime.endTurn();
921
+ } else {
922
+ this.classifyTurn();
923
+ this.answer();
924
+ }
925
+ }
735
926
  releaseHeldTurn() {
736
927
  if (!this.heldTurnTimer) return;
737
928
  clearTimeout(this.heldTurnTimer);
@@ -739,16 +930,16 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
739
930
  }
740
931
  /** Adds a message to the conversation and sends it to the client */
741
932
  addUserMessage(text, metadata) {
742
- if (this.config?.agent) {
743
- this.config.agent.addUserMessage(text, metadata);
933
+ if (this.agent) {
934
+ this.agent.addUserMessage(text, metadata);
744
935
  } else {
745
936
  this.addOwnMessage("user", text, metadata);
746
937
  }
747
938
  }
748
939
  /** Adds a message to the conversation and sends it to the client */
749
940
  addAssistantMessage(text, metadata) {
750
- if (this.config?.agent) {
751
- this.config.agent.addAssistantMessage(text, metadata);
941
+ if (this.agent) {
942
+ this.agent.addAssistantMessage(text, metadata);
752
943
  } else {
753
944
  this.addOwnMessage("assistant", text, metadata);
754
945
  }
@@ -772,6 +963,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
772
963
  `${"Message" /* Message */} ${JSON.stringify(message)}`
773
964
  );
774
965
  this.emit("Message", message);
966
+ if (message.role === "user" && this.config?.realtime) this.classifyTurn();
775
967
  }
776
968
  sendPartialAnswer(content) {
777
969
  if (!this.socket || content === "") return;
@@ -783,21 +975,28 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
783
975
  sendFirstMessage() {
784
976
  if (!this.config) return;
785
977
  if (this.config.firstMessage) {
786
- this.config.agent?.addAssistantMessage(this.config.firstMessage);
978
+ this.agent?.addAssistantMessage(this.config.firstMessage);
787
979
  this.speak(this.config.firstMessage);
788
- } else if (this.config.generateFirstMessage && this.config.agent) {
980
+ } else if (this.config.generateFirstMessage && this.agent) {
789
981
  this.answer();
790
982
  } else {
791
983
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
792
984
  }
793
985
  }
794
986
  answer() {
987
+ if (this.config?.classifierOptions?.waitBeforeAnswer) {
988
+ this.queueOperation(() => this.waitForClassification());
989
+ }
795
990
  this.queueOperation(async () => {
796
991
  await this._answer();
797
992
  });
798
993
  }
799
994
  async _answer() {
800
995
  if (!this.config) return;
996
+ if (this.config.realtime) {
997
+ this.config.realtime.respond();
998
+ return;
999
+ }
801
1000
  if (!this.config.agent) {
802
1001
  this.log("No agent, skipping answer");
803
1002
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -837,6 +1036,38 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
837
1036
  });
838
1037
  return stream.pipe(forward);
839
1038
  }
1039
+ /**
1040
+ * Classifies the turn that just ended, with the one before, and keeps the
1041
+ * result in the metadata of its last message
1042
+ */
1043
+ classifyTurn() {
1044
+ const classifier = this.config?.classifier;
1045
+ if (!classifier) return;
1046
+ const { input, message } = turnInput(
1047
+ this.conversation,
1048
+ this.config?.classifierOptions?.history
1049
+ );
1050
+ if (!message || input.turn.trim() === "") return;
1051
+ classifier.classify(input).then((classification) => {
1052
+ if (!classification) return;
1053
+ message.metadata = { ...message.metadata, classification };
1054
+ });
1055
+ }
1056
+ /** Waits for the classification in progress, never longer than its limit */
1057
+ async waitForClassification() {
1058
+ const pending = this.config?.classifier?.pending;
1059
+ if (!pending) return;
1060
+ const limit = this.config?.classifierOptions?.maxWait ?? DEFAULT_CLASSIFICATION_MAX_WAIT;
1061
+ let timer;
1062
+ const timeout = new Promise((resolve) => {
1063
+ timer = setTimeout(() => {
1064
+ this.log(`Classification took over ${limit} ms, answering without it`);
1065
+ resolve();
1066
+ }, limit);
1067
+ });
1068
+ await Promise.race([pending, timeout]);
1069
+ clearTimeout(timer);
1070
+ }
840
1071
  // Run text-to-speech and send to client
841
1072
  speak(message) {
842
1073
  this.queueOperation(async () => {
@@ -845,6 +1076,11 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
845
1076
  }
846
1077
  async _speak(message) {
847
1078
  if (!this.socket || !this.config) return;
1079
+ if (this.config.realtime) {
1080
+ const text = typeof message === "string" ? message : await readText(message);
1081
+ if (text.trim() !== "") this.config.realtime.say(text);
1082
+ return;
1083
+ }
848
1084
  let textStream;
849
1085
  if (typeof message === "string") {
850
1086
  if (!this.config.agent) {
@@ -883,6 +1119,14 @@ function hasContent(stream) {
883
1119
  stream.on("end", onEnd);
884
1120
  });
885
1121
  }
1122
+ function readText(stream) {
1123
+ return new Promise((resolve) => {
1124
+ let text = "";
1125
+ stream.on("data", (chunk) => text += chunk.toString());
1126
+ stream.once("end", () => resolve(text));
1127
+ stream.once("error", () => resolve(text));
1128
+ });
1129
+ }
886
1130
  function drain(stream) {
887
1131
  return new Promise((resolve) => {
888
1132
  stream.on("data", () => {
@@ -893,9 +1137,255 @@ function drain(stream) {
893
1137
  });
894
1138
  }
895
1139
 
1140
+ // src/realtime/Realtime.ts
1141
+ var USER_TRANSCRIPT_TIMEOUT = 3e3;
1142
+ function newAnswer(record) {
1143
+ return { transcript: "", hasAudio: false, record, resumed: false };
1144
+ }
1145
+ var Realtime = class extends Agent {
1146
+ constructor() {
1147
+ super(...arguments);
1148
+ // A turn opens with its first audio, so a blip without any sends nothing
1149
+ this.turn = "none";
1150
+ // Output of an interrupted answer can still be on its way
1151
+ this.discarding = false;
1152
+ // Answers asked for while another one was being given
1153
+ this.queue = [];
1154
+ this.answerWaiters = [];
1155
+ this.userTranscript = "";
1156
+ this.awaitingUserTranscript = false;
1157
+ this.afterUserTranscript = [];
1158
+ }
1159
+ /**
1160
+ * The user started speaking.
1161
+ *
1162
+ * The assistant stops, and the audio read from the stream goes to the model.
1163
+ * A turn left open, because the sentence sounded unfinished, carries on.
1164
+ */
1165
+ startTurn(audio) {
1166
+ this.cancel();
1167
+ if (this.turn === "none") this.turn = "pending";
1168
+ audio.on("data", (chunk) => this.sendAudio(chunk));
1169
+ }
1170
+ /** Closes the turn of the user, and has the model answer it */
1171
+ endTurn() {
1172
+ if (this.turn !== "open") return;
1173
+ this.turn = "none";
1174
+ this.awaitUserTranscript();
1175
+ this.beginAnswer(true);
1176
+ this.closeTurn();
1177
+ }
1178
+ /** Drops the turn of the user, without an answer */
1179
+ discardTurn() {
1180
+ this.cancel();
1181
+ if (this.turn === "open") {
1182
+ this.clearTurn();
1183
+ this.discarding = true;
1184
+ }
1185
+ this.turn = "none";
1186
+ }
1187
+ /** Has the model speak without waiting for the user, to open a call */
1188
+ respond() {
1189
+ this.whenIdle(() => {
1190
+ this.beginAnswer(true);
1191
+ this.generate();
1192
+ });
1193
+ }
1194
+ /**
1195
+ * Has the model say this text, word for word.
1196
+ *
1197
+ * The caller records the text in the conversation, as with a text to speech,
1198
+ * so the transcript of what the model says is not recorded again.
1199
+ */
1200
+ say(text) {
1201
+ this.whenIdle(() => {
1202
+ this.beginAnswer(false);
1203
+ this.speakText(text);
1204
+ });
1205
+ }
1206
+ cancel() {
1207
+ this.queue = [];
1208
+ const answer = this.currentAnswer;
1209
+ if (!answer) return;
1210
+ this.log("Cancel");
1211
+ this.currentAnswer = void 0;
1212
+ this.discarding = true;
1213
+ this.cancelAnswer();
1214
+ this.recordAnswer(answer);
1215
+ this.resolveAnswerWaiters();
1216
+ }
1217
+ /**
1218
+ * Adds a message the model did not hear, and tells the model about it.
1219
+ *
1220
+ * The transcripts of the call are recorded apart, so they are not sent back.
1221
+ */
1222
+ addMessage(role, text, metadata) {
1223
+ super.addMessage(role, text, metadata);
1224
+ if (text.trim() !== "") this.injectMessage(role, text);
1225
+ }
1226
+ addTool(tool) {
1227
+ super.addTool(tool);
1228
+ this.updateTools();
1229
+ }
1230
+ removeTool(name) {
1231
+ super.removeTool(name);
1232
+ this.updateTools();
1233
+ }
1234
+ /**
1235
+ * Lists the events that have listeners.
1236
+ *
1237
+ * Typed as the events of any agent, so a realtime model goes wherever an
1238
+ * agent is expected. The two extra events do not change what that code reads.
1239
+ */
1240
+ eventNames() {
1241
+ return super.eventNames();
1242
+ }
1243
+ destroy() {
1244
+ super.destroy();
1245
+ this.clearUserTranscriptTimer();
1246
+ this.afterUserTranscript = [];
1247
+ this.resolveAnswerWaiters();
1248
+ }
1249
+ generateAnswer(stream) {
1250
+ let written = 0;
1251
+ const onPartial = (transcript) => {
1252
+ stream.write(transcript.slice(written));
1253
+ written = transcript.length;
1254
+ };
1255
+ this.on("PartialMessage", onPartial);
1256
+ this.respond();
1257
+ return new Promise((resolve) => {
1258
+ this.answerWaiters.push(() => {
1259
+ this.off("PartialMessage", onPartial);
1260
+ resolve();
1261
+ });
1262
+ });
1263
+ }
1264
+ /** Nobody is speaking, nor about to, so the connection can be renewed */
1265
+ get isIdle() {
1266
+ return this.turn === "none" && !this.currentAnswer;
1267
+ }
1268
+ /** Voice of the answer, PCM16 at 16 kHz */
1269
+ emitAudio(chunk) {
1270
+ const answer = this.activeAnswer();
1271
+ if (!answer) return;
1272
+ answer.hasAudio = true;
1273
+ this.emit("Audio", chunk);
1274
+ }
1275
+ /** Transcript of the answer, as it is spoken */
1276
+ addAnswerTranscript(text) {
1277
+ const answer = this.activeAnswer();
1278
+ if (!answer) return;
1279
+ if (answer.resumed) {
1280
+ answer.resumed = false;
1281
+ if (answer.transcript !== "" && !/\s$/.test(answer.transcript)) {
1282
+ text = ` ${text.trimStart()}`;
1283
+ }
1284
+ }
1285
+ answer.transcript += text;
1286
+ if (answer.record) this.emit("PartialMessage", answer.transcript);
1287
+ }
1288
+ /**
1289
+ * The model finished its answer, or finished dropping an interrupted one.
1290
+ *
1291
+ * An answer without audio tells the client to stop waiting for one.
1292
+ */
1293
+ endAnswer() {
1294
+ if (this.discarding) {
1295
+ this.discarding = false;
1296
+ return;
1297
+ }
1298
+ const answer = this.currentAnswer;
1299
+ if (!answer) return;
1300
+ this.currentAnswer = void 0;
1301
+ if (!answer.hasAudio) this.emitAgentEvent("SkipAnswer");
1302
+ this.recordAnswer(answer);
1303
+ this.resolveAnswerWaiters();
1304
+ this.queue.shift()?.();
1305
+ }
1306
+ /** Runs a tool the model called, and records it in the conversation */
1307
+ async runTool(toolCall) {
1308
+ const result = await this.executeTool(toolCall);
1309
+ if (this.currentAnswer) this.currentAnswer.resumed = true;
1310
+ return result;
1311
+ }
1312
+ /** Transcript of the turn of the user, possibly in several pieces */
1313
+ addUserTranscript(text) {
1314
+ this.userTranscript += text;
1315
+ }
1316
+ /** The transcript of the turn is complete, it goes in the conversation */
1317
+ commitUserTranscript() {
1318
+ this.clearUserTranscriptTimer();
1319
+ this.awaitingUserTranscript = false;
1320
+ const text = this.userTranscript.trim();
1321
+ this.userTranscript = "";
1322
+ if (text !== "") super.addMessage("user", text);
1323
+ const pending = this.afterUserTranscript;
1324
+ this.afterUserTranscript = [];
1325
+ pending.forEach((record) => record());
1326
+ }
1327
+ sendAudio(chunk) {
1328
+ if (this.turn === "none") return;
1329
+ if (this.turn === "pending") {
1330
+ this.turn = "open";
1331
+ this.openTurn();
1332
+ }
1333
+ this.appendAudio(chunk);
1334
+ }
1335
+ beginAnswer(record) {
1336
+ this.discarding = false;
1337
+ this.currentAnswer = newAnswer(record);
1338
+ }
1339
+ /**
1340
+ * The answer output belongs to, if it is still wanted.
1341
+ *
1342
+ * The model can speak without being asked, to follow up on a tool call for
1343
+ * instance, which opens an answer of its own.
1344
+ */
1345
+ activeAnswer() {
1346
+ if (this.discarding) return void 0;
1347
+ if (!this.currentAnswer) this.currentAnswer = newAnswer(true);
1348
+ return this.currentAnswer;
1349
+ }
1350
+ whenIdle(start) {
1351
+ if (this.currentAnswer) {
1352
+ this.queue.push(start);
1353
+ } else {
1354
+ start();
1355
+ }
1356
+ }
1357
+ recordAnswer(answer) {
1358
+ if (!answer.record) return;
1359
+ const record = () => super.addMessage("assistant", answer.transcript.trim());
1360
+ if (this.awaitingUserTranscript) {
1361
+ this.afterUserTranscript.push(record);
1362
+ } else {
1363
+ record();
1364
+ }
1365
+ }
1366
+ awaitUserTranscript() {
1367
+ this.awaitingUserTranscript = true;
1368
+ this.clearUserTranscriptTimer();
1369
+ this.userTranscriptTimer = setTimeout(() => {
1370
+ this.log("No transcript of the turn, recording the answer anyway");
1371
+ this.commitUserTranscript();
1372
+ }, USER_TRANSCRIPT_TIMEOUT);
1373
+ }
1374
+ clearUserTranscriptTimer() {
1375
+ if (!this.userTranscriptTimer) return;
1376
+ clearTimeout(this.userTranscriptTimer);
1377
+ this.userTranscriptTimer = void 0;
1378
+ }
1379
+ resolveAnswerWaiters() {
1380
+ const waiters = this.answerWaiters;
1381
+ this.answerWaiters = [];
1382
+ waiters.forEach((resolve) => resolve());
1383
+ }
1384
+ };
1385
+
896
1386
  // src/recorder/MicdropRecorder.ts
897
- var import_eventemitter33 = require("eventemitter3");
898
- var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
1387
+ var import_eventemitter34 = require("eventemitter3");
1388
+ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
899
1389
  constructor(server) {
900
1390
  super();
901
1391
  this.server = server;
@@ -1052,8 +1542,8 @@ var SharedInstances = class {
1052
1542
  };
1053
1543
 
1054
1544
  // src/stt/STT.ts
1055
- var import_eventemitter34 = require("eventemitter3");
1056
- var STT = class extends import_eventemitter34.EventEmitter {
1545
+ var import_eventemitter35 = require("eventemitter3");
1546
+ var STT = class extends import_eventemitter35.EventEmitter {
1057
1547
  log(...message) {
1058
1548
  this.logger?.log(...message);
1059
1549
  }
@@ -1134,8 +1624,8 @@ var FallbackSTT = class extends STT {
1134
1624
  var import_stream4 = require("stream");
1135
1625
 
1136
1626
  // src/tts/TTS.ts
1137
- var import_eventemitter35 = require("eventemitter3");
1138
- var TTS = class extends import_eventemitter35.EventEmitter {
1627
+ var import_eventemitter36 = require("eventemitter3");
1628
+ var TTS = class extends import_eventemitter36.EventEmitter {
1139
1629
  log(...message) {
1140
1630
  this.logger?.log(...message);
1141
1631
  }
@@ -1387,6 +1877,7 @@ async function waitForParams(socket, validate) {
1387
1877
  AUTO_SEMANTIC_TURN_PROMPT,
1388
1878
  AUTO_SEMANTIC_TURN_TOOL_NAME,
1389
1879
  Agent,
1880
+ Classifier,
1390
1881
  FallbackAgent,
1391
1882
  FallbackSTT,
1392
1883
  FallbackTTS,
@@ -1401,14 +1892,18 @@ async function waitForParams(socket, validate) {
1401
1892
  MockSTT,
1402
1893
  MockTTS,
1403
1894
  Pcm16Resampler,
1895
+ Realtime,
1404
1896
  STT,
1405
1897
  SentenceSplitter,
1406
1898
  SentenceTTS,
1407
1899
  SharedInstances,
1408
1900
  TTS,
1901
+ currentTurn,
1409
1902
  float32ToPcm16,
1903
+ getTurnClassification,
1410
1904
  handleError,
1411
1905
  pcm16ToFloat32,
1906
+ turnInput,
1412
1907
  waitForParams
1413
1908
  });
1414
1909
  //# sourceMappingURL=index.js.map