@micdrop/server 2.7.0 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -25,8 +25,15 @@ var Agent = class extends EventEmitter {
25
25
  const answerCount = ++this.answerCount;
26
26
  const stream = new PassThrough();
27
27
  this.answering = true;
28
- Promise.resolve().then(() => this.options.onBeforeAnswer?.bind(this)(stream)).then((skip) => {
29
- if (skip) return;
28
+ Promise.resolve().then(
29
+ () => this.options.onBeforeAnswer?.bind(this)(stream)
30
+ ).then((result) => {
31
+ if (typeof result === "string") {
32
+ stream.write(result);
33
+ this.addAssistantMessage(result);
34
+ return;
35
+ }
36
+ if (result) return;
30
37
  return this.generateAnswer(stream);
31
38
  }).finally(() => {
32
39
  if (stream.writable) {
@@ -68,16 +75,16 @@ var Agent = class extends EventEmitter {
68
75
  metadata
69
76
  };
70
77
  this.conversation.push(message);
71
- this.emit("Message", message);
78
+ this.emitAgentEvent("Message", message);
72
79
  }
73
80
  addToolMessage(message) {
74
81
  this.log("Adding tool message:", message);
75
82
  this.conversation.push(message);
76
- this.emit("Message", message);
83
+ this.emitAgentEvent("Message", message);
77
84
  }
78
85
  endCall() {
79
86
  this.log("Ending call");
80
- this.emit("EndCall");
87
+ this.emitAgentEvent("EndCall");
81
88
  }
82
89
  cancelLastUserMessage() {
83
90
  this.log("Cancelling last user message");
@@ -87,11 +94,11 @@ var Agent = class extends EventEmitter {
87
94
  if (lastMessageIndex !== -1) {
88
95
  this.conversation.splice(lastMessageIndex, 1);
89
96
  }
90
- this.emit("CancelLastUserMessage");
97
+ this.emitAgentEvent("CancelLastUserMessage");
91
98
  }
92
99
  skipAnswer() {
93
100
  this.log("Skipping answer");
94
- this.emit("SkipAnswer");
101
+ this.emitAgentEvent("SkipAnswer");
95
102
  }
96
103
  getDefaultTools() {
97
104
  const tools = [];
@@ -137,7 +144,7 @@ var Agent = class extends EventEmitter {
137
144
  output: JSON.stringify(output ?? null)
138
145
  });
139
146
  if (tool.emitOutput) {
140
- this.emit("ToolCall", {
147
+ this.emitAgentEvent("ToolCall", {
141
148
  name: toolCall.toolName,
142
149
  parameters,
143
150
  output
@@ -196,6 +203,16 @@ var Agent = class extends EventEmitter {
196
203
  }
197
204
  return { message, metadata };
198
205
  }
206
+ /**
207
+ * Emits one of the events every agent has.
208
+ *
209
+ * A subclass can add events of its own, and TypeScript then no longer knows
210
+ * the shared ones are among them.
211
+ */
212
+ emitAgentEvent(event, ...args) {
213
+ const emitter = this;
214
+ emitter.emit(event, ...args);
215
+ }
199
216
  log(...message) {
200
217
  this.logger?.log(...message);
201
218
  }
@@ -340,6 +357,144 @@ var MockAgent = class extends Agent {
340
357
  }
341
358
  };
342
359
 
360
+ // src/classifier/Classifier.ts
361
+ import { EventEmitter as EventEmitter2 } from "eventemitter3";
362
+ var Classifier = class extends EventEmitter2 {
363
+ constructor() {
364
+ super(...arguments);
365
+ this.controllers = /* @__PURE__ */ new Set();
366
+ }
367
+ /**
368
+ * Classifies an input, and emits the result as `Classification`
369
+ * @param input - What to classify: a text, or JSON with named fields
370
+ * @returns The classification, or undefined when cancelled or failed
371
+ */
372
+ classify(input) {
373
+ const controller = new AbortController();
374
+ this.controllers.add(controller);
375
+ const pending = this.run(input, controller.signal).finally(() => {
376
+ this.controllers.delete(controller);
377
+ if (this._pending === pending) this._pending = void 0;
378
+ });
379
+ this._pending = pending;
380
+ return pending;
381
+ }
382
+ /** The last classification started, until it is done */
383
+ get pending() {
384
+ return this._pending;
385
+ }
386
+ /** Drops every classification in progress, which then emit nothing */
387
+ cancel() {
388
+ this.controllers.forEach((controller) => controller.abort());
389
+ this.controllers.clear();
390
+ this._pending = void 0;
391
+ }
392
+ log(...message) {
393
+ this.logger?.log(...message);
394
+ }
395
+ destroy() {
396
+ this.log("Destroyed");
397
+ this.cancel();
398
+ this.removeAllListeners();
399
+ }
400
+ async run(input, signal) {
401
+ const start = Date.now();
402
+ try {
403
+ const result = await this.evaluate(input, signal);
404
+ if (signal.aborted) return;
405
+ const classification = {
406
+ input,
407
+ result,
408
+ duration: Date.now() - start
409
+ };
410
+ this.log(
411
+ `Classified in ${classification.duration} ms:`,
412
+ JSON.stringify(input)
413
+ );
414
+ this.lastClassification = classification;
415
+ this.emit("Classification", classification);
416
+ return classification;
417
+ } catch (error) {
418
+ if (!signal.aborted) this.log("Classification failed:", error);
419
+ return;
420
+ }
421
+ }
422
+ };
423
+
424
+ // src/types.ts
425
+ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
426
+ MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
427
+ MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
428
+ MicdropClientCommands2["Mute"] = "Mute";
429
+ return MicdropClientCommands2;
430
+ })(MicdropClientCommands || {});
431
+ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
432
+ MicdropServerCommands2["Message"] = "Message";
433
+ MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
434
+ MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
435
+ MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
436
+ MicdropServerCommands2["EndCall"] = "EndCall";
437
+ MicdropServerCommands2["ToolCall"] = "ToolCall";
438
+ MicdropServerCommands2["Classification"] = "Classification";
439
+ return MicdropServerCommands2;
440
+ })(MicdropServerCommands || {});
441
+ function currentTurn(conversation) {
442
+ let start = conversation.length;
443
+ while (start > 0) {
444
+ const item = conversation[start - 1];
445
+ if (item.role !== "user" || item.metadata?.classification) break;
446
+ start--;
447
+ }
448
+ const messages = conversation.slice(start);
449
+ return {
450
+ transcript: messages.map((message) => message.content).join(" "),
451
+ messages,
452
+ before: conversation.slice(0, start)
453
+ };
454
+ }
455
+ function getTurnClassification(conversation) {
456
+ for (let index = conversation.length - 1; index >= 0; index--) {
457
+ const item = conversation[index];
458
+ if (item.role === "user") return item.metadata?.classification;
459
+ }
460
+ return void 0;
461
+ }
462
+
463
+ // src/classifier/turnInput.ts
464
+ function turnInput(conversation, history = 1) {
465
+ const turn = currentTurn(conversation);
466
+ const input = { history: [], turn: turn.transcript };
467
+ let rest = turn.before;
468
+ for (let n = 0; n < history && rest.length; n++) {
469
+ let end = rest.length;
470
+ const answers = [];
471
+ while (end > 0 && rest[end - 1].role !== "user") {
472
+ const item = rest[end - 1];
473
+ if (item.role === "assistant" && item.content.trim()) {
474
+ answers.unshift(item.content);
475
+ }
476
+ end--;
477
+ }
478
+ let start = end;
479
+ while (start > 0) {
480
+ const item = rest[start - 1];
481
+ if (item.role !== "user") break;
482
+ if (start < end && item.metadata?.classification)
483
+ break;
484
+ start--;
485
+ }
486
+ if (answers.length) {
487
+ input.history.unshift({ role: "assistant", text: answers.join(" ") });
488
+ }
489
+ if (start < end) {
490
+ const text = rest.slice(start, end).map((item) => "content" in item ? item.content : "").join(" ");
491
+ input.history.unshift({ role: "user", text });
492
+ }
493
+ rest = rest.slice(0, start);
494
+ }
495
+ return { input, message: turn.messages[turn.messages.length - 1] };
496
+ }
497
+
343
498
  // src/audio/Pcm16Resampler.ts
344
499
  var Pcm16Resampler = class {
345
500
  // Fractional position into the first sample of the buffer
@@ -427,30 +582,12 @@ function handleError(socket, error) {
427
582
  }
428
583
 
429
584
  // src/MicdropServer.ts
430
- import { EventEmitter as EventEmitter2 } from "eventemitter3";
585
+ import { EventEmitter as EventEmitter3 } from "eventemitter3";
431
586
  import { PassThrough as PassThrough2, Transform } from "stream";
432
-
433
- // src/types.ts
434
- var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
435
- MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
436
- MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
437
- MicdropClientCommands2["Mute"] = "Mute";
438
- return MicdropClientCommands2;
439
- })(MicdropClientCommands || {});
440
- var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
441
- MicdropServerCommands2["Message"] = "Message";
442
- MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
443
- MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
444
- MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
445
- MicdropServerCommands2["EndCall"] = "EndCall";
446
- MicdropServerCommands2["ToolCall"] = "ToolCall";
447
- return MicdropServerCommands2;
448
- })(MicdropServerCommands || {});
449
-
450
- // src/MicdropServer.ts
451
587
  var USER_SAMPLE_RATE = 16e3;
452
588
  var DEFAULT_TURN_MAX_WAIT = 4e3;
453
- var MicdropServer = class extends EventEmitter2 {
589
+ var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
590
+ var MicdropServer = class extends EventEmitter3 {
454
591
  constructor(socket, config) {
455
592
  super();
456
593
  this.socket = null;
@@ -469,9 +606,10 @@ var MicdropServer = class extends EventEmitter2 {
469
606
  this.log("Connection closed");
470
607
  const duration = Math.round((Date.now() - this.startTime) / 1e3);
471
608
  const conversation = this.conversation;
472
- this.config.agent?.destroy();
473
- this.config.stt.destroy();
609
+ this.agent?.destroy();
610
+ this.config.stt?.destroy();
474
611
  this.config.tts?.destroy();
612
+ this.config.classifier?.destroy();
475
613
  this.emit("End", {
476
614
  conversation,
477
615
  duration
@@ -513,7 +651,7 @@ var MicdropServer = class extends EventEmitter2 {
513
651
  this.answerUserTurn();
514
652
  }
515
653
  };
516
- this.onAudioTTS = (audio) => {
654
+ this.onAssistantAudio = (audio) => {
517
655
  if (!this.socket) return;
518
656
  this.log(`Send audio chunk (${audio.byteLength} bytes)`);
519
657
  this.socket.send(audio);
@@ -525,9 +663,23 @@ var MicdropServer = class extends EventEmitter2 {
525
663
  this.socket = socket;
526
664
  this.config = config;
527
665
  this.log(`Call started`);
528
- config.stt.on("Transcript", this.onTranscriptSTT);
529
- config.tts?.on("Audio", this.onAudioTTS);
530
- const agent = config.agent;
666
+ const { realtime } = config;
667
+ if (realtime) {
668
+ realtime.on("Audio", this.onAssistantAudio);
669
+ if (config.partialMessages) {
670
+ realtime.on(
671
+ "PartialMessage",
672
+ (content) => this.sendPartialAnswer(content)
673
+ );
674
+ }
675
+ } else {
676
+ if (!config.stt) {
677
+ throw new Error("MicdropServer needs an stt, or a realtime model");
678
+ }
679
+ config.stt.on("Transcript", this.onTranscriptSTT);
680
+ config.tts?.on("Audio", this.onAssistantAudio);
681
+ }
682
+ const agent = this.agent;
531
683
  if (agent) {
532
684
  agent.on("Message", this.onMessageAgent);
533
685
  agent.on(
@@ -549,13 +701,26 @@ var MicdropServer = class extends EventEmitter2 {
549
701
  )
550
702
  );
551
703
  }
704
+ if (config.classifierOptions?.sendToClient) {
705
+ config.classifier?.on(
706
+ "Classification",
707
+ (classification) => this.socket?.send(
708
+ `${"Classification" /* Classification */} ${JSON.stringify(classification)}`
709
+ )
710
+ );
711
+ }
552
712
  queueMicrotask(() => this.sendFirstMessage());
553
713
  socket.on("close", this.onClose);
554
714
  socket.on("message", this.onMessage);
555
715
  }
556
716
  /** Everything said so far, whether an agent or the server keeps it */
557
717
  get conversation() {
558
- return this.config?.agent?.conversation ?? this.ownConversation;
718
+ return this.agent?.conversation ?? this.ownConversation;
719
+ }
720
+ /** What holds the conversation and answers, a realtime model included */
721
+ get agent() {
722
+ if (!this.config) return void 0;
723
+ return this.config.realtime ?? this.config.agent;
559
724
  }
560
725
  log(...message) {
561
726
  this.logger?.log(...message);
@@ -580,8 +745,9 @@ var MicdropServer = class extends EventEmitter2 {
580
745
  this.processQueue();
581
746
  }
582
747
  cancel() {
748
+ this.config?.classifier?.cancel();
583
749
  this.config?.tts?.cancel();
584
- this.config?.agent?.cancel();
750
+ this.agent?.cancel();
585
751
  this.operationQueue = [];
586
752
  }
587
753
  onUserAudio(chunk) {
@@ -596,6 +762,7 @@ var MicdropServer = class extends EventEmitter2 {
596
762
  this.currentUserStream?.end();
597
763
  this.currentUserStream = void 0;
598
764
  this.cancel();
765
+ this.config?.realtime?.discardTurn();
599
766
  }
600
767
  onStartSpeaking() {
601
768
  if (!this.config) return;
@@ -605,8 +772,13 @@ var MicdropServer = class extends EventEmitter2 {
605
772
  this.config.turnDetector?.reset();
606
773
  this.turnComplete = void 0;
607
774
  this.releaseHeldTurn();
608
- this.config.stt.transcribe(this.currentUserStream);
609
- this.cancel();
775
+ if (this.config.realtime) {
776
+ this.cancel();
777
+ this.config.realtime.startTurn(this.currentUserStream);
778
+ } else {
779
+ this.config.stt?.transcribe(this.currentUserStream);
780
+ this.cancel();
781
+ }
610
782
  }
611
783
  onStopSpeaking() {
612
784
  const hasNoUserSpeech = !this.currentUserStream || this.userSpeechChunks === 0;
@@ -618,6 +790,10 @@ var MicdropServer = class extends EventEmitter2 {
618
790
  return;
619
791
  }
620
792
  this.turnComplete = this.predictTurnComplete();
793
+ if (this.config?.realtime) {
794
+ this.answerUserTurn();
795
+ return;
796
+ }
621
797
  const conversation = this.conversation;
622
798
  const lastMessage = conversation[conversation.length - 1];
623
799
  if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
@@ -650,7 +826,7 @@ var MicdropServer = class extends EventEmitter2 {
650
826
  }
651
827
  this.releaseHeldTurn();
652
828
  this.cancel();
653
- this.answer();
829
+ this.answerTurn();
654
830
  }
655
831
  /**
656
832
  * Answers anyway if the rest of the sentence never comes.
@@ -664,9 +840,19 @@ var MicdropServer = class extends EventEmitter2 {
664
840
  this.heldTurnTimer = void 0;
665
841
  this.log("Nothing more came, answering");
666
842
  this.cancel();
667
- this.answer();
843
+ this.answerTurn();
668
844
  }, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
669
845
  }
846
+ /** Answers what the user just said */
847
+ answerTurn() {
848
+ const realtime = this.config?.realtime;
849
+ if (realtime) {
850
+ realtime.endTurn();
851
+ } else {
852
+ this.classifyTurn();
853
+ this.answer();
854
+ }
855
+ }
670
856
  releaseHeldTurn() {
671
857
  if (!this.heldTurnTimer) return;
672
858
  clearTimeout(this.heldTurnTimer);
@@ -674,16 +860,16 @@ var MicdropServer = class extends EventEmitter2 {
674
860
  }
675
861
  /** Adds a message to the conversation and sends it to the client */
676
862
  addUserMessage(text, metadata) {
677
- if (this.config?.agent) {
678
- this.config.agent.addUserMessage(text, metadata);
863
+ if (this.agent) {
864
+ this.agent.addUserMessage(text, metadata);
679
865
  } else {
680
866
  this.addOwnMessage("user", text, metadata);
681
867
  }
682
868
  }
683
869
  /** Adds a message to the conversation and sends it to the client */
684
870
  addAssistantMessage(text, metadata) {
685
- if (this.config?.agent) {
686
- this.config.agent.addAssistantMessage(text, metadata);
871
+ if (this.agent) {
872
+ this.agent.addAssistantMessage(text, metadata);
687
873
  } else {
688
874
  this.addOwnMessage("assistant", text, metadata);
689
875
  }
@@ -707,6 +893,7 @@ var MicdropServer = class extends EventEmitter2 {
707
893
  `${"Message" /* Message */} ${JSON.stringify(message)}`
708
894
  );
709
895
  this.emit("Message", message);
896
+ if (message.role === "user" && this.config?.realtime) this.classifyTurn();
710
897
  }
711
898
  sendPartialAnswer(content) {
712
899
  if (!this.socket || content === "") return;
@@ -718,21 +905,28 @@ var MicdropServer = class extends EventEmitter2 {
718
905
  sendFirstMessage() {
719
906
  if (!this.config) return;
720
907
  if (this.config.firstMessage) {
721
- this.config.agent?.addAssistantMessage(this.config.firstMessage);
908
+ this.agent?.addAssistantMessage(this.config.firstMessage);
722
909
  this.speak(this.config.firstMessage);
723
- } else if (this.config.generateFirstMessage && this.config.agent) {
910
+ } else if (this.config.generateFirstMessage && this.agent) {
724
911
  this.answer();
725
912
  } else {
726
913
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
727
914
  }
728
915
  }
729
916
  answer() {
917
+ if (this.config?.classifierOptions?.waitBeforeAnswer) {
918
+ this.queueOperation(() => this.waitForClassification());
919
+ }
730
920
  this.queueOperation(async () => {
731
921
  await this._answer();
732
922
  });
733
923
  }
734
924
  async _answer() {
735
925
  if (!this.config) return;
926
+ if (this.config.realtime) {
927
+ this.config.realtime.respond();
928
+ return;
929
+ }
736
930
  if (!this.config.agent) {
737
931
  this.log("No agent, skipping answer");
738
932
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -772,6 +966,38 @@ var MicdropServer = class extends EventEmitter2 {
772
966
  });
773
967
  return stream.pipe(forward);
774
968
  }
969
+ /**
970
+ * Classifies the turn that just ended, with the one before, and keeps the
971
+ * result in the metadata of its last message
972
+ */
973
+ classifyTurn() {
974
+ const classifier = this.config?.classifier;
975
+ if (!classifier) return;
976
+ const { input, message } = turnInput(
977
+ this.conversation,
978
+ this.config?.classifierOptions?.history
979
+ );
980
+ if (!message || input.turn.trim() === "") return;
981
+ classifier.classify(input).then((classification) => {
982
+ if (!classification) return;
983
+ message.metadata = { ...message.metadata, classification };
984
+ });
985
+ }
986
+ /** Waits for the classification in progress, never longer than its limit */
987
+ async waitForClassification() {
988
+ const pending = this.config?.classifier?.pending;
989
+ if (!pending) return;
990
+ const limit = this.config?.classifierOptions?.maxWait ?? DEFAULT_CLASSIFICATION_MAX_WAIT;
991
+ let timer;
992
+ const timeout = new Promise((resolve) => {
993
+ timer = setTimeout(() => {
994
+ this.log(`Classification took over ${limit} ms, answering without it`);
995
+ resolve();
996
+ }, limit);
997
+ });
998
+ await Promise.race([pending, timeout]);
999
+ clearTimeout(timer);
1000
+ }
775
1001
  // Run text-to-speech and send to client
776
1002
  speak(message) {
777
1003
  this.queueOperation(async () => {
@@ -780,6 +1006,11 @@ var MicdropServer = class extends EventEmitter2 {
780
1006
  }
781
1007
  async _speak(message) {
782
1008
  if (!this.socket || !this.config) return;
1009
+ if (this.config.realtime) {
1010
+ const text = typeof message === "string" ? message : await readText(message);
1011
+ if (text.trim() !== "") this.config.realtime.say(text);
1012
+ return;
1013
+ }
783
1014
  let textStream;
784
1015
  if (typeof message === "string") {
785
1016
  if (!this.config.agent) {
@@ -818,6 +1049,14 @@ function hasContent(stream) {
818
1049
  stream.on("end", onEnd);
819
1050
  });
820
1051
  }
1052
+ function readText(stream) {
1053
+ return new Promise((resolve) => {
1054
+ let text = "";
1055
+ stream.on("data", (chunk) => text += chunk.toString());
1056
+ stream.once("end", () => resolve(text));
1057
+ stream.once("error", () => resolve(text));
1058
+ });
1059
+ }
821
1060
  function drain(stream) {
822
1061
  return new Promise((resolve) => {
823
1062
  stream.on("data", () => {
@@ -828,9 +1067,255 @@ function drain(stream) {
828
1067
  });
829
1068
  }
830
1069
 
1070
+ // src/realtime/Realtime.ts
1071
+ var USER_TRANSCRIPT_TIMEOUT = 3e3;
1072
+ function newAnswer(record) {
1073
+ return { transcript: "", hasAudio: false, record, resumed: false };
1074
+ }
1075
+ var Realtime = class extends Agent {
1076
+ constructor() {
1077
+ super(...arguments);
1078
+ // A turn opens with its first audio, so a blip without any sends nothing
1079
+ this.turn = "none";
1080
+ // Output of an interrupted answer can still be on its way
1081
+ this.discarding = false;
1082
+ // Answers asked for while another one was being given
1083
+ this.queue = [];
1084
+ this.answerWaiters = [];
1085
+ this.userTranscript = "";
1086
+ this.awaitingUserTranscript = false;
1087
+ this.afterUserTranscript = [];
1088
+ }
1089
+ /**
1090
+ * The user started speaking.
1091
+ *
1092
+ * The assistant stops, and the audio read from the stream goes to the model.
1093
+ * A turn left open, because the sentence sounded unfinished, carries on.
1094
+ */
1095
+ startTurn(audio) {
1096
+ this.cancel();
1097
+ if (this.turn === "none") this.turn = "pending";
1098
+ audio.on("data", (chunk) => this.sendAudio(chunk));
1099
+ }
1100
+ /** Closes the turn of the user, and has the model answer it */
1101
+ endTurn() {
1102
+ if (this.turn !== "open") return;
1103
+ this.turn = "none";
1104
+ this.awaitUserTranscript();
1105
+ this.beginAnswer(true);
1106
+ this.closeTurn();
1107
+ }
1108
+ /** Drops the turn of the user, without an answer */
1109
+ discardTurn() {
1110
+ this.cancel();
1111
+ if (this.turn === "open") {
1112
+ this.clearTurn();
1113
+ this.discarding = true;
1114
+ }
1115
+ this.turn = "none";
1116
+ }
1117
+ /** Has the model speak without waiting for the user, to open a call */
1118
+ respond() {
1119
+ this.whenIdle(() => {
1120
+ this.beginAnswer(true);
1121
+ this.generate();
1122
+ });
1123
+ }
1124
+ /**
1125
+ * Has the model say this text, word for word.
1126
+ *
1127
+ * The caller records the text in the conversation, as with a text to speech,
1128
+ * so the transcript of what the model says is not recorded again.
1129
+ */
1130
+ say(text) {
1131
+ this.whenIdle(() => {
1132
+ this.beginAnswer(false);
1133
+ this.speakText(text);
1134
+ });
1135
+ }
1136
+ cancel() {
1137
+ this.queue = [];
1138
+ const answer = this.currentAnswer;
1139
+ if (!answer) return;
1140
+ this.log("Cancel");
1141
+ this.currentAnswer = void 0;
1142
+ this.discarding = true;
1143
+ this.cancelAnswer();
1144
+ this.recordAnswer(answer);
1145
+ this.resolveAnswerWaiters();
1146
+ }
1147
+ /**
1148
+ * Adds a message the model did not hear, and tells the model about it.
1149
+ *
1150
+ * The transcripts of the call are recorded apart, so they are not sent back.
1151
+ */
1152
+ addMessage(role, text, metadata) {
1153
+ super.addMessage(role, text, metadata);
1154
+ if (text.trim() !== "") this.injectMessage(role, text);
1155
+ }
1156
+ addTool(tool) {
1157
+ super.addTool(tool);
1158
+ this.updateTools();
1159
+ }
1160
+ removeTool(name) {
1161
+ super.removeTool(name);
1162
+ this.updateTools();
1163
+ }
1164
+ /**
1165
+ * Lists the events that have listeners.
1166
+ *
1167
+ * Typed as the events of any agent, so a realtime model goes wherever an
1168
+ * agent is expected. The two extra events do not change what that code reads.
1169
+ */
1170
+ eventNames() {
1171
+ return super.eventNames();
1172
+ }
1173
+ destroy() {
1174
+ super.destroy();
1175
+ this.clearUserTranscriptTimer();
1176
+ this.afterUserTranscript = [];
1177
+ this.resolveAnswerWaiters();
1178
+ }
1179
+ generateAnswer(stream) {
1180
+ let written = 0;
1181
+ const onPartial = (transcript) => {
1182
+ stream.write(transcript.slice(written));
1183
+ written = transcript.length;
1184
+ };
1185
+ this.on("PartialMessage", onPartial);
1186
+ this.respond();
1187
+ return new Promise((resolve) => {
1188
+ this.answerWaiters.push(() => {
1189
+ this.off("PartialMessage", onPartial);
1190
+ resolve();
1191
+ });
1192
+ });
1193
+ }
1194
+ /** Nobody is speaking, nor about to, so the connection can be renewed */
1195
+ get isIdle() {
1196
+ return this.turn === "none" && !this.currentAnswer;
1197
+ }
1198
+ /** Voice of the answer, PCM16 at 16 kHz */
1199
+ emitAudio(chunk) {
1200
+ const answer = this.activeAnswer();
1201
+ if (!answer) return;
1202
+ answer.hasAudio = true;
1203
+ this.emit("Audio", chunk);
1204
+ }
1205
+ /** Transcript of the answer, as it is spoken */
1206
+ addAnswerTranscript(text) {
1207
+ const answer = this.activeAnswer();
1208
+ if (!answer) return;
1209
+ if (answer.resumed) {
1210
+ answer.resumed = false;
1211
+ if (answer.transcript !== "" && !/\s$/.test(answer.transcript)) {
1212
+ text = ` ${text.trimStart()}`;
1213
+ }
1214
+ }
1215
+ answer.transcript += text;
1216
+ if (answer.record) this.emit("PartialMessage", answer.transcript);
1217
+ }
1218
+ /**
1219
+ * The model finished its answer, or finished dropping an interrupted one.
1220
+ *
1221
+ * An answer without audio tells the client to stop waiting for one.
1222
+ */
1223
+ endAnswer() {
1224
+ if (this.discarding) {
1225
+ this.discarding = false;
1226
+ return;
1227
+ }
1228
+ const answer = this.currentAnswer;
1229
+ if (!answer) return;
1230
+ this.currentAnswer = void 0;
1231
+ if (!answer.hasAudio) this.emitAgentEvent("SkipAnswer");
1232
+ this.recordAnswer(answer);
1233
+ this.resolveAnswerWaiters();
1234
+ this.queue.shift()?.();
1235
+ }
1236
+ /** Runs a tool the model called, and records it in the conversation */
1237
+ async runTool(toolCall) {
1238
+ const result = await this.executeTool(toolCall);
1239
+ if (this.currentAnswer) this.currentAnswer.resumed = true;
1240
+ return result;
1241
+ }
1242
+ /** Transcript of the turn of the user, possibly in several pieces */
1243
+ addUserTranscript(text) {
1244
+ this.userTranscript += text;
1245
+ }
1246
+ /** The transcript of the turn is complete, it goes in the conversation */
1247
+ commitUserTranscript() {
1248
+ this.clearUserTranscriptTimer();
1249
+ this.awaitingUserTranscript = false;
1250
+ const text = this.userTranscript.trim();
1251
+ this.userTranscript = "";
1252
+ if (text !== "") super.addMessage("user", text);
1253
+ const pending = this.afterUserTranscript;
1254
+ this.afterUserTranscript = [];
1255
+ pending.forEach((record) => record());
1256
+ }
1257
+ sendAudio(chunk) {
1258
+ if (this.turn === "none") return;
1259
+ if (this.turn === "pending") {
1260
+ this.turn = "open";
1261
+ this.openTurn();
1262
+ }
1263
+ this.appendAudio(chunk);
1264
+ }
1265
+ beginAnswer(record) {
1266
+ this.discarding = false;
1267
+ this.currentAnswer = newAnswer(record);
1268
+ }
1269
+ /**
1270
+ * The answer output belongs to, if it is still wanted.
1271
+ *
1272
+ * The model can speak without being asked, to follow up on a tool call for
1273
+ * instance, which opens an answer of its own.
1274
+ */
1275
+ activeAnswer() {
1276
+ if (this.discarding) return void 0;
1277
+ if (!this.currentAnswer) this.currentAnswer = newAnswer(true);
1278
+ return this.currentAnswer;
1279
+ }
1280
+ whenIdle(start) {
1281
+ if (this.currentAnswer) {
1282
+ this.queue.push(start);
1283
+ } else {
1284
+ start();
1285
+ }
1286
+ }
1287
+ recordAnswer(answer) {
1288
+ if (!answer.record) return;
1289
+ const record = () => super.addMessage("assistant", answer.transcript.trim());
1290
+ if (this.awaitingUserTranscript) {
1291
+ this.afterUserTranscript.push(record);
1292
+ } else {
1293
+ record();
1294
+ }
1295
+ }
1296
+ awaitUserTranscript() {
1297
+ this.awaitingUserTranscript = true;
1298
+ this.clearUserTranscriptTimer();
1299
+ this.userTranscriptTimer = setTimeout(() => {
1300
+ this.log("No transcript of the turn, recording the answer anyway");
1301
+ this.commitUserTranscript();
1302
+ }, USER_TRANSCRIPT_TIMEOUT);
1303
+ }
1304
+ clearUserTranscriptTimer() {
1305
+ if (!this.userTranscriptTimer) return;
1306
+ clearTimeout(this.userTranscriptTimer);
1307
+ this.userTranscriptTimer = void 0;
1308
+ }
1309
+ resolveAnswerWaiters() {
1310
+ const waiters = this.answerWaiters;
1311
+ this.answerWaiters = [];
1312
+ waiters.forEach((resolve) => resolve());
1313
+ }
1314
+ };
1315
+
831
1316
  // src/recorder/MicdropRecorder.ts
832
- import { EventEmitter as EventEmitter3 } from "eventemitter3";
833
- var MicdropRecorder = class extends EventEmitter3 {
1317
+ import { EventEmitter as EventEmitter4 } from "eventemitter3";
1318
+ var MicdropRecorder = class extends EventEmitter4 {
834
1319
  constructor(server) {
835
1320
  super();
836
1321
  this.server = server;
@@ -987,8 +1472,8 @@ var SharedInstances = class {
987
1472
  };
988
1473
 
989
1474
  // src/stt/STT.ts
990
- import { EventEmitter as EventEmitter4 } from "eventemitter3";
991
- var STT = class extends EventEmitter4 {
1475
+ import { EventEmitter as EventEmitter5 } from "eventemitter3";
1476
+ var STT = class extends EventEmitter5 {
992
1477
  log(...message) {
993
1478
  this.logger?.log(...message);
994
1479
  }
@@ -1069,8 +1554,8 @@ var FallbackSTT = class extends STT {
1069
1554
  import { PassThrough as PassThrough4 } from "stream";
1070
1555
 
1071
1556
  // src/tts/TTS.ts
1072
- import { EventEmitter as EventEmitter5 } from "eventemitter3";
1073
- var TTS = class extends EventEmitter5 {
1557
+ import { EventEmitter as EventEmitter6 } from "eventemitter3";
1558
+ var TTS = class extends EventEmitter6 {
1074
1559
  log(...message) {
1075
1560
  this.logger?.log(...message);
1076
1561
  }
@@ -1321,6 +1806,7 @@ export {
1321
1806
  AUTO_SEMANTIC_TURN_PROMPT,
1322
1807
  AUTO_SEMANTIC_TURN_TOOL_NAME,
1323
1808
  Agent,
1809
+ Classifier,
1324
1810
  FallbackAgent,
1325
1811
  FallbackSTT,
1326
1812
  FallbackTTS,
@@ -1335,14 +1821,18 @@ export {
1335
1821
  MockSTT,
1336
1822
  MockTTS,
1337
1823
  Pcm16Resampler,
1824
+ Realtime,
1338
1825
  STT,
1339
1826
  SentenceSplitter,
1340
1827
  SentenceTTS,
1341
1828
  SharedInstances,
1342
1829
  TTS,
1830
+ currentTurn,
1343
1831
  float32ToPcm16,
1832
+ getTurnClassification,
1344
1833
  handleError,
1345
1834
  pcm16ToFloat32,
1835
+ turnInput,
1346
1836
  waitForParams
1347
1837
  };
1348
1838
  //# sourceMappingURL=index.mjs.map