@micdrop/server 2.4.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -503,6 +503,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
503
503
  })(MicdropClientCommands || {});
504
504
  var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
505
505
  MicdropServerCommands2["Message"] = "Message";
506
+ MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
506
507
  MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
507
508
  MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
508
509
  MicdropServerCommands2["EndCall"] = "EndCall";
@@ -511,27 +512,35 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
511
512
  })(MicdropServerCommands || {});
512
513
 
513
514
  // src/MicdropServer.ts
515
+ var USER_SAMPLE_RATE = 16e3;
516
+ var DEFAULT_TURN_MAX_WAIT = 4e3;
514
517
  var MicdropServer = class extends import_eventemitter32.EventEmitter {
515
518
  constructor(socket, config) {
516
519
  super();
517
520
  this.socket = null;
518
521
  this.config = null;
519
522
  this.startTime = Date.now();
523
+ // Holds the conversation when no agent does, and keeps it readable once the
524
+ // call is over and the config is gone
525
+ this.ownConversation = [];
520
526
  // Queue system for operations
521
527
  this.operationQueue = [];
522
528
  this.isProcessingQueue = false;
523
529
  this.userSpeechChunks = 0;
524
530
  this.onClose = () => {
531
+ this.releaseHeldTurn();
525
532
  if (!this.config) return;
526
533
  this.log("Connection closed");
527
534
  const duration = Math.round((Date.now() - this.startTime) / 1e3);
528
- this.config.agent.destroy();
535
+ const conversation = this.conversation;
536
+ this.config.agent?.destroy();
529
537
  this.config.stt.destroy();
530
- this.config.tts.destroy();
538
+ this.config.tts?.destroy();
531
539
  this.emit("End", {
532
- conversation: this.config.agent.conversation,
540
+ conversation,
533
541
  duration
534
542
  });
543
+ this.ownConversation = conversation;
535
544
  this.socket = null;
536
545
  this.config = null;
537
546
  };
@@ -562,11 +571,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
562
571
  return;
563
572
  }
564
573
  this.log(`User transcript: "${transcript}"`);
565
- this.config.agent.addUserMessage(transcript);
574
+ this.addUserMessage(transcript);
566
575
  if (!this.currentUserStream) {
567
576
  this.log("User stopped speaking, answering");
568
- this.cancel();
569
- this.answer();
577
+ this.answerUserTurn();
570
578
  }
571
579
  };
572
580
  this.onAudioTTS = (audio) => {
@@ -575,39 +583,44 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
575
583
  this.socket.send(audio);
576
584
  this.emit("AssistantAudio", audio);
577
585
  };
586
+ this.onMessageAgent = (message) => {
587
+ this.sendMessage(message);
588
+ };
578
589
  this.socket = socket;
579
590
  this.config = config;
580
591
  this.log(`Call started`);
581
- this.config.stt.on("Transcript", this.onTranscriptSTT);
582
- this.config.tts.on("Audio", this.onAudioTTS);
583
- this.config.agent.on(
584
- "Message",
585
- (message) => this.socket?.send(
586
- `${"Message" /* Message */} ${JSON.stringify(message)}`
587
- )
588
- );
589
- this.config.agent.on(
590
- "CancelLastUserMessage",
591
- () => this.socket?.send("CancelLastUserMessage" /* CancelLastUserMessage */)
592
- );
593
- this.config.agent.on(
594
- "SkipAnswer",
595
- () => this.socket?.send("SkipAnswer" /* SkipAnswer */)
596
- );
597
- this.config.agent.on(
598
- "EndCall",
599
- () => this.socket?.send("EndCall" /* EndCall */)
600
- );
601
- this.config.agent.on(
602
- "ToolCall",
603
- (toolCall) => this.socket?.send(
604
- `${"ToolCall" /* ToolCall */} ${JSON.stringify(toolCall)}`
605
- )
606
- );
592
+ config.stt.on("Transcript", this.onTranscriptSTT);
593
+ config.tts?.on("Audio", this.onAudioTTS);
594
+ const agent = config.agent;
595
+ if (agent) {
596
+ agent.on("Message", this.onMessageAgent);
597
+ agent.on(
598
+ "CancelLastUserMessage",
599
+ () => this.socket?.send("CancelLastUserMessage" /* CancelLastUserMessage */)
600
+ );
601
+ agent.on(
602
+ "SkipAnswer",
603
+ () => this.socket?.send("SkipAnswer" /* SkipAnswer */)
604
+ );
605
+ agent.on(
606
+ "EndCall",
607
+ () => this.socket?.send("EndCall" /* EndCall */)
608
+ );
609
+ agent.on(
610
+ "ToolCall",
611
+ (toolCall) => this.socket?.send(
612
+ `${"ToolCall" /* ToolCall */} ${JSON.stringify(toolCall)}`
613
+ )
614
+ );
615
+ }
607
616
  queueMicrotask(() => this.sendFirstMessage());
608
617
  socket.on("close", this.onClose);
609
618
  socket.on("message", this.onMessage);
610
619
  }
620
+ /** Everything said so far, whether an agent or the server keeps it */
621
+ get conversation() {
622
+ return this.config?.agent?.conversation ?? this.ownConversation;
623
+ }
611
624
  log(...message) {
612
625
  this.logger?.log(...message);
613
626
  }
@@ -631,14 +644,15 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
631
644
  this.processQueue();
632
645
  }
633
646
  cancel() {
634
- this.config?.tts.cancel();
635
- this.config?.agent.cancel();
647
+ this.config?.tts?.cancel();
648
+ this.config?.agent?.cancel();
636
649
  this.operationQueue = [];
637
650
  }
638
651
  onUserAudio(chunk) {
639
652
  this.log(`Received chunk (${chunk.byteLength} bytes)`);
640
653
  this.currentUserStream?.write(chunk);
641
654
  this.userSpeechChunks++;
655
+ this.config?.turnDetector?.push(pcm16ToFloat32(chunk), USER_SAMPLE_RATE);
642
656
  this.emit("UserAudio", chunk);
643
657
  }
644
658
  onMute() {
@@ -652,6 +666,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
652
666
  this.userSpeechChunks = 0;
653
667
  this.currentUserStream?.end();
654
668
  this.currentUserStream = new import_stream2.PassThrough();
669
+ this.config.turnDetector?.reset();
670
+ this.turnComplete = void 0;
671
+ this.releaseHeldTurn();
655
672
  this.config.stt.transcribe(this.currentUserStream);
656
673
  this.cancel();
657
674
  }
@@ -664,22 +681,110 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
664
681
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
665
682
  return;
666
683
  }
667
- const conversation = this.config?.agent.conversation;
668
- const lastMessage = conversation?.[conversation.length - 1];
684
+ this.turnComplete = this.predictTurnComplete();
685
+ const conversation = this.conversation;
686
+ const lastMessage = conversation[conversation.length - 1];
669
687
  if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
670
688
  this.log(
671
689
  "User stopped speaking and a transcript already exists, answering"
672
690
  );
691
+ this.answerUserTurn();
692
+ }
693
+ }
694
+ async predictTurnComplete() {
695
+ const detector = this.config?.turnDetector;
696
+ if (!detector) return true;
697
+ try {
698
+ const { complete } = await detector.predict();
699
+ this.log(`Turn sounds ${complete ? "finished" : "unfinished"}`);
700
+ return complete;
701
+ } catch (error) {
702
+ this.log(`Turn detection failed: ${error}`);
703
+ return true;
704
+ }
705
+ }
706
+ /** Answers, unless the sentence sounds like it has more coming */
707
+ async answerUserTurn() {
708
+ const complete = await (this.turnComplete ?? Promise.resolve(true));
709
+ if (!complete) {
710
+ this.log("Waiting for the rest of the sentence");
711
+ this.socket?.send("SkipAnswer" /* SkipAnswer */);
712
+ this.holdTurn();
713
+ return;
714
+ }
715
+ this.releaseHeldTurn();
716
+ this.cancel();
717
+ this.answer();
718
+ }
719
+ /**
720
+ * Answers anyway if the rest of the sentence never comes.
721
+ *
722
+ * Without it, a detector that hears an unfinished sentence where there is
723
+ * none leaves the call silent for good.
724
+ */
725
+ holdTurn() {
726
+ this.releaseHeldTurn();
727
+ this.heldTurnTimer = setTimeout(() => {
728
+ this.heldTurnTimer = void 0;
729
+ this.log("Nothing more came, answering");
673
730
  this.cancel();
674
731
  this.answer();
732
+ }, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
733
+ }
734
+ releaseHeldTurn() {
735
+ if (!this.heldTurnTimer) return;
736
+ clearTimeout(this.heldTurnTimer);
737
+ this.heldTurnTimer = void 0;
738
+ }
739
+ /** Adds a message to the conversation and sends it to the client */
740
+ addUserMessage(text, metadata) {
741
+ if (this.config?.agent) {
742
+ this.config.agent.addUserMessage(text, metadata);
743
+ } else {
744
+ this.addOwnMessage("user", text, metadata);
745
+ }
746
+ }
747
+ /** Adds a message to the conversation and sends it to the client */
748
+ addAssistantMessage(text, metadata) {
749
+ if (this.config?.agent) {
750
+ this.config.agent.addAssistantMessage(text, metadata);
751
+ } else {
752
+ this.addOwnMessage("assistant", text, metadata);
675
753
  }
676
754
  }
755
+ addOwnMessage(role, text, metadata) {
756
+ if (text.trim() === "") {
757
+ this.log(`Skipping empty ${role} message`);
758
+ return;
759
+ }
760
+ this.log(`Adding ${role} message to conversation: ${text}`);
761
+ const message = {
762
+ role,
763
+ content: text,
764
+ metadata
765
+ };
766
+ this.ownConversation.push(message);
767
+ this.sendMessage(message);
768
+ }
769
+ sendMessage(message) {
770
+ this.socket?.send(
771
+ `${"Message" /* Message */} ${JSON.stringify(message)}`
772
+ );
773
+ this.emit("Message", message);
774
+ }
775
+ sendPartialAnswer(content) {
776
+ if (!this.socket || content === "") return;
777
+ this.log(`Send partial answer: "${content}"`);
778
+ this.socket.send(
779
+ `${"PartialAssistantMessage" /* PartialAssistantMessage */} ${JSON.stringify(content)}`
780
+ );
781
+ }
677
782
  sendFirstMessage() {
678
783
  if (!this.config) return;
679
784
  if (this.config.firstMessage) {
680
- this.config.agent.addAssistantMessage(this.config.firstMessage);
785
+ this.config.agent?.addAssistantMessage(this.config.firstMessage);
681
786
  this.speak(this.config.firstMessage);
682
- } else if (this.config.generateFirstMessage) {
787
+ } else if (this.config.generateFirstMessage && this.config.agent) {
683
788
  this.answer();
684
789
  } else {
685
790
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -692,7 +797,13 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
692
797
  }
693
798
  async _answer() {
694
799
  if (!this.config) return;
695
- const lastMessage = this.config.agent.conversation[this.config.agent.conversation.length - 1];
800
+ if (!this.config.agent) {
801
+ this.log("No agent, skipping answer");
802
+ this.socket?.send("SkipAnswer" /* SkipAnswer */);
803
+ return;
804
+ }
805
+ const conversation = this.config.agent.conversation;
806
+ const lastMessage = conversation[conversation.length - 1];
696
807
  if (this.lastMessageSpeeched === lastMessage) {
697
808
  this.log("Already answered, skipping");
698
809
  return;
@@ -701,13 +812,30 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
701
812
  try {
702
813
  const stream = this.config.agent.answer();
703
814
  if (await hasContent(stream)) {
704
- await this._speak(stream);
815
+ await this._speak(
816
+ this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
817
+ );
818
+ } else {
819
+ this.socket?.send("SkipAnswer" /* SkipAnswer */);
705
820
  }
706
821
  } catch (error) {
707
822
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
708
823
  throw error;
709
824
  }
710
825
  }
826
+ /** Sends the answer to the client as it is written, ahead of the voice */
827
+ forwardPartialAnswer(stream) {
828
+ let content = "";
829
+ const send = (text) => this.sendPartialAnswer(text);
830
+ const forward = new import_stream2.Transform({
831
+ transform(chunk, _encoding, callback) {
832
+ content += chunk.toString();
833
+ send(content);
834
+ callback(null, chunk);
835
+ }
836
+ });
837
+ return stream.pipe(forward);
838
+ }
711
839
  // Run text-to-speech and send to client
712
840
  speak(message) {
713
841
  this.queueOperation(async () => {
@@ -718,6 +846,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
718
846
  if (!this.socket || !this.config) return;
719
847
  let textStream;
720
848
  if (typeof message === "string") {
849
+ if (!this.config.agent) {
850
+ this.addOwnMessage("assistant", message);
851
+ }
721
852
  const stream = new import_stream2.PassThrough();
722
853
  stream.write(message);
723
854
  stream.end();
@@ -725,7 +856,12 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
725
856
  } else {
726
857
  textStream = message;
727
858
  }
728
- this.config.tts.speak(textStream);
859
+ if (this.config.tts) {
860
+ this.config.tts.speak(textStream);
861
+ return;
862
+ }
863
+ await drain(textStream);
864
+ this.socket?.send("SkipAnswer" /* SkipAnswer */);
729
865
  }
730
866
  };
731
867
  function hasContent(stream) {
@@ -746,6 +882,15 @@ function hasContent(stream) {
746
882
  stream.on("end", onEnd);
747
883
  });
748
884
  }
885
+ function drain(stream) {
886
+ return new Promise((resolve) => {
887
+ stream.on("data", () => {
888
+ });
889
+ stream.once("end", () => resolve());
890
+ stream.once("error", () => resolve());
891
+ stream.once("close", () => resolve());
892
+ });
893
+ }
749
894
 
750
895
  // src/recorder/MicdropRecorder.ts
751
896
  var import_eventemitter33 = require("eventemitter3");
@@ -783,9 +928,7 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
783
928
  this.currentAssistantChunks.push(chunk);
784
929
  };
785
930
  this.onMessage = (message) => {
786
- const conversation = this.server.config?.agent.conversation;
787
- if (!conversation) return;
788
- const messageIndex = conversation.length - 1;
931
+ const messageIndex = this.server.conversation.length - 1;
789
932
  if (message.role === "user") {
790
933
  this.lastUserMessageIndex = messageIndex;
791
934
  if (this.currentUserChunks.length > 0) {
@@ -811,17 +954,13 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
811
954
  this.server.on("UserAudio", this.onUserAudio);
812
955
  this.server.on("AssistantAudio", this.onAssistantAudio);
813
956
  this.server.on("End", this.onEnd);
814
- const agent = this.server.config?.agent;
815
- if (agent) {
816
- agent.on("Message", this.onMessage);
817
- }
957
+ this.server.on("Message", this.onMessage);
818
958
  }
819
959
  finalizeUserAudio() {
820
960
  if (this.currentUserChunks.length === 0) return;
821
961
  if (this.lastUserMessageIndex < 0) return;
822
- const conversation = this.server.config?.agent.conversation;
823
- if (!conversation) return;
824
- const message = conversation[this.lastUserMessageIndex];
962
+ const message = this.server.conversation[this.lastUserMessageIndex];
963
+ if (!message) return;
825
964
  const buffer = Buffer.concat(this.currentUserChunks);
826
965
  const audioMessage = {
827
966
  buffer,
@@ -840,9 +979,8 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
840
979
  finalizeAssistantAudio() {
841
980
  if (this.currentAssistantChunks.length === 0) return;
842
981
  if (this.lastAssistantMessageIndex < 0) return;
843
- const conversation = this.server.config?.agent.conversation;
844
- if (!conversation) return;
845
- const message = conversation[this.lastAssistantMessageIndex];
982
+ const message = this.server.conversation[this.lastAssistantMessageIndex];
983
+ if (!message) return;
846
984
  const buffer = Buffer.concat(this.currentAssistantChunks);
847
985
  const audioMessage = {
848
986
  buffer,
@@ -866,10 +1004,7 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
866
1004
  this.server.off("UserAudio", this.onUserAudio);
867
1005
  this.server.off("AssistantAudio", this.onAssistantAudio);
868
1006
  this.server.off("End", this.onEnd);
869
- const agent = this.server.config?.agent;
870
- if (agent) {
871
- agent.off("Message", this.onMessage);
872
- }
1007
+ this.server.off("Message", this.onMessage);
873
1008
  this.removeAllListeners();
874
1009
  }
875
1010
  log(...message) {