@micdrop/server 2.4.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -428,7 +428,7 @@ function handleError(socket, error) {
428
428
 
429
429
  // src/MicdropServer.ts
430
430
  import { EventEmitter as EventEmitter2 } from "eventemitter3";
431
- import { PassThrough as PassThrough2 } from "stream";
431
+ import { PassThrough as PassThrough2, Transform } from "stream";
432
432
 
433
433
  // src/types.ts
434
434
  var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
@@ -439,6 +439,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
439
439
  })(MicdropClientCommands || {});
440
440
  var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
441
441
  MicdropServerCommands2["Message"] = "Message";
442
+ MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
442
443
  MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
443
444
  MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
444
445
  MicdropServerCommands2["EndCall"] = "EndCall";
@@ -447,27 +448,35 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
447
448
  })(MicdropServerCommands || {});
448
449
 
449
450
  // src/MicdropServer.ts
451
+ var USER_SAMPLE_RATE = 16e3;
452
+ var DEFAULT_TURN_MAX_WAIT = 4e3;
450
453
  var MicdropServer = class extends EventEmitter2 {
451
454
  constructor(socket, config) {
452
455
  super();
453
456
  this.socket = null;
454
457
  this.config = null;
455
458
  this.startTime = Date.now();
459
+ // Holds the conversation when no agent does, and keeps it readable once the
460
+ // call is over and the config is gone
461
+ this.ownConversation = [];
456
462
  // Queue system for operations
457
463
  this.operationQueue = [];
458
464
  this.isProcessingQueue = false;
459
465
  this.userSpeechChunks = 0;
460
466
  this.onClose = () => {
467
+ this.releaseHeldTurn();
461
468
  if (!this.config) return;
462
469
  this.log("Connection closed");
463
470
  const duration = Math.round((Date.now() - this.startTime) / 1e3);
464
- this.config.agent.destroy();
471
+ const conversation = this.conversation;
472
+ this.config.agent?.destroy();
465
473
  this.config.stt.destroy();
466
- this.config.tts.destroy();
474
+ this.config.tts?.destroy();
467
475
  this.emit("End", {
468
- conversation: this.config.agent.conversation,
476
+ conversation,
469
477
  duration
470
478
  });
479
+ this.ownConversation = conversation;
471
480
  this.socket = null;
472
481
  this.config = null;
473
482
  };
@@ -498,11 +507,10 @@ var MicdropServer = class extends EventEmitter2 {
498
507
  return;
499
508
  }
500
509
  this.log(`User transcript: "${transcript}"`);
501
- this.config.agent.addUserMessage(transcript);
510
+ this.addUserMessage(transcript);
502
511
  if (!this.currentUserStream) {
503
512
  this.log("User stopped speaking, answering");
504
- this.cancel();
505
- this.answer();
513
+ this.answerUserTurn();
506
514
  }
507
515
  };
508
516
  this.onAudioTTS = (audio) => {
@@ -511,39 +519,44 @@ var MicdropServer = class extends EventEmitter2 {
511
519
  this.socket.send(audio);
512
520
  this.emit("AssistantAudio", audio);
513
521
  };
522
+ this.onMessageAgent = (message) => {
523
+ this.sendMessage(message);
524
+ };
514
525
  this.socket = socket;
515
526
  this.config = config;
516
527
  this.log(`Call started`);
517
- this.config.stt.on("Transcript", this.onTranscriptSTT);
518
- this.config.tts.on("Audio", this.onAudioTTS);
519
- this.config.agent.on(
520
- "Message",
521
- (message) => this.socket?.send(
522
- `${"Message" /* Message */} ${JSON.stringify(message)}`
523
- )
524
- );
525
- this.config.agent.on(
526
- "CancelLastUserMessage",
527
- () => this.socket?.send("CancelLastUserMessage" /* CancelLastUserMessage */)
528
- );
529
- this.config.agent.on(
530
- "SkipAnswer",
531
- () => this.socket?.send("SkipAnswer" /* SkipAnswer */)
532
- );
533
- this.config.agent.on(
534
- "EndCall",
535
- () => this.socket?.send("EndCall" /* EndCall */)
536
- );
537
- this.config.agent.on(
538
- "ToolCall",
539
- (toolCall) => this.socket?.send(
540
- `${"ToolCall" /* ToolCall */} ${JSON.stringify(toolCall)}`
541
- )
542
- );
528
+ config.stt.on("Transcript", this.onTranscriptSTT);
529
+ config.tts?.on("Audio", this.onAudioTTS);
530
+ const agent = config.agent;
531
+ if (agent) {
532
+ agent.on("Message", this.onMessageAgent);
533
+ agent.on(
534
+ "CancelLastUserMessage",
535
+ () => this.socket?.send("CancelLastUserMessage" /* CancelLastUserMessage */)
536
+ );
537
+ agent.on(
538
+ "SkipAnswer",
539
+ () => this.socket?.send("SkipAnswer" /* SkipAnswer */)
540
+ );
541
+ agent.on(
542
+ "EndCall",
543
+ () => this.socket?.send("EndCall" /* EndCall */)
544
+ );
545
+ agent.on(
546
+ "ToolCall",
547
+ (toolCall) => this.socket?.send(
548
+ `${"ToolCall" /* ToolCall */} ${JSON.stringify(toolCall)}`
549
+ )
550
+ );
551
+ }
543
552
  queueMicrotask(() => this.sendFirstMessage());
544
553
  socket.on("close", this.onClose);
545
554
  socket.on("message", this.onMessage);
546
555
  }
556
+ /** Everything said so far, whether an agent or the server keeps it */
557
+ get conversation() {
558
+ return this.config?.agent?.conversation ?? this.ownConversation;
559
+ }
547
560
  log(...message) {
548
561
  this.logger?.log(...message);
549
562
  }
@@ -567,14 +580,15 @@ var MicdropServer = class extends EventEmitter2 {
567
580
  this.processQueue();
568
581
  }
569
582
  cancel() {
570
- this.config?.tts.cancel();
571
- this.config?.agent.cancel();
583
+ this.config?.tts?.cancel();
584
+ this.config?.agent?.cancel();
572
585
  this.operationQueue = [];
573
586
  }
574
587
  onUserAudio(chunk) {
575
588
  this.log(`Received chunk (${chunk.byteLength} bytes)`);
576
589
  this.currentUserStream?.write(chunk);
577
590
  this.userSpeechChunks++;
591
+ this.config?.turnDetector?.push(pcm16ToFloat32(chunk), USER_SAMPLE_RATE);
578
592
  this.emit("UserAudio", chunk);
579
593
  }
580
594
  onMute() {
@@ -588,6 +602,9 @@ var MicdropServer = class extends EventEmitter2 {
588
602
  this.userSpeechChunks = 0;
589
603
  this.currentUserStream?.end();
590
604
  this.currentUserStream = new PassThrough2();
605
+ this.config.turnDetector?.reset();
606
+ this.turnComplete = void 0;
607
+ this.releaseHeldTurn();
591
608
  this.config.stt.transcribe(this.currentUserStream);
592
609
  this.cancel();
593
610
  }
@@ -600,22 +617,110 @@ var MicdropServer = class extends EventEmitter2 {
600
617
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
601
618
  return;
602
619
  }
603
- const conversation = this.config?.agent.conversation;
604
- const lastMessage = conversation?.[conversation.length - 1];
620
+ this.turnComplete = this.predictTurnComplete();
621
+ const conversation = this.conversation;
622
+ const lastMessage = conversation[conversation.length - 1];
605
623
  if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
606
624
  this.log(
607
625
  "User stopped speaking and a transcript already exists, answering"
608
626
  );
627
+ this.answerUserTurn();
628
+ }
629
+ }
630
+ async predictTurnComplete() {
631
+ const detector = this.config?.turnDetector;
632
+ if (!detector) return true;
633
+ try {
634
+ const { complete } = await detector.predict();
635
+ this.log(`Turn sounds ${complete ? "finished" : "unfinished"}`);
636
+ return complete;
637
+ } catch (error) {
638
+ this.log(`Turn detection failed: ${error}`);
639
+ return true;
640
+ }
641
+ }
642
+ /** Answers, unless the sentence sounds like it has more coming */
643
+ async answerUserTurn() {
644
+ const complete = await (this.turnComplete ?? Promise.resolve(true));
645
+ if (!complete) {
646
+ this.log("Waiting for the rest of the sentence");
647
+ this.socket?.send("SkipAnswer" /* SkipAnswer */);
648
+ this.holdTurn();
649
+ return;
650
+ }
651
+ this.releaseHeldTurn();
652
+ this.cancel();
653
+ this.answer();
654
+ }
655
+ /**
656
+ * Answers anyway if the rest of the sentence never comes.
657
+ *
658
+ * Without it, a detector that hears an unfinished sentence where there is
659
+ * none leaves the call silent for good.
660
+ */
661
+ holdTurn() {
662
+ this.releaseHeldTurn();
663
+ this.heldTurnTimer = setTimeout(() => {
664
+ this.heldTurnTimer = void 0;
665
+ this.log("Nothing more came, answering");
609
666
  this.cancel();
610
667
  this.answer();
668
+ }, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
669
+ }
670
+ releaseHeldTurn() {
671
+ if (!this.heldTurnTimer) return;
672
+ clearTimeout(this.heldTurnTimer);
673
+ this.heldTurnTimer = void 0;
674
+ }
675
+ /** Adds a message to the conversation and sends it to the client */
676
+ addUserMessage(text, metadata) {
677
+ if (this.config?.agent) {
678
+ this.config.agent.addUserMessage(text, metadata);
679
+ } else {
680
+ this.addOwnMessage("user", text, metadata);
681
+ }
682
+ }
683
+ /** Adds a message to the conversation and sends it to the client */
684
+ addAssistantMessage(text, metadata) {
685
+ if (this.config?.agent) {
686
+ this.config.agent.addAssistantMessage(text, metadata);
687
+ } else {
688
+ this.addOwnMessage("assistant", text, metadata);
611
689
  }
612
690
  }
691
+ addOwnMessage(role, text, metadata) {
692
+ if (text.trim() === "") {
693
+ this.log(`Skipping empty ${role} message`);
694
+ return;
695
+ }
696
+ this.log(`Adding ${role} message to conversation: ${text}`);
697
+ const message = {
698
+ role,
699
+ content: text,
700
+ metadata
701
+ };
702
+ this.ownConversation.push(message);
703
+ this.sendMessage(message);
704
+ }
705
+ sendMessage(message) {
706
+ this.socket?.send(
707
+ `${"Message" /* Message */} ${JSON.stringify(message)}`
708
+ );
709
+ this.emit("Message", message);
710
+ }
711
+ sendPartialAnswer(content) {
712
+ if (!this.socket || content === "") return;
713
+ this.log(`Send partial answer: "${content}"`);
714
+ this.socket.send(
715
+ `${"PartialAssistantMessage" /* PartialAssistantMessage */} ${JSON.stringify(content)}`
716
+ );
717
+ }
613
718
  sendFirstMessage() {
614
719
  if (!this.config) return;
615
720
  if (this.config.firstMessage) {
616
- this.config.agent.addAssistantMessage(this.config.firstMessage);
721
+ this.config.agent?.addAssistantMessage(this.config.firstMessage);
617
722
  this.speak(this.config.firstMessage);
618
- } else if (this.config.generateFirstMessage) {
723
+ } else if (this.config.generateFirstMessage && this.config.agent) {
619
724
  this.answer();
620
725
  } else {
621
726
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -628,7 +733,13 @@ var MicdropServer = class extends EventEmitter2 {
628
733
  }
629
734
  async _answer() {
630
735
  if (!this.config) return;
631
- const lastMessage = this.config.agent.conversation[this.config.agent.conversation.length - 1];
736
+ if (!this.config.agent) {
737
+ this.log("No agent, skipping answer");
738
+ this.socket?.send("SkipAnswer" /* SkipAnswer */);
739
+ return;
740
+ }
741
+ const conversation = this.config.agent.conversation;
742
+ const lastMessage = conversation[conversation.length - 1];
632
743
  if (this.lastMessageSpeeched === lastMessage) {
633
744
  this.log("Already answered, skipping");
634
745
  return;
@@ -637,13 +748,30 @@ var MicdropServer = class extends EventEmitter2 {
637
748
  try {
638
749
  const stream = this.config.agent.answer();
639
750
  if (await hasContent(stream)) {
640
- await this._speak(stream);
751
+ await this._speak(
752
+ this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
753
+ );
754
+ } else {
755
+ this.socket?.send("SkipAnswer" /* SkipAnswer */);
641
756
  }
642
757
  } catch (error) {
643
758
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
644
759
  throw error;
645
760
  }
646
761
  }
762
+ /** Sends the answer to the client as it is written, ahead of the voice */
763
+ forwardPartialAnswer(stream) {
764
+ let content = "";
765
+ const send = (text) => this.sendPartialAnswer(text);
766
+ const forward = new Transform({
767
+ transform(chunk, _encoding, callback) {
768
+ content += chunk.toString();
769
+ send(content);
770
+ callback(null, chunk);
771
+ }
772
+ });
773
+ return stream.pipe(forward);
774
+ }
647
775
  // Run text-to-speech and send to client
648
776
  speak(message) {
649
777
  this.queueOperation(async () => {
@@ -654,6 +782,9 @@ var MicdropServer = class extends EventEmitter2 {
654
782
  if (!this.socket || !this.config) return;
655
783
  let textStream;
656
784
  if (typeof message === "string") {
785
+ if (!this.config.agent) {
786
+ this.addOwnMessage("assistant", message);
787
+ }
657
788
  const stream = new PassThrough2();
658
789
  stream.write(message);
659
790
  stream.end();
@@ -661,7 +792,12 @@ var MicdropServer = class extends EventEmitter2 {
661
792
  } else {
662
793
  textStream = message;
663
794
  }
664
- this.config.tts.speak(textStream);
795
+ if (this.config.tts) {
796
+ this.config.tts.speak(textStream);
797
+ return;
798
+ }
799
+ await drain(textStream);
800
+ this.socket?.send("SkipAnswer" /* SkipAnswer */);
665
801
  }
666
802
  };
667
803
  function hasContent(stream) {
@@ -682,6 +818,15 @@ function hasContent(stream) {
682
818
  stream.on("end", onEnd);
683
819
  });
684
820
  }
821
+ function drain(stream) {
822
+ return new Promise((resolve) => {
823
+ stream.on("data", () => {
824
+ });
825
+ stream.once("end", () => resolve());
826
+ stream.once("error", () => resolve());
827
+ stream.once("close", () => resolve());
828
+ });
829
+ }
685
830
 
686
831
  // src/recorder/MicdropRecorder.ts
687
832
  import { EventEmitter as EventEmitter3 } from "eventemitter3";
@@ -719,9 +864,7 @@ var MicdropRecorder = class extends EventEmitter3 {
719
864
  this.currentAssistantChunks.push(chunk);
720
865
  };
721
866
  this.onMessage = (message) => {
722
- const conversation = this.server.config?.agent.conversation;
723
- if (!conversation) return;
724
- const messageIndex = conversation.length - 1;
867
+ const messageIndex = this.server.conversation.length - 1;
725
868
  if (message.role === "user") {
726
869
  this.lastUserMessageIndex = messageIndex;
727
870
  if (this.currentUserChunks.length > 0) {
@@ -747,17 +890,13 @@ var MicdropRecorder = class extends EventEmitter3 {
747
890
  this.server.on("UserAudio", this.onUserAudio);
748
891
  this.server.on("AssistantAudio", this.onAssistantAudio);
749
892
  this.server.on("End", this.onEnd);
750
- const agent = this.server.config?.agent;
751
- if (agent) {
752
- agent.on("Message", this.onMessage);
753
- }
893
+ this.server.on("Message", this.onMessage);
754
894
  }
755
895
  finalizeUserAudio() {
756
896
  if (this.currentUserChunks.length === 0) return;
757
897
  if (this.lastUserMessageIndex < 0) return;
758
- const conversation = this.server.config?.agent.conversation;
759
- if (!conversation) return;
760
- const message = conversation[this.lastUserMessageIndex];
898
+ const message = this.server.conversation[this.lastUserMessageIndex];
899
+ if (!message) return;
761
900
  const buffer = Buffer.concat(this.currentUserChunks);
762
901
  const audioMessage = {
763
902
  buffer,
@@ -776,9 +915,8 @@ var MicdropRecorder = class extends EventEmitter3 {
776
915
  finalizeAssistantAudio() {
777
916
  if (this.currentAssistantChunks.length === 0) return;
778
917
  if (this.lastAssistantMessageIndex < 0) return;
779
- const conversation = this.server.config?.agent.conversation;
780
- if (!conversation) return;
781
- const message = conversation[this.lastAssistantMessageIndex];
918
+ const message = this.server.conversation[this.lastAssistantMessageIndex];
919
+ if (!message) return;
782
920
  const buffer = Buffer.concat(this.currentAssistantChunks);
783
921
  const audioMessage = {
784
922
  buffer,
@@ -802,10 +940,7 @@ var MicdropRecorder = class extends EventEmitter3 {
802
940
  this.server.off("UserAudio", this.onUserAudio);
803
941
  this.server.off("AssistantAudio", this.onAssistantAudio);
804
942
  this.server.off("End", this.onEnd);
805
- const agent = this.server.config?.agent;
806
- if (agent) {
807
- agent.off("Message", this.onMessage);
808
- }
943
+ this.server.off("Message", this.onMessage);
809
944
  this.removeAllListeners();
810
945
  }
811
946
  log(...message) {