@micdrop/server 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/index.d.mts +106 -3
- package/dist/index.d.ts +106 -3
- package/dist/index.js +193 -58
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +194 -59
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -3
package/dist/index.js
CHANGED
|
@@ -503,6 +503,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
|
503
503
|
})(MicdropClientCommands || {});
|
|
504
504
|
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
505
505
|
MicdropServerCommands2["Message"] = "Message";
|
|
506
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
506
507
|
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
507
508
|
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
508
509
|
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
@@ -511,27 +512,35 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
|
511
512
|
})(MicdropServerCommands || {});
|
|
512
513
|
|
|
513
514
|
// src/MicdropServer.ts
|
|
515
|
+
var USER_SAMPLE_RATE = 16e3;
|
|
516
|
+
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
514
517
|
var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
515
518
|
constructor(socket, config) {
|
|
516
519
|
super();
|
|
517
520
|
this.socket = null;
|
|
518
521
|
this.config = null;
|
|
519
522
|
this.startTime = Date.now();
|
|
523
|
+
// Holds the conversation when no agent does, and keeps it readable once the
|
|
524
|
+
// call is over and the config is gone
|
|
525
|
+
this.ownConversation = [];
|
|
520
526
|
// Queue system for operations
|
|
521
527
|
this.operationQueue = [];
|
|
522
528
|
this.isProcessingQueue = false;
|
|
523
529
|
this.userSpeechChunks = 0;
|
|
524
530
|
this.onClose = () => {
|
|
531
|
+
this.releaseHeldTurn();
|
|
525
532
|
if (!this.config) return;
|
|
526
533
|
this.log("Connection closed");
|
|
527
534
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
528
|
-
this.
|
|
535
|
+
const conversation = this.conversation;
|
|
536
|
+
this.config.agent?.destroy();
|
|
529
537
|
this.config.stt.destroy();
|
|
530
|
-
this.config.tts
|
|
538
|
+
this.config.tts?.destroy();
|
|
531
539
|
this.emit("End", {
|
|
532
|
-
conversation
|
|
540
|
+
conversation,
|
|
533
541
|
duration
|
|
534
542
|
});
|
|
543
|
+
this.ownConversation = conversation;
|
|
535
544
|
this.socket = null;
|
|
536
545
|
this.config = null;
|
|
537
546
|
};
|
|
@@ -562,11 +571,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
562
571
|
return;
|
|
563
572
|
}
|
|
564
573
|
this.log(`User transcript: "${transcript}"`);
|
|
565
|
-
this.
|
|
574
|
+
this.addUserMessage(transcript);
|
|
566
575
|
if (!this.currentUserStream) {
|
|
567
576
|
this.log("User stopped speaking, answering");
|
|
568
|
-
this.
|
|
569
|
-
this.answer();
|
|
577
|
+
this.answerUserTurn();
|
|
570
578
|
}
|
|
571
579
|
};
|
|
572
580
|
this.onAudioTTS = (audio) => {
|
|
@@ -575,39 +583,44 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
575
583
|
this.socket.send(audio);
|
|
576
584
|
this.emit("AssistantAudio", audio);
|
|
577
585
|
};
|
|
586
|
+
this.onMessageAgent = (message) => {
|
|
587
|
+
this.sendMessage(message);
|
|
588
|
+
};
|
|
578
589
|
this.socket = socket;
|
|
579
590
|
this.config = config;
|
|
580
591
|
this.log(`Call started`);
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
(
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
)
|
|
606
|
-
);
|
|
592
|
+
config.stt.on("Transcript", this.onTranscriptSTT);
|
|
593
|
+
config.tts?.on("Audio", this.onAudioTTS);
|
|
594
|
+
const agent = config.agent;
|
|
595
|
+
if (agent) {
|
|
596
|
+
agent.on("Message", this.onMessageAgent);
|
|
597
|
+
agent.on(
|
|
598
|
+
"CancelLastUserMessage",
|
|
599
|
+
() => this.socket?.send("CancelLastUserMessage" /* CancelLastUserMessage */)
|
|
600
|
+
);
|
|
601
|
+
agent.on(
|
|
602
|
+
"SkipAnswer",
|
|
603
|
+
() => this.socket?.send("SkipAnswer" /* SkipAnswer */)
|
|
604
|
+
);
|
|
605
|
+
agent.on(
|
|
606
|
+
"EndCall",
|
|
607
|
+
() => this.socket?.send("EndCall" /* EndCall */)
|
|
608
|
+
);
|
|
609
|
+
agent.on(
|
|
610
|
+
"ToolCall",
|
|
611
|
+
(toolCall) => this.socket?.send(
|
|
612
|
+
`${"ToolCall" /* ToolCall */} ${JSON.stringify(toolCall)}`
|
|
613
|
+
)
|
|
614
|
+
);
|
|
615
|
+
}
|
|
607
616
|
queueMicrotask(() => this.sendFirstMessage());
|
|
608
617
|
socket.on("close", this.onClose);
|
|
609
618
|
socket.on("message", this.onMessage);
|
|
610
619
|
}
|
|
620
|
+
/** Everything said so far, whether an agent or the server keeps it */
|
|
621
|
+
get conversation() {
|
|
622
|
+
return this.config?.agent?.conversation ?? this.ownConversation;
|
|
623
|
+
}
|
|
611
624
|
log(...message) {
|
|
612
625
|
this.logger?.log(...message);
|
|
613
626
|
}
|
|
@@ -631,14 +644,15 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
631
644
|
this.processQueue();
|
|
632
645
|
}
|
|
633
646
|
cancel() {
|
|
634
|
-
this.config?.tts
|
|
635
|
-
this.config?.agent
|
|
647
|
+
this.config?.tts?.cancel();
|
|
648
|
+
this.config?.agent?.cancel();
|
|
636
649
|
this.operationQueue = [];
|
|
637
650
|
}
|
|
638
651
|
onUserAudio(chunk) {
|
|
639
652
|
this.log(`Received chunk (${chunk.byteLength} bytes)`);
|
|
640
653
|
this.currentUserStream?.write(chunk);
|
|
641
654
|
this.userSpeechChunks++;
|
|
655
|
+
this.config?.turnDetector?.push(pcm16ToFloat32(chunk), USER_SAMPLE_RATE);
|
|
642
656
|
this.emit("UserAudio", chunk);
|
|
643
657
|
}
|
|
644
658
|
onMute() {
|
|
@@ -652,6 +666,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
652
666
|
this.userSpeechChunks = 0;
|
|
653
667
|
this.currentUserStream?.end();
|
|
654
668
|
this.currentUserStream = new import_stream2.PassThrough();
|
|
669
|
+
this.config.turnDetector?.reset();
|
|
670
|
+
this.turnComplete = void 0;
|
|
671
|
+
this.releaseHeldTurn();
|
|
655
672
|
this.config.stt.transcribe(this.currentUserStream);
|
|
656
673
|
this.cancel();
|
|
657
674
|
}
|
|
@@ -664,22 +681,110 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
664
681
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
665
682
|
return;
|
|
666
683
|
}
|
|
667
|
-
|
|
668
|
-
const
|
|
684
|
+
this.turnComplete = this.predictTurnComplete();
|
|
685
|
+
const conversation = this.conversation;
|
|
686
|
+
const lastMessage = conversation[conversation.length - 1];
|
|
669
687
|
if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
|
|
670
688
|
this.log(
|
|
671
689
|
"User stopped speaking and a transcript already exists, answering"
|
|
672
690
|
);
|
|
691
|
+
this.answerUserTurn();
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
async predictTurnComplete() {
|
|
695
|
+
const detector = this.config?.turnDetector;
|
|
696
|
+
if (!detector) return true;
|
|
697
|
+
try {
|
|
698
|
+
const { complete } = await detector.predict();
|
|
699
|
+
this.log(`Turn sounds ${complete ? "finished" : "unfinished"}`);
|
|
700
|
+
return complete;
|
|
701
|
+
} catch (error) {
|
|
702
|
+
this.log(`Turn detection failed: ${error}`);
|
|
703
|
+
return true;
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
/** Answers, unless the sentence sounds like it has more coming */
|
|
707
|
+
async answerUserTurn() {
|
|
708
|
+
const complete = await (this.turnComplete ?? Promise.resolve(true));
|
|
709
|
+
if (!complete) {
|
|
710
|
+
this.log("Waiting for the rest of the sentence");
|
|
711
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
712
|
+
this.holdTurn();
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
this.releaseHeldTurn();
|
|
716
|
+
this.cancel();
|
|
717
|
+
this.answer();
|
|
718
|
+
}
|
|
719
|
+
/**
|
|
720
|
+
* Answers anyway if the rest of the sentence never comes.
|
|
721
|
+
*
|
|
722
|
+
* Without it, a detector that hears an unfinished sentence where there is
|
|
723
|
+
* none leaves the call silent for good.
|
|
724
|
+
*/
|
|
725
|
+
holdTurn() {
|
|
726
|
+
this.releaseHeldTurn();
|
|
727
|
+
this.heldTurnTimer = setTimeout(() => {
|
|
728
|
+
this.heldTurnTimer = void 0;
|
|
729
|
+
this.log("Nothing more came, answering");
|
|
673
730
|
this.cancel();
|
|
674
731
|
this.answer();
|
|
732
|
+
}, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
|
|
733
|
+
}
|
|
734
|
+
releaseHeldTurn() {
|
|
735
|
+
if (!this.heldTurnTimer) return;
|
|
736
|
+
clearTimeout(this.heldTurnTimer);
|
|
737
|
+
this.heldTurnTimer = void 0;
|
|
738
|
+
}
|
|
739
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
740
|
+
addUserMessage(text, metadata) {
|
|
741
|
+
if (this.config?.agent) {
|
|
742
|
+
this.config.agent.addUserMessage(text, metadata);
|
|
743
|
+
} else {
|
|
744
|
+
this.addOwnMessage("user", text, metadata);
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
748
|
+
addAssistantMessage(text, metadata) {
|
|
749
|
+
if (this.config?.agent) {
|
|
750
|
+
this.config.agent.addAssistantMessage(text, metadata);
|
|
751
|
+
} else {
|
|
752
|
+
this.addOwnMessage("assistant", text, metadata);
|
|
675
753
|
}
|
|
676
754
|
}
|
|
755
|
+
addOwnMessage(role, text, metadata) {
|
|
756
|
+
if (text.trim() === "") {
|
|
757
|
+
this.log(`Skipping empty ${role} message`);
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
this.log(`Adding ${role} message to conversation: ${text}`);
|
|
761
|
+
const message = {
|
|
762
|
+
role,
|
|
763
|
+
content: text,
|
|
764
|
+
metadata
|
|
765
|
+
};
|
|
766
|
+
this.ownConversation.push(message);
|
|
767
|
+
this.sendMessage(message);
|
|
768
|
+
}
|
|
769
|
+
sendMessage(message) {
|
|
770
|
+
this.socket?.send(
|
|
771
|
+
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
772
|
+
);
|
|
773
|
+
this.emit("Message", message);
|
|
774
|
+
}
|
|
775
|
+
sendPartialAnswer(content) {
|
|
776
|
+
if (!this.socket || content === "") return;
|
|
777
|
+
this.log(`Send partial answer: "${content}"`);
|
|
778
|
+
this.socket.send(
|
|
779
|
+
`${"PartialAssistantMessage" /* PartialAssistantMessage */} ${JSON.stringify(content)}`
|
|
780
|
+
);
|
|
781
|
+
}
|
|
677
782
|
sendFirstMessage() {
|
|
678
783
|
if (!this.config) return;
|
|
679
784
|
if (this.config.firstMessage) {
|
|
680
|
-
this.config.agent
|
|
785
|
+
this.config.agent?.addAssistantMessage(this.config.firstMessage);
|
|
681
786
|
this.speak(this.config.firstMessage);
|
|
682
|
-
} else if (this.config.generateFirstMessage) {
|
|
787
|
+
} else if (this.config.generateFirstMessage && this.config.agent) {
|
|
683
788
|
this.answer();
|
|
684
789
|
} else {
|
|
685
790
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -692,7 +797,13 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
692
797
|
}
|
|
693
798
|
async _answer() {
|
|
694
799
|
if (!this.config) return;
|
|
695
|
-
|
|
800
|
+
if (!this.config.agent) {
|
|
801
|
+
this.log("No agent, skipping answer");
|
|
802
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
803
|
+
return;
|
|
804
|
+
}
|
|
805
|
+
const conversation = this.config.agent.conversation;
|
|
806
|
+
const lastMessage = conversation[conversation.length - 1];
|
|
696
807
|
if (this.lastMessageSpeeched === lastMessage) {
|
|
697
808
|
this.log("Already answered, skipping");
|
|
698
809
|
return;
|
|
@@ -701,13 +812,30 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
701
812
|
try {
|
|
702
813
|
const stream = this.config.agent.answer();
|
|
703
814
|
if (await hasContent(stream)) {
|
|
704
|
-
await this._speak(
|
|
815
|
+
await this._speak(
|
|
816
|
+
this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
|
|
817
|
+
);
|
|
818
|
+
} else {
|
|
819
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
705
820
|
}
|
|
706
821
|
} catch (error) {
|
|
707
822
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
708
823
|
throw error;
|
|
709
824
|
}
|
|
710
825
|
}
|
|
826
|
+
/** Sends the answer to the client as it is written, ahead of the voice */
|
|
827
|
+
forwardPartialAnswer(stream) {
|
|
828
|
+
let content = "";
|
|
829
|
+
const send = (text) => this.sendPartialAnswer(text);
|
|
830
|
+
const forward = new import_stream2.Transform({
|
|
831
|
+
transform(chunk, _encoding, callback) {
|
|
832
|
+
content += chunk.toString();
|
|
833
|
+
send(content);
|
|
834
|
+
callback(null, chunk);
|
|
835
|
+
}
|
|
836
|
+
});
|
|
837
|
+
return stream.pipe(forward);
|
|
838
|
+
}
|
|
711
839
|
// Run text-to-speech and send to client
|
|
712
840
|
speak(message) {
|
|
713
841
|
this.queueOperation(async () => {
|
|
@@ -718,6 +846,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
718
846
|
if (!this.socket || !this.config) return;
|
|
719
847
|
let textStream;
|
|
720
848
|
if (typeof message === "string") {
|
|
849
|
+
if (!this.config.agent) {
|
|
850
|
+
this.addOwnMessage("assistant", message);
|
|
851
|
+
}
|
|
721
852
|
const stream = new import_stream2.PassThrough();
|
|
722
853
|
stream.write(message);
|
|
723
854
|
stream.end();
|
|
@@ -725,7 +856,12 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
725
856
|
} else {
|
|
726
857
|
textStream = message;
|
|
727
858
|
}
|
|
728
|
-
this.config.tts
|
|
859
|
+
if (this.config.tts) {
|
|
860
|
+
this.config.tts.speak(textStream);
|
|
861
|
+
return;
|
|
862
|
+
}
|
|
863
|
+
await drain(textStream);
|
|
864
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
729
865
|
}
|
|
730
866
|
};
|
|
731
867
|
function hasContent(stream) {
|
|
@@ -746,6 +882,15 @@ function hasContent(stream) {
|
|
|
746
882
|
stream.on("end", onEnd);
|
|
747
883
|
});
|
|
748
884
|
}
|
|
885
|
+
function drain(stream) {
|
|
886
|
+
return new Promise((resolve) => {
|
|
887
|
+
stream.on("data", () => {
|
|
888
|
+
});
|
|
889
|
+
stream.once("end", () => resolve());
|
|
890
|
+
stream.once("error", () => resolve());
|
|
891
|
+
stream.once("close", () => resolve());
|
|
892
|
+
});
|
|
893
|
+
}
|
|
749
894
|
|
|
750
895
|
// src/recorder/MicdropRecorder.ts
|
|
751
896
|
var import_eventemitter33 = require("eventemitter3");
|
|
@@ -783,9 +928,7 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
783
928
|
this.currentAssistantChunks.push(chunk);
|
|
784
929
|
};
|
|
785
930
|
this.onMessage = (message) => {
|
|
786
|
-
const
|
|
787
|
-
if (!conversation) return;
|
|
788
|
-
const messageIndex = conversation.length - 1;
|
|
931
|
+
const messageIndex = this.server.conversation.length - 1;
|
|
789
932
|
if (message.role === "user") {
|
|
790
933
|
this.lastUserMessageIndex = messageIndex;
|
|
791
934
|
if (this.currentUserChunks.length > 0) {
|
|
@@ -811,17 +954,13 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
811
954
|
this.server.on("UserAudio", this.onUserAudio);
|
|
812
955
|
this.server.on("AssistantAudio", this.onAssistantAudio);
|
|
813
956
|
this.server.on("End", this.onEnd);
|
|
814
|
-
|
|
815
|
-
if (agent) {
|
|
816
|
-
agent.on("Message", this.onMessage);
|
|
817
|
-
}
|
|
957
|
+
this.server.on("Message", this.onMessage);
|
|
818
958
|
}
|
|
819
959
|
finalizeUserAudio() {
|
|
820
960
|
if (this.currentUserChunks.length === 0) return;
|
|
821
961
|
if (this.lastUserMessageIndex < 0) return;
|
|
822
|
-
const
|
|
823
|
-
if (!
|
|
824
|
-
const message = conversation[this.lastUserMessageIndex];
|
|
962
|
+
const message = this.server.conversation[this.lastUserMessageIndex];
|
|
963
|
+
if (!message) return;
|
|
825
964
|
const buffer = Buffer.concat(this.currentUserChunks);
|
|
826
965
|
const audioMessage = {
|
|
827
966
|
buffer,
|
|
@@ -840,9 +979,8 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
840
979
|
finalizeAssistantAudio() {
|
|
841
980
|
if (this.currentAssistantChunks.length === 0) return;
|
|
842
981
|
if (this.lastAssistantMessageIndex < 0) return;
|
|
843
|
-
const
|
|
844
|
-
if (!
|
|
845
|
-
const message = conversation[this.lastAssistantMessageIndex];
|
|
982
|
+
const message = this.server.conversation[this.lastAssistantMessageIndex];
|
|
983
|
+
if (!message) return;
|
|
846
984
|
const buffer = Buffer.concat(this.currentAssistantChunks);
|
|
847
985
|
const audioMessage = {
|
|
848
986
|
buffer,
|
|
@@ -866,10 +1004,7 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
866
1004
|
this.server.off("UserAudio", this.onUserAudio);
|
|
867
1005
|
this.server.off("AssistantAudio", this.onAssistantAudio);
|
|
868
1006
|
this.server.off("End", this.onEnd);
|
|
869
|
-
|
|
870
|
-
if (agent) {
|
|
871
|
-
agent.off("Message", this.onMessage);
|
|
872
|
-
}
|
|
1007
|
+
this.server.off("Message", this.onMessage);
|
|
873
1008
|
this.removeAllListeners();
|
|
874
1009
|
}
|
|
875
1010
|
log(...message) {
|