@micdrop/server 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/index.d.mts +59 -2
- package/dist/index.d.ts +59 -2
- package/dist/index.js +140 -56
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +141 -57
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -3
package/dist/index.mjs
CHANGED
|
@@ -428,7 +428,7 @@ function handleError(socket, error) {
|
|
|
428
428
|
|
|
429
429
|
// src/MicdropServer.ts
|
|
430
430
|
import { EventEmitter as EventEmitter2 } from "eventemitter3";
|
|
431
|
-
import { PassThrough as PassThrough2 } from "stream";
|
|
431
|
+
import { PassThrough as PassThrough2, Transform } from "stream";
|
|
432
432
|
|
|
433
433
|
// src/types.ts
|
|
434
434
|
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
@@ -439,6 +439,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
|
439
439
|
})(MicdropClientCommands || {});
|
|
440
440
|
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
441
441
|
MicdropServerCommands2["Message"] = "Message";
|
|
442
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
442
443
|
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
443
444
|
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
444
445
|
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
@@ -455,6 +456,9 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
455
456
|
this.socket = null;
|
|
456
457
|
this.config = null;
|
|
457
458
|
this.startTime = Date.now();
|
|
459
|
+
// Holds the conversation when no agent does, and keeps it readable once the
|
|
460
|
+
// call is over and the config is gone
|
|
461
|
+
this.ownConversation = [];
|
|
458
462
|
// Queue system for operations
|
|
459
463
|
this.operationQueue = [];
|
|
460
464
|
this.isProcessingQueue = false;
|
|
@@ -464,13 +468,15 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
464
468
|
if (!this.config) return;
|
|
465
469
|
this.log("Connection closed");
|
|
466
470
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
467
|
-
this.
|
|
471
|
+
const conversation = this.conversation;
|
|
472
|
+
this.config.agent?.destroy();
|
|
468
473
|
this.config.stt.destroy();
|
|
469
|
-
this.config.tts
|
|
474
|
+
this.config.tts?.destroy();
|
|
470
475
|
this.emit("End", {
|
|
471
|
-
conversation
|
|
476
|
+
conversation,
|
|
472
477
|
duration
|
|
473
478
|
});
|
|
479
|
+
this.ownConversation = conversation;
|
|
474
480
|
this.socket = null;
|
|
475
481
|
this.config = null;
|
|
476
482
|
};
|
|
@@ -501,7 +507,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
501
507
|
return;
|
|
502
508
|
}
|
|
503
509
|
this.log(`User transcript: "${transcript}"`);
|
|
504
|
-
this.
|
|
510
|
+
this.addUserMessage(transcript);
|
|
505
511
|
if (!this.currentUserStream) {
|
|
506
512
|
this.log("User stopped speaking, answering");
|
|
507
513
|
this.answerUserTurn();
|
|
@@ -513,39 +519,44 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
513
519
|
this.socket.send(audio);
|
|
514
520
|
this.emit("AssistantAudio", audio);
|
|
515
521
|
};
|
|
522
|
+
this.onMessageAgent = (message) => {
|
|
523
|
+
this.sendMessage(message);
|
|
524
|
+
};
|
|
516
525
|
this.socket = socket;
|
|
517
526
|
this.config = config;
|
|
518
527
|
this.log(`Call started`);
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
(
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
)
|
|
544
|
-
);
|
|
528
|
+
config.stt.on("Transcript", this.onTranscriptSTT);
|
|
529
|
+
config.tts?.on("Audio", this.onAudioTTS);
|
|
530
|
+
const agent = config.agent;
|
|
531
|
+
if (agent) {
|
|
532
|
+
agent.on("Message", this.onMessageAgent);
|
|
533
|
+
agent.on(
|
|
534
|
+
"CancelLastUserMessage",
|
|
535
|
+
() => this.socket?.send("CancelLastUserMessage" /* CancelLastUserMessage */)
|
|
536
|
+
);
|
|
537
|
+
agent.on(
|
|
538
|
+
"SkipAnswer",
|
|
539
|
+
() => this.socket?.send("SkipAnswer" /* SkipAnswer */)
|
|
540
|
+
);
|
|
541
|
+
agent.on(
|
|
542
|
+
"EndCall",
|
|
543
|
+
() => this.socket?.send("EndCall" /* EndCall */)
|
|
544
|
+
);
|
|
545
|
+
agent.on(
|
|
546
|
+
"ToolCall",
|
|
547
|
+
(toolCall) => this.socket?.send(
|
|
548
|
+
`${"ToolCall" /* ToolCall */} ${JSON.stringify(toolCall)}`
|
|
549
|
+
)
|
|
550
|
+
);
|
|
551
|
+
}
|
|
545
552
|
queueMicrotask(() => this.sendFirstMessage());
|
|
546
553
|
socket.on("close", this.onClose);
|
|
547
554
|
socket.on("message", this.onMessage);
|
|
548
555
|
}
|
|
556
|
+
/** Everything said so far, whether an agent or the server keeps it */
|
|
557
|
+
get conversation() {
|
|
558
|
+
return this.config?.agent?.conversation ?? this.ownConversation;
|
|
559
|
+
}
|
|
549
560
|
log(...message) {
|
|
550
561
|
this.logger?.log(...message);
|
|
551
562
|
}
|
|
@@ -569,8 +580,8 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
569
580
|
this.processQueue();
|
|
570
581
|
}
|
|
571
582
|
cancel() {
|
|
572
|
-
this.config?.tts
|
|
573
|
-
this.config?.agent
|
|
583
|
+
this.config?.tts?.cancel();
|
|
584
|
+
this.config?.agent?.cancel();
|
|
574
585
|
this.operationQueue = [];
|
|
575
586
|
}
|
|
576
587
|
onUserAudio(chunk) {
|
|
@@ -607,8 +618,8 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
607
618
|
return;
|
|
608
619
|
}
|
|
609
620
|
this.turnComplete = this.predictTurnComplete();
|
|
610
|
-
const conversation = this.
|
|
611
|
-
const lastMessage = conversation
|
|
621
|
+
const conversation = this.conversation;
|
|
622
|
+
const lastMessage = conversation[conversation.length - 1];
|
|
612
623
|
if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
|
|
613
624
|
this.log(
|
|
614
625
|
"User stopped speaking and a transcript already exists, answering"
|
|
@@ -661,12 +672,55 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
661
672
|
clearTimeout(this.heldTurnTimer);
|
|
662
673
|
this.heldTurnTimer = void 0;
|
|
663
674
|
}
|
|
675
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
676
|
+
addUserMessage(text, metadata) {
|
|
677
|
+
if (this.config?.agent) {
|
|
678
|
+
this.config.agent.addUserMessage(text, metadata);
|
|
679
|
+
} else {
|
|
680
|
+
this.addOwnMessage("user", text, metadata);
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
684
|
+
addAssistantMessage(text, metadata) {
|
|
685
|
+
if (this.config?.agent) {
|
|
686
|
+
this.config.agent.addAssistantMessage(text, metadata);
|
|
687
|
+
} else {
|
|
688
|
+
this.addOwnMessage("assistant", text, metadata);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
addOwnMessage(role, text, metadata) {
|
|
692
|
+
if (text.trim() === "") {
|
|
693
|
+
this.log(`Skipping empty ${role} message`);
|
|
694
|
+
return;
|
|
695
|
+
}
|
|
696
|
+
this.log(`Adding ${role} message to conversation: ${text}`);
|
|
697
|
+
const message = {
|
|
698
|
+
role,
|
|
699
|
+
content: text,
|
|
700
|
+
metadata
|
|
701
|
+
};
|
|
702
|
+
this.ownConversation.push(message);
|
|
703
|
+
this.sendMessage(message);
|
|
704
|
+
}
|
|
705
|
+
sendMessage(message) {
|
|
706
|
+
this.socket?.send(
|
|
707
|
+
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
708
|
+
);
|
|
709
|
+
this.emit("Message", message);
|
|
710
|
+
}
|
|
711
|
+
sendPartialAnswer(content) {
|
|
712
|
+
if (!this.socket || content === "") return;
|
|
713
|
+
this.log(`Send partial answer: "${content}"`);
|
|
714
|
+
this.socket.send(
|
|
715
|
+
`${"PartialAssistantMessage" /* PartialAssistantMessage */} ${JSON.stringify(content)}`
|
|
716
|
+
);
|
|
717
|
+
}
|
|
664
718
|
sendFirstMessage() {
|
|
665
719
|
if (!this.config) return;
|
|
666
720
|
if (this.config.firstMessage) {
|
|
667
|
-
this.config.agent
|
|
721
|
+
this.config.agent?.addAssistantMessage(this.config.firstMessage);
|
|
668
722
|
this.speak(this.config.firstMessage);
|
|
669
|
-
} else if (this.config.generateFirstMessage) {
|
|
723
|
+
} else if (this.config.generateFirstMessage && this.config.agent) {
|
|
670
724
|
this.answer();
|
|
671
725
|
} else {
|
|
672
726
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -679,7 +733,13 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
679
733
|
}
|
|
680
734
|
async _answer() {
|
|
681
735
|
if (!this.config) return;
|
|
682
|
-
|
|
736
|
+
if (!this.config.agent) {
|
|
737
|
+
this.log("No agent, skipping answer");
|
|
738
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
const conversation = this.config.agent.conversation;
|
|
742
|
+
const lastMessage = conversation[conversation.length - 1];
|
|
683
743
|
if (this.lastMessageSpeeched === lastMessage) {
|
|
684
744
|
this.log("Already answered, skipping");
|
|
685
745
|
return;
|
|
@@ -688,13 +748,30 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
688
748
|
try {
|
|
689
749
|
const stream = this.config.agent.answer();
|
|
690
750
|
if (await hasContent(stream)) {
|
|
691
|
-
await this._speak(
|
|
751
|
+
await this._speak(
|
|
752
|
+
this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
|
|
753
|
+
);
|
|
754
|
+
} else {
|
|
755
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
692
756
|
}
|
|
693
757
|
} catch (error) {
|
|
694
758
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
695
759
|
throw error;
|
|
696
760
|
}
|
|
697
761
|
}
|
|
762
|
+
/** Sends the answer to the client as it is written, ahead of the voice */
|
|
763
|
+
forwardPartialAnswer(stream) {
|
|
764
|
+
let content = "";
|
|
765
|
+
const send = (text) => this.sendPartialAnswer(text);
|
|
766
|
+
const forward = new Transform({
|
|
767
|
+
transform(chunk, _encoding, callback) {
|
|
768
|
+
content += chunk.toString();
|
|
769
|
+
send(content);
|
|
770
|
+
callback(null, chunk);
|
|
771
|
+
}
|
|
772
|
+
});
|
|
773
|
+
return stream.pipe(forward);
|
|
774
|
+
}
|
|
698
775
|
// Run text-to-speech and send to client
|
|
699
776
|
speak(message) {
|
|
700
777
|
this.queueOperation(async () => {
|
|
@@ -705,6 +782,9 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
705
782
|
if (!this.socket || !this.config) return;
|
|
706
783
|
let textStream;
|
|
707
784
|
if (typeof message === "string") {
|
|
785
|
+
if (!this.config.agent) {
|
|
786
|
+
this.addOwnMessage("assistant", message);
|
|
787
|
+
}
|
|
708
788
|
const stream = new PassThrough2();
|
|
709
789
|
stream.write(message);
|
|
710
790
|
stream.end();
|
|
@@ -712,7 +792,12 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
712
792
|
} else {
|
|
713
793
|
textStream = message;
|
|
714
794
|
}
|
|
715
|
-
this.config.tts
|
|
795
|
+
if (this.config.tts) {
|
|
796
|
+
this.config.tts.speak(textStream);
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
await drain(textStream);
|
|
800
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
716
801
|
}
|
|
717
802
|
};
|
|
718
803
|
function hasContent(stream) {
|
|
@@ -733,6 +818,15 @@ function hasContent(stream) {
|
|
|
733
818
|
stream.on("end", onEnd);
|
|
734
819
|
});
|
|
735
820
|
}
|
|
821
|
+
function drain(stream) {
|
|
822
|
+
return new Promise((resolve) => {
|
|
823
|
+
stream.on("data", () => {
|
|
824
|
+
});
|
|
825
|
+
stream.once("end", () => resolve());
|
|
826
|
+
stream.once("error", () => resolve());
|
|
827
|
+
stream.once("close", () => resolve());
|
|
828
|
+
});
|
|
829
|
+
}
|
|
736
830
|
|
|
737
831
|
// src/recorder/MicdropRecorder.ts
|
|
738
832
|
import { EventEmitter as EventEmitter3 } from "eventemitter3";
|
|
@@ -770,9 +864,7 @@ var MicdropRecorder = class extends EventEmitter3 {
|
|
|
770
864
|
this.currentAssistantChunks.push(chunk);
|
|
771
865
|
};
|
|
772
866
|
this.onMessage = (message) => {
|
|
773
|
-
const
|
|
774
|
-
if (!conversation) return;
|
|
775
|
-
const messageIndex = conversation.length - 1;
|
|
867
|
+
const messageIndex = this.server.conversation.length - 1;
|
|
776
868
|
if (message.role === "user") {
|
|
777
869
|
this.lastUserMessageIndex = messageIndex;
|
|
778
870
|
if (this.currentUserChunks.length > 0) {
|
|
@@ -798,17 +890,13 @@ var MicdropRecorder = class extends EventEmitter3 {
|
|
|
798
890
|
this.server.on("UserAudio", this.onUserAudio);
|
|
799
891
|
this.server.on("AssistantAudio", this.onAssistantAudio);
|
|
800
892
|
this.server.on("End", this.onEnd);
|
|
801
|
-
|
|
802
|
-
if (agent) {
|
|
803
|
-
agent.on("Message", this.onMessage);
|
|
804
|
-
}
|
|
893
|
+
this.server.on("Message", this.onMessage);
|
|
805
894
|
}
|
|
806
895
|
finalizeUserAudio() {
|
|
807
896
|
if (this.currentUserChunks.length === 0) return;
|
|
808
897
|
if (this.lastUserMessageIndex < 0) return;
|
|
809
|
-
const
|
|
810
|
-
if (!
|
|
811
|
-
const message = conversation[this.lastUserMessageIndex];
|
|
898
|
+
const message = this.server.conversation[this.lastUserMessageIndex];
|
|
899
|
+
if (!message) return;
|
|
812
900
|
const buffer = Buffer.concat(this.currentUserChunks);
|
|
813
901
|
const audioMessage = {
|
|
814
902
|
buffer,
|
|
@@ -827,9 +915,8 @@ var MicdropRecorder = class extends EventEmitter3 {
|
|
|
827
915
|
finalizeAssistantAudio() {
|
|
828
916
|
if (this.currentAssistantChunks.length === 0) return;
|
|
829
917
|
if (this.lastAssistantMessageIndex < 0) return;
|
|
830
|
-
const
|
|
831
|
-
if (!
|
|
832
|
-
const message = conversation[this.lastAssistantMessageIndex];
|
|
918
|
+
const message = this.server.conversation[this.lastAssistantMessageIndex];
|
|
919
|
+
if (!message) return;
|
|
833
920
|
const buffer = Buffer.concat(this.currentAssistantChunks);
|
|
834
921
|
const audioMessage = {
|
|
835
922
|
buffer,
|
|
@@ -853,10 +940,7 @@ var MicdropRecorder = class extends EventEmitter3 {
|
|
|
853
940
|
this.server.off("UserAudio", this.onUserAudio);
|
|
854
941
|
this.server.off("AssistantAudio", this.onAssistantAudio);
|
|
855
942
|
this.server.off("End", this.onEnd);
|
|
856
|
-
|
|
857
|
-
if (agent) {
|
|
858
|
-
agent.off("Message", this.onMessage);
|
|
859
|
-
}
|
|
943
|
+
this.server.off("Message", this.onMessage);
|
|
860
944
|
this.removeAllListeners();
|
|
861
945
|
}
|
|
862
946
|
log(...message) {
|