@micdrop/server 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/index.d.mts +106 -3
- package/dist/index.d.ts +106 -3
- package/dist/index.js +193 -58
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +194 -59
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -3
package/dist/index.mjs
CHANGED
|
@@ -428,7 +428,7 @@ function handleError(socket, error) {
|
|
|
428
428
|
|
|
429
429
|
// src/MicdropServer.ts
|
|
430
430
|
import { EventEmitter as EventEmitter2 } from "eventemitter3";
|
|
431
|
-
import { PassThrough as PassThrough2 } from "stream";
|
|
431
|
+
import { PassThrough as PassThrough2, Transform } from "stream";
|
|
432
432
|
|
|
433
433
|
// src/types.ts
|
|
434
434
|
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
@@ -439,6 +439,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
|
439
439
|
})(MicdropClientCommands || {});
|
|
440
440
|
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
441
441
|
MicdropServerCommands2["Message"] = "Message";
|
|
442
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
442
443
|
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
443
444
|
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
444
445
|
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
@@ -447,27 +448,35 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
|
447
448
|
})(MicdropServerCommands || {});
|
|
448
449
|
|
|
449
450
|
// src/MicdropServer.ts
|
|
451
|
+
var USER_SAMPLE_RATE = 16e3;
|
|
452
|
+
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
450
453
|
var MicdropServer = class extends EventEmitter2 {
|
|
451
454
|
constructor(socket, config) {
|
|
452
455
|
super();
|
|
453
456
|
this.socket = null;
|
|
454
457
|
this.config = null;
|
|
455
458
|
this.startTime = Date.now();
|
|
459
|
+
// Holds the conversation when no agent does, and keeps it readable once the
|
|
460
|
+
// call is over and the config is gone
|
|
461
|
+
this.ownConversation = [];
|
|
456
462
|
// Queue system for operations
|
|
457
463
|
this.operationQueue = [];
|
|
458
464
|
this.isProcessingQueue = false;
|
|
459
465
|
this.userSpeechChunks = 0;
|
|
460
466
|
this.onClose = () => {
|
|
467
|
+
this.releaseHeldTurn();
|
|
461
468
|
if (!this.config) return;
|
|
462
469
|
this.log("Connection closed");
|
|
463
470
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
464
|
-
this.
|
|
471
|
+
const conversation = this.conversation;
|
|
472
|
+
this.config.agent?.destroy();
|
|
465
473
|
this.config.stt.destroy();
|
|
466
|
-
this.config.tts
|
|
474
|
+
this.config.tts?.destroy();
|
|
467
475
|
this.emit("End", {
|
|
468
|
-
conversation
|
|
476
|
+
conversation,
|
|
469
477
|
duration
|
|
470
478
|
});
|
|
479
|
+
this.ownConversation = conversation;
|
|
471
480
|
this.socket = null;
|
|
472
481
|
this.config = null;
|
|
473
482
|
};
|
|
@@ -498,11 +507,10 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
498
507
|
return;
|
|
499
508
|
}
|
|
500
509
|
this.log(`User transcript: "${transcript}"`);
|
|
501
|
-
this.
|
|
510
|
+
this.addUserMessage(transcript);
|
|
502
511
|
if (!this.currentUserStream) {
|
|
503
512
|
this.log("User stopped speaking, answering");
|
|
504
|
-
this.
|
|
505
|
-
this.answer();
|
|
513
|
+
this.answerUserTurn();
|
|
506
514
|
}
|
|
507
515
|
};
|
|
508
516
|
this.onAudioTTS = (audio) => {
|
|
@@ -511,39 +519,44 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
511
519
|
this.socket.send(audio);
|
|
512
520
|
this.emit("AssistantAudio", audio);
|
|
513
521
|
};
|
|
522
|
+
this.onMessageAgent = (message) => {
|
|
523
|
+
this.sendMessage(message);
|
|
524
|
+
};
|
|
514
525
|
this.socket = socket;
|
|
515
526
|
this.config = config;
|
|
516
527
|
this.log(`Call started`);
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
(
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
)
|
|
542
|
-
);
|
|
528
|
+
config.stt.on("Transcript", this.onTranscriptSTT);
|
|
529
|
+
config.tts?.on("Audio", this.onAudioTTS);
|
|
530
|
+
const agent = config.agent;
|
|
531
|
+
if (agent) {
|
|
532
|
+
agent.on("Message", this.onMessageAgent);
|
|
533
|
+
agent.on(
|
|
534
|
+
"CancelLastUserMessage",
|
|
535
|
+
() => this.socket?.send("CancelLastUserMessage" /* CancelLastUserMessage */)
|
|
536
|
+
);
|
|
537
|
+
agent.on(
|
|
538
|
+
"SkipAnswer",
|
|
539
|
+
() => this.socket?.send("SkipAnswer" /* SkipAnswer */)
|
|
540
|
+
);
|
|
541
|
+
agent.on(
|
|
542
|
+
"EndCall",
|
|
543
|
+
() => this.socket?.send("EndCall" /* EndCall */)
|
|
544
|
+
);
|
|
545
|
+
agent.on(
|
|
546
|
+
"ToolCall",
|
|
547
|
+
(toolCall) => this.socket?.send(
|
|
548
|
+
`${"ToolCall" /* ToolCall */} ${JSON.stringify(toolCall)}`
|
|
549
|
+
)
|
|
550
|
+
);
|
|
551
|
+
}
|
|
543
552
|
queueMicrotask(() => this.sendFirstMessage());
|
|
544
553
|
socket.on("close", this.onClose);
|
|
545
554
|
socket.on("message", this.onMessage);
|
|
546
555
|
}
|
|
556
|
+
/** Everything said so far, whether an agent or the server keeps it */
|
|
557
|
+
get conversation() {
|
|
558
|
+
return this.config?.agent?.conversation ?? this.ownConversation;
|
|
559
|
+
}
|
|
547
560
|
log(...message) {
|
|
548
561
|
this.logger?.log(...message);
|
|
549
562
|
}
|
|
@@ -567,14 +580,15 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
567
580
|
this.processQueue();
|
|
568
581
|
}
|
|
569
582
|
cancel() {
|
|
570
|
-
this.config?.tts
|
|
571
|
-
this.config?.agent
|
|
583
|
+
this.config?.tts?.cancel();
|
|
584
|
+
this.config?.agent?.cancel();
|
|
572
585
|
this.operationQueue = [];
|
|
573
586
|
}
|
|
574
587
|
onUserAudio(chunk) {
|
|
575
588
|
this.log(`Received chunk (${chunk.byteLength} bytes)`);
|
|
576
589
|
this.currentUserStream?.write(chunk);
|
|
577
590
|
this.userSpeechChunks++;
|
|
591
|
+
this.config?.turnDetector?.push(pcm16ToFloat32(chunk), USER_SAMPLE_RATE);
|
|
578
592
|
this.emit("UserAudio", chunk);
|
|
579
593
|
}
|
|
580
594
|
onMute() {
|
|
@@ -588,6 +602,9 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
588
602
|
this.userSpeechChunks = 0;
|
|
589
603
|
this.currentUserStream?.end();
|
|
590
604
|
this.currentUserStream = new PassThrough2();
|
|
605
|
+
this.config.turnDetector?.reset();
|
|
606
|
+
this.turnComplete = void 0;
|
|
607
|
+
this.releaseHeldTurn();
|
|
591
608
|
this.config.stt.transcribe(this.currentUserStream);
|
|
592
609
|
this.cancel();
|
|
593
610
|
}
|
|
@@ -600,22 +617,110 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
600
617
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
601
618
|
return;
|
|
602
619
|
}
|
|
603
|
-
|
|
604
|
-
const
|
|
620
|
+
this.turnComplete = this.predictTurnComplete();
|
|
621
|
+
const conversation = this.conversation;
|
|
622
|
+
const lastMessage = conversation[conversation.length - 1];
|
|
605
623
|
if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
|
|
606
624
|
this.log(
|
|
607
625
|
"User stopped speaking and a transcript already exists, answering"
|
|
608
626
|
);
|
|
627
|
+
this.answerUserTurn();
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
async predictTurnComplete() {
|
|
631
|
+
const detector = this.config?.turnDetector;
|
|
632
|
+
if (!detector) return true;
|
|
633
|
+
try {
|
|
634
|
+
const { complete } = await detector.predict();
|
|
635
|
+
this.log(`Turn sounds ${complete ? "finished" : "unfinished"}`);
|
|
636
|
+
return complete;
|
|
637
|
+
} catch (error) {
|
|
638
|
+
this.log(`Turn detection failed: ${error}`);
|
|
639
|
+
return true;
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
/** Answers, unless the sentence sounds like it has more coming */
|
|
643
|
+
async answerUserTurn() {
|
|
644
|
+
const complete = await (this.turnComplete ?? Promise.resolve(true));
|
|
645
|
+
if (!complete) {
|
|
646
|
+
this.log("Waiting for the rest of the sentence");
|
|
647
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
648
|
+
this.holdTurn();
|
|
649
|
+
return;
|
|
650
|
+
}
|
|
651
|
+
this.releaseHeldTurn();
|
|
652
|
+
this.cancel();
|
|
653
|
+
this.answer();
|
|
654
|
+
}
|
|
655
|
+
/**
|
|
656
|
+
* Answers anyway if the rest of the sentence never comes.
|
|
657
|
+
*
|
|
658
|
+
* Without it, a detector that hears an unfinished sentence where there is
|
|
659
|
+
* none leaves the call silent for good.
|
|
660
|
+
*/
|
|
661
|
+
holdTurn() {
|
|
662
|
+
this.releaseHeldTurn();
|
|
663
|
+
this.heldTurnTimer = setTimeout(() => {
|
|
664
|
+
this.heldTurnTimer = void 0;
|
|
665
|
+
this.log("Nothing more came, answering");
|
|
609
666
|
this.cancel();
|
|
610
667
|
this.answer();
|
|
668
|
+
}, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
|
|
669
|
+
}
|
|
670
|
+
releaseHeldTurn() {
|
|
671
|
+
if (!this.heldTurnTimer) return;
|
|
672
|
+
clearTimeout(this.heldTurnTimer);
|
|
673
|
+
this.heldTurnTimer = void 0;
|
|
674
|
+
}
|
|
675
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
676
|
+
addUserMessage(text, metadata) {
|
|
677
|
+
if (this.config?.agent) {
|
|
678
|
+
this.config.agent.addUserMessage(text, metadata);
|
|
679
|
+
} else {
|
|
680
|
+
this.addOwnMessage("user", text, metadata);
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
684
|
+
addAssistantMessage(text, metadata) {
|
|
685
|
+
if (this.config?.agent) {
|
|
686
|
+
this.config.agent.addAssistantMessage(text, metadata);
|
|
687
|
+
} else {
|
|
688
|
+
this.addOwnMessage("assistant", text, metadata);
|
|
611
689
|
}
|
|
612
690
|
}
|
|
691
|
+
addOwnMessage(role, text, metadata) {
|
|
692
|
+
if (text.trim() === "") {
|
|
693
|
+
this.log(`Skipping empty ${role} message`);
|
|
694
|
+
return;
|
|
695
|
+
}
|
|
696
|
+
this.log(`Adding ${role} message to conversation: ${text}`);
|
|
697
|
+
const message = {
|
|
698
|
+
role,
|
|
699
|
+
content: text,
|
|
700
|
+
metadata
|
|
701
|
+
};
|
|
702
|
+
this.ownConversation.push(message);
|
|
703
|
+
this.sendMessage(message);
|
|
704
|
+
}
|
|
705
|
+
sendMessage(message) {
|
|
706
|
+
this.socket?.send(
|
|
707
|
+
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
708
|
+
);
|
|
709
|
+
this.emit("Message", message);
|
|
710
|
+
}
|
|
711
|
+
sendPartialAnswer(content) {
|
|
712
|
+
if (!this.socket || content === "") return;
|
|
713
|
+
this.log(`Send partial answer: "${content}"`);
|
|
714
|
+
this.socket.send(
|
|
715
|
+
`${"PartialAssistantMessage" /* PartialAssistantMessage */} ${JSON.stringify(content)}`
|
|
716
|
+
);
|
|
717
|
+
}
|
|
613
718
|
sendFirstMessage() {
|
|
614
719
|
if (!this.config) return;
|
|
615
720
|
if (this.config.firstMessage) {
|
|
616
|
-
this.config.agent
|
|
721
|
+
this.config.agent?.addAssistantMessage(this.config.firstMessage);
|
|
617
722
|
this.speak(this.config.firstMessage);
|
|
618
|
-
} else if (this.config.generateFirstMessage) {
|
|
723
|
+
} else if (this.config.generateFirstMessage && this.config.agent) {
|
|
619
724
|
this.answer();
|
|
620
725
|
} else {
|
|
621
726
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -628,7 +733,13 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
628
733
|
}
|
|
629
734
|
async _answer() {
|
|
630
735
|
if (!this.config) return;
|
|
631
|
-
|
|
736
|
+
if (!this.config.agent) {
|
|
737
|
+
this.log("No agent, skipping answer");
|
|
738
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
const conversation = this.config.agent.conversation;
|
|
742
|
+
const lastMessage = conversation[conversation.length - 1];
|
|
632
743
|
if (this.lastMessageSpeeched === lastMessage) {
|
|
633
744
|
this.log("Already answered, skipping");
|
|
634
745
|
return;
|
|
@@ -637,13 +748,30 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
637
748
|
try {
|
|
638
749
|
const stream = this.config.agent.answer();
|
|
639
750
|
if (await hasContent(stream)) {
|
|
640
|
-
await this._speak(
|
|
751
|
+
await this._speak(
|
|
752
|
+
this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
|
|
753
|
+
);
|
|
754
|
+
} else {
|
|
755
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
641
756
|
}
|
|
642
757
|
} catch (error) {
|
|
643
758
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
644
759
|
throw error;
|
|
645
760
|
}
|
|
646
761
|
}
|
|
762
|
+
/** Sends the answer to the client as it is written, ahead of the voice */
|
|
763
|
+
forwardPartialAnswer(stream) {
|
|
764
|
+
let content = "";
|
|
765
|
+
const send = (text) => this.sendPartialAnswer(text);
|
|
766
|
+
const forward = new Transform({
|
|
767
|
+
transform(chunk, _encoding, callback) {
|
|
768
|
+
content += chunk.toString();
|
|
769
|
+
send(content);
|
|
770
|
+
callback(null, chunk);
|
|
771
|
+
}
|
|
772
|
+
});
|
|
773
|
+
return stream.pipe(forward);
|
|
774
|
+
}
|
|
647
775
|
// Run text-to-speech and send to client
|
|
648
776
|
speak(message) {
|
|
649
777
|
this.queueOperation(async () => {
|
|
@@ -654,6 +782,9 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
654
782
|
if (!this.socket || !this.config) return;
|
|
655
783
|
let textStream;
|
|
656
784
|
if (typeof message === "string") {
|
|
785
|
+
if (!this.config.agent) {
|
|
786
|
+
this.addOwnMessage("assistant", message);
|
|
787
|
+
}
|
|
657
788
|
const stream = new PassThrough2();
|
|
658
789
|
stream.write(message);
|
|
659
790
|
stream.end();
|
|
@@ -661,7 +792,12 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
661
792
|
} else {
|
|
662
793
|
textStream = message;
|
|
663
794
|
}
|
|
664
|
-
this.config.tts
|
|
795
|
+
if (this.config.tts) {
|
|
796
|
+
this.config.tts.speak(textStream);
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
await drain(textStream);
|
|
800
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
665
801
|
}
|
|
666
802
|
};
|
|
667
803
|
function hasContent(stream) {
|
|
@@ -682,6 +818,15 @@ function hasContent(stream) {
|
|
|
682
818
|
stream.on("end", onEnd);
|
|
683
819
|
});
|
|
684
820
|
}
|
|
821
|
+
function drain(stream) {
|
|
822
|
+
return new Promise((resolve) => {
|
|
823
|
+
stream.on("data", () => {
|
|
824
|
+
});
|
|
825
|
+
stream.once("end", () => resolve());
|
|
826
|
+
stream.once("error", () => resolve());
|
|
827
|
+
stream.once("close", () => resolve());
|
|
828
|
+
});
|
|
829
|
+
}
|
|
685
830
|
|
|
686
831
|
// src/recorder/MicdropRecorder.ts
|
|
687
832
|
import { EventEmitter as EventEmitter3 } from "eventemitter3";
|
|
@@ -719,9 +864,7 @@ var MicdropRecorder = class extends EventEmitter3 {
|
|
|
719
864
|
this.currentAssistantChunks.push(chunk);
|
|
720
865
|
};
|
|
721
866
|
this.onMessage = (message) => {
|
|
722
|
-
const
|
|
723
|
-
if (!conversation) return;
|
|
724
|
-
const messageIndex = conversation.length - 1;
|
|
867
|
+
const messageIndex = this.server.conversation.length - 1;
|
|
725
868
|
if (message.role === "user") {
|
|
726
869
|
this.lastUserMessageIndex = messageIndex;
|
|
727
870
|
if (this.currentUserChunks.length > 0) {
|
|
@@ -747,17 +890,13 @@ var MicdropRecorder = class extends EventEmitter3 {
|
|
|
747
890
|
this.server.on("UserAudio", this.onUserAudio);
|
|
748
891
|
this.server.on("AssistantAudio", this.onAssistantAudio);
|
|
749
892
|
this.server.on("End", this.onEnd);
|
|
750
|
-
|
|
751
|
-
if (agent) {
|
|
752
|
-
agent.on("Message", this.onMessage);
|
|
753
|
-
}
|
|
893
|
+
this.server.on("Message", this.onMessage);
|
|
754
894
|
}
|
|
755
895
|
finalizeUserAudio() {
|
|
756
896
|
if (this.currentUserChunks.length === 0) return;
|
|
757
897
|
if (this.lastUserMessageIndex < 0) return;
|
|
758
|
-
const
|
|
759
|
-
if (!
|
|
760
|
-
const message = conversation[this.lastUserMessageIndex];
|
|
898
|
+
const message = this.server.conversation[this.lastUserMessageIndex];
|
|
899
|
+
if (!message) return;
|
|
761
900
|
const buffer = Buffer.concat(this.currentUserChunks);
|
|
762
901
|
const audioMessage = {
|
|
763
902
|
buffer,
|
|
@@ -776,9 +915,8 @@ var MicdropRecorder = class extends EventEmitter3 {
|
|
|
776
915
|
finalizeAssistantAudio() {
|
|
777
916
|
if (this.currentAssistantChunks.length === 0) return;
|
|
778
917
|
if (this.lastAssistantMessageIndex < 0) return;
|
|
779
|
-
const
|
|
780
|
-
if (!
|
|
781
|
-
const message = conversation[this.lastAssistantMessageIndex];
|
|
918
|
+
const message = this.server.conversation[this.lastAssistantMessageIndex];
|
|
919
|
+
if (!message) return;
|
|
782
920
|
const buffer = Buffer.concat(this.currentAssistantChunks);
|
|
783
921
|
const audioMessage = {
|
|
784
922
|
buffer,
|
|
@@ -802,10 +940,7 @@ var MicdropRecorder = class extends EventEmitter3 {
|
|
|
802
940
|
this.server.off("UserAudio", this.onUserAudio);
|
|
803
941
|
this.server.off("AssistantAudio", this.onAssistantAudio);
|
|
804
942
|
this.server.off("End", this.onEnd);
|
|
805
|
-
|
|
806
|
-
if (agent) {
|
|
807
|
-
agent.off("Message", this.onMessage);
|
|
808
|
-
}
|
|
943
|
+
this.server.off("Message", this.onMessage);
|
|
809
944
|
this.removeAllListeners();
|
|
810
945
|
}
|
|
811
946
|
log(...message) {
|