@micdrop/server 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/index.d.mts +99 -3
- package/dist/index.d.ts +99 -3
- package/dist/index.js +180 -56
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +180 -57
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -3
package/dist/index.js
CHANGED
|
@@ -54,6 +54,7 @@ __export(index_exports, {
|
|
|
54
54
|
STT: () => STT,
|
|
55
55
|
SentenceSplitter: () => SentenceSplitter,
|
|
56
56
|
SentenceTTS: () => SentenceTTS,
|
|
57
|
+
SharedInstances: () => SharedInstances,
|
|
57
58
|
TTS: () => TTS,
|
|
58
59
|
float32ToPcm16: () => float32ToPcm16,
|
|
59
60
|
handleError: () => handleError,
|
|
@@ -503,6 +504,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
|
503
504
|
})(MicdropClientCommands || {});
|
|
504
505
|
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
505
506
|
MicdropServerCommands2["Message"] = "Message";
|
|
507
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
506
508
|
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
507
509
|
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
508
510
|
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
@@ -519,6 +521,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
519
521
|
this.socket = null;
|
|
520
522
|
this.config = null;
|
|
521
523
|
this.startTime = Date.now();
|
|
524
|
+
// Holds the conversation when no agent does, and keeps it readable once the
|
|
525
|
+
// call is over and the config is gone
|
|
526
|
+
this.ownConversation = [];
|
|
522
527
|
// Queue system for operations
|
|
523
528
|
this.operationQueue = [];
|
|
524
529
|
this.isProcessingQueue = false;
|
|
@@ -528,13 +533,15 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
528
533
|
if (!this.config) return;
|
|
529
534
|
this.log("Connection closed");
|
|
530
535
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
531
|
-
this.
|
|
536
|
+
const conversation = this.conversation;
|
|
537
|
+
this.config.agent?.destroy();
|
|
532
538
|
this.config.stt.destroy();
|
|
533
|
-
this.config.tts
|
|
539
|
+
this.config.tts?.destroy();
|
|
534
540
|
this.emit("End", {
|
|
535
|
-
conversation
|
|
541
|
+
conversation,
|
|
536
542
|
duration
|
|
537
543
|
});
|
|
544
|
+
this.ownConversation = conversation;
|
|
538
545
|
this.socket = null;
|
|
539
546
|
this.config = null;
|
|
540
547
|
};
|
|
@@ -565,7 +572,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
565
572
|
return;
|
|
566
573
|
}
|
|
567
574
|
this.log(`User transcript: "${transcript}"`);
|
|
568
|
-
this.
|
|
575
|
+
this.addUserMessage(transcript);
|
|
569
576
|
if (!this.currentUserStream) {
|
|
570
577
|
this.log("User stopped speaking, answering");
|
|
571
578
|
this.answerUserTurn();
|
|
@@ -577,39 +584,44 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
577
584
|
this.socket.send(audio);
|
|
578
585
|
this.emit("AssistantAudio", audio);
|
|
579
586
|
};
|
|
587
|
+
this.onMessageAgent = (message) => {
|
|
588
|
+
this.sendMessage(message);
|
|
589
|
+
};
|
|
580
590
|
this.socket = socket;
|
|
581
591
|
this.config = config;
|
|
582
592
|
this.log(`Call started`);
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
(
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
)
|
|
608
|
-
);
|
|
593
|
+
config.stt.on("Transcript", this.onTranscriptSTT);
|
|
594
|
+
config.tts?.on("Audio", this.onAudioTTS);
|
|
595
|
+
const agent = config.agent;
|
|
596
|
+
if (agent) {
|
|
597
|
+
agent.on("Message", this.onMessageAgent);
|
|
598
|
+
agent.on(
|
|
599
|
+
"CancelLastUserMessage",
|
|
600
|
+
() => this.socket?.send("CancelLastUserMessage" /* CancelLastUserMessage */)
|
|
601
|
+
);
|
|
602
|
+
agent.on(
|
|
603
|
+
"SkipAnswer",
|
|
604
|
+
() => this.socket?.send("SkipAnswer" /* SkipAnswer */)
|
|
605
|
+
);
|
|
606
|
+
agent.on(
|
|
607
|
+
"EndCall",
|
|
608
|
+
() => this.socket?.send("EndCall" /* EndCall */)
|
|
609
|
+
);
|
|
610
|
+
agent.on(
|
|
611
|
+
"ToolCall",
|
|
612
|
+
(toolCall) => this.socket?.send(
|
|
613
|
+
`${"ToolCall" /* ToolCall */} ${JSON.stringify(toolCall)}`
|
|
614
|
+
)
|
|
615
|
+
);
|
|
616
|
+
}
|
|
609
617
|
queueMicrotask(() => this.sendFirstMessage());
|
|
610
618
|
socket.on("close", this.onClose);
|
|
611
619
|
socket.on("message", this.onMessage);
|
|
612
620
|
}
|
|
621
|
+
/** Everything said so far, whether an agent or the server keeps it */
|
|
622
|
+
get conversation() {
|
|
623
|
+
return this.config?.agent?.conversation ?? this.ownConversation;
|
|
624
|
+
}
|
|
613
625
|
log(...message) {
|
|
614
626
|
this.logger?.log(...message);
|
|
615
627
|
}
|
|
@@ -633,8 +645,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
633
645
|
this.processQueue();
|
|
634
646
|
}
|
|
635
647
|
cancel() {
|
|
636
|
-
this.config?.tts
|
|
637
|
-
this.config?.agent
|
|
648
|
+
this.config?.tts?.cancel();
|
|
649
|
+
this.config?.agent?.cancel();
|
|
638
650
|
this.operationQueue = [];
|
|
639
651
|
}
|
|
640
652
|
onUserAudio(chunk) {
|
|
@@ -671,8 +683,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
671
683
|
return;
|
|
672
684
|
}
|
|
673
685
|
this.turnComplete = this.predictTurnComplete();
|
|
674
|
-
const conversation = this.
|
|
675
|
-
const lastMessage = conversation
|
|
686
|
+
const conversation = this.conversation;
|
|
687
|
+
const lastMessage = conversation[conversation.length - 1];
|
|
676
688
|
if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
|
|
677
689
|
this.log(
|
|
678
690
|
"User stopped speaking and a transcript already exists, answering"
|
|
@@ -725,12 +737,55 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
725
737
|
clearTimeout(this.heldTurnTimer);
|
|
726
738
|
this.heldTurnTimer = void 0;
|
|
727
739
|
}
|
|
740
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
741
|
+
addUserMessage(text, metadata) {
|
|
742
|
+
if (this.config?.agent) {
|
|
743
|
+
this.config.agent.addUserMessage(text, metadata);
|
|
744
|
+
} else {
|
|
745
|
+
this.addOwnMessage("user", text, metadata);
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
749
|
+
addAssistantMessage(text, metadata) {
|
|
750
|
+
if (this.config?.agent) {
|
|
751
|
+
this.config.agent.addAssistantMessage(text, metadata);
|
|
752
|
+
} else {
|
|
753
|
+
this.addOwnMessage("assistant", text, metadata);
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
addOwnMessage(role, text, metadata) {
|
|
757
|
+
if (text.trim() === "") {
|
|
758
|
+
this.log(`Skipping empty ${role} message`);
|
|
759
|
+
return;
|
|
760
|
+
}
|
|
761
|
+
this.log(`Adding ${role} message to conversation: ${text}`);
|
|
762
|
+
const message = {
|
|
763
|
+
role,
|
|
764
|
+
content: text,
|
|
765
|
+
metadata
|
|
766
|
+
};
|
|
767
|
+
this.ownConversation.push(message);
|
|
768
|
+
this.sendMessage(message);
|
|
769
|
+
}
|
|
770
|
+
sendMessage(message) {
|
|
771
|
+
this.socket?.send(
|
|
772
|
+
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
773
|
+
);
|
|
774
|
+
this.emit("Message", message);
|
|
775
|
+
}
|
|
776
|
+
sendPartialAnswer(content) {
|
|
777
|
+
if (!this.socket || content === "") return;
|
|
778
|
+
this.log(`Send partial answer: "${content}"`);
|
|
779
|
+
this.socket.send(
|
|
780
|
+
`${"PartialAssistantMessage" /* PartialAssistantMessage */} ${JSON.stringify(content)}`
|
|
781
|
+
);
|
|
782
|
+
}
|
|
728
783
|
sendFirstMessage() {
|
|
729
784
|
if (!this.config) return;
|
|
730
785
|
if (this.config.firstMessage) {
|
|
731
|
-
this.config.agent
|
|
786
|
+
this.config.agent?.addAssistantMessage(this.config.firstMessage);
|
|
732
787
|
this.speak(this.config.firstMessage);
|
|
733
|
-
} else if (this.config.generateFirstMessage) {
|
|
788
|
+
} else if (this.config.generateFirstMessage && this.config.agent) {
|
|
734
789
|
this.answer();
|
|
735
790
|
} else {
|
|
736
791
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -743,7 +798,13 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
743
798
|
}
|
|
744
799
|
async _answer() {
|
|
745
800
|
if (!this.config) return;
|
|
746
|
-
|
|
801
|
+
if (!this.config.agent) {
|
|
802
|
+
this.log("No agent, skipping answer");
|
|
803
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
804
|
+
return;
|
|
805
|
+
}
|
|
806
|
+
const conversation = this.config.agent.conversation;
|
|
807
|
+
const lastMessage = conversation[conversation.length - 1];
|
|
747
808
|
if (this.lastMessageSpeeched === lastMessage) {
|
|
748
809
|
this.log("Already answered, skipping");
|
|
749
810
|
return;
|
|
@@ -752,13 +813,30 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
752
813
|
try {
|
|
753
814
|
const stream = this.config.agent.answer();
|
|
754
815
|
if (await hasContent(stream)) {
|
|
755
|
-
await this._speak(
|
|
816
|
+
await this._speak(
|
|
817
|
+
this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
|
|
818
|
+
);
|
|
819
|
+
} else {
|
|
820
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
756
821
|
}
|
|
757
822
|
} catch (error) {
|
|
758
823
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
759
824
|
throw error;
|
|
760
825
|
}
|
|
761
826
|
}
|
|
827
|
+
/** Sends the answer to the client as it is written, ahead of the voice */
|
|
828
|
+
forwardPartialAnswer(stream) {
|
|
829
|
+
let content = "";
|
|
830
|
+
const send = (text) => this.sendPartialAnswer(text);
|
|
831
|
+
const forward = new import_stream2.Transform({
|
|
832
|
+
transform(chunk, _encoding, callback) {
|
|
833
|
+
content += chunk.toString();
|
|
834
|
+
send(content);
|
|
835
|
+
callback(null, chunk);
|
|
836
|
+
}
|
|
837
|
+
});
|
|
838
|
+
return stream.pipe(forward);
|
|
839
|
+
}
|
|
762
840
|
// Run text-to-speech and send to client
|
|
763
841
|
speak(message) {
|
|
764
842
|
this.queueOperation(async () => {
|
|
@@ -769,6 +847,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
769
847
|
if (!this.socket || !this.config) return;
|
|
770
848
|
let textStream;
|
|
771
849
|
if (typeof message === "string") {
|
|
850
|
+
if (!this.config.agent) {
|
|
851
|
+
this.addOwnMessage("assistant", message);
|
|
852
|
+
}
|
|
772
853
|
const stream = new import_stream2.PassThrough();
|
|
773
854
|
stream.write(message);
|
|
774
855
|
stream.end();
|
|
@@ -776,7 +857,12 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
776
857
|
} else {
|
|
777
858
|
textStream = message;
|
|
778
859
|
}
|
|
779
|
-
this.config.tts
|
|
860
|
+
if (this.config.tts) {
|
|
861
|
+
this.config.tts.speak(textStream);
|
|
862
|
+
return;
|
|
863
|
+
}
|
|
864
|
+
await drain(textStream);
|
|
865
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
780
866
|
}
|
|
781
867
|
};
|
|
782
868
|
function hasContent(stream) {
|
|
@@ -797,6 +883,15 @@ function hasContent(stream) {
|
|
|
797
883
|
stream.on("end", onEnd);
|
|
798
884
|
});
|
|
799
885
|
}
|
|
886
|
+
function drain(stream) {
|
|
887
|
+
return new Promise((resolve) => {
|
|
888
|
+
stream.on("data", () => {
|
|
889
|
+
});
|
|
890
|
+
stream.once("end", () => resolve());
|
|
891
|
+
stream.once("error", () => resolve());
|
|
892
|
+
stream.once("close", () => resolve());
|
|
893
|
+
});
|
|
894
|
+
}
|
|
800
895
|
|
|
801
896
|
// src/recorder/MicdropRecorder.ts
|
|
802
897
|
var import_eventemitter33 = require("eventemitter3");
|
|
@@ -834,9 +929,7 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
834
929
|
this.currentAssistantChunks.push(chunk);
|
|
835
930
|
};
|
|
836
931
|
this.onMessage = (message) => {
|
|
837
|
-
const
|
|
838
|
-
if (!conversation) return;
|
|
839
|
-
const messageIndex = conversation.length - 1;
|
|
932
|
+
const messageIndex = this.server.conversation.length - 1;
|
|
840
933
|
if (message.role === "user") {
|
|
841
934
|
this.lastUserMessageIndex = messageIndex;
|
|
842
935
|
if (this.currentUserChunks.length > 0) {
|
|
@@ -862,17 +955,13 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
862
955
|
this.server.on("UserAudio", this.onUserAudio);
|
|
863
956
|
this.server.on("AssistantAudio", this.onAssistantAudio);
|
|
864
957
|
this.server.on("End", this.onEnd);
|
|
865
|
-
|
|
866
|
-
if (agent) {
|
|
867
|
-
agent.on("Message", this.onMessage);
|
|
868
|
-
}
|
|
958
|
+
this.server.on("Message", this.onMessage);
|
|
869
959
|
}
|
|
870
960
|
finalizeUserAudio() {
|
|
871
961
|
if (this.currentUserChunks.length === 0) return;
|
|
872
962
|
if (this.lastUserMessageIndex < 0) return;
|
|
873
|
-
const
|
|
874
|
-
if (!
|
|
875
|
-
const message = conversation[this.lastUserMessageIndex];
|
|
963
|
+
const message = this.server.conversation[this.lastUserMessageIndex];
|
|
964
|
+
if (!message) return;
|
|
876
965
|
const buffer = Buffer.concat(this.currentUserChunks);
|
|
877
966
|
const audioMessage = {
|
|
878
967
|
buffer,
|
|
@@ -891,9 +980,8 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
891
980
|
finalizeAssistantAudio() {
|
|
892
981
|
if (this.currentAssistantChunks.length === 0) return;
|
|
893
982
|
if (this.lastAssistantMessageIndex < 0) return;
|
|
894
|
-
const
|
|
895
|
-
if (!
|
|
896
|
-
const message = conversation[this.lastAssistantMessageIndex];
|
|
983
|
+
const message = this.server.conversation[this.lastAssistantMessageIndex];
|
|
984
|
+
if (!message) return;
|
|
897
985
|
const buffer = Buffer.concat(this.currentAssistantChunks);
|
|
898
986
|
const audioMessage = {
|
|
899
987
|
buffer,
|
|
@@ -917,10 +1005,7 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
917
1005
|
this.server.off("UserAudio", this.onUserAudio);
|
|
918
1006
|
this.server.off("AssistantAudio", this.onAssistantAudio);
|
|
919
1007
|
this.server.off("End", this.onEnd);
|
|
920
|
-
|
|
921
|
-
if (agent) {
|
|
922
|
-
agent.off("Message", this.onMessage);
|
|
923
|
-
}
|
|
1008
|
+
this.server.off("Message", this.onMessage);
|
|
924
1009
|
this.removeAllListeners();
|
|
925
1010
|
}
|
|
926
1011
|
log(...message) {
|
|
@@ -928,6 +1013,44 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
928
1013
|
}
|
|
929
1014
|
};
|
|
930
1015
|
|
|
1016
|
+
// src/SharedInstances.ts
|
|
1017
|
+
var SharedInstances = class {
|
|
1018
|
+
constructor() {
|
|
1019
|
+
this.entries = /* @__PURE__ */ new Map();
|
|
1020
|
+
}
|
|
1021
|
+
/**
|
|
1022
|
+
* Returns the instance for this key, calling `create` the first time.
|
|
1023
|
+
*
|
|
1024
|
+
* The key can be any JSON-serializable value, usually an array of the
|
|
1025
|
+
* options that define the instance. Concurrent callers get the same promise
|
|
1026
|
+
* instead of loading the model twice.
|
|
1027
|
+
*/
|
|
1028
|
+
load(key, create) {
|
|
1029
|
+
const id = typeof key === "string" ? key : JSON.stringify(key);
|
|
1030
|
+
const existing = this.entries.get(id);
|
|
1031
|
+
if (existing) return existing;
|
|
1032
|
+
let loading;
|
|
1033
|
+
try {
|
|
1034
|
+
loading = create();
|
|
1035
|
+
} catch (error) {
|
|
1036
|
+
return Promise.reject(error);
|
|
1037
|
+
}
|
|
1038
|
+
loading.catch(() => {
|
|
1039
|
+
if (this.entries.get(id) === loading) this.entries.delete(id);
|
|
1040
|
+
});
|
|
1041
|
+
this.entries.set(id, loading);
|
|
1042
|
+
return loading;
|
|
1043
|
+
}
|
|
1044
|
+
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
1045
|
+
has(key) {
|
|
1046
|
+
return this.entries.has(typeof key === "string" ? key : JSON.stringify(key));
|
|
1047
|
+
}
|
|
1048
|
+
/** Drops every instance, so the next call loads again. */
|
|
1049
|
+
clear() {
|
|
1050
|
+
this.entries.clear();
|
|
1051
|
+
}
|
|
1052
|
+
};
|
|
1053
|
+
|
|
931
1054
|
// src/stt/STT.ts
|
|
932
1055
|
var import_eventemitter34 = require("eventemitter3");
|
|
933
1056
|
var STT = class extends import_eventemitter34.EventEmitter {
|
|
@@ -1281,6 +1404,7 @@ async function waitForParams(socket, validate) {
|
|
|
1281
1404
|
STT,
|
|
1282
1405
|
SentenceSplitter,
|
|
1283
1406
|
SentenceTTS,
|
|
1407
|
+
SharedInstances,
|
|
1284
1408
|
TTS,
|
|
1285
1409
|
float32ToPcm16,
|
|
1286
1410
|
handleError,
|