@micdrop/server 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/index.d.mts +295 -7
- package/dist/index.d.ts +295 -7
- package/dist/index.js +548 -53
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +543 -53
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -37,6 +37,7 @@ __export(index_exports, {
|
|
|
37
37
|
AUTO_SEMANTIC_TURN_PROMPT: () => AUTO_SEMANTIC_TURN_PROMPT,
|
|
38
38
|
AUTO_SEMANTIC_TURN_TOOL_NAME: () => AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
39
39
|
Agent: () => Agent,
|
|
40
|
+
Classifier: () => Classifier,
|
|
40
41
|
FallbackAgent: () => FallbackAgent,
|
|
41
42
|
FallbackSTT: () => FallbackSTT,
|
|
42
43
|
FallbackTTS: () => FallbackTTS,
|
|
@@ -51,14 +52,18 @@ __export(index_exports, {
|
|
|
51
52
|
MockSTT: () => MockSTT,
|
|
52
53
|
MockTTS: () => MockTTS,
|
|
53
54
|
Pcm16Resampler: () => Pcm16Resampler,
|
|
55
|
+
Realtime: () => Realtime,
|
|
54
56
|
STT: () => STT,
|
|
55
57
|
SentenceSplitter: () => SentenceSplitter,
|
|
56
58
|
SentenceTTS: () => SentenceTTS,
|
|
57
59
|
SharedInstances: () => SharedInstances,
|
|
58
60
|
TTS: () => TTS,
|
|
61
|
+
currentTurn: () => currentTurn,
|
|
59
62
|
float32ToPcm16: () => float32ToPcm16,
|
|
63
|
+
getTurnClassification: () => getTurnClassification,
|
|
60
64
|
handleError: () => handleError,
|
|
61
65
|
pcm16ToFloat32: () => pcm16ToFloat32,
|
|
66
|
+
turnInput: () => turnInput,
|
|
62
67
|
waitForParams: () => waitForParams
|
|
63
68
|
});
|
|
64
69
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -90,8 +95,15 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
90
95
|
const answerCount = ++this.answerCount;
|
|
91
96
|
const stream = new import_stream.PassThrough();
|
|
92
97
|
this.answering = true;
|
|
93
|
-
Promise.resolve().then(
|
|
94
|
-
|
|
98
|
+
Promise.resolve().then(
|
|
99
|
+
() => this.options.onBeforeAnswer?.bind(this)(stream)
|
|
100
|
+
).then((result) => {
|
|
101
|
+
if (typeof result === "string") {
|
|
102
|
+
stream.write(result);
|
|
103
|
+
this.addAssistantMessage(result);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
if (result) return;
|
|
95
107
|
return this.generateAnswer(stream);
|
|
96
108
|
}).finally(() => {
|
|
97
109
|
if (stream.writable) {
|
|
@@ -133,16 +145,16 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
133
145
|
metadata
|
|
134
146
|
};
|
|
135
147
|
this.conversation.push(message);
|
|
136
|
-
this.
|
|
148
|
+
this.emitAgentEvent("Message", message);
|
|
137
149
|
}
|
|
138
150
|
addToolMessage(message) {
|
|
139
151
|
this.log("Adding tool message:", message);
|
|
140
152
|
this.conversation.push(message);
|
|
141
|
-
this.
|
|
153
|
+
this.emitAgentEvent("Message", message);
|
|
142
154
|
}
|
|
143
155
|
endCall() {
|
|
144
156
|
this.log("Ending call");
|
|
145
|
-
this.
|
|
157
|
+
this.emitAgentEvent("EndCall");
|
|
146
158
|
}
|
|
147
159
|
cancelLastUserMessage() {
|
|
148
160
|
this.log("Cancelling last user message");
|
|
@@ -152,11 +164,11 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
152
164
|
if (lastMessageIndex !== -1) {
|
|
153
165
|
this.conversation.splice(lastMessageIndex, 1);
|
|
154
166
|
}
|
|
155
|
-
this.
|
|
167
|
+
this.emitAgentEvent("CancelLastUserMessage");
|
|
156
168
|
}
|
|
157
169
|
skipAnswer() {
|
|
158
170
|
this.log("Skipping answer");
|
|
159
|
-
this.
|
|
171
|
+
this.emitAgentEvent("SkipAnswer");
|
|
160
172
|
}
|
|
161
173
|
getDefaultTools() {
|
|
162
174
|
const tools = [];
|
|
@@ -202,7 +214,7 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
202
214
|
output: JSON.stringify(output ?? null)
|
|
203
215
|
});
|
|
204
216
|
if (tool.emitOutput) {
|
|
205
|
-
this.
|
|
217
|
+
this.emitAgentEvent("ToolCall", {
|
|
206
218
|
name: toolCall.toolName,
|
|
207
219
|
parameters,
|
|
208
220
|
output
|
|
@@ -261,6 +273,16 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
261
273
|
}
|
|
262
274
|
return { message, metadata };
|
|
263
275
|
}
|
|
276
|
+
/**
|
|
277
|
+
* Emits one of the events every agent has.
|
|
278
|
+
*
|
|
279
|
+
* A subclass can add events of its own, and TypeScript then no longer knows
|
|
280
|
+
* the shared ones are among them.
|
|
281
|
+
*/
|
|
282
|
+
emitAgentEvent(event, ...args) {
|
|
283
|
+
const emitter = this;
|
|
284
|
+
emitter.emit(event, ...args);
|
|
285
|
+
}
|
|
264
286
|
log(...message) {
|
|
265
287
|
this.logger?.log(...message);
|
|
266
288
|
}
|
|
@@ -405,6 +427,144 @@ var MockAgent = class extends Agent {
|
|
|
405
427
|
}
|
|
406
428
|
};
|
|
407
429
|
|
|
430
|
+
// src/classifier/Classifier.ts
|
|
431
|
+
var import_eventemitter32 = require("eventemitter3");
|
|
432
|
+
var Classifier = class extends import_eventemitter32.EventEmitter {
|
|
433
|
+
constructor() {
|
|
434
|
+
super(...arguments);
|
|
435
|
+
this.controllers = /* @__PURE__ */ new Set();
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Classifies an input, and emits the result as `Classification`
|
|
439
|
+
* @param input - What to classify: a text, or JSON with named fields
|
|
440
|
+
* @returns The classification, or undefined when cancelled or failed
|
|
441
|
+
*/
|
|
442
|
+
classify(input) {
|
|
443
|
+
const controller = new AbortController();
|
|
444
|
+
this.controllers.add(controller);
|
|
445
|
+
const pending = this.run(input, controller.signal).finally(() => {
|
|
446
|
+
this.controllers.delete(controller);
|
|
447
|
+
if (this._pending === pending) this._pending = void 0;
|
|
448
|
+
});
|
|
449
|
+
this._pending = pending;
|
|
450
|
+
return pending;
|
|
451
|
+
}
|
|
452
|
+
/** The last classification started, until it is done */
|
|
453
|
+
get pending() {
|
|
454
|
+
return this._pending;
|
|
455
|
+
}
|
|
456
|
+
/** Drops every classification in progress, which then emit nothing */
|
|
457
|
+
cancel() {
|
|
458
|
+
this.controllers.forEach((controller) => controller.abort());
|
|
459
|
+
this.controllers.clear();
|
|
460
|
+
this._pending = void 0;
|
|
461
|
+
}
|
|
462
|
+
log(...message) {
|
|
463
|
+
this.logger?.log(...message);
|
|
464
|
+
}
|
|
465
|
+
destroy() {
|
|
466
|
+
this.log("Destroyed");
|
|
467
|
+
this.cancel();
|
|
468
|
+
this.removeAllListeners();
|
|
469
|
+
}
|
|
470
|
+
async run(input, signal) {
|
|
471
|
+
const start = Date.now();
|
|
472
|
+
try {
|
|
473
|
+
const result = await this.evaluate(input, signal);
|
|
474
|
+
if (signal.aborted) return;
|
|
475
|
+
const classification = {
|
|
476
|
+
input,
|
|
477
|
+
result,
|
|
478
|
+
duration: Date.now() - start
|
|
479
|
+
};
|
|
480
|
+
this.log(
|
|
481
|
+
`Classified in ${classification.duration} ms:`,
|
|
482
|
+
JSON.stringify(input)
|
|
483
|
+
);
|
|
484
|
+
this.lastClassification = classification;
|
|
485
|
+
this.emit("Classification", classification);
|
|
486
|
+
return classification;
|
|
487
|
+
} catch (error) {
|
|
488
|
+
if (!signal.aborted) this.log("Classification failed:", error);
|
|
489
|
+
return;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
};
|
|
493
|
+
|
|
494
|
+
// src/types.ts
|
|
495
|
+
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
496
|
+
MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
497
|
+
MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
498
|
+
MicdropClientCommands2["Mute"] = "Mute";
|
|
499
|
+
return MicdropClientCommands2;
|
|
500
|
+
})(MicdropClientCommands || {});
|
|
501
|
+
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
502
|
+
MicdropServerCommands2["Message"] = "Message";
|
|
503
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
504
|
+
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
505
|
+
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
506
|
+
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
507
|
+
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
508
|
+
MicdropServerCommands2["Classification"] = "Classification";
|
|
509
|
+
return MicdropServerCommands2;
|
|
510
|
+
})(MicdropServerCommands || {});
|
|
511
|
+
function currentTurn(conversation) {
|
|
512
|
+
let start = conversation.length;
|
|
513
|
+
while (start > 0) {
|
|
514
|
+
const item = conversation[start - 1];
|
|
515
|
+
if (item.role !== "user" || item.metadata?.classification) break;
|
|
516
|
+
start--;
|
|
517
|
+
}
|
|
518
|
+
const messages = conversation.slice(start);
|
|
519
|
+
return {
|
|
520
|
+
transcript: messages.map((message) => message.content).join(" "),
|
|
521
|
+
messages,
|
|
522
|
+
before: conversation.slice(0, start)
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
function getTurnClassification(conversation) {
|
|
526
|
+
for (let index = conversation.length - 1; index >= 0; index--) {
|
|
527
|
+
const item = conversation[index];
|
|
528
|
+
if (item.role === "user") return item.metadata?.classification;
|
|
529
|
+
}
|
|
530
|
+
return void 0;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
// src/classifier/turnInput.ts
|
|
534
|
+
function turnInput(conversation, history = 1) {
|
|
535
|
+
const turn = currentTurn(conversation);
|
|
536
|
+
const input = { history: [], turn: turn.transcript };
|
|
537
|
+
let rest = turn.before;
|
|
538
|
+
for (let n = 0; n < history && rest.length; n++) {
|
|
539
|
+
let end = rest.length;
|
|
540
|
+
const answers = [];
|
|
541
|
+
while (end > 0 && rest[end - 1].role !== "user") {
|
|
542
|
+
const item = rest[end - 1];
|
|
543
|
+
if (item.role === "assistant" && item.content.trim()) {
|
|
544
|
+
answers.unshift(item.content);
|
|
545
|
+
}
|
|
546
|
+
end--;
|
|
547
|
+
}
|
|
548
|
+
let start = end;
|
|
549
|
+
while (start > 0) {
|
|
550
|
+
const item = rest[start - 1];
|
|
551
|
+
if (item.role !== "user") break;
|
|
552
|
+
if (start < end && item.metadata?.classification)
|
|
553
|
+
break;
|
|
554
|
+
start--;
|
|
555
|
+
}
|
|
556
|
+
if (answers.length) {
|
|
557
|
+
input.history.unshift({ role: "assistant", text: answers.join(" ") });
|
|
558
|
+
}
|
|
559
|
+
if (start < end) {
|
|
560
|
+
const text = rest.slice(start, end).map((item) => "content" in item ? item.content : "").join(" ");
|
|
561
|
+
input.history.unshift({ role: "user", text });
|
|
562
|
+
}
|
|
563
|
+
rest = rest.slice(0, start);
|
|
564
|
+
}
|
|
565
|
+
return { input, message: turn.messages[turn.messages.length - 1] };
|
|
566
|
+
}
|
|
567
|
+
|
|
408
568
|
// src/audio/Pcm16Resampler.ts
|
|
409
569
|
var Pcm16Resampler = class {
|
|
410
570
|
// Fractional position into the first sample of the buffer
|
|
@@ -492,30 +652,12 @@ function handleError(socket, error) {
|
|
|
492
652
|
}
|
|
493
653
|
|
|
494
654
|
// src/MicdropServer.ts
|
|
495
|
-
var
|
|
655
|
+
var import_eventemitter33 = require("eventemitter3");
|
|
496
656
|
var import_stream2 = require("stream");
|
|
497
|
-
|
|
498
|
-
// src/types.ts
|
|
499
|
-
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
500
|
-
MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
501
|
-
MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
502
|
-
MicdropClientCommands2["Mute"] = "Mute";
|
|
503
|
-
return MicdropClientCommands2;
|
|
504
|
-
})(MicdropClientCommands || {});
|
|
505
|
-
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
506
|
-
MicdropServerCommands2["Message"] = "Message";
|
|
507
|
-
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
508
|
-
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
509
|
-
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
510
|
-
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
511
|
-
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
512
|
-
return MicdropServerCommands2;
|
|
513
|
-
})(MicdropServerCommands || {});
|
|
514
|
-
|
|
515
|
-
// src/MicdropServer.ts
|
|
516
657
|
var USER_SAMPLE_RATE = 16e3;
|
|
517
658
|
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
518
|
-
var
|
|
659
|
+
var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
|
|
660
|
+
var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
519
661
|
constructor(socket, config) {
|
|
520
662
|
super();
|
|
521
663
|
this.socket = null;
|
|
@@ -534,9 +676,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
534
676
|
this.log("Connection closed");
|
|
535
677
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
536
678
|
const conversation = this.conversation;
|
|
537
|
-
this.
|
|
538
|
-
this.config.stt
|
|
679
|
+
this.agent?.destroy();
|
|
680
|
+
this.config.stt?.destroy();
|
|
539
681
|
this.config.tts?.destroy();
|
|
682
|
+
this.config.classifier?.destroy();
|
|
540
683
|
this.emit("End", {
|
|
541
684
|
conversation,
|
|
542
685
|
duration
|
|
@@ -578,7 +721,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
578
721
|
this.answerUserTurn();
|
|
579
722
|
}
|
|
580
723
|
};
|
|
581
|
-
this.
|
|
724
|
+
this.onAssistantAudio = (audio) => {
|
|
582
725
|
if (!this.socket) return;
|
|
583
726
|
this.log(`Send audio chunk (${audio.byteLength} bytes)`);
|
|
584
727
|
this.socket.send(audio);
|
|
@@ -590,9 +733,23 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
590
733
|
this.socket = socket;
|
|
591
734
|
this.config = config;
|
|
592
735
|
this.log(`Call started`);
|
|
593
|
-
config
|
|
594
|
-
|
|
595
|
-
|
|
736
|
+
const { realtime } = config;
|
|
737
|
+
if (realtime) {
|
|
738
|
+
realtime.on("Audio", this.onAssistantAudio);
|
|
739
|
+
if (config.partialMessages) {
|
|
740
|
+
realtime.on(
|
|
741
|
+
"PartialMessage",
|
|
742
|
+
(content) => this.sendPartialAnswer(content)
|
|
743
|
+
);
|
|
744
|
+
}
|
|
745
|
+
} else {
|
|
746
|
+
if (!config.stt) {
|
|
747
|
+
throw new Error("MicdropServer needs an stt, or a realtime model");
|
|
748
|
+
}
|
|
749
|
+
config.stt.on("Transcript", this.onTranscriptSTT);
|
|
750
|
+
config.tts?.on("Audio", this.onAssistantAudio);
|
|
751
|
+
}
|
|
752
|
+
const agent = this.agent;
|
|
596
753
|
if (agent) {
|
|
597
754
|
agent.on("Message", this.onMessageAgent);
|
|
598
755
|
agent.on(
|
|
@@ -614,13 +771,26 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
614
771
|
)
|
|
615
772
|
);
|
|
616
773
|
}
|
|
774
|
+
if (config.classifierOptions?.sendToClient) {
|
|
775
|
+
config.classifier?.on(
|
|
776
|
+
"Classification",
|
|
777
|
+
(classification) => this.socket?.send(
|
|
778
|
+
`${"Classification" /* Classification */} ${JSON.stringify(classification)}`
|
|
779
|
+
)
|
|
780
|
+
);
|
|
781
|
+
}
|
|
617
782
|
queueMicrotask(() => this.sendFirstMessage());
|
|
618
783
|
socket.on("close", this.onClose);
|
|
619
784
|
socket.on("message", this.onMessage);
|
|
620
785
|
}
|
|
621
786
|
/** Everything said so far, whether an agent or the server keeps it */
|
|
622
787
|
get conversation() {
|
|
623
|
-
return this.
|
|
788
|
+
return this.agent?.conversation ?? this.ownConversation;
|
|
789
|
+
}
|
|
790
|
+
/** What holds the conversation and answers, a realtime model included */
|
|
791
|
+
get agent() {
|
|
792
|
+
if (!this.config) return void 0;
|
|
793
|
+
return this.config.realtime ?? this.config.agent;
|
|
624
794
|
}
|
|
625
795
|
log(...message) {
|
|
626
796
|
this.logger?.log(...message);
|
|
@@ -645,8 +815,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
645
815
|
this.processQueue();
|
|
646
816
|
}
|
|
647
817
|
cancel() {
|
|
818
|
+
this.config?.classifier?.cancel();
|
|
648
819
|
this.config?.tts?.cancel();
|
|
649
|
-
this.
|
|
820
|
+
this.agent?.cancel();
|
|
650
821
|
this.operationQueue = [];
|
|
651
822
|
}
|
|
652
823
|
onUserAudio(chunk) {
|
|
@@ -661,6 +832,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
661
832
|
this.currentUserStream?.end();
|
|
662
833
|
this.currentUserStream = void 0;
|
|
663
834
|
this.cancel();
|
|
835
|
+
this.config?.realtime?.discardTurn();
|
|
664
836
|
}
|
|
665
837
|
onStartSpeaking() {
|
|
666
838
|
if (!this.config) return;
|
|
@@ -670,8 +842,13 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
670
842
|
this.config.turnDetector?.reset();
|
|
671
843
|
this.turnComplete = void 0;
|
|
672
844
|
this.releaseHeldTurn();
|
|
673
|
-
this.config.
|
|
674
|
-
|
|
845
|
+
if (this.config.realtime) {
|
|
846
|
+
this.cancel();
|
|
847
|
+
this.config.realtime.startTurn(this.currentUserStream);
|
|
848
|
+
} else {
|
|
849
|
+
this.config.stt?.transcribe(this.currentUserStream);
|
|
850
|
+
this.cancel();
|
|
851
|
+
}
|
|
675
852
|
}
|
|
676
853
|
onStopSpeaking() {
|
|
677
854
|
const hasNoUserSpeech = !this.currentUserStream || this.userSpeechChunks === 0;
|
|
@@ -683,6 +860,10 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
683
860
|
return;
|
|
684
861
|
}
|
|
685
862
|
this.turnComplete = this.predictTurnComplete();
|
|
863
|
+
if (this.config?.realtime) {
|
|
864
|
+
this.answerUserTurn();
|
|
865
|
+
return;
|
|
866
|
+
}
|
|
686
867
|
const conversation = this.conversation;
|
|
687
868
|
const lastMessage = conversation[conversation.length - 1];
|
|
688
869
|
if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
|
|
@@ -715,7 +896,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
715
896
|
}
|
|
716
897
|
this.releaseHeldTurn();
|
|
717
898
|
this.cancel();
|
|
718
|
-
this.
|
|
899
|
+
this.answerTurn();
|
|
719
900
|
}
|
|
720
901
|
/**
|
|
721
902
|
* Answers anyway if the rest of the sentence never comes.
|
|
@@ -729,9 +910,19 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
729
910
|
this.heldTurnTimer = void 0;
|
|
730
911
|
this.log("Nothing more came, answering");
|
|
731
912
|
this.cancel();
|
|
732
|
-
this.
|
|
913
|
+
this.answerTurn();
|
|
733
914
|
}, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
|
|
734
915
|
}
|
|
916
|
+
/** Answers what the user just said */
|
|
917
|
+
answerTurn() {
|
|
918
|
+
const realtime = this.config?.realtime;
|
|
919
|
+
if (realtime) {
|
|
920
|
+
realtime.endTurn();
|
|
921
|
+
} else {
|
|
922
|
+
this.classifyTurn();
|
|
923
|
+
this.answer();
|
|
924
|
+
}
|
|
925
|
+
}
|
|
735
926
|
releaseHeldTurn() {
|
|
736
927
|
if (!this.heldTurnTimer) return;
|
|
737
928
|
clearTimeout(this.heldTurnTimer);
|
|
@@ -739,16 +930,16 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
739
930
|
}
|
|
740
931
|
/** Adds a message to the conversation and sends it to the client */
|
|
741
932
|
addUserMessage(text, metadata) {
|
|
742
|
-
if (this.
|
|
743
|
-
this.
|
|
933
|
+
if (this.agent) {
|
|
934
|
+
this.agent.addUserMessage(text, metadata);
|
|
744
935
|
} else {
|
|
745
936
|
this.addOwnMessage("user", text, metadata);
|
|
746
937
|
}
|
|
747
938
|
}
|
|
748
939
|
/** Adds a message to the conversation and sends it to the client */
|
|
749
940
|
addAssistantMessage(text, metadata) {
|
|
750
|
-
if (this.
|
|
751
|
-
this.
|
|
941
|
+
if (this.agent) {
|
|
942
|
+
this.agent.addAssistantMessage(text, metadata);
|
|
752
943
|
} else {
|
|
753
944
|
this.addOwnMessage("assistant", text, metadata);
|
|
754
945
|
}
|
|
@@ -772,6 +963,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
772
963
|
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
773
964
|
);
|
|
774
965
|
this.emit("Message", message);
|
|
966
|
+
if (message.role === "user" && this.config?.realtime) this.classifyTurn();
|
|
775
967
|
}
|
|
776
968
|
sendPartialAnswer(content) {
|
|
777
969
|
if (!this.socket || content === "") return;
|
|
@@ -783,21 +975,28 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
783
975
|
sendFirstMessage() {
|
|
784
976
|
if (!this.config) return;
|
|
785
977
|
if (this.config.firstMessage) {
|
|
786
|
-
this.
|
|
978
|
+
this.agent?.addAssistantMessage(this.config.firstMessage);
|
|
787
979
|
this.speak(this.config.firstMessage);
|
|
788
|
-
} else if (this.config.generateFirstMessage && this.
|
|
980
|
+
} else if (this.config.generateFirstMessage && this.agent) {
|
|
789
981
|
this.answer();
|
|
790
982
|
} else {
|
|
791
983
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
792
984
|
}
|
|
793
985
|
}
|
|
794
986
|
answer() {
|
|
987
|
+
if (this.config?.classifierOptions?.waitBeforeAnswer) {
|
|
988
|
+
this.queueOperation(() => this.waitForClassification());
|
|
989
|
+
}
|
|
795
990
|
this.queueOperation(async () => {
|
|
796
991
|
await this._answer();
|
|
797
992
|
});
|
|
798
993
|
}
|
|
799
994
|
async _answer() {
|
|
800
995
|
if (!this.config) return;
|
|
996
|
+
if (this.config.realtime) {
|
|
997
|
+
this.config.realtime.respond();
|
|
998
|
+
return;
|
|
999
|
+
}
|
|
801
1000
|
if (!this.config.agent) {
|
|
802
1001
|
this.log("No agent, skipping answer");
|
|
803
1002
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -837,6 +1036,38 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
837
1036
|
});
|
|
838
1037
|
return stream.pipe(forward);
|
|
839
1038
|
}
|
|
1039
|
+
/**
|
|
1040
|
+
* Classifies the turn that just ended, with the one before, and keeps the
|
|
1041
|
+
* result in the metadata of its last message
|
|
1042
|
+
*/
|
|
1043
|
+
classifyTurn() {
|
|
1044
|
+
const classifier = this.config?.classifier;
|
|
1045
|
+
if (!classifier) return;
|
|
1046
|
+
const { input, message } = turnInput(
|
|
1047
|
+
this.conversation,
|
|
1048
|
+
this.config?.classifierOptions?.history
|
|
1049
|
+
);
|
|
1050
|
+
if (!message || input.turn.trim() === "") return;
|
|
1051
|
+
classifier.classify(input).then((classification) => {
|
|
1052
|
+
if (!classification) return;
|
|
1053
|
+
message.metadata = { ...message.metadata, classification };
|
|
1054
|
+
});
|
|
1055
|
+
}
|
|
1056
|
+
/** Waits for the classification in progress, never longer than its limit */
|
|
1057
|
+
async waitForClassification() {
|
|
1058
|
+
const pending = this.config?.classifier?.pending;
|
|
1059
|
+
if (!pending) return;
|
|
1060
|
+
const limit = this.config?.classifierOptions?.maxWait ?? DEFAULT_CLASSIFICATION_MAX_WAIT;
|
|
1061
|
+
let timer;
|
|
1062
|
+
const timeout = new Promise((resolve) => {
|
|
1063
|
+
timer = setTimeout(() => {
|
|
1064
|
+
this.log(`Classification took over ${limit} ms, answering without it`);
|
|
1065
|
+
resolve();
|
|
1066
|
+
}, limit);
|
|
1067
|
+
});
|
|
1068
|
+
await Promise.race([pending, timeout]);
|
|
1069
|
+
clearTimeout(timer);
|
|
1070
|
+
}
|
|
840
1071
|
// Run text-to-speech and send to client
|
|
841
1072
|
speak(message) {
|
|
842
1073
|
this.queueOperation(async () => {
|
|
@@ -845,6 +1076,11 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
845
1076
|
}
|
|
846
1077
|
async _speak(message) {
|
|
847
1078
|
if (!this.socket || !this.config) return;
|
|
1079
|
+
if (this.config.realtime) {
|
|
1080
|
+
const text = typeof message === "string" ? message : await readText(message);
|
|
1081
|
+
if (text.trim() !== "") this.config.realtime.say(text);
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
848
1084
|
let textStream;
|
|
849
1085
|
if (typeof message === "string") {
|
|
850
1086
|
if (!this.config.agent) {
|
|
@@ -883,6 +1119,14 @@ function hasContent(stream) {
|
|
|
883
1119
|
stream.on("end", onEnd);
|
|
884
1120
|
});
|
|
885
1121
|
}
|
|
1122
|
+
function readText(stream) {
|
|
1123
|
+
return new Promise((resolve) => {
|
|
1124
|
+
let text = "";
|
|
1125
|
+
stream.on("data", (chunk) => text += chunk.toString());
|
|
1126
|
+
stream.once("end", () => resolve(text));
|
|
1127
|
+
stream.once("error", () => resolve(text));
|
|
1128
|
+
});
|
|
1129
|
+
}
|
|
886
1130
|
function drain(stream) {
|
|
887
1131
|
return new Promise((resolve) => {
|
|
888
1132
|
stream.on("data", () => {
|
|
@@ -893,9 +1137,255 @@ function drain(stream) {
|
|
|
893
1137
|
});
|
|
894
1138
|
}
|
|
895
1139
|
|
|
1140
|
+
// src/realtime/Realtime.ts
|
|
1141
|
+
var USER_TRANSCRIPT_TIMEOUT = 3e3;
|
|
1142
|
+
function newAnswer(record) {
|
|
1143
|
+
return { transcript: "", hasAudio: false, record, resumed: false };
|
|
1144
|
+
}
|
|
1145
|
+
var Realtime = class extends Agent {
|
|
1146
|
+
constructor() {
|
|
1147
|
+
super(...arguments);
|
|
1148
|
+
// A turn opens with its first audio, so a blip without any sends nothing
|
|
1149
|
+
this.turn = "none";
|
|
1150
|
+
// Output of an interrupted answer can still be on its way
|
|
1151
|
+
this.discarding = false;
|
|
1152
|
+
// Answers asked for while another one was being given
|
|
1153
|
+
this.queue = [];
|
|
1154
|
+
this.answerWaiters = [];
|
|
1155
|
+
this.userTranscript = "";
|
|
1156
|
+
this.awaitingUserTranscript = false;
|
|
1157
|
+
this.afterUserTranscript = [];
|
|
1158
|
+
}
|
|
1159
|
+
/**
|
|
1160
|
+
* The user started speaking.
|
|
1161
|
+
*
|
|
1162
|
+
* The assistant stops, and the audio read from the stream goes to the model.
|
|
1163
|
+
* A turn left open, because the sentence sounded unfinished, carries on.
|
|
1164
|
+
*/
|
|
1165
|
+
startTurn(audio) {
|
|
1166
|
+
this.cancel();
|
|
1167
|
+
if (this.turn === "none") this.turn = "pending";
|
|
1168
|
+
audio.on("data", (chunk) => this.sendAudio(chunk));
|
|
1169
|
+
}
|
|
1170
|
+
/** Closes the turn of the user, and has the model answer it */
|
|
1171
|
+
endTurn() {
|
|
1172
|
+
if (this.turn !== "open") return;
|
|
1173
|
+
this.turn = "none";
|
|
1174
|
+
this.awaitUserTranscript();
|
|
1175
|
+
this.beginAnswer(true);
|
|
1176
|
+
this.closeTurn();
|
|
1177
|
+
}
|
|
1178
|
+
/** Drops the turn of the user, without an answer */
|
|
1179
|
+
discardTurn() {
|
|
1180
|
+
this.cancel();
|
|
1181
|
+
if (this.turn === "open") {
|
|
1182
|
+
this.clearTurn();
|
|
1183
|
+
this.discarding = true;
|
|
1184
|
+
}
|
|
1185
|
+
this.turn = "none";
|
|
1186
|
+
}
|
|
1187
|
+
/** Has the model speak without waiting for the user, to open a call */
|
|
1188
|
+
respond() {
|
|
1189
|
+
this.whenIdle(() => {
|
|
1190
|
+
this.beginAnswer(true);
|
|
1191
|
+
this.generate();
|
|
1192
|
+
});
|
|
1193
|
+
}
|
|
1194
|
+
/**
|
|
1195
|
+
* Has the model say this text, word for word.
|
|
1196
|
+
*
|
|
1197
|
+
* The caller records the text in the conversation, as with a text to speech,
|
|
1198
|
+
* so the transcript of what the model says is not recorded again.
|
|
1199
|
+
*/
|
|
1200
|
+
say(text) {
|
|
1201
|
+
this.whenIdle(() => {
|
|
1202
|
+
this.beginAnswer(false);
|
|
1203
|
+
this.speakText(text);
|
|
1204
|
+
});
|
|
1205
|
+
}
|
|
1206
|
+
cancel() {
|
|
1207
|
+
this.queue = [];
|
|
1208
|
+
const answer = this.currentAnswer;
|
|
1209
|
+
if (!answer) return;
|
|
1210
|
+
this.log("Cancel");
|
|
1211
|
+
this.currentAnswer = void 0;
|
|
1212
|
+
this.discarding = true;
|
|
1213
|
+
this.cancelAnswer();
|
|
1214
|
+
this.recordAnswer(answer);
|
|
1215
|
+
this.resolveAnswerWaiters();
|
|
1216
|
+
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Adds a message the model did not hear, and tells the model about it.
|
|
1219
|
+
*
|
|
1220
|
+
* The transcripts of the call are recorded apart, so they are not sent back.
|
|
1221
|
+
*/
|
|
1222
|
+
addMessage(role, text, metadata) {
|
|
1223
|
+
super.addMessage(role, text, metadata);
|
|
1224
|
+
if (text.trim() !== "") this.injectMessage(role, text);
|
|
1225
|
+
}
|
|
1226
|
+
addTool(tool) {
|
|
1227
|
+
super.addTool(tool);
|
|
1228
|
+
this.updateTools();
|
|
1229
|
+
}
|
|
1230
|
+
removeTool(name) {
|
|
1231
|
+
super.removeTool(name);
|
|
1232
|
+
this.updateTools();
|
|
1233
|
+
}
|
|
1234
|
+
/**
|
|
1235
|
+
* Lists the events that have listeners.
|
|
1236
|
+
*
|
|
1237
|
+
* Typed as the events of any agent, so a realtime model goes wherever an
|
|
1238
|
+
* agent is expected. The two extra events do not change what that code reads.
|
|
1239
|
+
*/
|
|
1240
|
+
eventNames() {
|
|
1241
|
+
return super.eventNames();
|
|
1242
|
+
}
|
|
1243
|
+
destroy() {
|
|
1244
|
+
super.destroy();
|
|
1245
|
+
this.clearUserTranscriptTimer();
|
|
1246
|
+
this.afterUserTranscript = [];
|
|
1247
|
+
this.resolveAnswerWaiters();
|
|
1248
|
+
}
|
|
1249
|
+
generateAnswer(stream) {
|
|
1250
|
+
let written = 0;
|
|
1251
|
+
const onPartial = (transcript) => {
|
|
1252
|
+
stream.write(transcript.slice(written));
|
|
1253
|
+
written = transcript.length;
|
|
1254
|
+
};
|
|
1255
|
+
this.on("PartialMessage", onPartial);
|
|
1256
|
+
this.respond();
|
|
1257
|
+
return new Promise((resolve) => {
|
|
1258
|
+
this.answerWaiters.push(() => {
|
|
1259
|
+
this.off("PartialMessage", onPartial);
|
|
1260
|
+
resolve();
|
|
1261
|
+
});
|
|
1262
|
+
});
|
|
1263
|
+
}
|
|
1264
|
+
/** Nobody is speaking, nor about to, so the connection can be renewed */
|
|
1265
|
+
get isIdle() {
|
|
1266
|
+
return this.turn === "none" && !this.currentAnswer;
|
|
1267
|
+
}
|
|
1268
|
+
/** Voice of the answer, PCM16 at 16 kHz */
|
|
1269
|
+
emitAudio(chunk) {
|
|
1270
|
+
const answer = this.activeAnswer();
|
|
1271
|
+
if (!answer) return;
|
|
1272
|
+
answer.hasAudio = true;
|
|
1273
|
+
this.emit("Audio", chunk);
|
|
1274
|
+
}
|
|
1275
|
+
/** Transcript of the answer, as it is spoken */
|
|
1276
|
+
addAnswerTranscript(text) {
|
|
1277
|
+
const answer = this.activeAnswer();
|
|
1278
|
+
if (!answer) return;
|
|
1279
|
+
if (answer.resumed) {
|
|
1280
|
+
answer.resumed = false;
|
|
1281
|
+
if (answer.transcript !== "" && !/\s$/.test(answer.transcript)) {
|
|
1282
|
+
text = ` ${text.trimStart()}`;
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
answer.transcript += text;
|
|
1286
|
+
if (answer.record) this.emit("PartialMessage", answer.transcript);
|
|
1287
|
+
}
|
|
1288
|
+
/**
|
|
1289
|
+
* The model finished its answer, or finished dropping an interrupted one.
|
|
1290
|
+
*
|
|
1291
|
+
* An answer without audio tells the client to stop waiting for one.
|
|
1292
|
+
*/
|
|
1293
|
+
endAnswer() {
|
|
1294
|
+
if (this.discarding) {
|
|
1295
|
+
this.discarding = false;
|
|
1296
|
+
return;
|
|
1297
|
+
}
|
|
1298
|
+
const answer = this.currentAnswer;
|
|
1299
|
+
if (!answer) return;
|
|
1300
|
+
this.currentAnswer = void 0;
|
|
1301
|
+
if (!answer.hasAudio) this.emitAgentEvent("SkipAnswer");
|
|
1302
|
+
this.recordAnswer(answer);
|
|
1303
|
+
this.resolveAnswerWaiters();
|
|
1304
|
+
this.queue.shift()?.();
|
|
1305
|
+
}
|
|
1306
|
+
/** Runs a tool the model called, and records it in the conversation */
|
|
1307
|
+
async runTool(toolCall) {
|
|
1308
|
+
const result = await this.executeTool(toolCall);
|
|
1309
|
+
if (this.currentAnswer) this.currentAnswer.resumed = true;
|
|
1310
|
+
return result;
|
|
1311
|
+
}
|
|
1312
|
+
/** Transcript of the turn of the user, possibly in several pieces */
|
|
1313
|
+
addUserTranscript(text) {
|
|
1314
|
+
this.userTranscript += text;
|
|
1315
|
+
}
|
|
1316
|
+
/** The transcript of the turn is complete, it goes in the conversation */
|
|
1317
|
+
commitUserTranscript() {
|
|
1318
|
+
this.clearUserTranscriptTimer();
|
|
1319
|
+
this.awaitingUserTranscript = false;
|
|
1320
|
+
const text = this.userTranscript.trim();
|
|
1321
|
+
this.userTranscript = "";
|
|
1322
|
+
if (text !== "") super.addMessage("user", text);
|
|
1323
|
+
const pending = this.afterUserTranscript;
|
|
1324
|
+
this.afterUserTranscript = [];
|
|
1325
|
+
pending.forEach((record) => record());
|
|
1326
|
+
}
|
|
1327
|
+
sendAudio(chunk) {
|
|
1328
|
+
if (this.turn === "none") return;
|
|
1329
|
+
if (this.turn === "pending") {
|
|
1330
|
+
this.turn = "open";
|
|
1331
|
+
this.openTurn();
|
|
1332
|
+
}
|
|
1333
|
+
this.appendAudio(chunk);
|
|
1334
|
+
}
|
|
1335
|
+
beginAnswer(record) {
|
|
1336
|
+
this.discarding = false;
|
|
1337
|
+
this.currentAnswer = newAnswer(record);
|
|
1338
|
+
}
|
|
1339
|
+
/**
|
|
1340
|
+
* The answer output belongs to, if it is still wanted.
|
|
1341
|
+
*
|
|
1342
|
+
* The model can speak without being asked, to follow up on a tool call for
|
|
1343
|
+
* instance, which opens an answer of its own.
|
|
1344
|
+
*/
|
|
1345
|
+
activeAnswer() {
|
|
1346
|
+
if (this.discarding) return void 0;
|
|
1347
|
+
if (!this.currentAnswer) this.currentAnswer = newAnswer(true);
|
|
1348
|
+
return this.currentAnswer;
|
|
1349
|
+
}
|
|
1350
|
+
whenIdle(start) {
|
|
1351
|
+
if (this.currentAnswer) {
|
|
1352
|
+
this.queue.push(start);
|
|
1353
|
+
} else {
|
|
1354
|
+
start();
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
recordAnswer(answer) {
|
|
1358
|
+
if (!answer.record) return;
|
|
1359
|
+
const record = () => super.addMessage("assistant", answer.transcript.trim());
|
|
1360
|
+
if (this.awaitingUserTranscript) {
|
|
1361
|
+
this.afterUserTranscript.push(record);
|
|
1362
|
+
} else {
|
|
1363
|
+
record();
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
awaitUserTranscript() {
|
|
1367
|
+
this.awaitingUserTranscript = true;
|
|
1368
|
+
this.clearUserTranscriptTimer();
|
|
1369
|
+
this.userTranscriptTimer = setTimeout(() => {
|
|
1370
|
+
this.log("No transcript of the turn, recording the answer anyway");
|
|
1371
|
+
this.commitUserTranscript();
|
|
1372
|
+
}, USER_TRANSCRIPT_TIMEOUT);
|
|
1373
|
+
}
|
|
1374
|
+
clearUserTranscriptTimer() {
|
|
1375
|
+
if (!this.userTranscriptTimer) return;
|
|
1376
|
+
clearTimeout(this.userTranscriptTimer);
|
|
1377
|
+
this.userTranscriptTimer = void 0;
|
|
1378
|
+
}
|
|
1379
|
+
resolveAnswerWaiters() {
|
|
1380
|
+
const waiters = this.answerWaiters;
|
|
1381
|
+
this.answerWaiters = [];
|
|
1382
|
+
waiters.forEach((resolve) => resolve());
|
|
1383
|
+
}
|
|
1384
|
+
};
|
|
1385
|
+
|
|
896
1386
|
// src/recorder/MicdropRecorder.ts
|
|
897
|
-
var
|
|
898
|
-
var MicdropRecorder = class extends
|
|
1387
|
+
var import_eventemitter34 = require("eventemitter3");
|
|
1388
|
+
var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
899
1389
|
constructor(server) {
|
|
900
1390
|
super();
|
|
901
1391
|
this.server = server;
|
|
@@ -1052,8 +1542,8 @@ var SharedInstances = class {
|
|
|
1052
1542
|
};
|
|
1053
1543
|
|
|
1054
1544
|
// src/stt/STT.ts
|
|
1055
|
-
var
|
|
1056
|
-
var STT = class extends
|
|
1545
|
+
var import_eventemitter35 = require("eventemitter3");
|
|
1546
|
+
var STT = class extends import_eventemitter35.EventEmitter {
|
|
1057
1547
|
log(...message) {
|
|
1058
1548
|
this.logger?.log(...message);
|
|
1059
1549
|
}
|
|
@@ -1134,8 +1624,8 @@ var FallbackSTT = class extends STT {
|
|
|
1134
1624
|
var import_stream4 = require("stream");
|
|
1135
1625
|
|
|
1136
1626
|
// src/tts/TTS.ts
|
|
1137
|
-
var
|
|
1138
|
-
var TTS = class extends
|
|
1627
|
+
var import_eventemitter36 = require("eventemitter3");
|
|
1628
|
+
var TTS = class extends import_eventemitter36.EventEmitter {
|
|
1139
1629
|
log(...message) {
|
|
1140
1630
|
this.logger?.log(...message);
|
|
1141
1631
|
}
|
|
@@ -1387,6 +1877,7 @@ async function waitForParams(socket, validate) {
|
|
|
1387
1877
|
AUTO_SEMANTIC_TURN_PROMPT,
|
|
1388
1878
|
AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
1389
1879
|
Agent,
|
|
1880
|
+
Classifier,
|
|
1390
1881
|
FallbackAgent,
|
|
1391
1882
|
FallbackSTT,
|
|
1392
1883
|
FallbackTTS,
|
|
@@ -1401,14 +1892,18 @@ async function waitForParams(socket, validate) {
|
|
|
1401
1892
|
MockSTT,
|
|
1402
1893
|
MockTTS,
|
|
1403
1894
|
Pcm16Resampler,
|
|
1895
|
+
Realtime,
|
|
1404
1896
|
STT,
|
|
1405
1897
|
SentenceSplitter,
|
|
1406
1898
|
SentenceTTS,
|
|
1407
1899
|
SharedInstances,
|
|
1408
1900
|
TTS,
|
|
1901
|
+
currentTurn,
|
|
1409
1902
|
float32ToPcm16,
|
|
1903
|
+
getTurnClassification,
|
|
1410
1904
|
handleError,
|
|
1411
1905
|
pcm16ToFloat32,
|
|
1906
|
+
turnInput,
|
|
1412
1907
|
waitForParams
|
|
1413
1908
|
});
|
|
1414
1909
|
//# sourceMappingURL=index.js.map
|