@micdrop/server 2.7.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/index.d.mts +295 -7
- package/dist/index.d.ts +295 -7
- package/dist/index.js +548 -53
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +543 -53
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -25,8 +25,15 @@ var Agent = class extends EventEmitter {
|
|
|
25
25
|
const answerCount = ++this.answerCount;
|
|
26
26
|
const stream = new PassThrough();
|
|
27
27
|
this.answering = true;
|
|
28
|
-
Promise.resolve().then(
|
|
29
|
-
|
|
28
|
+
Promise.resolve().then(
|
|
29
|
+
() => this.options.onBeforeAnswer?.bind(this)(stream)
|
|
30
|
+
).then((result) => {
|
|
31
|
+
if (typeof result === "string") {
|
|
32
|
+
stream.write(result);
|
|
33
|
+
this.addAssistantMessage(result);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
if (result) return;
|
|
30
37
|
return this.generateAnswer(stream);
|
|
31
38
|
}).finally(() => {
|
|
32
39
|
if (stream.writable) {
|
|
@@ -68,16 +75,16 @@ var Agent = class extends EventEmitter {
|
|
|
68
75
|
metadata
|
|
69
76
|
};
|
|
70
77
|
this.conversation.push(message);
|
|
71
|
-
this.
|
|
78
|
+
this.emitAgentEvent("Message", message);
|
|
72
79
|
}
|
|
73
80
|
addToolMessage(message) {
|
|
74
81
|
this.log("Adding tool message:", message);
|
|
75
82
|
this.conversation.push(message);
|
|
76
|
-
this.
|
|
83
|
+
this.emitAgentEvent("Message", message);
|
|
77
84
|
}
|
|
78
85
|
endCall() {
|
|
79
86
|
this.log("Ending call");
|
|
80
|
-
this.
|
|
87
|
+
this.emitAgentEvent("EndCall");
|
|
81
88
|
}
|
|
82
89
|
cancelLastUserMessage() {
|
|
83
90
|
this.log("Cancelling last user message");
|
|
@@ -87,11 +94,11 @@ var Agent = class extends EventEmitter {
|
|
|
87
94
|
if (lastMessageIndex !== -1) {
|
|
88
95
|
this.conversation.splice(lastMessageIndex, 1);
|
|
89
96
|
}
|
|
90
|
-
this.
|
|
97
|
+
this.emitAgentEvent("CancelLastUserMessage");
|
|
91
98
|
}
|
|
92
99
|
skipAnswer() {
|
|
93
100
|
this.log("Skipping answer");
|
|
94
|
-
this.
|
|
101
|
+
this.emitAgentEvent("SkipAnswer");
|
|
95
102
|
}
|
|
96
103
|
getDefaultTools() {
|
|
97
104
|
const tools = [];
|
|
@@ -137,7 +144,7 @@ var Agent = class extends EventEmitter {
|
|
|
137
144
|
output: JSON.stringify(output ?? null)
|
|
138
145
|
});
|
|
139
146
|
if (tool.emitOutput) {
|
|
140
|
-
this.
|
|
147
|
+
this.emitAgentEvent("ToolCall", {
|
|
141
148
|
name: toolCall.toolName,
|
|
142
149
|
parameters,
|
|
143
150
|
output
|
|
@@ -196,6 +203,16 @@ var Agent = class extends EventEmitter {
|
|
|
196
203
|
}
|
|
197
204
|
return { message, metadata };
|
|
198
205
|
}
|
|
206
|
+
/**
|
|
207
|
+
* Emits one of the events every agent has.
|
|
208
|
+
*
|
|
209
|
+
* A subclass can add events of its own, and TypeScript then no longer knows
|
|
210
|
+
* the shared ones are among them.
|
|
211
|
+
*/
|
|
212
|
+
emitAgentEvent(event, ...args) {
|
|
213
|
+
const emitter = this;
|
|
214
|
+
emitter.emit(event, ...args);
|
|
215
|
+
}
|
|
199
216
|
log(...message) {
|
|
200
217
|
this.logger?.log(...message);
|
|
201
218
|
}
|
|
@@ -340,6 +357,144 @@ var MockAgent = class extends Agent {
|
|
|
340
357
|
}
|
|
341
358
|
};
|
|
342
359
|
|
|
360
|
+
// src/classifier/Classifier.ts
|
|
361
|
+
import { EventEmitter as EventEmitter2 } from "eventemitter3";
|
|
362
|
+
var Classifier = class extends EventEmitter2 {
|
|
363
|
+
constructor() {
|
|
364
|
+
super(...arguments);
|
|
365
|
+
this.controllers = /* @__PURE__ */ new Set();
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Classifies an input, and emits the result as `Classification`
|
|
369
|
+
* @param input - What to classify: a text, or JSON with named fields
|
|
370
|
+
* @returns The classification, or undefined when cancelled or failed
|
|
371
|
+
*/
|
|
372
|
+
classify(input) {
|
|
373
|
+
const controller = new AbortController();
|
|
374
|
+
this.controllers.add(controller);
|
|
375
|
+
const pending = this.run(input, controller.signal).finally(() => {
|
|
376
|
+
this.controllers.delete(controller);
|
|
377
|
+
if (this._pending === pending) this._pending = void 0;
|
|
378
|
+
});
|
|
379
|
+
this._pending = pending;
|
|
380
|
+
return pending;
|
|
381
|
+
}
|
|
382
|
+
/** The last classification started, until it is done */
|
|
383
|
+
get pending() {
|
|
384
|
+
return this._pending;
|
|
385
|
+
}
|
|
386
|
+
/** Drops every classification in progress, which then emit nothing */
|
|
387
|
+
cancel() {
|
|
388
|
+
this.controllers.forEach((controller) => controller.abort());
|
|
389
|
+
this.controllers.clear();
|
|
390
|
+
this._pending = void 0;
|
|
391
|
+
}
|
|
392
|
+
log(...message) {
|
|
393
|
+
this.logger?.log(...message);
|
|
394
|
+
}
|
|
395
|
+
destroy() {
|
|
396
|
+
this.log("Destroyed");
|
|
397
|
+
this.cancel();
|
|
398
|
+
this.removeAllListeners();
|
|
399
|
+
}
|
|
400
|
+
async run(input, signal) {
|
|
401
|
+
const start = Date.now();
|
|
402
|
+
try {
|
|
403
|
+
const result = await this.evaluate(input, signal);
|
|
404
|
+
if (signal.aborted) return;
|
|
405
|
+
const classification = {
|
|
406
|
+
input,
|
|
407
|
+
result,
|
|
408
|
+
duration: Date.now() - start
|
|
409
|
+
};
|
|
410
|
+
this.log(
|
|
411
|
+
`Classified in ${classification.duration} ms:`,
|
|
412
|
+
JSON.stringify(input)
|
|
413
|
+
);
|
|
414
|
+
this.lastClassification = classification;
|
|
415
|
+
this.emit("Classification", classification);
|
|
416
|
+
return classification;
|
|
417
|
+
} catch (error) {
|
|
418
|
+
if (!signal.aborted) this.log("Classification failed:", error);
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
};
|
|
423
|
+
|
|
424
|
+
// src/types.ts
|
|
425
|
+
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
426
|
+
MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
427
|
+
MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
428
|
+
MicdropClientCommands2["Mute"] = "Mute";
|
|
429
|
+
return MicdropClientCommands2;
|
|
430
|
+
})(MicdropClientCommands || {});
|
|
431
|
+
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
432
|
+
MicdropServerCommands2["Message"] = "Message";
|
|
433
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
434
|
+
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
435
|
+
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
436
|
+
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
437
|
+
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
438
|
+
MicdropServerCommands2["Classification"] = "Classification";
|
|
439
|
+
return MicdropServerCommands2;
|
|
440
|
+
})(MicdropServerCommands || {});
|
|
441
|
+
function currentTurn(conversation) {
|
|
442
|
+
let start = conversation.length;
|
|
443
|
+
while (start > 0) {
|
|
444
|
+
const item = conversation[start - 1];
|
|
445
|
+
if (item.role !== "user" || item.metadata?.classification) break;
|
|
446
|
+
start--;
|
|
447
|
+
}
|
|
448
|
+
const messages = conversation.slice(start);
|
|
449
|
+
return {
|
|
450
|
+
transcript: messages.map((message) => message.content).join(" "),
|
|
451
|
+
messages,
|
|
452
|
+
before: conversation.slice(0, start)
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
function getTurnClassification(conversation) {
|
|
456
|
+
for (let index = conversation.length - 1; index >= 0; index--) {
|
|
457
|
+
const item = conversation[index];
|
|
458
|
+
if (item.role === "user") return item.metadata?.classification;
|
|
459
|
+
}
|
|
460
|
+
return void 0;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// src/classifier/turnInput.ts
|
|
464
|
+
function turnInput(conversation, history = 1) {
|
|
465
|
+
const turn = currentTurn(conversation);
|
|
466
|
+
const input = { history: [], turn: turn.transcript };
|
|
467
|
+
let rest = turn.before;
|
|
468
|
+
for (let n = 0; n < history && rest.length; n++) {
|
|
469
|
+
let end = rest.length;
|
|
470
|
+
const answers = [];
|
|
471
|
+
while (end > 0 && rest[end - 1].role !== "user") {
|
|
472
|
+
const item = rest[end - 1];
|
|
473
|
+
if (item.role === "assistant" && item.content.trim()) {
|
|
474
|
+
answers.unshift(item.content);
|
|
475
|
+
}
|
|
476
|
+
end--;
|
|
477
|
+
}
|
|
478
|
+
let start = end;
|
|
479
|
+
while (start > 0) {
|
|
480
|
+
const item = rest[start - 1];
|
|
481
|
+
if (item.role !== "user") break;
|
|
482
|
+
if (start < end && item.metadata?.classification)
|
|
483
|
+
break;
|
|
484
|
+
start--;
|
|
485
|
+
}
|
|
486
|
+
if (answers.length) {
|
|
487
|
+
input.history.unshift({ role: "assistant", text: answers.join(" ") });
|
|
488
|
+
}
|
|
489
|
+
if (start < end) {
|
|
490
|
+
const text = rest.slice(start, end).map((item) => "content" in item ? item.content : "").join(" ");
|
|
491
|
+
input.history.unshift({ role: "user", text });
|
|
492
|
+
}
|
|
493
|
+
rest = rest.slice(0, start);
|
|
494
|
+
}
|
|
495
|
+
return { input, message: turn.messages[turn.messages.length - 1] };
|
|
496
|
+
}
|
|
497
|
+
|
|
343
498
|
// src/audio/Pcm16Resampler.ts
|
|
344
499
|
var Pcm16Resampler = class {
|
|
345
500
|
// Fractional position into the first sample of the buffer
|
|
@@ -427,30 +582,12 @@ function handleError(socket, error) {
|
|
|
427
582
|
}
|
|
428
583
|
|
|
429
584
|
// src/MicdropServer.ts
|
|
430
|
-
import { EventEmitter as
|
|
585
|
+
import { EventEmitter as EventEmitter3 } from "eventemitter3";
|
|
431
586
|
import { PassThrough as PassThrough2, Transform } from "stream";
|
|
432
|
-
|
|
433
|
-
// src/types.ts
|
|
434
|
-
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
435
|
-
MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
436
|
-
MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
437
|
-
MicdropClientCommands2["Mute"] = "Mute";
|
|
438
|
-
return MicdropClientCommands2;
|
|
439
|
-
})(MicdropClientCommands || {});
|
|
440
|
-
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
441
|
-
MicdropServerCommands2["Message"] = "Message";
|
|
442
|
-
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
443
|
-
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
444
|
-
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
445
|
-
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
446
|
-
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
447
|
-
return MicdropServerCommands2;
|
|
448
|
-
})(MicdropServerCommands || {});
|
|
449
|
-
|
|
450
|
-
// src/MicdropServer.ts
|
|
451
587
|
var USER_SAMPLE_RATE = 16e3;
|
|
452
588
|
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
453
|
-
var
|
|
589
|
+
var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
|
|
590
|
+
var MicdropServer = class extends EventEmitter3 {
|
|
454
591
|
constructor(socket, config) {
|
|
455
592
|
super();
|
|
456
593
|
this.socket = null;
|
|
@@ -469,9 +606,10 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
469
606
|
this.log("Connection closed");
|
|
470
607
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
471
608
|
const conversation = this.conversation;
|
|
472
|
-
this.
|
|
473
|
-
this.config.stt
|
|
609
|
+
this.agent?.destroy();
|
|
610
|
+
this.config.stt?.destroy();
|
|
474
611
|
this.config.tts?.destroy();
|
|
612
|
+
this.config.classifier?.destroy();
|
|
475
613
|
this.emit("End", {
|
|
476
614
|
conversation,
|
|
477
615
|
duration
|
|
@@ -513,7 +651,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
513
651
|
this.answerUserTurn();
|
|
514
652
|
}
|
|
515
653
|
};
|
|
516
|
-
this.
|
|
654
|
+
this.onAssistantAudio = (audio) => {
|
|
517
655
|
if (!this.socket) return;
|
|
518
656
|
this.log(`Send audio chunk (${audio.byteLength} bytes)`);
|
|
519
657
|
this.socket.send(audio);
|
|
@@ -525,9 +663,23 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
525
663
|
this.socket = socket;
|
|
526
664
|
this.config = config;
|
|
527
665
|
this.log(`Call started`);
|
|
528
|
-
config
|
|
529
|
-
|
|
530
|
-
|
|
666
|
+
const { realtime } = config;
|
|
667
|
+
if (realtime) {
|
|
668
|
+
realtime.on("Audio", this.onAssistantAudio);
|
|
669
|
+
if (config.partialMessages) {
|
|
670
|
+
realtime.on(
|
|
671
|
+
"PartialMessage",
|
|
672
|
+
(content) => this.sendPartialAnswer(content)
|
|
673
|
+
);
|
|
674
|
+
}
|
|
675
|
+
} else {
|
|
676
|
+
if (!config.stt) {
|
|
677
|
+
throw new Error("MicdropServer needs an stt, or a realtime model");
|
|
678
|
+
}
|
|
679
|
+
config.stt.on("Transcript", this.onTranscriptSTT);
|
|
680
|
+
config.tts?.on("Audio", this.onAssistantAudio);
|
|
681
|
+
}
|
|
682
|
+
const agent = this.agent;
|
|
531
683
|
if (agent) {
|
|
532
684
|
agent.on("Message", this.onMessageAgent);
|
|
533
685
|
agent.on(
|
|
@@ -549,13 +701,26 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
549
701
|
)
|
|
550
702
|
);
|
|
551
703
|
}
|
|
704
|
+
if (config.classifierOptions?.sendToClient) {
|
|
705
|
+
config.classifier?.on(
|
|
706
|
+
"Classification",
|
|
707
|
+
(classification) => this.socket?.send(
|
|
708
|
+
`${"Classification" /* Classification */} ${JSON.stringify(classification)}`
|
|
709
|
+
)
|
|
710
|
+
);
|
|
711
|
+
}
|
|
552
712
|
queueMicrotask(() => this.sendFirstMessage());
|
|
553
713
|
socket.on("close", this.onClose);
|
|
554
714
|
socket.on("message", this.onMessage);
|
|
555
715
|
}
|
|
556
716
|
/** Everything said so far, whether an agent or the server keeps it */
|
|
557
717
|
get conversation() {
|
|
558
|
-
return this.
|
|
718
|
+
return this.agent?.conversation ?? this.ownConversation;
|
|
719
|
+
}
|
|
720
|
+
/** What holds the conversation and answers, a realtime model included */
|
|
721
|
+
get agent() {
|
|
722
|
+
if (!this.config) return void 0;
|
|
723
|
+
return this.config.realtime ?? this.config.agent;
|
|
559
724
|
}
|
|
560
725
|
log(...message) {
|
|
561
726
|
this.logger?.log(...message);
|
|
@@ -580,8 +745,9 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
580
745
|
this.processQueue();
|
|
581
746
|
}
|
|
582
747
|
cancel() {
|
|
748
|
+
this.config?.classifier?.cancel();
|
|
583
749
|
this.config?.tts?.cancel();
|
|
584
|
-
this.
|
|
750
|
+
this.agent?.cancel();
|
|
585
751
|
this.operationQueue = [];
|
|
586
752
|
}
|
|
587
753
|
onUserAudio(chunk) {
|
|
@@ -596,6 +762,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
596
762
|
this.currentUserStream?.end();
|
|
597
763
|
this.currentUserStream = void 0;
|
|
598
764
|
this.cancel();
|
|
765
|
+
this.config?.realtime?.discardTurn();
|
|
599
766
|
}
|
|
600
767
|
onStartSpeaking() {
|
|
601
768
|
if (!this.config) return;
|
|
@@ -605,8 +772,13 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
605
772
|
this.config.turnDetector?.reset();
|
|
606
773
|
this.turnComplete = void 0;
|
|
607
774
|
this.releaseHeldTurn();
|
|
608
|
-
this.config.
|
|
609
|
-
|
|
775
|
+
if (this.config.realtime) {
|
|
776
|
+
this.cancel();
|
|
777
|
+
this.config.realtime.startTurn(this.currentUserStream);
|
|
778
|
+
} else {
|
|
779
|
+
this.config.stt?.transcribe(this.currentUserStream);
|
|
780
|
+
this.cancel();
|
|
781
|
+
}
|
|
610
782
|
}
|
|
611
783
|
onStopSpeaking() {
|
|
612
784
|
const hasNoUserSpeech = !this.currentUserStream || this.userSpeechChunks === 0;
|
|
@@ -618,6 +790,10 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
618
790
|
return;
|
|
619
791
|
}
|
|
620
792
|
this.turnComplete = this.predictTurnComplete();
|
|
793
|
+
if (this.config?.realtime) {
|
|
794
|
+
this.answerUserTurn();
|
|
795
|
+
return;
|
|
796
|
+
}
|
|
621
797
|
const conversation = this.conversation;
|
|
622
798
|
const lastMessage = conversation[conversation.length - 1];
|
|
623
799
|
if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
|
|
@@ -650,7 +826,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
650
826
|
}
|
|
651
827
|
this.releaseHeldTurn();
|
|
652
828
|
this.cancel();
|
|
653
|
-
this.
|
|
829
|
+
this.answerTurn();
|
|
654
830
|
}
|
|
655
831
|
/**
|
|
656
832
|
* Answers anyway if the rest of the sentence never comes.
|
|
@@ -664,9 +840,19 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
664
840
|
this.heldTurnTimer = void 0;
|
|
665
841
|
this.log("Nothing more came, answering");
|
|
666
842
|
this.cancel();
|
|
667
|
-
this.
|
|
843
|
+
this.answerTurn();
|
|
668
844
|
}, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
|
|
669
845
|
}
|
|
846
|
+
/** Answers what the user just said */
|
|
847
|
+
answerTurn() {
|
|
848
|
+
const realtime = this.config?.realtime;
|
|
849
|
+
if (realtime) {
|
|
850
|
+
realtime.endTurn();
|
|
851
|
+
} else {
|
|
852
|
+
this.classifyTurn();
|
|
853
|
+
this.answer();
|
|
854
|
+
}
|
|
855
|
+
}
|
|
670
856
|
releaseHeldTurn() {
|
|
671
857
|
if (!this.heldTurnTimer) return;
|
|
672
858
|
clearTimeout(this.heldTurnTimer);
|
|
@@ -674,16 +860,16 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
674
860
|
}
|
|
675
861
|
/** Adds a message to the conversation and sends it to the client */
|
|
676
862
|
addUserMessage(text, metadata) {
|
|
677
|
-
if (this.
|
|
678
|
-
this.
|
|
863
|
+
if (this.agent) {
|
|
864
|
+
this.agent.addUserMessage(text, metadata);
|
|
679
865
|
} else {
|
|
680
866
|
this.addOwnMessage("user", text, metadata);
|
|
681
867
|
}
|
|
682
868
|
}
|
|
683
869
|
/** Adds a message to the conversation and sends it to the client */
|
|
684
870
|
addAssistantMessage(text, metadata) {
|
|
685
|
-
if (this.
|
|
686
|
-
this.
|
|
871
|
+
if (this.agent) {
|
|
872
|
+
this.agent.addAssistantMessage(text, metadata);
|
|
687
873
|
} else {
|
|
688
874
|
this.addOwnMessage("assistant", text, metadata);
|
|
689
875
|
}
|
|
@@ -707,6 +893,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
707
893
|
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
708
894
|
);
|
|
709
895
|
this.emit("Message", message);
|
|
896
|
+
if (message.role === "user" && this.config?.realtime) this.classifyTurn();
|
|
710
897
|
}
|
|
711
898
|
sendPartialAnswer(content) {
|
|
712
899
|
if (!this.socket || content === "") return;
|
|
@@ -718,21 +905,28 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
718
905
|
sendFirstMessage() {
|
|
719
906
|
if (!this.config) return;
|
|
720
907
|
if (this.config.firstMessage) {
|
|
721
|
-
this.
|
|
908
|
+
this.agent?.addAssistantMessage(this.config.firstMessage);
|
|
722
909
|
this.speak(this.config.firstMessage);
|
|
723
|
-
} else if (this.config.generateFirstMessage && this.
|
|
910
|
+
} else if (this.config.generateFirstMessage && this.agent) {
|
|
724
911
|
this.answer();
|
|
725
912
|
} else {
|
|
726
913
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
727
914
|
}
|
|
728
915
|
}
|
|
729
916
|
answer() {
|
|
917
|
+
if (this.config?.classifierOptions?.waitBeforeAnswer) {
|
|
918
|
+
this.queueOperation(() => this.waitForClassification());
|
|
919
|
+
}
|
|
730
920
|
this.queueOperation(async () => {
|
|
731
921
|
await this._answer();
|
|
732
922
|
});
|
|
733
923
|
}
|
|
734
924
|
async _answer() {
|
|
735
925
|
if (!this.config) return;
|
|
926
|
+
if (this.config.realtime) {
|
|
927
|
+
this.config.realtime.respond();
|
|
928
|
+
return;
|
|
929
|
+
}
|
|
736
930
|
if (!this.config.agent) {
|
|
737
931
|
this.log("No agent, skipping answer");
|
|
738
932
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -772,6 +966,38 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
772
966
|
});
|
|
773
967
|
return stream.pipe(forward);
|
|
774
968
|
}
|
|
969
|
+
/**
|
|
970
|
+
* Classifies the turn that just ended, with the one before, and keeps the
|
|
971
|
+
* result in the metadata of its last message
|
|
972
|
+
*/
|
|
973
|
+
classifyTurn() {
|
|
974
|
+
const classifier = this.config?.classifier;
|
|
975
|
+
if (!classifier) return;
|
|
976
|
+
const { input, message } = turnInput(
|
|
977
|
+
this.conversation,
|
|
978
|
+
this.config?.classifierOptions?.history
|
|
979
|
+
);
|
|
980
|
+
if (!message || input.turn.trim() === "") return;
|
|
981
|
+
classifier.classify(input).then((classification) => {
|
|
982
|
+
if (!classification) return;
|
|
983
|
+
message.metadata = { ...message.metadata, classification };
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
/** Waits for the classification in progress, never longer than its limit */
|
|
987
|
+
async waitForClassification() {
|
|
988
|
+
const pending = this.config?.classifier?.pending;
|
|
989
|
+
if (!pending) return;
|
|
990
|
+
const limit = this.config?.classifierOptions?.maxWait ?? DEFAULT_CLASSIFICATION_MAX_WAIT;
|
|
991
|
+
let timer;
|
|
992
|
+
const timeout = new Promise((resolve) => {
|
|
993
|
+
timer = setTimeout(() => {
|
|
994
|
+
this.log(`Classification took over ${limit} ms, answering without it`);
|
|
995
|
+
resolve();
|
|
996
|
+
}, limit);
|
|
997
|
+
});
|
|
998
|
+
await Promise.race([pending, timeout]);
|
|
999
|
+
clearTimeout(timer);
|
|
1000
|
+
}
|
|
775
1001
|
// Run text-to-speech and send to client
|
|
776
1002
|
speak(message) {
|
|
777
1003
|
this.queueOperation(async () => {
|
|
@@ -780,6 +1006,11 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
780
1006
|
}
|
|
781
1007
|
async _speak(message) {
|
|
782
1008
|
if (!this.socket || !this.config) return;
|
|
1009
|
+
if (this.config.realtime) {
|
|
1010
|
+
const text = typeof message === "string" ? message : await readText(message);
|
|
1011
|
+
if (text.trim() !== "") this.config.realtime.say(text);
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
783
1014
|
let textStream;
|
|
784
1015
|
if (typeof message === "string") {
|
|
785
1016
|
if (!this.config.agent) {
|
|
@@ -818,6 +1049,14 @@ function hasContent(stream) {
|
|
|
818
1049
|
stream.on("end", onEnd);
|
|
819
1050
|
});
|
|
820
1051
|
}
|
|
1052
|
+
function readText(stream) {
|
|
1053
|
+
return new Promise((resolve) => {
|
|
1054
|
+
let text = "";
|
|
1055
|
+
stream.on("data", (chunk) => text += chunk.toString());
|
|
1056
|
+
stream.once("end", () => resolve(text));
|
|
1057
|
+
stream.once("error", () => resolve(text));
|
|
1058
|
+
});
|
|
1059
|
+
}
|
|
821
1060
|
function drain(stream) {
|
|
822
1061
|
return new Promise((resolve) => {
|
|
823
1062
|
stream.on("data", () => {
|
|
@@ -828,9 +1067,255 @@ function drain(stream) {
|
|
|
828
1067
|
});
|
|
829
1068
|
}
|
|
830
1069
|
|
|
1070
|
+
// src/realtime/Realtime.ts
|
|
1071
|
+
var USER_TRANSCRIPT_TIMEOUT = 3e3;
|
|
1072
|
+
function newAnswer(record) {
|
|
1073
|
+
return { transcript: "", hasAudio: false, record, resumed: false };
|
|
1074
|
+
}
|
|
1075
|
+
var Realtime = class extends Agent {
|
|
1076
|
+
constructor() {
|
|
1077
|
+
super(...arguments);
|
|
1078
|
+
// A turn opens with its first audio, so a blip without any sends nothing
|
|
1079
|
+
this.turn = "none";
|
|
1080
|
+
// Output of an interrupted answer can still be on its way
|
|
1081
|
+
this.discarding = false;
|
|
1082
|
+
// Answers asked for while another one was being given
|
|
1083
|
+
this.queue = [];
|
|
1084
|
+
this.answerWaiters = [];
|
|
1085
|
+
this.userTranscript = "";
|
|
1086
|
+
this.awaitingUserTranscript = false;
|
|
1087
|
+
this.afterUserTranscript = [];
|
|
1088
|
+
}
|
|
1089
|
+
/**
|
|
1090
|
+
* The user started speaking.
|
|
1091
|
+
*
|
|
1092
|
+
* The assistant stops, and the audio read from the stream goes to the model.
|
|
1093
|
+
* A turn left open, because the sentence sounded unfinished, carries on.
|
|
1094
|
+
*/
|
|
1095
|
+
startTurn(audio) {
|
|
1096
|
+
this.cancel();
|
|
1097
|
+
if (this.turn === "none") this.turn = "pending";
|
|
1098
|
+
audio.on("data", (chunk) => this.sendAudio(chunk));
|
|
1099
|
+
}
|
|
1100
|
+
/** Closes the turn of the user, and has the model answer it */
|
|
1101
|
+
endTurn() {
|
|
1102
|
+
if (this.turn !== "open") return;
|
|
1103
|
+
this.turn = "none";
|
|
1104
|
+
this.awaitUserTranscript();
|
|
1105
|
+
this.beginAnswer(true);
|
|
1106
|
+
this.closeTurn();
|
|
1107
|
+
}
|
|
1108
|
+
/** Drops the turn of the user, without an answer */
|
|
1109
|
+
discardTurn() {
|
|
1110
|
+
this.cancel();
|
|
1111
|
+
if (this.turn === "open") {
|
|
1112
|
+
this.clearTurn();
|
|
1113
|
+
this.discarding = true;
|
|
1114
|
+
}
|
|
1115
|
+
this.turn = "none";
|
|
1116
|
+
}
|
|
1117
|
+
/** Has the model speak without waiting for the user, to open a call */
|
|
1118
|
+
respond() {
|
|
1119
|
+
this.whenIdle(() => {
|
|
1120
|
+
this.beginAnswer(true);
|
|
1121
|
+
this.generate();
|
|
1122
|
+
});
|
|
1123
|
+
}
|
|
1124
|
+
/**
|
|
1125
|
+
* Has the model say this text, word for word.
|
|
1126
|
+
*
|
|
1127
|
+
* The caller records the text in the conversation, as with a text to speech,
|
|
1128
|
+
* so the transcript of what the model says is not recorded again.
|
|
1129
|
+
*/
|
|
1130
|
+
say(text) {
|
|
1131
|
+
this.whenIdle(() => {
|
|
1132
|
+
this.beginAnswer(false);
|
|
1133
|
+
this.speakText(text);
|
|
1134
|
+
});
|
|
1135
|
+
}
|
|
1136
|
+
cancel() {
|
|
1137
|
+
this.queue = [];
|
|
1138
|
+
const answer = this.currentAnswer;
|
|
1139
|
+
if (!answer) return;
|
|
1140
|
+
this.log("Cancel");
|
|
1141
|
+
this.currentAnswer = void 0;
|
|
1142
|
+
this.discarding = true;
|
|
1143
|
+
this.cancelAnswer();
|
|
1144
|
+
this.recordAnswer(answer);
|
|
1145
|
+
this.resolveAnswerWaiters();
|
|
1146
|
+
}
|
|
1147
|
+
/**
|
|
1148
|
+
* Adds a message the model did not hear, and tells the model about it.
|
|
1149
|
+
*
|
|
1150
|
+
* The transcripts of the call are recorded apart, so they are not sent back.
|
|
1151
|
+
*/
|
|
1152
|
+
addMessage(role, text, metadata) {
|
|
1153
|
+
super.addMessage(role, text, metadata);
|
|
1154
|
+
if (text.trim() !== "") this.injectMessage(role, text);
|
|
1155
|
+
}
|
|
1156
|
+
addTool(tool) {
|
|
1157
|
+
super.addTool(tool);
|
|
1158
|
+
this.updateTools();
|
|
1159
|
+
}
|
|
1160
|
+
removeTool(name) {
|
|
1161
|
+
super.removeTool(name);
|
|
1162
|
+
this.updateTools();
|
|
1163
|
+
}
|
|
1164
|
+
/**
|
|
1165
|
+
* Lists the events that have listeners.
|
|
1166
|
+
*
|
|
1167
|
+
* Typed as the events of any agent, so a realtime model goes wherever an
|
|
1168
|
+
* agent is expected. The two extra events do not change what that code reads.
|
|
1169
|
+
*/
|
|
1170
|
+
eventNames() {
|
|
1171
|
+
return super.eventNames();
|
|
1172
|
+
}
|
|
1173
|
+
destroy() {
|
|
1174
|
+
super.destroy();
|
|
1175
|
+
this.clearUserTranscriptTimer();
|
|
1176
|
+
this.afterUserTranscript = [];
|
|
1177
|
+
this.resolveAnswerWaiters();
|
|
1178
|
+
}
|
|
1179
|
+
generateAnswer(stream) {
|
|
1180
|
+
let written = 0;
|
|
1181
|
+
const onPartial = (transcript) => {
|
|
1182
|
+
stream.write(transcript.slice(written));
|
|
1183
|
+
written = transcript.length;
|
|
1184
|
+
};
|
|
1185
|
+
this.on("PartialMessage", onPartial);
|
|
1186
|
+
this.respond();
|
|
1187
|
+
return new Promise((resolve) => {
|
|
1188
|
+
this.answerWaiters.push(() => {
|
|
1189
|
+
this.off("PartialMessage", onPartial);
|
|
1190
|
+
resolve();
|
|
1191
|
+
});
|
|
1192
|
+
});
|
|
1193
|
+
}
|
|
1194
|
+
/** Nobody is speaking, nor about to, so the connection can be renewed */
|
|
1195
|
+
get isIdle() {
|
|
1196
|
+
return this.turn === "none" && !this.currentAnswer;
|
|
1197
|
+
}
|
|
1198
|
+
/** Voice of the answer, PCM16 at 16 kHz */
|
|
1199
|
+
emitAudio(chunk) {
|
|
1200
|
+
const answer = this.activeAnswer();
|
|
1201
|
+
if (!answer) return;
|
|
1202
|
+
answer.hasAudio = true;
|
|
1203
|
+
this.emit("Audio", chunk);
|
|
1204
|
+
}
|
|
1205
|
+
/** Transcript of the answer, as it is spoken */
|
|
1206
|
+
addAnswerTranscript(text) {
|
|
1207
|
+
const answer = this.activeAnswer();
|
|
1208
|
+
if (!answer) return;
|
|
1209
|
+
if (answer.resumed) {
|
|
1210
|
+
answer.resumed = false;
|
|
1211
|
+
if (answer.transcript !== "" && !/\s$/.test(answer.transcript)) {
|
|
1212
|
+
text = ` ${text.trimStart()}`;
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
answer.transcript += text;
|
|
1216
|
+
if (answer.record) this.emit("PartialMessage", answer.transcript);
|
|
1217
|
+
}
|
|
1218
|
+
/**
|
|
1219
|
+
* The model finished its answer, or finished dropping an interrupted one.
|
|
1220
|
+
*
|
|
1221
|
+
* An answer without audio tells the client to stop waiting for one.
|
|
1222
|
+
*/
|
|
1223
|
+
endAnswer() {
|
|
1224
|
+
if (this.discarding) {
|
|
1225
|
+
this.discarding = false;
|
|
1226
|
+
return;
|
|
1227
|
+
}
|
|
1228
|
+
const answer = this.currentAnswer;
|
|
1229
|
+
if (!answer) return;
|
|
1230
|
+
this.currentAnswer = void 0;
|
|
1231
|
+
if (!answer.hasAudio) this.emitAgentEvent("SkipAnswer");
|
|
1232
|
+
this.recordAnswer(answer);
|
|
1233
|
+
this.resolveAnswerWaiters();
|
|
1234
|
+
this.queue.shift()?.();
|
|
1235
|
+
}
|
|
1236
|
+
/** Runs a tool the model called, and records it in the conversation */
|
|
1237
|
+
async runTool(toolCall) {
|
|
1238
|
+
const result = await this.executeTool(toolCall);
|
|
1239
|
+
if (this.currentAnswer) this.currentAnswer.resumed = true;
|
|
1240
|
+
return result;
|
|
1241
|
+
}
|
|
1242
|
+
/** Transcript of the turn of the user, possibly in several pieces */
|
|
1243
|
+
addUserTranscript(text) {
|
|
1244
|
+
this.userTranscript += text;
|
|
1245
|
+
}
|
|
1246
|
+
/** The transcript of the turn is complete, it goes in the conversation */
|
|
1247
|
+
commitUserTranscript() {
|
|
1248
|
+
this.clearUserTranscriptTimer();
|
|
1249
|
+
this.awaitingUserTranscript = false;
|
|
1250
|
+
const text = this.userTranscript.trim();
|
|
1251
|
+
this.userTranscript = "";
|
|
1252
|
+
if (text !== "") super.addMessage("user", text);
|
|
1253
|
+
const pending = this.afterUserTranscript;
|
|
1254
|
+
this.afterUserTranscript = [];
|
|
1255
|
+
pending.forEach((record) => record());
|
|
1256
|
+
}
|
|
1257
|
+
sendAudio(chunk) {
|
|
1258
|
+
if (this.turn === "none") return;
|
|
1259
|
+
if (this.turn === "pending") {
|
|
1260
|
+
this.turn = "open";
|
|
1261
|
+
this.openTurn();
|
|
1262
|
+
}
|
|
1263
|
+
this.appendAudio(chunk);
|
|
1264
|
+
}
|
|
1265
|
+
beginAnswer(record) {
|
|
1266
|
+
this.discarding = false;
|
|
1267
|
+
this.currentAnswer = newAnswer(record);
|
|
1268
|
+
}
|
|
1269
|
+
/**
|
|
1270
|
+
* The answer output belongs to, if it is still wanted.
|
|
1271
|
+
*
|
|
1272
|
+
* The model can speak without being asked, to follow up on a tool call for
|
|
1273
|
+
* instance, which opens an answer of its own.
|
|
1274
|
+
*/
|
|
1275
|
+
activeAnswer() {
|
|
1276
|
+
if (this.discarding) return void 0;
|
|
1277
|
+
if (!this.currentAnswer) this.currentAnswer = newAnswer(true);
|
|
1278
|
+
return this.currentAnswer;
|
|
1279
|
+
}
|
|
1280
|
+
whenIdle(start) {
|
|
1281
|
+
if (this.currentAnswer) {
|
|
1282
|
+
this.queue.push(start);
|
|
1283
|
+
} else {
|
|
1284
|
+
start();
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
recordAnswer(answer) {
|
|
1288
|
+
if (!answer.record) return;
|
|
1289
|
+
const record = () => super.addMessage("assistant", answer.transcript.trim());
|
|
1290
|
+
if (this.awaitingUserTranscript) {
|
|
1291
|
+
this.afterUserTranscript.push(record);
|
|
1292
|
+
} else {
|
|
1293
|
+
record();
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
awaitUserTranscript() {
|
|
1297
|
+
this.awaitingUserTranscript = true;
|
|
1298
|
+
this.clearUserTranscriptTimer();
|
|
1299
|
+
this.userTranscriptTimer = setTimeout(() => {
|
|
1300
|
+
this.log("No transcript of the turn, recording the answer anyway");
|
|
1301
|
+
this.commitUserTranscript();
|
|
1302
|
+
}, USER_TRANSCRIPT_TIMEOUT);
|
|
1303
|
+
}
|
|
1304
|
+
clearUserTranscriptTimer() {
|
|
1305
|
+
if (!this.userTranscriptTimer) return;
|
|
1306
|
+
clearTimeout(this.userTranscriptTimer);
|
|
1307
|
+
this.userTranscriptTimer = void 0;
|
|
1308
|
+
}
|
|
1309
|
+
resolveAnswerWaiters() {
|
|
1310
|
+
const waiters = this.answerWaiters;
|
|
1311
|
+
this.answerWaiters = [];
|
|
1312
|
+
waiters.forEach((resolve) => resolve());
|
|
1313
|
+
}
|
|
1314
|
+
};
|
|
1315
|
+
|
|
831
1316
|
// src/recorder/MicdropRecorder.ts
|
|
832
|
-
import { EventEmitter as
|
|
833
|
-
var MicdropRecorder = class extends
|
|
1317
|
+
import { EventEmitter as EventEmitter4 } from "eventemitter3";
|
|
1318
|
+
var MicdropRecorder = class extends EventEmitter4 {
|
|
834
1319
|
constructor(server) {
|
|
835
1320
|
super();
|
|
836
1321
|
this.server = server;
|
|
@@ -987,8 +1472,8 @@ var SharedInstances = class {
|
|
|
987
1472
|
};
|
|
988
1473
|
|
|
989
1474
|
// src/stt/STT.ts
|
|
990
|
-
import { EventEmitter as
|
|
991
|
-
var STT = class extends
|
|
1475
|
+
import { EventEmitter as EventEmitter5 } from "eventemitter3";
|
|
1476
|
+
var STT = class extends EventEmitter5 {
|
|
992
1477
|
log(...message) {
|
|
993
1478
|
this.logger?.log(...message);
|
|
994
1479
|
}
|
|
@@ -1069,8 +1554,8 @@ var FallbackSTT = class extends STT {
|
|
|
1069
1554
|
import { PassThrough as PassThrough4 } from "stream";
|
|
1070
1555
|
|
|
1071
1556
|
// src/tts/TTS.ts
|
|
1072
|
-
import { EventEmitter as
|
|
1073
|
-
var TTS = class extends
|
|
1557
|
+
import { EventEmitter as EventEmitter6 } from "eventemitter3";
|
|
1558
|
+
var TTS = class extends EventEmitter6 {
|
|
1074
1559
|
log(...message) {
|
|
1075
1560
|
this.logger?.log(...message);
|
|
1076
1561
|
}
|
|
@@ -1321,6 +1806,7 @@ export {
|
|
|
1321
1806
|
AUTO_SEMANTIC_TURN_PROMPT,
|
|
1322
1807
|
AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
1323
1808
|
Agent,
|
|
1809
|
+
Classifier,
|
|
1324
1810
|
FallbackAgent,
|
|
1325
1811
|
FallbackSTT,
|
|
1326
1812
|
FallbackTTS,
|
|
@@ -1335,14 +1821,18 @@ export {
|
|
|
1335
1821
|
MockSTT,
|
|
1336
1822
|
MockTTS,
|
|
1337
1823
|
Pcm16Resampler,
|
|
1824
|
+
Realtime,
|
|
1338
1825
|
STT,
|
|
1339
1826
|
SentenceSplitter,
|
|
1340
1827
|
SentenceTTS,
|
|
1341
1828
|
SharedInstances,
|
|
1342
1829
|
TTS,
|
|
1830
|
+
currentTurn,
|
|
1343
1831
|
float32ToPcm16,
|
|
1832
|
+
getTurnClassification,
|
|
1344
1833
|
handleError,
|
|
1345
1834
|
pcm16ToFloat32,
|
|
1835
|
+
turnInput,
|
|
1346
1836
|
waitForParams
|
|
1347
1837
|
};
|
|
1348
1838
|
//# sourceMappingURL=index.mjs.map
|