@micdrop/server 2.8.0 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +348 -10
- package/dist/index.d.ts +348 -10
- package/dist/index.js +624 -70
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +619 -70
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -37,6 +37,7 @@ __export(index_exports, {
|
|
|
37
37
|
AUTO_SEMANTIC_TURN_PROMPT: () => AUTO_SEMANTIC_TURN_PROMPT,
|
|
38
38
|
AUTO_SEMANTIC_TURN_TOOL_NAME: () => AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
39
39
|
Agent: () => Agent,
|
|
40
|
+
Classifier: () => Classifier,
|
|
40
41
|
FallbackAgent: () => FallbackAgent,
|
|
41
42
|
FallbackSTT: () => FallbackSTT,
|
|
42
43
|
FallbackTTS: () => FallbackTTS,
|
|
@@ -52,14 +53,18 @@ __export(index_exports, {
|
|
|
52
53
|
MockTTS: () => MockTTS,
|
|
53
54
|
Pcm16Resampler: () => Pcm16Resampler,
|
|
54
55
|
Realtime: () => Realtime,
|
|
56
|
+
ReconnectingSocket: () => ReconnectingSocket,
|
|
55
57
|
STT: () => STT,
|
|
56
58
|
SentenceSplitter: () => SentenceSplitter,
|
|
57
59
|
SentenceTTS: () => SentenceTTS,
|
|
58
60
|
SharedInstances: () => SharedInstances,
|
|
59
61
|
TTS: () => TTS,
|
|
62
|
+
currentTurn: () => currentTurn,
|
|
60
63
|
float32ToPcm16: () => float32ToPcm16,
|
|
64
|
+
getTurnClassification: () => getTurnClassification,
|
|
61
65
|
handleError: () => handleError,
|
|
62
66
|
pcm16ToFloat32: () => pcm16ToFloat32,
|
|
67
|
+
turnInput: () => turnInput,
|
|
63
68
|
waitForParams: () => waitForParams
|
|
64
69
|
});
|
|
65
70
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -81,27 +86,65 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
81
86
|
constructor(options) {
|
|
82
87
|
super();
|
|
83
88
|
this.options = options;
|
|
89
|
+
/**
|
|
90
|
+
* @deprecated Read the signal given to `generateAnswer` instead. Still
|
|
91
|
+
* bumped by every answer and every cancel, for agents written before it.
|
|
92
|
+
*/
|
|
84
93
|
this.answerCount = 0;
|
|
85
|
-
this.answering = false;
|
|
86
94
|
this.conversation = [{ role: "system", content: options.systemPrompt }];
|
|
87
95
|
this.tools = this.getDefaultTools();
|
|
88
96
|
}
|
|
97
|
+
/** Whether an answer is being generated */
|
|
98
|
+
get isAnswering() {
|
|
99
|
+
return !!this.answerController;
|
|
100
|
+
}
|
|
101
|
+
/** @deprecated Read `isAnswering` instead */
|
|
102
|
+
get answering() {
|
|
103
|
+
return this.isAnswering;
|
|
104
|
+
}
|
|
105
|
+
/** @deprecated The answer is over once its stream ends or it is cancelled */
|
|
106
|
+
set answering(_value) {
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Stops the answer being generated, by aborting its signal.
|
|
110
|
+
*
|
|
111
|
+
* An agent overriding it to stop more than the signal does calls
|
|
112
|
+
* `super.cancel()`.
|
|
113
|
+
*/
|
|
114
|
+
cancel() {
|
|
115
|
+
this.answerCount++;
|
|
116
|
+
if (!this.answerController) return;
|
|
117
|
+
this.log("Cancel");
|
|
118
|
+
this.answerController.abort();
|
|
119
|
+
this.answerController = void 0;
|
|
120
|
+
}
|
|
89
121
|
answer() {
|
|
90
122
|
this.log("Start answering");
|
|
91
|
-
|
|
123
|
+
this.answerCount++;
|
|
124
|
+
this.answerController?.abort();
|
|
125
|
+
const controller = new AbortController();
|
|
126
|
+
this.answerController = controller;
|
|
127
|
+
const { signal } = controller;
|
|
92
128
|
const stream = new import_stream.PassThrough();
|
|
93
|
-
this.answering = true;
|
|
94
129
|
Promise.resolve().then(
|
|
95
130
|
() => this.options.onBeforeAnswer?.bind(this)(stream)
|
|
96
|
-
).then((
|
|
97
|
-
if (
|
|
98
|
-
|
|
131
|
+
).then((result) => {
|
|
132
|
+
if (signal.aborted) return;
|
|
133
|
+
if (typeof result === "string") {
|
|
134
|
+
stream.write(result);
|
|
135
|
+
this.addAssistantMessage(result);
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
if (result) return;
|
|
139
|
+
return this.generateAnswer(stream, signal);
|
|
140
|
+
}).catch((error) => {
|
|
141
|
+
if (!signal.aborted) this.log("Error answering:", error);
|
|
99
142
|
}).finally(() => {
|
|
100
143
|
if (stream.writable) {
|
|
101
144
|
stream.end();
|
|
102
145
|
}
|
|
103
|
-
if (
|
|
104
|
-
this.
|
|
146
|
+
if (this.answerController === controller) {
|
|
147
|
+
this.answerController = void 0;
|
|
105
148
|
}
|
|
106
149
|
});
|
|
107
150
|
return stream;
|
|
@@ -327,10 +370,10 @@ var FallbackAgent = class extends Agent {
|
|
|
327
370
|
extract(message) {
|
|
328
371
|
return this.agent ? this.agent.extract(message) : super.extract(message);
|
|
329
372
|
}
|
|
330
|
-
async generateAnswer(stream) {
|
|
373
|
+
async generateAnswer(stream, signal) {
|
|
331
374
|
for (let attempt = 0; attempt < this.fallbackOptions.factories.length; attempt++) {
|
|
332
375
|
const agent = this.agent;
|
|
333
|
-
if (!agent) return;
|
|
376
|
+
if (!agent || signal.aborted) return;
|
|
334
377
|
let failed = false;
|
|
335
378
|
const onFailed = () => {
|
|
336
379
|
failed = true;
|
|
@@ -341,7 +384,7 @@ var FallbackAgent = class extends Agent {
|
|
|
341
384
|
} finally {
|
|
342
385
|
agent.off("Failed", onFailed);
|
|
343
386
|
}
|
|
344
|
-
if (!failed) return;
|
|
387
|
+
if (!failed || signal.aborted) return;
|
|
345
388
|
this.log("Agent failed, trying next agent");
|
|
346
389
|
this.startNextAgent();
|
|
347
390
|
}
|
|
@@ -349,6 +392,7 @@ var FallbackAgent = class extends Agent {
|
|
|
349
392
|
this.emit("Failed");
|
|
350
393
|
}
|
|
351
394
|
cancel() {
|
|
395
|
+
super.cancel();
|
|
352
396
|
this.agent?.cancel();
|
|
353
397
|
}
|
|
354
398
|
destroy() {
|
|
@@ -411,13 +455,149 @@ var MockAgent = class extends Agent {
|
|
|
411
455
|
}
|
|
412
456
|
async generateAnswer(stream) {
|
|
413
457
|
const message = `Assistant Message ${this.i++}`;
|
|
414
|
-
this.addAssistantMessage(message);
|
|
415
458
|
stream.write(message);
|
|
459
|
+
this.addAssistantMessage(message);
|
|
460
|
+
}
|
|
461
|
+
};
|
|
462
|
+
|
|
463
|
+
// src/classifier/Classifier.ts
|
|
464
|
+
var import_eventemitter32 = require("eventemitter3");
|
|
465
|
+
var Classifier = class extends import_eventemitter32.EventEmitter {
|
|
466
|
+
constructor() {
|
|
467
|
+
super(...arguments);
|
|
468
|
+
this.controllers = /* @__PURE__ */ new Set();
|
|
469
|
+
}
|
|
470
|
+
/**
|
|
471
|
+
* Classifies an input, and emits the result as `Classification`
|
|
472
|
+
* @param input - What to classify: a text, or JSON with named fields
|
|
473
|
+
* @returns The classification, or undefined when cancelled or failed
|
|
474
|
+
*/
|
|
475
|
+
classify(input) {
|
|
476
|
+
const controller = new AbortController();
|
|
477
|
+
this.controllers.add(controller);
|
|
478
|
+
const pending = this.run(input, controller.signal).finally(() => {
|
|
479
|
+
this.controllers.delete(controller);
|
|
480
|
+
if (this._pending === pending) this._pending = void 0;
|
|
481
|
+
});
|
|
482
|
+
this._pending = pending;
|
|
483
|
+
return pending;
|
|
484
|
+
}
|
|
485
|
+
/** The last classification started, until it is done */
|
|
486
|
+
get pending() {
|
|
487
|
+
return this._pending;
|
|
416
488
|
}
|
|
489
|
+
/** Drops every classification in progress, which then emit nothing */
|
|
417
490
|
cancel() {
|
|
491
|
+
this.controllers.forEach((controller) => controller.abort());
|
|
492
|
+
this.controllers.clear();
|
|
493
|
+
this._pending = void 0;
|
|
494
|
+
}
|
|
495
|
+
log(...message) {
|
|
496
|
+
this.logger?.log(...message);
|
|
497
|
+
}
|
|
498
|
+
destroy() {
|
|
499
|
+
this.log("Destroyed");
|
|
500
|
+
this.cancel();
|
|
501
|
+
this.removeAllListeners();
|
|
502
|
+
}
|
|
503
|
+
async run(input, signal) {
|
|
504
|
+
const start = Date.now();
|
|
505
|
+
try {
|
|
506
|
+
const result = await this.evaluate(input, signal);
|
|
507
|
+
if (signal.aborted) return;
|
|
508
|
+
const classification = {
|
|
509
|
+
input,
|
|
510
|
+
result,
|
|
511
|
+
duration: Date.now() - start
|
|
512
|
+
};
|
|
513
|
+
this.log(
|
|
514
|
+
`Classified in ${classification.duration} ms:`,
|
|
515
|
+
JSON.stringify(input)
|
|
516
|
+
);
|
|
517
|
+
this.lastClassification = classification;
|
|
518
|
+
this.emit("Classification", classification);
|
|
519
|
+
return classification;
|
|
520
|
+
} catch (error) {
|
|
521
|
+
if (!signal.aborted) this.log("Classification failed:", error);
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
418
524
|
}
|
|
419
525
|
};
|
|
420
526
|
|
|
527
|
+
// src/types.ts
|
|
528
|
+
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
529
|
+
MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
530
|
+
MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
531
|
+
MicdropClientCommands2["Mute"] = "Mute";
|
|
532
|
+
return MicdropClientCommands2;
|
|
533
|
+
})(MicdropClientCommands || {});
|
|
534
|
+
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
535
|
+
MicdropServerCommands2["Message"] = "Message";
|
|
536
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
537
|
+
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
538
|
+
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
539
|
+
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
540
|
+
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
541
|
+
MicdropServerCommands2["Classification"] = "Classification";
|
|
542
|
+
return MicdropServerCommands2;
|
|
543
|
+
})(MicdropServerCommands || {});
|
|
544
|
+
function currentTurn(conversation) {
|
|
545
|
+
let start = conversation.length;
|
|
546
|
+
while (start > 0) {
|
|
547
|
+
const item = conversation[start - 1];
|
|
548
|
+
if (item.role !== "user" || item.metadata?.classification) break;
|
|
549
|
+
start--;
|
|
550
|
+
}
|
|
551
|
+
const messages = conversation.slice(start);
|
|
552
|
+
return {
|
|
553
|
+
transcript: messages.map((message) => message.content).join(" "),
|
|
554
|
+
messages,
|
|
555
|
+
before: conversation.slice(0, start)
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
function getTurnClassification(conversation) {
|
|
559
|
+
for (let index = conversation.length - 1; index >= 0; index--) {
|
|
560
|
+
const item = conversation[index];
|
|
561
|
+
if (item.role === "user") return item.metadata?.classification;
|
|
562
|
+
}
|
|
563
|
+
return void 0;
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
// src/classifier/turnInput.ts
|
|
567
|
+
function turnInput(conversation, history = 1) {
|
|
568
|
+
const turn = currentTurn(conversation);
|
|
569
|
+
const input = { history: [], turn: turn.transcript };
|
|
570
|
+
let rest = turn.before;
|
|
571
|
+
for (let n = 0; n < history && rest.length; n++) {
|
|
572
|
+
let end = rest.length;
|
|
573
|
+
const answers = [];
|
|
574
|
+
while (end > 0 && rest[end - 1].role !== "user") {
|
|
575
|
+
const item = rest[end - 1];
|
|
576
|
+
if (item.role === "assistant" && item.content.trim()) {
|
|
577
|
+
answers.unshift(item.content);
|
|
578
|
+
}
|
|
579
|
+
end--;
|
|
580
|
+
}
|
|
581
|
+
let start = end;
|
|
582
|
+
while (start > 0) {
|
|
583
|
+
const item = rest[start - 1];
|
|
584
|
+
if (item.role !== "user") break;
|
|
585
|
+
if (start < end && item.metadata?.classification)
|
|
586
|
+
break;
|
|
587
|
+
start--;
|
|
588
|
+
}
|
|
589
|
+
if (answers.length) {
|
|
590
|
+
input.history.unshift({ role: "assistant", text: answers.join(" ") });
|
|
591
|
+
}
|
|
592
|
+
if (start < end) {
|
|
593
|
+
const text = rest.slice(start, end).map((item) => "content" in item ? item.content : "").join(" ");
|
|
594
|
+
input.history.unshift({ role: "user", text });
|
|
595
|
+
}
|
|
596
|
+
rest = rest.slice(0, start);
|
|
597
|
+
}
|
|
598
|
+
return { input, message: turn.messages[turn.messages.length - 1] };
|
|
599
|
+
}
|
|
600
|
+
|
|
421
601
|
// src/audio/Pcm16Resampler.ts
|
|
422
602
|
var Pcm16Resampler = class {
|
|
423
603
|
// Fractional position into the first sample of the buffer
|
|
@@ -505,35 +685,23 @@ function handleError(socket, error) {
|
|
|
505
685
|
}
|
|
506
686
|
|
|
507
687
|
// src/MicdropServer.ts
|
|
508
|
-
var
|
|
688
|
+
var import_eventemitter33 = require("eventemitter3");
|
|
509
689
|
var import_stream2 = require("stream");
|
|
510
|
-
|
|
511
|
-
// src/types.ts
|
|
512
|
-
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
513
|
-
MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
514
|
-
MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
515
|
-
MicdropClientCommands2["Mute"] = "Mute";
|
|
516
|
-
return MicdropClientCommands2;
|
|
517
|
-
})(MicdropClientCommands || {});
|
|
518
|
-
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
519
|
-
MicdropServerCommands2["Message"] = "Message";
|
|
520
|
-
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
521
|
-
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
522
|
-
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
523
|
-
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
524
|
-
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
525
|
-
return MicdropServerCommands2;
|
|
526
|
-
})(MicdropServerCommands || {});
|
|
527
|
-
|
|
528
|
-
// src/MicdropServer.ts
|
|
529
690
|
var USER_SAMPLE_RATE = 16e3;
|
|
530
691
|
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
531
|
-
var
|
|
692
|
+
var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
|
|
693
|
+
var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
532
694
|
constructor(socket, config) {
|
|
533
695
|
super();
|
|
534
696
|
this.socket = null;
|
|
535
697
|
this.config = null;
|
|
536
698
|
this.startTime = Date.now();
|
|
699
|
+
// Bumped by every cancel, so an answer cancelled before its first word can
|
|
700
|
+
// tell, and leave its message to the answer that replaces it
|
|
701
|
+
this.cancelRound = 0;
|
|
702
|
+
// Counts the assistant messages sent, so the partial answer of one already
|
|
703
|
+
// settled is not sent after it
|
|
704
|
+
this.assistantMessages = 0;
|
|
537
705
|
// Holds the conversation when no agent does, and keeps it readable once the
|
|
538
706
|
// call is over and the config is gone
|
|
539
707
|
this.ownConversation = [];
|
|
@@ -541,15 +709,22 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
541
709
|
this.operationQueue = [];
|
|
542
710
|
this.isProcessingQueue = false;
|
|
543
711
|
this.userSpeechChunks = 0;
|
|
544
|
-
|
|
712
|
+
// Bumped whenever the user speaks again or mutes, so a decision taken on an
|
|
713
|
+
// earlier stretch of speech is dropped rather than answering over them
|
|
714
|
+
this.turnRound = 0;
|
|
715
|
+
this.onClose = (code, reason) => {
|
|
545
716
|
this.releaseHeldTurn();
|
|
546
717
|
if (!this.config) return;
|
|
547
|
-
this.log("Connection closed"
|
|
718
|
+
this.log("Connection closed", {
|
|
719
|
+
code,
|
|
720
|
+
reason: reason?.toString() || void 0
|
|
721
|
+
});
|
|
548
722
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
549
723
|
const conversation = this.conversation;
|
|
550
724
|
this.agent?.destroy();
|
|
551
725
|
this.config.stt?.destroy();
|
|
552
726
|
this.config.tts?.destroy();
|
|
727
|
+
this.config.classifier?.destroy();
|
|
553
728
|
this.emit("End", {
|
|
554
729
|
conversation,
|
|
555
730
|
duration
|
|
@@ -641,6 +816,14 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
641
816
|
)
|
|
642
817
|
);
|
|
643
818
|
}
|
|
819
|
+
if (config.classifierOptions?.sendToClient) {
|
|
820
|
+
config.classifier?.on(
|
|
821
|
+
"Classification",
|
|
822
|
+
(classification) => this.socket?.send(
|
|
823
|
+
`${"Classification" /* Classification */} ${JSON.stringify(classification)}`
|
|
824
|
+
)
|
|
825
|
+
);
|
|
826
|
+
}
|
|
644
827
|
queueMicrotask(() => this.sendFirstMessage());
|
|
645
828
|
socket.on("close", this.onClose);
|
|
646
829
|
socket.on("message", this.onMessage);
|
|
@@ -677,6 +860,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
677
860
|
this.processQueue();
|
|
678
861
|
}
|
|
679
862
|
cancel() {
|
|
863
|
+
this.cancelRound++;
|
|
864
|
+
this.config?.classifier?.cancel();
|
|
680
865
|
this.config?.tts?.cancel();
|
|
681
866
|
this.agent?.cancel();
|
|
682
867
|
this.operationQueue = [];
|
|
@@ -689,6 +874,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
689
874
|
this.emit("UserAudio", chunk);
|
|
690
875
|
}
|
|
691
876
|
onMute() {
|
|
877
|
+
this.turnRound++;
|
|
878
|
+
this.releaseHeldTurn();
|
|
692
879
|
this.userSpeechChunks = 0;
|
|
693
880
|
this.currentUserStream?.end();
|
|
694
881
|
this.currentUserStream = void 0;
|
|
@@ -697,6 +884,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
697
884
|
}
|
|
698
885
|
onStartSpeaking() {
|
|
699
886
|
if (!this.config) return;
|
|
887
|
+
this.turnRound++;
|
|
700
888
|
this.userSpeechChunks = 0;
|
|
701
889
|
this.currentUserStream?.end();
|
|
702
890
|
this.currentUserStream = new import_stream2.PassThrough();
|
|
@@ -748,7 +936,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
748
936
|
}
|
|
749
937
|
/** Answers, unless the sentence sounds like it has more coming */
|
|
750
938
|
async answerUserTurn() {
|
|
939
|
+
const round = this.turnRound;
|
|
751
940
|
const complete = await (this.turnComplete ?? Promise.resolve(true));
|
|
941
|
+
if (round !== this.turnRound || !this.config) return;
|
|
752
942
|
if (!complete) {
|
|
753
943
|
this.log("Waiting for the rest of the sentence");
|
|
754
944
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -780,6 +970,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
780
970
|
if (realtime) {
|
|
781
971
|
realtime.endTurn();
|
|
782
972
|
} else {
|
|
973
|
+
this.classifyTurn();
|
|
783
974
|
this.answer();
|
|
784
975
|
}
|
|
785
976
|
}
|
|
@@ -823,6 +1014,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
823
1014
|
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
824
1015
|
);
|
|
825
1016
|
this.emit("Message", message);
|
|
1017
|
+
if (message.role === "assistant") this.assistantMessages++;
|
|
1018
|
+
if (message.role === "user" && this.config?.realtime) this.classifyTurn();
|
|
826
1019
|
}
|
|
827
1020
|
sendPartialAnswer(content) {
|
|
828
1021
|
if (!this.socket || content === "") return;
|
|
@@ -843,6 +1036,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
843
1036
|
}
|
|
844
1037
|
}
|
|
845
1038
|
answer() {
|
|
1039
|
+
if (this.config?.classifierOptions?.waitBeforeAnswer) {
|
|
1040
|
+
this.queueOperation(() => this.waitForClassification());
|
|
1041
|
+
}
|
|
846
1042
|
this.queueOperation(async () => {
|
|
847
1043
|
await this._answer();
|
|
848
1044
|
});
|
|
@@ -865,12 +1061,18 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
865
1061
|
return;
|
|
866
1062
|
}
|
|
867
1063
|
this.lastMessageSpeeched = lastMessage;
|
|
1064
|
+
const round = this.cancelRound;
|
|
868
1065
|
try {
|
|
1066
|
+
const sentBefore = this.assistantMessages;
|
|
869
1067
|
const stream = this.config.agent.answer();
|
|
870
1068
|
if (await hasContent(stream)) {
|
|
871
1069
|
await this._speak(
|
|
872
|
-
this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
|
|
1070
|
+
this.config.partialMessages ? this.forwardPartialAnswer(stream, sentBefore) : stream
|
|
873
1071
|
);
|
|
1072
|
+
} else if (round !== this.cancelRound) {
|
|
1073
|
+
if (this.lastMessageSpeeched === lastMessage) {
|
|
1074
|
+
this.lastMessageSpeeched = void 0;
|
|
1075
|
+
}
|
|
874
1076
|
} else {
|
|
875
1077
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
876
1078
|
}
|
|
@@ -879,10 +1081,19 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
879
1081
|
throw error;
|
|
880
1082
|
}
|
|
881
1083
|
}
|
|
882
|
-
/**
|
|
883
|
-
|
|
1084
|
+
/**
|
|
1085
|
+
* Sends the answer to the client as it is written, ahead of the voice.
|
|
1086
|
+
*
|
|
1087
|
+
* The stream reaches here a moment after the agent wrote it, so the agent
|
|
1088
|
+
* can record the complete message first, which the client then shows. A
|
|
1089
|
+
* partial answer arriving after it would show the answer twice, so the
|
|
1090
|
+
* forwarding stops once the message is sent.
|
|
1091
|
+
*/
|
|
1092
|
+
forwardPartialAnswer(stream, sentBefore) {
|
|
884
1093
|
let content = "";
|
|
885
|
-
const send = (text) =>
|
|
1094
|
+
const send = (text) => {
|
|
1095
|
+
if (this.assistantMessages === sentBefore) this.sendPartialAnswer(text);
|
|
1096
|
+
};
|
|
886
1097
|
const forward = new import_stream2.Transform({
|
|
887
1098
|
transform(chunk, _encoding, callback) {
|
|
888
1099
|
content += chunk.toString();
|
|
@@ -892,6 +1103,38 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
892
1103
|
});
|
|
893
1104
|
return stream.pipe(forward);
|
|
894
1105
|
}
|
|
1106
|
+
/**
|
|
1107
|
+
* Classifies the turn that just ended, with the one before, and keeps the
|
|
1108
|
+
* result in the metadata of its last message
|
|
1109
|
+
*/
|
|
1110
|
+
classifyTurn() {
|
|
1111
|
+
const classifier = this.config?.classifier;
|
|
1112
|
+
if (!classifier) return;
|
|
1113
|
+
const { input, message } = turnInput(
|
|
1114
|
+
this.conversation,
|
|
1115
|
+
this.config?.classifierOptions?.history
|
|
1116
|
+
);
|
|
1117
|
+
if (!message || input.turn.trim() === "") return;
|
|
1118
|
+
classifier.classify(input).then((classification) => {
|
|
1119
|
+
if (!classification) return;
|
|
1120
|
+
message.metadata = { ...message.metadata, classification };
|
|
1121
|
+
});
|
|
1122
|
+
}
|
|
1123
|
+
/** Waits for the classification in progress, never longer than its limit */
|
|
1124
|
+
async waitForClassification() {
|
|
1125
|
+
const pending = this.config?.classifier?.pending;
|
|
1126
|
+
if (!pending) return;
|
|
1127
|
+
const limit = this.config?.classifierOptions?.maxWait ?? DEFAULT_CLASSIFICATION_MAX_WAIT;
|
|
1128
|
+
let timer;
|
|
1129
|
+
const timeout = new Promise((resolve) => {
|
|
1130
|
+
timer = setTimeout(() => {
|
|
1131
|
+
this.log(`Classification took over ${limit} ms, answering without it`);
|
|
1132
|
+
resolve();
|
|
1133
|
+
}, limit);
|
|
1134
|
+
});
|
|
1135
|
+
await Promise.race([pending, timeout]);
|
|
1136
|
+
clearTimeout(timer);
|
|
1137
|
+
}
|
|
895
1138
|
// Run text-to-speech and send to client
|
|
896
1139
|
speak(message) {
|
|
897
1140
|
this.queueOperation(async () => {
|
|
@@ -1028,6 +1271,7 @@ var Realtime = class extends Agent {
|
|
|
1028
1271
|
});
|
|
1029
1272
|
}
|
|
1030
1273
|
cancel() {
|
|
1274
|
+
super.cancel();
|
|
1031
1275
|
this.queue = [];
|
|
1032
1276
|
const answer = this.currentAnswer;
|
|
1033
1277
|
if (!answer) return;
|
|
@@ -1208,8 +1452,8 @@ var Realtime = class extends Agent {
|
|
|
1208
1452
|
};
|
|
1209
1453
|
|
|
1210
1454
|
// src/recorder/MicdropRecorder.ts
|
|
1211
|
-
var
|
|
1212
|
-
var MicdropRecorder = class extends
|
|
1455
|
+
var import_eventemitter34 = require("eventemitter3");
|
|
1456
|
+
var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
1213
1457
|
constructor(server) {
|
|
1214
1458
|
super();
|
|
1215
1459
|
this.server = server;
|
|
@@ -1355,19 +1599,281 @@ var SharedInstances = class {
|
|
|
1355
1599
|
this.entries.set(id, loading);
|
|
1356
1600
|
return loading;
|
|
1357
1601
|
}
|
|
1602
|
+
/** The instance for this key, loaded or being loaded, without loading it. */
|
|
1603
|
+
get(key) {
|
|
1604
|
+
return this.entries.get(typeof key === "string" ? key : JSON.stringify(key));
|
|
1605
|
+
}
|
|
1358
1606
|
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
1359
1607
|
has(key) {
|
|
1360
1608
|
return this.entries.has(typeof key === "string" ? key : JSON.stringify(key));
|
|
1361
1609
|
}
|
|
1610
|
+
/**
|
|
1611
|
+
* Forgets the instance for this key, so the next call loads a new one. For
|
|
1612
|
+
* an instance that broke, a crashed worker for instance. Releasing what it
|
|
1613
|
+
* holds is left to the caller.
|
|
1614
|
+
*/
|
|
1615
|
+
delete(key) {
|
|
1616
|
+
this.entries.delete(typeof key === "string" ? key : JSON.stringify(key));
|
|
1617
|
+
}
|
|
1362
1618
|
/** Drops every instance, so the next call loads again. */
|
|
1363
1619
|
clear() {
|
|
1364
1620
|
this.entries.clear();
|
|
1365
1621
|
}
|
|
1366
1622
|
};
|
|
1367
1623
|
|
|
1624
|
+
// src/socket/ReconnectingSocket.ts
|
|
1625
|
+
var import_eventemitter35 = require("eventemitter3");
|
|
1626
|
+
var import_ws = __toESM(require("ws"));
|
|
1627
|
+
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
1628
|
+
var DEFAULT_RETRY_DELAY = 1e3;
|
|
1629
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
1630
|
+
var ReconnectingSocket = class extends import_eventemitter35.EventEmitter {
|
|
1631
|
+
constructor(options) {
|
|
1632
|
+
super();
|
|
1633
|
+
this.options = options;
|
|
1634
|
+
this._status = "idle";
|
|
1635
|
+
this.everOpened = false;
|
|
1636
|
+
this.retryCount = 0;
|
|
1637
|
+
// Numbers the attempts, so one overtaken while it awaited its address stops
|
|
1638
|
+
this.attemptId = 0;
|
|
1639
|
+
this.waiters = [];
|
|
1640
|
+
}
|
|
1641
|
+
get status() {
|
|
1642
|
+
return this._status;
|
|
1643
|
+
}
|
|
1644
|
+
/** Whether a message sent now goes out */
|
|
1645
|
+
get isOpen() {
|
|
1646
|
+
return this._status === "open";
|
|
1647
|
+
}
|
|
1648
|
+
/**
|
|
1649
|
+
* Opens the connection. Called again after `failed`, it starts over with a
|
|
1650
|
+
* full retry budget, and does nothing in any other status.
|
|
1651
|
+
*/
|
|
1652
|
+
connect() {
|
|
1653
|
+
if (this._status !== "idle" && this._status !== "failed") return;
|
|
1654
|
+
this.retryCount = 0;
|
|
1655
|
+
this.attempt();
|
|
1656
|
+
}
|
|
1657
|
+
/**
|
|
1658
|
+
* Waits for the connection to be open.
|
|
1659
|
+
* @returns True once open, false if it failed or was destroyed meanwhile
|
|
1660
|
+
*/
|
|
1661
|
+
ready() {
|
|
1662
|
+
if (this._status === "open") return Promise.resolve(true);
|
|
1663
|
+
if (this._status === "destroyed" || this._status === "failed") {
|
|
1664
|
+
return Promise.resolve(false);
|
|
1665
|
+
}
|
|
1666
|
+
return new Promise((resolve) => this.waiters.push(resolve));
|
|
1667
|
+
}
|
|
1668
|
+
/**
|
|
1669
|
+
* Sends a message if the connection is open
|
|
1670
|
+
* @returns False when it did not go out
|
|
1671
|
+
*/
|
|
1672
|
+
send(data) {
|
|
1673
|
+
if (this._status !== "open" || !this.socket) return false;
|
|
1674
|
+
this.socket.send(data);
|
|
1675
|
+
return true;
|
|
1676
|
+
}
|
|
1677
|
+
/**
|
|
1678
|
+
* Drops the connection and opens a new one right away, for a provider that
|
|
1679
|
+
* can only stop what it is doing by closing. It costs no retry, and the new
|
|
1680
|
+
* connection is not a reconnection.
|
|
1681
|
+
*/
|
|
1682
|
+
restart() {
|
|
1683
|
+
if (this._status === "destroyed") return;
|
|
1684
|
+
this.log("Restarting the connection");
|
|
1685
|
+
this.clearRetryTimer();
|
|
1686
|
+
this.drop();
|
|
1687
|
+
this.everOpened = false;
|
|
1688
|
+
this.retryCount = 0;
|
|
1689
|
+
this.attempt();
|
|
1690
|
+
}
|
|
1691
|
+
/** Closes the connection for good */
|
|
1692
|
+
destroy() {
|
|
1693
|
+
if (this._status === "destroyed") return;
|
|
1694
|
+
this.clearRetryTimer();
|
|
1695
|
+
this.drop();
|
|
1696
|
+
this.setStatus("destroyed");
|
|
1697
|
+
this.settleWaiters(false);
|
|
1698
|
+
this.removeAllListeners();
|
|
1699
|
+
}
|
|
1700
|
+
async attempt() {
|
|
1701
|
+
const id = ++this.attemptId;
|
|
1702
|
+
const isOvertaken = () => id !== this.attemptId || this._status !== "connecting";
|
|
1703
|
+
this.setStatus("connecting");
|
|
1704
|
+
let url;
|
|
1705
|
+
try {
|
|
1706
|
+
const { url: getUrl } = this.options;
|
|
1707
|
+
url = typeof getUrl === "string" ? getUrl : await getUrl();
|
|
1708
|
+
} catch (error) {
|
|
1709
|
+
if (isOvertaken()) return;
|
|
1710
|
+
this.onAttemptFailed(error);
|
|
1711
|
+
return;
|
|
1712
|
+
}
|
|
1713
|
+
if (isOvertaken()) return;
|
|
1714
|
+
const socket = new import_ws.default(url, this.options.clientOptions);
|
|
1715
|
+
this.socket = socket;
|
|
1716
|
+
let settled = false;
|
|
1717
|
+
const fail = (error) => {
|
|
1718
|
+
if (settled || this.socket !== socket) return;
|
|
1719
|
+
settled = true;
|
|
1720
|
+
clearTimeout(timeout);
|
|
1721
|
+
this.drop();
|
|
1722
|
+
this.onAttemptFailed(error);
|
|
1723
|
+
};
|
|
1724
|
+
const timeout = setTimeout(
|
|
1725
|
+
() => fail(new Error("Connection timeout")),
|
|
1726
|
+
this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
|
|
1727
|
+
);
|
|
1728
|
+
socket.on("error", (error) => {
|
|
1729
|
+
this.log("WebSocket error:", error.message);
|
|
1730
|
+
fail(error);
|
|
1731
|
+
});
|
|
1732
|
+
const { handshake } = this.options;
|
|
1733
|
+
const open = () => {
|
|
1734
|
+
settled = true;
|
|
1735
|
+
clearTimeout(timeout);
|
|
1736
|
+
this.onOpen();
|
|
1737
|
+
};
|
|
1738
|
+
socket.on("open", () => {
|
|
1739
|
+
if (this.socket !== socket) return;
|
|
1740
|
+
try {
|
|
1741
|
+
handshake?.start?.(socket);
|
|
1742
|
+
} catch (error) {
|
|
1743
|
+
fail(error);
|
|
1744
|
+
return;
|
|
1745
|
+
}
|
|
1746
|
+
if (!handshake?.read) open();
|
|
1747
|
+
});
|
|
1748
|
+
socket.on("message", (data, isBinary) => {
|
|
1749
|
+
if (this.socket !== socket) return;
|
|
1750
|
+
if (this._status === "open") {
|
|
1751
|
+
this.prove();
|
|
1752
|
+
this.emit("Message", data, isBinary);
|
|
1753
|
+
return;
|
|
1754
|
+
}
|
|
1755
|
+
if (settled || !handshake?.read) return;
|
|
1756
|
+
try {
|
|
1757
|
+
if (handshake.read(data, isBinary)) open();
|
|
1758
|
+
} catch (error) {
|
|
1759
|
+
fail(error);
|
|
1760
|
+
}
|
|
1761
|
+
});
|
|
1762
|
+
socket.on("close", (code, reasonBuffer) => {
|
|
1763
|
+
const reason = reasonBuffer.toString();
|
|
1764
|
+
if (!settled) {
|
|
1765
|
+
fail(new Error(`Connection closed (${code}) ${reason}`));
|
|
1766
|
+
return;
|
|
1767
|
+
}
|
|
1768
|
+
if (this.socket !== socket) return;
|
|
1769
|
+
this.log("Connection closed", { code, reason });
|
|
1770
|
+
this.drop();
|
|
1771
|
+
this.setStatus("reconnecting");
|
|
1772
|
+
this.emit("Close", code, reason);
|
|
1773
|
+
if (this._status !== "reconnecting") return;
|
|
1774
|
+
this.scheduleRetry(new Error(`Connection closed (${code}) ${reason}`));
|
|
1775
|
+
});
|
|
1776
|
+
}
|
|
1777
|
+
onOpen() {
|
|
1778
|
+
this.log("Connection opened");
|
|
1779
|
+
const reconnected = this.everOpened;
|
|
1780
|
+
this.everOpened = true;
|
|
1781
|
+
if (this.options.handshake?.read) {
|
|
1782
|
+
this.prove();
|
|
1783
|
+
} else {
|
|
1784
|
+
this.proofTimer = setTimeout(
|
|
1785
|
+
() => this.prove(),
|
|
1786
|
+
this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
|
|
1787
|
+
);
|
|
1788
|
+
}
|
|
1789
|
+
this.startKeepAlive();
|
|
1790
|
+
this.setStatus("open");
|
|
1791
|
+
this.emit("Open", { reconnected });
|
|
1792
|
+
this.settleWaiters(true);
|
|
1793
|
+
}
|
|
1794
|
+
/** The connection works, the retry budget is whole again */
|
|
1795
|
+
prove() {
|
|
1796
|
+
this.clearProofTimer();
|
|
1797
|
+
this.retryCount = 0;
|
|
1798
|
+
}
|
|
1799
|
+
clearProofTimer() {
|
|
1800
|
+
if (!this.proofTimer) return;
|
|
1801
|
+
clearTimeout(this.proofTimer);
|
|
1802
|
+
this.proofTimer = void 0;
|
|
1803
|
+
}
|
|
1804
|
+
onAttemptFailed(error) {
|
|
1805
|
+
this.log("Connection failed:", error.message);
|
|
1806
|
+
this.scheduleRetry(error);
|
|
1807
|
+
}
|
|
1808
|
+
scheduleRetry(error) {
|
|
1809
|
+
if (this._status === "destroyed") return;
|
|
1810
|
+
this.retryCount++;
|
|
1811
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
1812
|
+
this.log("Max retries reached, giving up");
|
|
1813
|
+
this.setStatus("failed");
|
|
1814
|
+
this.settleWaiters(false);
|
|
1815
|
+
this.emit("Failed", error);
|
|
1816
|
+
return;
|
|
1817
|
+
}
|
|
1818
|
+
this.setStatus("reconnecting");
|
|
1819
|
+
this.log(`Reconnecting (attempt ${this.retryCount})`);
|
|
1820
|
+
this.clearRetryTimer();
|
|
1821
|
+
this.retryTimer = setTimeout(() => {
|
|
1822
|
+
this.retryTimer = void 0;
|
|
1823
|
+
if (this._status === "reconnecting") this.attempt();
|
|
1824
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
|
|
1825
|
+
}
|
|
1826
|
+
/** Lets go of the current socket, which will never be heard from again */
|
|
1827
|
+
drop() {
|
|
1828
|
+
this.stopKeepAlive();
|
|
1829
|
+
this.clearProofTimer();
|
|
1830
|
+
const socket = this.socket;
|
|
1831
|
+
this.socket = void 0;
|
|
1832
|
+
if (!socket) return;
|
|
1833
|
+
socket.removeAllListeners();
|
|
1834
|
+
socket.on("error", () => {
|
|
1835
|
+
});
|
|
1836
|
+
if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
|
|
1837
|
+
socket.close(1e3);
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
startKeepAlive() {
|
|
1841
|
+
const keepAlive = this.options.keepAlive;
|
|
1842
|
+
if (!keepAlive) return;
|
|
1843
|
+
this.stopKeepAlive();
|
|
1844
|
+
this.keepAliveTimer = setInterval(() => {
|
|
1845
|
+
const { message } = keepAlive;
|
|
1846
|
+
this.send(typeof message === "string" ? message : message());
|
|
1847
|
+
}, keepAlive.interval);
|
|
1848
|
+
}
|
|
1849
|
+
stopKeepAlive() {
|
|
1850
|
+
if (!this.keepAliveTimer) return;
|
|
1851
|
+
clearInterval(this.keepAliveTimer);
|
|
1852
|
+
this.keepAliveTimer = void 0;
|
|
1853
|
+
}
|
|
1854
|
+
clearRetryTimer() {
|
|
1855
|
+
if (!this.retryTimer) return;
|
|
1856
|
+
clearTimeout(this.retryTimer);
|
|
1857
|
+
this.retryTimer = void 0;
|
|
1858
|
+
}
|
|
1859
|
+
settleWaiters(open) {
|
|
1860
|
+
const waiters = this.waiters;
|
|
1861
|
+
this.waiters = [];
|
|
1862
|
+
waiters.forEach((resolve) => resolve(open));
|
|
1863
|
+
}
|
|
1864
|
+
setStatus(status) {
|
|
1865
|
+
if (this._status === status) return;
|
|
1866
|
+
this._status = status;
|
|
1867
|
+
this.emit("StatusChange", status);
|
|
1868
|
+
}
|
|
1869
|
+
log(...message) {
|
|
1870
|
+
this.options.log?.(...message);
|
|
1871
|
+
}
|
|
1872
|
+
};
|
|
1873
|
+
|
|
1368
1874
|
// src/stt/STT.ts
|
|
1369
|
-
var
|
|
1370
|
-
var STT = class extends
|
|
1875
|
+
var import_eventemitter36 = require("eventemitter3");
|
|
1876
|
+
var STT = class extends import_eventemitter36.EventEmitter {
|
|
1371
1877
|
log(...message) {
|
|
1372
1878
|
this.logger?.log(...message);
|
|
1373
1879
|
}
|
|
@@ -1393,24 +1899,31 @@ var MockSTT = class extends STT {
|
|
|
1393
1899
|
// src/stt/FallbackSTT.ts
|
|
1394
1900
|
var import_stream3 = require("stream");
|
|
1395
1901
|
var FallbackSTT = class extends STT {
|
|
1396
|
-
// Start at -1 because we need to increment it before using it
|
|
1397
1902
|
constructor(options) {
|
|
1398
1903
|
super();
|
|
1399
1904
|
this.options = options;
|
|
1400
1905
|
this.stt = null;
|
|
1401
1906
|
this.sttIndex = -1;
|
|
1907
|
+
// Start at -1 because we need to increment it before using it
|
|
1908
|
+
// In the order spoken, each STT giving one transcript per utterance in turn
|
|
1909
|
+
this.pending = [];
|
|
1402
1910
|
this.onTranscript = (transcript) => {
|
|
1911
|
+
this.pending.shift();
|
|
1403
1912
|
this.emit("Transcript", transcript);
|
|
1404
1913
|
};
|
|
1405
|
-
|
|
1914
|
+
/**
|
|
1915
|
+
* Hands every utterance still waiting to the next STT, one stream each so
|
|
1916
|
+
* each gets its own transcript. One still being spoken keeps going there.
|
|
1917
|
+
*/
|
|
1918
|
+
this.onFailed = () => {
|
|
1406
1919
|
this.log("STT failed, trying next STT");
|
|
1407
1920
|
this.startNextSTT();
|
|
1408
|
-
if (
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
this.stt?.transcribe(stream);
|
|
1412
|
-
chunks.forEach((chunk) => stream.write(chunk));
|
|
1413
|
-
stream.end();
|
|
1921
|
+
if (this.pending.length > 0) this.log("Sending audio chunks again");
|
|
1922
|
+
for (const utterance of this.pending) {
|
|
1923
|
+
utterance.stream = new import_stream3.PassThrough();
|
|
1924
|
+
this.stt?.transcribe(utterance.stream);
|
|
1925
|
+
utterance.chunks.forEach((chunk) => utterance.stream.write(chunk));
|
|
1926
|
+
if (utterance.ended) utterance.stream.end();
|
|
1414
1927
|
}
|
|
1415
1928
|
};
|
|
1416
1929
|
if (this.options.factories.length === 0) {
|
|
@@ -1419,13 +1932,28 @@ var FallbackSTT = class extends STT {
|
|
|
1419
1932
|
this.startNextSTT();
|
|
1420
1933
|
}
|
|
1421
1934
|
transcribe(audioStream) {
|
|
1422
|
-
|
|
1935
|
+
const utterance = {
|
|
1936
|
+
chunks: [],
|
|
1937
|
+
ended: false,
|
|
1938
|
+
stream: new import_stream3.PassThrough()
|
|
1939
|
+
};
|
|
1940
|
+
this.pending.push(utterance);
|
|
1941
|
+
audioStream.on("data", (chunk) => {
|
|
1942
|
+
utterance.chunks.push(chunk);
|
|
1943
|
+
utterance.stream.write(chunk);
|
|
1944
|
+
});
|
|
1945
|
+
audioStream.on("end", () => {
|
|
1946
|
+
utterance.ended = true;
|
|
1947
|
+
utterance.stream.end();
|
|
1948
|
+
});
|
|
1949
|
+
this.stt?.transcribe(utterance.stream);
|
|
1423
1950
|
}
|
|
1424
1951
|
destroy() {
|
|
1425
1952
|
super.destroy();
|
|
1426
1953
|
this.stt?.destroy();
|
|
1427
1954
|
this.stt = null;
|
|
1428
1955
|
this.sttIndex = -1;
|
|
1956
|
+
this.pending = [];
|
|
1429
1957
|
}
|
|
1430
1958
|
startNextSTT() {
|
|
1431
1959
|
this.sttIndex++;
|
|
@@ -1448,8 +1976,8 @@ var FallbackSTT = class extends STT {
|
|
|
1448
1976
|
var import_stream4 = require("stream");
|
|
1449
1977
|
|
|
1450
1978
|
// src/tts/TTS.ts
|
|
1451
|
-
var
|
|
1452
|
-
var TTS = class extends
|
|
1979
|
+
var import_eventemitter37 = require("eventemitter3");
|
|
1980
|
+
var TTS = class extends import_eventemitter37.EventEmitter {
|
|
1453
1981
|
log(...message) {
|
|
1454
1982
|
this.logger?.log(...message);
|
|
1455
1983
|
}
|
|
@@ -1461,7 +1989,6 @@ var TTS = class extends import_eventemitter35.EventEmitter {
|
|
|
1461
1989
|
|
|
1462
1990
|
// src/tts/FallbackTTS.ts
|
|
1463
1991
|
var FallbackTTS = class extends TTS {
|
|
1464
|
-
// Start at -1 because we need to increment it before using it
|
|
1465
1992
|
constructor(options) {
|
|
1466
1993
|
super();
|
|
1467
1994
|
this.options = options;
|
|
@@ -1470,16 +1997,22 @@ var FallbackTTS = class extends TTS {
|
|
|
1470
1997
|
this.onAudio = (audio) => {
|
|
1471
1998
|
this.emit("Audio", audio);
|
|
1472
1999
|
};
|
|
2000
|
+
/**
|
|
2001
|
+
* Hands the unspoken text to the next TTS. An utterance whose text is still
|
|
2002
|
+
* being written goes on there, so its end is spoken too.
|
|
2003
|
+
*/
|
|
1473
2004
|
this.onFailed = (chunks) => {
|
|
1474
2005
|
this.log("TTS failed, trying next TTS");
|
|
1475
2006
|
this.startNextTTS();
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
2007
|
+
const utterance = this.current;
|
|
2008
|
+
const continues = !!utterance && !utterance.ended;
|
|
2009
|
+
if (chunks.length === 0 && !continues) return;
|
|
2010
|
+
this.log("Sending text chunks again");
|
|
2011
|
+
const stream = new import_stream4.PassThrough();
|
|
2012
|
+
this.tts?.speak(stream);
|
|
2013
|
+
chunks.forEach((chunk) => stream.write(chunk));
|
|
2014
|
+
if (continues) utterance.stream = stream;
|
|
2015
|
+
else stream.end();
|
|
1483
2016
|
};
|
|
1484
2017
|
if (this.options.factories.length === 0) {
|
|
1485
2018
|
throw new Error("FallbackTTS: No factories provided");
|
|
@@ -1487,9 +2020,20 @@ var FallbackTTS = class extends TTS {
|
|
|
1487
2020
|
this.startNextTTS();
|
|
1488
2021
|
}
|
|
1489
2022
|
speak(textStream) {
|
|
1490
|
-
|
|
2023
|
+
const utterance = {
|
|
2024
|
+
ended: false,
|
|
2025
|
+
stream: new import_stream4.PassThrough()
|
|
2026
|
+
};
|
|
2027
|
+
this.current = utterance;
|
|
2028
|
+
textStream.on("data", (chunk) => utterance.stream.write(chunk));
|
|
2029
|
+
textStream.on("end", () => {
|
|
2030
|
+
utterance.ended = true;
|
|
2031
|
+
utterance.stream.end();
|
|
2032
|
+
});
|
|
2033
|
+
this.tts?.speak(utterance.stream);
|
|
1491
2034
|
}
|
|
1492
2035
|
cancel() {
|
|
2036
|
+
this.current = void 0;
|
|
1493
2037
|
this.tts?.cancel();
|
|
1494
2038
|
}
|
|
1495
2039
|
destroy() {
|
|
@@ -1517,26 +2061,28 @@ var FallbackTTS = class extends TTS {
|
|
|
1517
2061
|
|
|
1518
2062
|
// src/tts/MockTTS.ts
|
|
1519
2063
|
var fs = __toESM(require("fs"));
|
|
1520
|
-
var import_stream5 = require("stream");
|
|
1521
2064
|
var MockTTS = class extends TTS {
|
|
1522
2065
|
constructor(audioFilePaths) {
|
|
1523
2066
|
super();
|
|
1524
2067
|
this.audioFilePaths = audioFilePaths;
|
|
2068
|
+
// Bumped by every answer and every cancel, so the chunks of an answer that
|
|
2069
|
+
// is no longer wanted stop there
|
|
2070
|
+
this.generation = 0;
|
|
1525
2071
|
}
|
|
1526
2072
|
speak(textStream) {
|
|
1527
|
-
const
|
|
2073
|
+
const generation = ++this.generation;
|
|
1528
2074
|
textStream.once("data", async () => {
|
|
1529
2075
|
for (const filePath of this.audioFilePaths) {
|
|
1530
2076
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
2077
|
+
if (generation !== this.generation) return;
|
|
2078
|
+
const audio = fs.readFileSync(filePath);
|
|
2079
|
+
this.log(`Loaded chunk (${audio.length} bytes)`);
|
|
2080
|
+
this.emit("Audio", audio);
|
|
1534
2081
|
}
|
|
1535
|
-
audioStream.end();
|
|
1536
2082
|
});
|
|
1537
|
-
return audioStream;
|
|
1538
2083
|
}
|
|
1539
2084
|
cancel() {
|
|
2085
|
+
this.generation++;
|
|
1540
2086
|
}
|
|
1541
2087
|
};
|
|
1542
2088
|
|
|
@@ -1615,6 +2161,9 @@ var SentenceTTS = class extends TTS {
|
|
|
1615
2161
|
this.counter++;
|
|
1616
2162
|
counter = this.counter;
|
|
1617
2163
|
this.splitter.reset();
|
|
2164
|
+
this.queue = [];
|
|
2165
|
+
this.controller?.abort();
|
|
2166
|
+
this.controller = void 0;
|
|
1618
2167
|
return true;
|
|
1619
2168
|
};
|
|
1620
2169
|
textStream.on("data", (chunk) => {
|
|
@@ -1701,6 +2250,7 @@ async function waitForParams(socket, validate) {
|
|
|
1701
2250
|
AUTO_SEMANTIC_TURN_PROMPT,
|
|
1702
2251
|
AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
1703
2252
|
Agent,
|
|
2253
|
+
Classifier,
|
|
1704
2254
|
FallbackAgent,
|
|
1705
2255
|
FallbackSTT,
|
|
1706
2256
|
FallbackTTS,
|
|
@@ -1716,14 +2266,18 @@ async function waitForParams(socket, validate) {
|
|
|
1716
2266
|
MockTTS,
|
|
1717
2267
|
Pcm16Resampler,
|
|
1718
2268
|
Realtime,
|
|
2269
|
+
ReconnectingSocket,
|
|
1719
2270
|
STT,
|
|
1720
2271
|
SentenceSplitter,
|
|
1721
2272
|
SentenceTTS,
|
|
1722
2273
|
SharedInstances,
|
|
1723
2274
|
TTS,
|
|
2275
|
+
currentTurn,
|
|
1724
2276
|
float32ToPcm16,
|
|
2277
|
+
getTurnClassification,
|
|
1725
2278
|
handleError,
|
|
1726
2279
|
pcm16ToFloat32,
|
|
2280
|
+
turnInput,
|
|
1727
2281
|
waitForParams
|
|
1728
2282
|
});
|
|
1729
2283
|
//# sourceMappingURL=index.js.map
|