@micdrop/server 2.8.0 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +348 -10
- package/dist/index.d.ts +348 -10
- package/dist/index.js +624 -70
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +619 -70
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -3
package/dist/index.mjs
CHANGED
|
@@ -15,27 +15,65 @@ var Agent = class extends EventEmitter {
|
|
|
15
15
|
constructor(options) {
|
|
16
16
|
super();
|
|
17
17
|
this.options = options;
|
|
18
|
+
/**
|
|
19
|
+
* @deprecated Read the signal given to `generateAnswer` instead. Still
|
|
20
|
+
* bumped by every answer and every cancel, for agents written before it.
|
|
21
|
+
*/
|
|
18
22
|
this.answerCount = 0;
|
|
19
|
-
this.answering = false;
|
|
20
23
|
this.conversation = [{ role: "system", content: options.systemPrompt }];
|
|
21
24
|
this.tools = this.getDefaultTools();
|
|
22
25
|
}
|
|
26
|
+
/** Whether an answer is being generated */
|
|
27
|
+
get isAnswering() {
|
|
28
|
+
return !!this.answerController;
|
|
29
|
+
}
|
|
30
|
+
/** @deprecated Read `isAnswering` instead */
|
|
31
|
+
get answering() {
|
|
32
|
+
return this.isAnswering;
|
|
33
|
+
}
|
|
34
|
+
/** @deprecated The answer is over once its stream ends or it is cancelled */
|
|
35
|
+
set answering(_value) {
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Stops the answer being generated, by aborting its signal.
|
|
39
|
+
*
|
|
40
|
+
* An agent overriding it to stop more than the signal does calls
|
|
41
|
+
* `super.cancel()`.
|
|
42
|
+
*/
|
|
43
|
+
cancel() {
|
|
44
|
+
this.answerCount++;
|
|
45
|
+
if (!this.answerController) return;
|
|
46
|
+
this.log("Cancel");
|
|
47
|
+
this.answerController.abort();
|
|
48
|
+
this.answerController = void 0;
|
|
49
|
+
}
|
|
23
50
|
answer() {
|
|
24
51
|
this.log("Start answering");
|
|
25
|
-
|
|
52
|
+
this.answerCount++;
|
|
53
|
+
this.answerController?.abort();
|
|
54
|
+
const controller = new AbortController();
|
|
55
|
+
this.answerController = controller;
|
|
56
|
+
const { signal } = controller;
|
|
26
57
|
const stream = new PassThrough();
|
|
27
|
-
this.answering = true;
|
|
28
58
|
Promise.resolve().then(
|
|
29
59
|
() => this.options.onBeforeAnswer?.bind(this)(stream)
|
|
30
|
-
).then((
|
|
31
|
-
if (
|
|
32
|
-
|
|
60
|
+
).then((result) => {
|
|
61
|
+
if (signal.aborted) return;
|
|
62
|
+
if (typeof result === "string") {
|
|
63
|
+
stream.write(result);
|
|
64
|
+
this.addAssistantMessage(result);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
if (result) return;
|
|
68
|
+
return this.generateAnswer(stream, signal);
|
|
69
|
+
}).catch((error) => {
|
|
70
|
+
if (!signal.aborted) this.log("Error answering:", error);
|
|
33
71
|
}).finally(() => {
|
|
34
72
|
if (stream.writable) {
|
|
35
73
|
stream.end();
|
|
36
74
|
}
|
|
37
|
-
if (
|
|
38
|
-
this.
|
|
75
|
+
if (this.answerController === controller) {
|
|
76
|
+
this.answerController = void 0;
|
|
39
77
|
}
|
|
40
78
|
});
|
|
41
79
|
return stream;
|
|
@@ -261,10 +299,10 @@ var FallbackAgent = class extends Agent {
|
|
|
261
299
|
extract(message) {
|
|
262
300
|
return this.agent ? this.agent.extract(message) : super.extract(message);
|
|
263
301
|
}
|
|
264
|
-
async generateAnswer(stream) {
|
|
302
|
+
async generateAnswer(stream, signal) {
|
|
265
303
|
for (let attempt = 0; attempt < this.fallbackOptions.factories.length; attempt++) {
|
|
266
304
|
const agent = this.agent;
|
|
267
|
-
if (!agent) return;
|
|
305
|
+
if (!agent || signal.aborted) return;
|
|
268
306
|
let failed = false;
|
|
269
307
|
const onFailed = () => {
|
|
270
308
|
failed = true;
|
|
@@ -275,7 +313,7 @@ var FallbackAgent = class extends Agent {
|
|
|
275
313
|
} finally {
|
|
276
314
|
agent.off("Failed", onFailed);
|
|
277
315
|
}
|
|
278
|
-
if (!failed) return;
|
|
316
|
+
if (!failed || signal.aborted) return;
|
|
279
317
|
this.log("Agent failed, trying next agent");
|
|
280
318
|
this.startNextAgent();
|
|
281
319
|
}
|
|
@@ -283,6 +321,7 @@ var FallbackAgent = class extends Agent {
|
|
|
283
321
|
this.emit("Failed");
|
|
284
322
|
}
|
|
285
323
|
cancel() {
|
|
324
|
+
super.cancel();
|
|
286
325
|
this.agent?.cancel();
|
|
287
326
|
}
|
|
288
327
|
destroy() {
|
|
@@ -345,13 +384,149 @@ var MockAgent = class extends Agent {
|
|
|
345
384
|
}
|
|
346
385
|
async generateAnswer(stream) {
|
|
347
386
|
const message = `Assistant Message ${this.i++}`;
|
|
348
|
-
this.addAssistantMessage(message);
|
|
349
387
|
stream.write(message);
|
|
388
|
+
this.addAssistantMessage(message);
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
|
|
392
|
+
// src/classifier/Classifier.ts
|
|
393
|
+
import { EventEmitter as EventEmitter2 } from "eventemitter3";
|
|
394
|
+
var Classifier = class extends EventEmitter2 {
|
|
395
|
+
constructor() {
|
|
396
|
+
super(...arguments);
|
|
397
|
+
this.controllers = /* @__PURE__ */ new Set();
|
|
398
|
+
}
|
|
399
|
+
/**
|
|
400
|
+
* Classifies an input, and emits the result as `Classification`
|
|
401
|
+
* @param input - What to classify: a text, or JSON with named fields
|
|
402
|
+
* @returns The classification, or undefined when cancelled or failed
|
|
403
|
+
*/
|
|
404
|
+
classify(input) {
|
|
405
|
+
const controller = new AbortController();
|
|
406
|
+
this.controllers.add(controller);
|
|
407
|
+
const pending = this.run(input, controller.signal).finally(() => {
|
|
408
|
+
this.controllers.delete(controller);
|
|
409
|
+
if (this._pending === pending) this._pending = void 0;
|
|
410
|
+
});
|
|
411
|
+
this._pending = pending;
|
|
412
|
+
return pending;
|
|
413
|
+
}
|
|
414
|
+
/** The last classification started, until it is done */
|
|
415
|
+
get pending() {
|
|
416
|
+
return this._pending;
|
|
350
417
|
}
|
|
418
|
+
/** Drops every classification in progress, which then emit nothing */
|
|
351
419
|
cancel() {
|
|
420
|
+
this.controllers.forEach((controller) => controller.abort());
|
|
421
|
+
this.controllers.clear();
|
|
422
|
+
this._pending = void 0;
|
|
423
|
+
}
|
|
424
|
+
log(...message) {
|
|
425
|
+
this.logger?.log(...message);
|
|
426
|
+
}
|
|
427
|
+
destroy() {
|
|
428
|
+
this.log("Destroyed");
|
|
429
|
+
this.cancel();
|
|
430
|
+
this.removeAllListeners();
|
|
431
|
+
}
|
|
432
|
+
async run(input, signal) {
|
|
433
|
+
const start = Date.now();
|
|
434
|
+
try {
|
|
435
|
+
const result = await this.evaluate(input, signal);
|
|
436
|
+
if (signal.aborted) return;
|
|
437
|
+
const classification = {
|
|
438
|
+
input,
|
|
439
|
+
result,
|
|
440
|
+
duration: Date.now() - start
|
|
441
|
+
};
|
|
442
|
+
this.log(
|
|
443
|
+
`Classified in ${classification.duration} ms:`,
|
|
444
|
+
JSON.stringify(input)
|
|
445
|
+
);
|
|
446
|
+
this.lastClassification = classification;
|
|
447
|
+
this.emit("Classification", classification);
|
|
448
|
+
return classification;
|
|
449
|
+
} catch (error) {
|
|
450
|
+
if (!signal.aborted) this.log("Classification failed:", error);
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
352
453
|
}
|
|
353
454
|
};
|
|
354
455
|
|
|
456
|
+
// src/types.ts
|
|
457
|
+
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
458
|
+
MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
459
|
+
MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
460
|
+
MicdropClientCommands2["Mute"] = "Mute";
|
|
461
|
+
return MicdropClientCommands2;
|
|
462
|
+
})(MicdropClientCommands || {});
|
|
463
|
+
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
464
|
+
MicdropServerCommands2["Message"] = "Message";
|
|
465
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
466
|
+
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
467
|
+
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
468
|
+
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
469
|
+
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
470
|
+
MicdropServerCommands2["Classification"] = "Classification";
|
|
471
|
+
return MicdropServerCommands2;
|
|
472
|
+
})(MicdropServerCommands || {});
|
|
473
|
+
function currentTurn(conversation) {
|
|
474
|
+
let start = conversation.length;
|
|
475
|
+
while (start > 0) {
|
|
476
|
+
const item = conversation[start - 1];
|
|
477
|
+
if (item.role !== "user" || item.metadata?.classification) break;
|
|
478
|
+
start--;
|
|
479
|
+
}
|
|
480
|
+
const messages = conversation.slice(start);
|
|
481
|
+
return {
|
|
482
|
+
transcript: messages.map((message) => message.content).join(" "),
|
|
483
|
+
messages,
|
|
484
|
+
before: conversation.slice(0, start)
|
|
485
|
+
};
|
|
486
|
+
}
|
|
487
|
+
function getTurnClassification(conversation) {
|
|
488
|
+
for (let index = conversation.length - 1; index >= 0; index--) {
|
|
489
|
+
const item = conversation[index];
|
|
490
|
+
if (item.role === "user") return item.metadata?.classification;
|
|
491
|
+
}
|
|
492
|
+
return void 0;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// src/classifier/turnInput.ts
|
|
496
|
+
function turnInput(conversation, history = 1) {
|
|
497
|
+
const turn = currentTurn(conversation);
|
|
498
|
+
const input = { history: [], turn: turn.transcript };
|
|
499
|
+
let rest = turn.before;
|
|
500
|
+
for (let n = 0; n < history && rest.length; n++) {
|
|
501
|
+
let end = rest.length;
|
|
502
|
+
const answers = [];
|
|
503
|
+
while (end > 0 && rest[end - 1].role !== "user") {
|
|
504
|
+
const item = rest[end - 1];
|
|
505
|
+
if (item.role === "assistant" && item.content.trim()) {
|
|
506
|
+
answers.unshift(item.content);
|
|
507
|
+
}
|
|
508
|
+
end--;
|
|
509
|
+
}
|
|
510
|
+
let start = end;
|
|
511
|
+
while (start > 0) {
|
|
512
|
+
const item = rest[start - 1];
|
|
513
|
+
if (item.role !== "user") break;
|
|
514
|
+
if (start < end && item.metadata?.classification)
|
|
515
|
+
break;
|
|
516
|
+
start--;
|
|
517
|
+
}
|
|
518
|
+
if (answers.length) {
|
|
519
|
+
input.history.unshift({ role: "assistant", text: answers.join(" ") });
|
|
520
|
+
}
|
|
521
|
+
if (start < end) {
|
|
522
|
+
const text = rest.slice(start, end).map((item) => "content" in item ? item.content : "").join(" ");
|
|
523
|
+
input.history.unshift({ role: "user", text });
|
|
524
|
+
}
|
|
525
|
+
rest = rest.slice(0, start);
|
|
526
|
+
}
|
|
527
|
+
return { input, message: turn.messages[turn.messages.length - 1] };
|
|
528
|
+
}
|
|
529
|
+
|
|
355
530
|
// src/audio/Pcm16Resampler.ts
|
|
356
531
|
var Pcm16Resampler = class {
|
|
357
532
|
// Fractional position into the first sample of the buffer
|
|
@@ -439,35 +614,23 @@ function handleError(socket, error) {
|
|
|
439
614
|
}
|
|
440
615
|
|
|
441
616
|
// src/MicdropServer.ts
|
|
442
|
-
import { EventEmitter as
|
|
617
|
+
import { EventEmitter as EventEmitter3 } from "eventemitter3";
|
|
443
618
|
import { PassThrough as PassThrough2, Transform } from "stream";
|
|
444
|
-
|
|
445
|
-
// src/types.ts
|
|
446
|
-
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
447
|
-
MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
448
|
-
MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
449
|
-
MicdropClientCommands2["Mute"] = "Mute";
|
|
450
|
-
return MicdropClientCommands2;
|
|
451
|
-
})(MicdropClientCommands || {});
|
|
452
|
-
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
453
|
-
MicdropServerCommands2["Message"] = "Message";
|
|
454
|
-
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
455
|
-
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
456
|
-
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
457
|
-
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
458
|
-
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
459
|
-
return MicdropServerCommands2;
|
|
460
|
-
})(MicdropServerCommands || {});
|
|
461
|
-
|
|
462
|
-
// src/MicdropServer.ts
|
|
463
619
|
var USER_SAMPLE_RATE = 16e3;
|
|
464
620
|
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
465
|
-
var
|
|
621
|
+
var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
|
|
622
|
+
var MicdropServer = class extends EventEmitter3 {
|
|
466
623
|
constructor(socket, config) {
|
|
467
624
|
super();
|
|
468
625
|
this.socket = null;
|
|
469
626
|
this.config = null;
|
|
470
627
|
this.startTime = Date.now();
|
|
628
|
+
// Bumped by every cancel, so an answer cancelled before its first word can
|
|
629
|
+
// tell, and leave its message to the answer that replaces it
|
|
630
|
+
this.cancelRound = 0;
|
|
631
|
+
// Counts the assistant messages sent, so the partial answer of one already
|
|
632
|
+
// settled is not sent after it
|
|
633
|
+
this.assistantMessages = 0;
|
|
471
634
|
// Holds the conversation when no agent does, and keeps it readable once the
|
|
472
635
|
// call is over and the config is gone
|
|
473
636
|
this.ownConversation = [];
|
|
@@ -475,15 +638,22 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
475
638
|
this.operationQueue = [];
|
|
476
639
|
this.isProcessingQueue = false;
|
|
477
640
|
this.userSpeechChunks = 0;
|
|
478
|
-
|
|
641
|
+
// Bumped whenever the user speaks again or mutes, so a decision taken on an
|
|
642
|
+
// earlier stretch of speech is dropped rather than answering over them
|
|
643
|
+
this.turnRound = 0;
|
|
644
|
+
this.onClose = (code, reason) => {
|
|
479
645
|
this.releaseHeldTurn();
|
|
480
646
|
if (!this.config) return;
|
|
481
|
-
this.log("Connection closed"
|
|
647
|
+
this.log("Connection closed", {
|
|
648
|
+
code,
|
|
649
|
+
reason: reason?.toString() || void 0
|
|
650
|
+
});
|
|
482
651
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
483
652
|
const conversation = this.conversation;
|
|
484
653
|
this.agent?.destroy();
|
|
485
654
|
this.config.stt?.destroy();
|
|
486
655
|
this.config.tts?.destroy();
|
|
656
|
+
this.config.classifier?.destroy();
|
|
487
657
|
this.emit("End", {
|
|
488
658
|
conversation,
|
|
489
659
|
duration
|
|
@@ -575,6 +745,14 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
575
745
|
)
|
|
576
746
|
);
|
|
577
747
|
}
|
|
748
|
+
if (config.classifierOptions?.sendToClient) {
|
|
749
|
+
config.classifier?.on(
|
|
750
|
+
"Classification",
|
|
751
|
+
(classification) => this.socket?.send(
|
|
752
|
+
`${"Classification" /* Classification */} ${JSON.stringify(classification)}`
|
|
753
|
+
)
|
|
754
|
+
);
|
|
755
|
+
}
|
|
578
756
|
queueMicrotask(() => this.sendFirstMessage());
|
|
579
757
|
socket.on("close", this.onClose);
|
|
580
758
|
socket.on("message", this.onMessage);
|
|
@@ -611,6 +789,8 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
611
789
|
this.processQueue();
|
|
612
790
|
}
|
|
613
791
|
cancel() {
|
|
792
|
+
this.cancelRound++;
|
|
793
|
+
this.config?.classifier?.cancel();
|
|
614
794
|
this.config?.tts?.cancel();
|
|
615
795
|
this.agent?.cancel();
|
|
616
796
|
this.operationQueue = [];
|
|
@@ -623,6 +803,8 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
623
803
|
this.emit("UserAudio", chunk);
|
|
624
804
|
}
|
|
625
805
|
onMute() {
|
|
806
|
+
this.turnRound++;
|
|
807
|
+
this.releaseHeldTurn();
|
|
626
808
|
this.userSpeechChunks = 0;
|
|
627
809
|
this.currentUserStream?.end();
|
|
628
810
|
this.currentUserStream = void 0;
|
|
@@ -631,6 +813,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
631
813
|
}
|
|
632
814
|
onStartSpeaking() {
|
|
633
815
|
if (!this.config) return;
|
|
816
|
+
this.turnRound++;
|
|
634
817
|
this.userSpeechChunks = 0;
|
|
635
818
|
this.currentUserStream?.end();
|
|
636
819
|
this.currentUserStream = new PassThrough2();
|
|
@@ -682,7 +865,9 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
682
865
|
}
|
|
683
866
|
/** Answers, unless the sentence sounds like it has more coming */
|
|
684
867
|
async answerUserTurn() {
|
|
868
|
+
const round = this.turnRound;
|
|
685
869
|
const complete = await (this.turnComplete ?? Promise.resolve(true));
|
|
870
|
+
if (round !== this.turnRound || !this.config) return;
|
|
686
871
|
if (!complete) {
|
|
687
872
|
this.log("Waiting for the rest of the sentence");
|
|
688
873
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -714,6 +899,7 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
714
899
|
if (realtime) {
|
|
715
900
|
realtime.endTurn();
|
|
716
901
|
} else {
|
|
902
|
+
this.classifyTurn();
|
|
717
903
|
this.answer();
|
|
718
904
|
}
|
|
719
905
|
}
|
|
@@ -757,6 +943,8 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
757
943
|
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
758
944
|
);
|
|
759
945
|
this.emit("Message", message);
|
|
946
|
+
if (message.role === "assistant") this.assistantMessages++;
|
|
947
|
+
if (message.role === "user" && this.config?.realtime) this.classifyTurn();
|
|
760
948
|
}
|
|
761
949
|
sendPartialAnswer(content) {
|
|
762
950
|
if (!this.socket || content === "") return;
|
|
@@ -777,6 +965,9 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
777
965
|
}
|
|
778
966
|
}
|
|
779
967
|
answer() {
|
|
968
|
+
if (this.config?.classifierOptions?.waitBeforeAnswer) {
|
|
969
|
+
this.queueOperation(() => this.waitForClassification());
|
|
970
|
+
}
|
|
780
971
|
this.queueOperation(async () => {
|
|
781
972
|
await this._answer();
|
|
782
973
|
});
|
|
@@ -799,12 +990,18 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
799
990
|
return;
|
|
800
991
|
}
|
|
801
992
|
this.lastMessageSpeeched = lastMessage;
|
|
993
|
+
const round = this.cancelRound;
|
|
802
994
|
try {
|
|
995
|
+
const sentBefore = this.assistantMessages;
|
|
803
996
|
const stream = this.config.agent.answer();
|
|
804
997
|
if (await hasContent(stream)) {
|
|
805
998
|
await this._speak(
|
|
806
|
-
this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
|
|
999
|
+
this.config.partialMessages ? this.forwardPartialAnswer(stream, sentBefore) : stream
|
|
807
1000
|
);
|
|
1001
|
+
} else if (round !== this.cancelRound) {
|
|
1002
|
+
if (this.lastMessageSpeeched === lastMessage) {
|
|
1003
|
+
this.lastMessageSpeeched = void 0;
|
|
1004
|
+
}
|
|
808
1005
|
} else {
|
|
809
1006
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
810
1007
|
}
|
|
@@ -813,10 +1010,19 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
813
1010
|
throw error;
|
|
814
1011
|
}
|
|
815
1012
|
}
|
|
816
|
-
/**
|
|
817
|
-
|
|
1013
|
+
/**
|
|
1014
|
+
* Sends the answer to the client as it is written, ahead of the voice.
|
|
1015
|
+
*
|
|
1016
|
+
* The stream reaches here a moment after the agent wrote it, so the agent
|
|
1017
|
+
* can record the complete message first, which the client then shows. A
|
|
1018
|
+
* partial answer arriving after it would show the answer twice, so the
|
|
1019
|
+
* forwarding stops once the message is sent.
|
|
1020
|
+
*/
|
|
1021
|
+
forwardPartialAnswer(stream, sentBefore) {
|
|
818
1022
|
let content = "";
|
|
819
|
-
const send = (text) =>
|
|
1023
|
+
const send = (text) => {
|
|
1024
|
+
if (this.assistantMessages === sentBefore) this.sendPartialAnswer(text);
|
|
1025
|
+
};
|
|
820
1026
|
const forward = new Transform({
|
|
821
1027
|
transform(chunk, _encoding, callback) {
|
|
822
1028
|
content += chunk.toString();
|
|
@@ -826,6 +1032,38 @@ var MicdropServer = class extends EventEmitter2 {
|
|
|
826
1032
|
});
|
|
827
1033
|
return stream.pipe(forward);
|
|
828
1034
|
}
|
|
1035
|
+
/**
|
|
1036
|
+
* Classifies the turn that just ended, with the one before, and keeps the
|
|
1037
|
+
* result in the metadata of its last message
|
|
1038
|
+
*/
|
|
1039
|
+
classifyTurn() {
|
|
1040
|
+
const classifier = this.config?.classifier;
|
|
1041
|
+
if (!classifier) return;
|
|
1042
|
+
const { input, message } = turnInput(
|
|
1043
|
+
this.conversation,
|
|
1044
|
+
this.config?.classifierOptions?.history
|
|
1045
|
+
);
|
|
1046
|
+
if (!message || input.turn.trim() === "") return;
|
|
1047
|
+
classifier.classify(input).then((classification) => {
|
|
1048
|
+
if (!classification) return;
|
|
1049
|
+
message.metadata = { ...message.metadata, classification };
|
|
1050
|
+
});
|
|
1051
|
+
}
|
|
1052
|
+
/** Waits for the classification in progress, never longer than its limit */
|
|
1053
|
+
async waitForClassification() {
|
|
1054
|
+
const pending = this.config?.classifier?.pending;
|
|
1055
|
+
if (!pending) return;
|
|
1056
|
+
const limit = this.config?.classifierOptions?.maxWait ?? DEFAULT_CLASSIFICATION_MAX_WAIT;
|
|
1057
|
+
let timer;
|
|
1058
|
+
const timeout = new Promise((resolve) => {
|
|
1059
|
+
timer = setTimeout(() => {
|
|
1060
|
+
this.log(`Classification took over ${limit} ms, answering without it`);
|
|
1061
|
+
resolve();
|
|
1062
|
+
}, limit);
|
|
1063
|
+
});
|
|
1064
|
+
await Promise.race([pending, timeout]);
|
|
1065
|
+
clearTimeout(timer);
|
|
1066
|
+
}
|
|
829
1067
|
// Run text-to-speech and send to client
|
|
830
1068
|
speak(message) {
|
|
831
1069
|
this.queueOperation(async () => {
|
|
@@ -962,6 +1200,7 @@ var Realtime = class extends Agent {
|
|
|
962
1200
|
});
|
|
963
1201
|
}
|
|
964
1202
|
cancel() {
|
|
1203
|
+
super.cancel();
|
|
965
1204
|
this.queue = [];
|
|
966
1205
|
const answer = this.currentAnswer;
|
|
967
1206
|
if (!answer) return;
|
|
@@ -1142,8 +1381,8 @@ var Realtime = class extends Agent {
|
|
|
1142
1381
|
};
|
|
1143
1382
|
|
|
1144
1383
|
// src/recorder/MicdropRecorder.ts
|
|
1145
|
-
import { EventEmitter as
|
|
1146
|
-
var MicdropRecorder = class extends
|
|
1384
|
+
import { EventEmitter as EventEmitter4 } from "eventemitter3";
|
|
1385
|
+
var MicdropRecorder = class extends EventEmitter4 {
|
|
1147
1386
|
constructor(server) {
|
|
1148
1387
|
super();
|
|
1149
1388
|
this.server = server;
|
|
@@ -1289,19 +1528,281 @@ var SharedInstances = class {
|
|
|
1289
1528
|
this.entries.set(id, loading);
|
|
1290
1529
|
return loading;
|
|
1291
1530
|
}
|
|
1531
|
+
/** The instance for this key, loaded or being loaded, without loading it. */
|
|
1532
|
+
get(key) {
|
|
1533
|
+
return this.entries.get(typeof key === "string" ? key : JSON.stringify(key));
|
|
1534
|
+
}
|
|
1292
1535
|
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
1293
1536
|
has(key) {
|
|
1294
1537
|
return this.entries.has(typeof key === "string" ? key : JSON.stringify(key));
|
|
1295
1538
|
}
|
|
1539
|
+
/**
|
|
1540
|
+
* Forgets the instance for this key, so the next call loads a new one. For
|
|
1541
|
+
* an instance that broke, a crashed worker for instance. Releasing what it
|
|
1542
|
+
* holds is left to the caller.
|
|
1543
|
+
*/
|
|
1544
|
+
delete(key) {
|
|
1545
|
+
this.entries.delete(typeof key === "string" ? key : JSON.stringify(key));
|
|
1546
|
+
}
|
|
1296
1547
|
/** Drops every instance, so the next call loads again. */
|
|
1297
1548
|
clear() {
|
|
1298
1549
|
this.entries.clear();
|
|
1299
1550
|
}
|
|
1300
1551
|
};
|
|
1301
1552
|
|
|
1553
|
+
// src/socket/ReconnectingSocket.ts
|
|
1554
|
+
import { EventEmitter as EventEmitter5 } from "eventemitter3";
|
|
1555
|
+
import WebSocket from "ws";
|
|
1556
|
+
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
1557
|
+
var DEFAULT_RETRY_DELAY = 1e3;
|
|
1558
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
1559
|
+
var ReconnectingSocket = class extends EventEmitter5 {
|
|
1560
|
+
constructor(options) {
|
|
1561
|
+
super();
|
|
1562
|
+
this.options = options;
|
|
1563
|
+
this._status = "idle";
|
|
1564
|
+
this.everOpened = false;
|
|
1565
|
+
this.retryCount = 0;
|
|
1566
|
+
// Numbers the attempts, so one overtaken while it awaited its address stops
|
|
1567
|
+
this.attemptId = 0;
|
|
1568
|
+
this.waiters = [];
|
|
1569
|
+
}
|
|
1570
|
+
get status() {
|
|
1571
|
+
return this._status;
|
|
1572
|
+
}
|
|
1573
|
+
/** Whether a message sent now goes out */
|
|
1574
|
+
get isOpen() {
|
|
1575
|
+
return this._status === "open";
|
|
1576
|
+
}
|
|
1577
|
+
/**
|
|
1578
|
+
* Opens the connection. Called again after `failed`, it starts over with a
|
|
1579
|
+
* full retry budget, and does nothing in any other status.
|
|
1580
|
+
*/
|
|
1581
|
+
connect() {
|
|
1582
|
+
if (this._status !== "idle" && this._status !== "failed") return;
|
|
1583
|
+
this.retryCount = 0;
|
|
1584
|
+
this.attempt();
|
|
1585
|
+
}
|
|
1586
|
+
/**
|
|
1587
|
+
* Waits for the connection to be open.
|
|
1588
|
+
* @returns True once open, false if it failed or was destroyed meanwhile
|
|
1589
|
+
*/
|
|
1590
|
+
ready() {
|
|
1591
|
+
if (this._status === "open") return Promise.resolve(true);
|
|
1592
|
+
if (this._status === "destroyed" || this._status === "failed") {
|
|
1593
|
+
return Promise.resolve(false);
|
|
1594
|
+
}
|
|
1595
|
+
return new Promise((resolve) => this.waiters.push(resolve));
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* Sends a message if the connection is open
|
|
1599
|
+
* @returns False when it did not go out
|
|
1600
|
+
*/
|
|
1601
|
+
send(data) {
|
|
1602
|
+
if (this._status !== "open" || !this.socket) return false;
|
|
1603
|
+
this.socket.send(data);
|
|
1604
|
+
return true;
|
|
1605
|
+
}
|
|
1606
|
+
/**
|
|
1607
|
+
* Drops the connection and opens a new one right away, for a provider that
|
|
1608
|
+
* can only stop what it is doing by closing. It costs no retry, and the new
|
|
1609
|
+
* connection is not a reconnection.
|
|
1610
|
+
*/
|
|
1611
|
+
restart() {
|
|
1612
|
+
if (this._status === "destroyed") return;
|
|
1613
|
+
this.log("Restarting the connection");
|
|
1614
|
+
this.clearRetryTimer();
|
|
1615
|
+
this.drop();
|
|
1616
|
+
this.everOpened = false;
|
|
1617
|
+
this.retryCount = 0;
|
|
1618
|
+
this.attempt();
|
|
1619
|
+
}
|
|
1620
|
+
/** Closes the connection for good */
|
|
1621
|
+
destroy() {
|
|
1622
|
+
if (this._status === "destroyed") return;
|
|
1623
|
+
this.clearRetryTimer();
|
|
1624
|
+
this.drop();
|
|
1625
|
+
this.setStatus("destroyed");
|
|
1626
|
+
this.settleWaiters(false);
|
|
1627
|
+
this.removeAllListeners();
|
|
1628
|
+
}
|
|
1629
|
+
async attempt() {
|
|
1630
|
+
const id = ++this.attemptId;
|
|
1631
|
+
const isOvertaken = () => id !== this.attemptId || this._status !== "connecting";
|
|
1632
|
+
this.setStatus("connecting");
|
|
1633
|
+
let url;
|
|
1634
|
+
try {
|
|
1635
|
+
const { url: getUrl } = this.options;
|
|
1636
|
+
url = typeof getUrl === "string" ? getUrl : await getUrl();
|
|
1637
|
+
} catch (error) {
|
|
1638
|
+
if (isOvertaken()) return;
|
|
1639
|
+
this.onAttemptFailed(error);
|
|
1640
|
+
return;
|
|
1641
|
+
}
|
|
1642
|
+
if (isOvertaken()) return;
|
|
1643
|
+
const socket = new WebSocket(url, this.options.clientOptions);
|
|
1644
|
+
this.socket = socket;
|
|
1645
|
+
let settled = false;
|
|
1646
|
+
const fail = (error) => {
|
|
1647
|
+
if (settled || this.socket !== socket) return;
|
|
1648
|
+
settled = true;
|
|
1649
|
+
clearTimeout(timeout);
|
|
1650
|
+
this.drop();
|
|
1651
|
+
this.onAttemptFailed(error);
|
|
1652
|
+
};
|
|
1653
|
+
const timeout = setTimeout(
|
|
1654
|
+
() => fail(new Error("Connection timeout")),
|
|
1655
|
+
this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
|
|
1656
|
+
);
|
|
1657
|
+
socket.on("error", (error) => {
|
|
1658
|
+
this.log("WebSocket error:", error.message);
|
|
1659
|
+
fail(error);
|
|
1660
|
+
});
|
|
1661
|
+
const { handshake } = this.options;
|
|
1662
|
+
const open = () => {
|
|
1663
|
+
settled = true;
|
|
1664
|
+
clearTimeout(timeout);
|
|
1665
|
+
this.onOpen();
|
|
1666
|
+
};
|
|
1667
|
+
socket.on("open", () => {
|
|
1668
|
+
if (this.socket !== socket) return;
|
|
1669
|
+
try {
|
|
1670
|
+
handshake?.start?.(socket);
|
|
1671
|
+
} catch (error) {
|
|
1672
|
+
fail(error);
|
|
1673
|
+
return;
|
|
1674
|
+
}
|
|
1675
|
+
if (!handshake?.read) open();
|
|
1676
|
+
});
|
|
1677
|
+
socket.on("message", (data, isBinary) => {
|
|
1678
|
+
if (this.socket !== socket) return;
|
|
1679
|
+
if (this._status === "open") {
|
|
1680
|
+
this.prove();
|
|
1681
|
+
this.emit("Message", data, isBinary);
|
|
1682
|
+
return;
|
|
1683
|
+
}
|
|
1684
|
+
if (settled || !handshake?.read) return;
|
|
1685
|
+
try {
|
|
1686
|
+
if (handshake.read(data, isBinary)) open();
|
|
1687
|
+
} catch (error) {
|
|
1688
|
+
fail(error);
|
|
1689
|
+
}
|
|
1690
|
+
});
|
|
1691
|
+
socket.on("close", (code, reasonBuffer) => {
|
|
1692
|
+
const reason = reasonBuffer.toString();
|
|
1693
|
+
if (!settled) {
|
|
1694
|
+
fail(new Error(`Connection closed (${code}) ${reason}`));
|
|
1695
|
+
return;
|
|
1696
|
+
}
|
|
1697
|
+
if (this.socket !== socket) return;
|
|
1698
|
+
this.log("Connection closed", { code, reason });
|
|
1699
|
+
this.drop();
|
|
1700
|
+
this.setStatus("reconnecting");
|
|
1701
|
+
this.emit("Close", code, reason);
|
|
1702
|
+
if (this._status !== "reconnecting") return;
|
|
1703
|
+
this.scheduleRetry(new Error(`Connection closed (${code}) ${reason}`));
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
onOpen() {
|
|
1707
|
+
this.log("Connection opened");
|
|
1708
|
+
const reconnected = this.everOpened;
|
|
1709
|
+
this.everOpened = true;
|
|
1710
|
+
if (this.options.handshake?.read) {
|
|
1711
|
+
this.prove();
|
|
1712
|
+
} else {
|
|
1713
|
+
this.proofTimer = setTimeout(
|
|
1714
|
+
() => this.prove(),
|
|
1715
|
+
this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
|
|
1716
|
+
);
|
|
1717
|
+
}
|
|
1718
|
+
this.startKeepAlive();
|
|
1719
|
+
this.setStatus("open");
|
|
1720
|
+
this.emit("Open", { reconnected });
|
|
1721
|
+
this.settleWaiters(true);
|
|
1722
|
+
}
|
|
1723
|
+
/** The connection works, the retry budget is whole again */
|
|
1724
|
+
prove() {
|
|
1725
|
+
this.clearProofTimer();
|
|
1726
|
+
this.retryCount = 0;
|
|
1727
|
+
}
|
|
1728
|
+
clearProofTimer() {
|
|
1729
|
+
if (!this.proofTimer) return;
|
|
1730
|
+
clearTimeout(this.proofTimer);
|
|
1731
|
+
this.proofTimer = void 0;
|
|
1732
|
+
}
|
|
1733
|
+
onAttemptFailed(error) {
|
|
1734
|
+
this.log("Connection failed:", error.message);
|
|
1735
|
+
this.scheduleRetry(error);
|
|
1736
|
+
}
|
|
1737
|
+
scheduleRetry(error) {
|
|
1738
|
+
if (this._status === "destroyed") return;
|
|
1739
|
+
this.retryCount++;
|
|
1740
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
1741
|
+
this.log("Max retries reached, giving up");
|
|
1742
|
+
this.setStatus("failed");
|
|
1743
|
+
this.settleWaiters(false);
|
|
1744
|
+
this.emit("Failed", error);
|
|
1745
|
+
return;
|
|
1746
|
+
}
|
|
1747
|
+
this.setStatus("reconnecting");
|
|
1748
|
+
this.log(`Reconnecting (attempt ${this.retryCount})`);
|
|
1749
|
+
this.clearRetryTimer();
|
|
1750
|
+
this.retryTimer = setTimeout(() => {
|
|
1751
|
+
this.retryTimer = void 0;
|
|
1752
|
+
if (this._status === "reconnecting") this.attempt();
|
|
1753
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
|
|
1754
|
+
}
|
|
1755
|
+
/** Lets go of the current socket, which will never be heard from again */
|
|
1756
|
+
drop() {
|
|
1757
|
+
this.stopKeepAlive();
|
|
1758
|
+
this.clearProofTimer();
|
|
1759
|
+
const socket = this.socket;
|
|
1760
|
+
this.socket = void 0;
|
|
1761
|
+
if (!socket) return;
|
|
1762
|
+
socket.removeAllListeners();
|
|
1763
|
+
socket.on("error", () => {
|
|
1764
|
+
});
|
|
1765
|
+
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
|
|
1766
|
+
socket.close(1e3);
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
startKeepAlive() {
|
|
1770
|
+
const keepAlive = this.options.keepAlive;
|
|
1771
|
+
if (!keepAlive) return;
|
|
1772
|
+
this.stopKeepAlive();
|
|
1773
|
+
this.keepAliveTimer = setInterval(() => {
|
|
1774
|
+
const { message } = keepAlive;
|
|
1775
|
+
this.send(typeof message === "string" ? message : message());
|
|
1776
|
+
}, keepAlive.interval);
|
|
1777
|
+
}
|
|
1778
|
+
stopKeepAlive() {
|
|
1779
|
+
if (!this.keepAliveTimer) return;
|
|
1780
|
+
clearInterval(this.keepAliveTimer);
|
|
1781
|
+
this.keepAliveTimer = void 0;
|
|
1782
|
+
}
|
|
1783
|
+
clearRetryTimer() {
|
|
1784
|
+
if (!this.retryTimer) return;
|
|
1785
|
+
clearTimeout(this.retryTimer);
|
|
1786
|
+
this.retryTimer = void 0;
|
|
1787
|
+
}
|
|
1788
|
+
settleWaiters(open) {
|
|
1789
|
+
const waiters = this.waiters;
|
|
1790
|
+
this.waiters = [];
|
|
1791
|
+
waiters.forEach((resolve) => resolve(open));
|
|
1792
|
+
}
|
|
1793
|
+
setStatus(status) {
|
|
1794
|
+
if (this._status === status) return;
|
|
1795
|
+
this._status = status;
|
|
1796
|
+
this.emit("StatusChange", status);
|
|
1797
|
+
}
|
|
1798
|
+
log(...message) {
|
|
1799
|
+
this.options.log?.(...message);
|
|
1800
|
+
}
|
|
1801
|
+
};
|
|
1802
|
+
|
|
1302
1803
|
// src/stt/STT.ts
|
|
1303
|
-
import { EventEmitter as
|
|
1304
|
-
var STT = class extends
|
|
1804
|
+
import { EventEmitter as EventEmitter6 } from "eventemitter3";
|
|
1805
|
+
var STT = class extends EventEmitter6 {
|
|
1305
1806
|
log(...message) {
|
|
1306
1807
|
this.logger?.log(...message);
|
|
1307
1808
|
}
|
|
@@ -1327,24 +1828,31 @@ var MockSTT = class extends STT {
|
|
|
1327
1828
|
// src/stt/FallbackSTT.ts
|
|
1328
1829
|
import { PassThrough as PassThrough3 } from "stream";
|
|
1329
1830
|
var FallbackSTT = class extends STT {
|
|
1330
|
-
// Start at -1 because we need to increment it before using it
|
|
1331
1831
|
constructor(options) {
|
|
1332
1832
|
super();
|
|
1333
1833
|
this.options = options;
|
|
1334
1834
|
this.stt = null;
|
|
1335
1835
|
this.sttIndex = -1;
|
|
1836
|
+
// Start at -1 because we need to increment it before using it
|
|
1837
|
+
// In the order spoken, each STT giving one transcript per utterance in turn
|
|
1838
|
+
this.pending = [];
|
|
1336
1839
|
this.onTranscript = (transcript) => {
|
|
1840
|
+
this.pending.shift();
|
|
1337
1841
|
this.emit("Transcript", transcript);
|
|
1338
1842
|
};
|
|
1339
|
-
|
|
1843
|
+
/**
|
|
1844
|
+
* Hands every utterance still waiting to the next STT, one stream each so
|
|
1845
|
+
* each gets its own transcript. One still being spoken keeps going there.
|
|
1846
|
+
*/
|
|
1847
|
+
this.onFailed = () => {
|
|
1340
1848
|
this.log("STT failed, trying next STT");
|
|
1341
1849
|
this.startNextSTT();
|
|
1342
|
-
if (
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
this.stt?.transcribe(stream);
|
|
1346
|
-
chunks.forEach((chunk) => stream.write(chunk));
|
|
1347
|
-
stream.end();
|
|
1850
|
+
if (this.pending.length > 0) this.log("Sending audio chunks again");
|
|
1851
|
+
for (const utterance of this.pending) {
|
|
1852
|
+
utterance.stream = new PassThrough3();
|
|
1853
|
+
this.stt?.transcribe(utterance.stream);
|
|
1854
|
+
utterance.chunks.forEach((chunk) => utterance.stream.write(chunk));
|
|
1855
|
+
if (utterance.ended) utterance.stream.end();
|
|
1348
1856
|
}
|
|
1349
1857
|
};
|
|
1350
1858
|
if (this.options.factories.length === 0) {
|
|
@@ -1353,13 +1861,28 @@ var FallbackSTT = class extends STT {
|
|
|
1353
1861
|
this.startNextSTT();
|
|
1354
1862
|
}
|
|
1355
1863
|
transcribe(audioStream) {
|
|
1356
|
-
|
|
1864
|
+
const utterance = {
|
|
1865
|
+
chunks: [],
|
|
1866
|
+
ended: false,
|
|
1867
|
+
stream: new PassThrough3()
|
|
1868
|
+
};
|
|
1869
|
+
this.pending.push(utterance);
|
|
1870
|
+
audioStream.on("data", (chunk) => {
|
|
1871
|
+
utterance.chunks.push(chunk);
|
|
1872
|
+
utterance.stream.write(chunk);
|
|
1873
|
+
});
|
|
1874
|
+
audioStream.on("end", () => {
|
|
1875
|
+
utterance.ended = true;
|
|
1876
|
+
utterance.stream.end();
|
|
1877
|
+
});
|
|
1878
|
+
this.stt?.transcribe(utterance.stream);
|
|
1357
1879
|
}
|
|
1358
1880
|
destroy() {
|
|
1359
1881
|
super.destroy();
|
|
1360
1882
|
this.stt?.destroy();
|
|
1361
1883
|
this.stt = null;
|
|
1362
1884
|
this.sttIndex = -1;
|
|
1885
|
+
this.pending = [];
|
|
1363
1886
|
}
|
|
1364
1887
|
startNextSTT() {
|
|
1365
1888
|
this.sttIndex++;
|
|
@@ -1382,8 +1905,8 @@ var FallbackSTT = class extends STT {
|
|
|
1382
1905
|
import { PassThrough as PassThrough4 } from "stream";
|
|
1383
1906
|
|
|
1384
1907
|
// src/tts/TTS.ts
|
|
1385
|
-
import { EventEmitter as
|
|
1386
|
-
var TTS = class extends
|
|
1908
|
+
import { EventEmitter as EventEmitter7 } from "eventemitter3";
|
|
1909
|
+
var TTS = class extends EventEmitter7 {
|
|
1387
1910
|
log(...message) {
|
|
1388
1911
|
this.logger?.log(...message);
|
|
1389
1912
|
}
|
|
@@ -1395,7 +1918,6 @@ var TTS = class extends EventEmitter5 {
|
|
|
1395
1918
|
|
|
1396
1919
|
// src/tts/FallbackTTS.ts
|
|
1397
1920
|
var FallbackTTS = class extends TTS {
|
|
1398
|
-
// Start at -1 because we need to increment it before using it
|
|
1399
1921
|
constructor(options) {
|
|
1400
1922
|
super();
|
|
1401
1923
|
this.options = options;
|
|
@@ -1404,16 +1926,22 @@ var FallbackTTS = class extends TTS {
|
|
|
1404
1926
|
this.onAudio = (audio) => {
|
|
1405
1927
|
this.emit("Audio", audio);
|
|
1406
1928
|
};
|
|
1929
|
+
/**
|
|
1930
|
+
* Hands the unspoken text to the next TTS. An utterance whose text is still
|
|
1931
|
+
* being written goes on there, so its end is spoken too.
|
|
1932
|
+
*/
|
|
1407
1933
|
this.onFailed = (chunks) => {
|
|
1408
1934
|
this.log("TTS failed, trying next TTS");
|
|
1409
1935
|
this.startNextTTS();
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1936
|
+
const utterance = this.current;
|
|
1937
|
+
const continues = !!utterance && !utterance.ended;
|
|
1938
|
+
if (chunks.length === 0 && !continues) return;
|
|
1939
|
+
this.log("Sending text chunks again");
|
|
1940
|
+
const stream = new PassThrough4();
|
|
1941
|
+
this.tts?.speak(stream);
|
|
1942
|
+
chunks.forEach((chunk) => stream.write(chunk));
|
|
1943
|
+
if (continues) utterance.stream = stream;
|
|
1944
|
+
else stream.end();
|
|
1417
1945
|
};
|
|
1418
1946
|
if (this.options.factories.length === 0) {
|
|
1419
1947
|
throw new Error("FallbackTTS: No factories provided");
|
|
@@ -1421,9 +1949,20 @@ var FallbackTTS = class extends TTS {
|
|
|
1421
1949
|
this.startNextTTS();
|
|
1422
1950
|
}
|
|
1423
1951
|
speak(textStream) {
|
|
1424
|
-
|
|
1952
|
+
const utterance = {
|
|
1953
|
+
ended: false,
|
|
1954
|
+
stream: new PassThrough4()
|
|
1955
|
+
};
|
|
1956
|
+
this.current = utterance;
|
|
1957
|
+
textStream.on("data", (chunk) => utterance.stream.write(chunk));
|
|
1958
|
+
textStream.on("end", () => {
|
|
1959
|
+
utterance.ended = true;
|
|
1960
|
+
utterance.stream.end();
|
|
1961
|
+
});
|
|
1962
|
+
this.tts?.speak(utterance.stream);
|
|
1425
1963
|
}
|
|
1426
1964
|
cancel() {
|
|
1965
|
+
this.current = void 0;
|
|
1427
1966
|
this.tts?.cancel();
|
|
1428
1967
|
}
|
|
1429
1968
|
destroy() {
|
|
@@ -1451,26 +1990,28 @@ var FallbackTTS = class extends TTS {
|
|
|
1451
1990
|
|
|
1452
1991
|
// src/tts/MockTTS.ts
|
|
1453
1992
|
import * as fs from "fs";
|
|
1454
|
-
import { PassThrough as PassThrough5 } from "stream";
|
|
1455
1993
|
var MockTTS = class extends TTS {
|
|
1456
1994
|
constructor(audioFilePaths) {
|
|
1457
1995
|
super();
|
|
1458
1996
|
this.audioFilePaths = audioFilePaths;
|
|
1997
|
+
// Bumped by every answer and every cancel, so the chunks of an answer that
|
|
1998
|
+
// is no longer wanted stop there
|
|
1999
|
+
this.generation = 0;
|
|
1459
2000
|
}
|
|
1460
2001
|
speak(textStream) {
|
|
1461
|
-
const
|
|
2002
|
+
const generation = ++this.generation;
|
|
1462
2003
|
textStream.once("data", async () => {
|
|
1463
2004
|
for (const filePath of this.audioFilePaths) {
|
|
1464
2005
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
2006
|
+
if (generation !== this.generation) return;
|
|
2007
|
+
const audio = fs.readFileSync(filePath);
|
|
2008
|
+
this.log(`Loaded chunk (${audio.length} bytes)`);
|
|
2009
|
+
this.emit("Audio", audio);
|
|
1468
2010
|
}
|
|
1469
|
-
audioStream.end();
|
|
1470
2011
|
});
|
|
1471
|
-
return audioStream;
|
|
1472
2012
|
}
|
|
1473
2013
|
cancel() {
|
|
2014
|
+
this.generation++;
|
|
1474
2015
|
}
|
|
1475
2016
|
};
|
|
1476
2017
|
|
|
@@ -1549,6 +2090,9 @@ var SentenceTTS = class extends TTS {
|
|
|
1549
2090
|
this.counter++;
|
|
1550
2091
|
counter = this.counter;
|
|
1551
2092
|
this.splitter.reset();
|
|
2093
|
+
this.queue = [];
|
|
2094
|
+
this.controller?.abort();
|
|
2095
|
+
this.controller = void 0;
|
|
1552
2096
|
return true;
|
|
1553
2097
|
};
|
|
1554
2098
|
textStream.on("data", (chunk) => {
|
|
@@ -1634,6 +2178,7 @@ export {
|
|
|
1634
2178
|
AUTO_SEMANTIC_TURN_PROMPT,
|
|
1635
2179
|
AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
1636
2180
|
Agent,
|
|
2181
|
+
Classifier,
|
|
1637
2182
|
FallbackAgent,
|
|
1638
2183
|
FallbackSTT,
|
|
1639
2184
|
FallbackTTS,
|
|
@@ -1649,14 +2194,18 @@ export {
|
|
|
1649
2194
|
MockTTS,
|
|
1650
2195
|
Pcm16Resampler,
|
|
1651
2196
|
Realtime,
|
|
2197
|
+
ReconnectingSocket,
|
|
1652
2198
|
STT,
|
|
1653
2199
|
SentenceSplitter,
|
|
1654
2200
|
SentenceTTS,
|
|
1655
2201
|
SharedInstances,
|
|
1656
2202
|
TTS,
|
|
2203
|
+
currentTurn,
|
|
1657
2204
|
float32ToPcm16,
|
|
2205
|
+
getTurnClassification,
|
|
1658
2206
|
handleError,
|
|
1659
2207
|
pcm16ToFloat32,
|
|
2208
|
+
turnInput,
|
|
1660
2209
|
waitForParams
|
|
1661
2210
|
};
|
|
1662
2211
|
//# sourceMappingURL=index.mjs.map
|