@micdrop/server 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/index.d.mts +59 -2
- package/dist/index.d.ts +59 -2
- package/dist/index.js +140 -56
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +141 -57
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -3
package/README.md
CHANGED
|
@@ -64,9 +64,13 @@ wss.on('connection', (socket) => {
|
|
|
64
64
|
|
|
65
65
|
Micdrop server has 3 main components:
|
|
66
66
|
|
|
67
|
-
- `Agent` - AI agent using LLM
|
|
67
|
+
- `Agent` - AI agent using LLM, optional
|
|
68
68
|
- `STT` - Speech-to-text
|
|
69
|
-
- `TTS` - Text-to-speech
|
|
69
|
+
- `TTS` - Text-to-speech, optional
|
|
70
|
+
|
|
71
|
+
A server given a speech to text alone transcribes and stays quiet, which is all
|
|
72
|
+
a dictation tool needs. One given an agent and no voice answers in writing. See
|
|
73
|
+
[dictation and text-only calls](https://micdrop.dev/docs/server/dictation).
|
|
70
74
|
|
|
71
75
|
### Available implementations
|
|
72
76
|
|
package/dist/index.d.mts
CHANGED
|
@@ -16,6 +16,7 @@ declare enum MicdropClientCommands {
|
|
|
16
16
|
}
|
|
17
17
|
declare enum MicdropServerCommands {
|
|
18
18
|
Message = "Message",
|
|
19
|
+
PartialAssistantMessage = "PartialAssistantMessage",
|
|
19
20
|
CancelLastUserMessage = "CancelLastUserMessage",
|
|
20
21
|
SkipAnswer = "SkipAnswer",
|
|
21
22
|
EndCall = "EndCall",
|
|
@@ -376,15 +377,58 @@ declare abstract class SentenceTTS extends TTS {
|
|
|
376
377
|
|
|
377
378
|
interface MicdropServerEvents {
|
|
378
379
|
End: [MicdropCallSummary];
|
|
380
|
+
/** A message was added to the conversation, by the agent or by the server */
|
|
381
|
+
Message: [MicdropConversationItem];
|
|
379
382
|
UserAudio: [Buffer];
|
|
380
383
|
AssistantAudio: [Buffer];
|
|
381
384
|
}
|
|
382
385
|
interface MicdropConfig {
|
|
386
|
+
/**
|
|
387
|
+
* Opening line, spoken before the user has said anything.
|
|
388
|
+
*
|
|
389
|
+
* It goes into the conversation like any other assistant message. Leaving it
|
|
390
|
+
* out, along with `generateFirstMessage`, hands the first turn to the user.
|
|
391
|
+
*/
|
|
383
392
|
firstMessage?: string;
|
|
393
|
+
/**
|
|
394
|
+
* Asks the agent for its opening line instead of fixing one, so the greeting
|
|
395
|
+
* can lean on the system prompt and on whatever the call was given.
|
|
396
|
+
*
|
|
397
|
+
* Ignored when `firstMessage` is set, and out of reach without an agent.
|
|
398
|
+
*/
|
|
384
399
|
generateFirstMessage?: boolean;
|
|
385
|
-
|
|
400
|
+
/**
|
|
401
|
+
* Generates the answers.
|
|
402
|
+
*
|
|
403
|
+
* Leaving it out turns the call into a one-way one: what the user says is
|
|
404
|
+
* transcribed and sent to the client, and the assistant stays quiet unless
|
|
405
|
+
* the application calls `speak()` itself.
|
|
406
|
+
*/
|
|
407
|
+
agent?: Agent;
|
|
408
|
+
/**
|
|
409
|
+
* Turns what the user says into text, the one component a call cannot do
|
|
410
|
+
* without.
|
|
411
|
+
*
|
|
412
|
+
* It is fed the audio of each turn as it arrives, and the transcripts it
|
|
413
|
+
* returns drive everything that follows.
|
|
414
|
+
*/
|
|
386
415
|
stt: STT;
|
|
387
|
-
|
|
416
|
+
/**
|
|
417
|
+
* Gives the answers a voice.
|
|
418
|
+
*
|
|
419
|
+
* Leaving it out keeps the call textual: the answer reaches the client as a
|
|
420
|
+
* message and nothing is synthesized.
|
|
421
|
+
*/
|
|
422
|
+
tts?: TTS;
|
|
423
|
+
/**
|
|
424
|
+
* Sends the answer to the client while it is still being written, on top of
|
|
425
|
+
* the complete message that follows.
|
|
426
|
+
*
|
|
427
|
+
* Works with any agent, since the text is read from the stream it writes to.
|
|
428
|
+
* Off by default, since a client that only displays finished messages has
|
|
429
|
+
* nothing to do with it.
|
|
430
|
+
*/
|
|
431
|
+
partialMessages?: boolean;
|
|
388
432
|
/**
|
|
389
433
|
* Waits for the rest of the sentence when the speaker paused in the middle
|
|
390
434
|
* of one, instead of answering an unfinished question.
|
|
@@ -403,6 +447,7 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
403
447
|
logger?: Logger;
|
|
404
448
|
private startTime;
|
|
405
449
|
private lastMessageSpeeched?;
|
|
450
|
+
private ownConversation;
|
|
406
451
|
private operationQueue;
|
|
407
452
|
private isProcessingQueue;
|
|
408
453
|
private currentUserStream?;
|
|
@@ -410,6 +455,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
410
455
|
private turnComplete?;
|
|
411
456
|
private heldTurnTimer?;
|
|
412
457
|
constructor(socket: WebSocket$1, config: MicdropConfig);
|
|
458
|
+
/** Everything said so far, whether an agent or the server keeps it */
|
|
459
|
+
get conversation(): MicdropConversation;
|
|
413
460
|
private log;
|
|
414
461
|
private processQueue;
|
|
415
462
|
private queueOperation;
|
|
@@ -433,9 +480,19 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
433
480
|
private releaseHeldTurn;
|
|
434
481
|
private onTranscriptSTT;
|
|
435
482
|
private onAudioTTS;
|
|
483
|
+
private onMessageAgent;
|
|
484
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
485
|
+
addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
486
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
487
|
+
addAssistantMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
488
|
+
private addOwnMessage;
|
|
489
|
+
private sendMessage;
|
|
490
|
+
private sendPartialAnswer;
|
|
436
491
|
private sendFirstMessage;
|
|
437
492
|
answer(): void;
|
|
438
493
|
private _answer;
|
|
494
|
+
/** Sends the answer to the client as it is written, ahead of the voice */
|
|
495
|
+
private forwardPartialAnswer;
|
|
439
496
|
speak(message: string | Readable): void;
|
|
440
497
|
private _speak;
|
|
441
498
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -16,6 +16,7 @@ declare enum MicdropClientCommands {
|
|
|
16
16
|
}
|
|
17
17
|
declare enum MicdropServerCommands {
|
|
18
18
|
Message = "Message",
|
|
19
|
+
PartialAssistantMessage = "PartialAssistantMessage",
|
|
19
20
|
CancelLastUserMessage = "CancelLastUserMessage",
|
|
20
21
|
SkipAnswer = "SkipAnswer",
|
|
21
22
|
EndCall = "EndCall",
|
|
@@ -376,15 +377,58 @@ declare abstract class SentenceTTS extends TTS {
|
|
|
376
377
|
|
|
377
378
|
interface MicdropServerEvents {
|
|
378
379
|
End: [MicdropCallSummary];
|
|
380
|
+
/** A message was added to the conversation, by the agent or by the server */
|
|
381
|
+
Message: [MicdropConversationItem];
|
|
379
382
|
UserAudio: [Buffer];
|
|
380
383
|
AssistantAudio: [Buffer];
|
|
381
384
|
}
|
|
382
385
|
interface MicdropConfig {
|
|
386
|
+
/**
|
|
387
|
+
* Opening line, spoken before the user has said anything.
|
|
388
|
+
*
|
|
389
|
+
* It goes into the conversation like any other assistant message. Leaving it
|
|
390
|
+
* out, along with `generateFirstMessage`, hands the first turn to the user.
|
|
391
|
+
*/
|
|
383
392
|
firstMessage?: string;
|
|
393
|
+
/**
|
|
394
|
+
* Asks the agent for its opening line instead of fixing one, so the greeting
|
|
395
|
+
* can lean on the system prompt and on whatever the call was given.
|
|
396
|
+
*
|
|
397
|
+
* Ignored when `firstMessage` is set, and out of reach without an agent.
|
|
398
|
+
*/
|
|
384
399
|
generateFirstMessage?: boolean;
|
|
385
|
-
|
|
400
|
+
/**
|
|
401
|
+
* Generates the answers.
|
|
402
|
+
*
|
|
403
|
+
* Leaving it out turns the call into a one-way one: what the user says is
|
|
404
|
+
* transcribed and sent to the client, and the assistant stays quiet unless
|
|
405
|
+
* the application calls `speak()` itself.
|
|
406
|
+
*/
|
|
407
|
+
agent?: Agent;
|
|
408
|
+
/**
|
|
409
|
+
* Turns what the user says into text, the one component a call cannot do
|
|
410
|
+
* without.
|
|
411
|
+
*
|
|
412
|
+
* It is fed the audio of each turn as it arrives, and the transcripts it
|
|
413
|
+
* returns drive everything that follows.
|
|
414
|
+
*/
|
|
386
415
|
stt: STT;
|
|
387
|
-
|
|
416
|
+
/**
|
|
417
|
+
* Gives the answers a voice.
|
|
418
|
+
*
|
|
419
|
+
* Leaving it out keeps the call textual: the answer reaches the client as a
|
|
420
|
+
* message and nothing is synthesized.
|
|
421
|
+
*/
|
|
422
|
+
tts?: TTS;
|
|
423
|
+
/**
|
|
424
|
+
* Sends the answer to the client while it is still being written, on top of
|
|
425
|
+
* the complete message that follows.
|
|
426
|
+
*
|
|
427
|
+
* Works with any agent, since the text is read from the stream it writes to.
|
|
428
|
+
* Off by default, since a client that only displays finished messages has
|
|
429
|
+
* nothing to do with it.
|
|
430
|
+
*/
|
|
431
|
+
partialMessages?: boolean;
|
|
388
432
|
/**
|
|
389
433
|
* Waits for the rest of the sentence when the speaker paused in the middle
|
|
390
434
|
* of one, instead of answering an unfinished question.
|
|
@@ -403,6 +447,7 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
403
447
|
logger?: Logger;
|
|
404
448
|
private startTime;
|
|
405
449
|
private lastMessageSpeeched?;
|
|
450
|
+
private ownConversation;
|
|
406
451
|
private operationQueue;
|
|
407
452
|
private isProcessingQueue;
|
|
408
453
|
private currentUserStream?;
|
|
@@ -410,6 +455,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
410
455
|
private turnComplete?;
|
|
411
456
|
private heldTurnTimer?;
|
|
412
457
|
constructor(socket: WebSocket$1, config: MicdropConfig);
|
|
458
|
+
/** Everything said so far, whether an agent or the server keeps it */
|
|
459
|
+
get conversation(): MicdropConversation;
|
|
413
460
|
private log;
|
|
414
461
|
private processQueue;
|
|
415
462
|
private queueOperation;
|
|
@@ -433,9 +480,19 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
433
480
|
private releaseHeldTurn;
|
|
434
481
|
private onTranscriptSTT;
|
|
435
482
|
private onAudioTTS;
|
|
483
|
+
private onMessageAgent;
|
|
484
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
485
|
+
addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
486
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
487
|
+
addAssistantMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
488
|
+
private addOwnMessage;
|
|
489
|
+
private sendMessage;
|
|
490
|
+
private sendPartialAnswer;
|
|
436
491
|
private sendFirstMessage;
|
|
437
492
|
answer(): void;
|
|
438
493
|
private _answer;
|
|
494
|
+
/** Sends the answer to the client as it is written, ahead of the voice */
|
|
495
|
+
private forwardPartialAnswer;
|
|
439
496
|
speak(message: string | Readable): void;
|
|
440
497
|
private _speak;
|
|
441
498
|
}
|
package/dist/index.js
CHANGED
|
@@ -503,6 +503,7 @@ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
|
503
503
|
})(MicdropClientCommands || {});
|
|
504
504
|
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
505
505
|
MicdropServerCommands2["Message"] = "Message";
|
|
506
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
506
507
|
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
507
508
|
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
508
509
|
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
@@ -519,6 +520,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
519
520
|
this.socket = null;
|
|
520
521
|
this.config = null;
|
|
521
522
|
this.startTime = Date.now();
|
|
523
|
+
// Holds the conversation when no agent does, and keeps it readable once the
|
|
524
|
+
// call is over and the config is gone
|
|
525
|
+
this.ownConversation = [];
|
|
522
526
|
// Queue system for operations
|
|
523
527
|
this.operationQueue = [];
|
|
524
528
|
this.isProcessingQueue = false;
|
|
@@ -528,13 +532,15 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
528
532
|
if (!this.config) return;
|
|
529
533
|
this.log("Connection closed");
|
|
530
534
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
531
|
-
this.
|
|
535
|
+
const conversation = this.conversation;
|
|
536
|
+
this.config.agent?.destroy();
|
|
532
537
|
this.config.stt.destroy();
|
|
533
|
-
this.config.tts
|
|
538
|
+
this.config.tts?.destroy();
|
|
534
539
|
this.emit("End", {
|
|
535
|
-
conversation
|
|
540
|
+
conversation,
|
|
536
541
|
duration
|
|
537
542
|
});
|
|
543
|
+
this.ownConversation = conversation;
|
|
538
544
|
this.socket = null;
|
|
539
545
|
this.config = null;
|
|
540
546
|
};
|
|
@@ -565,7 +571,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
565
571
|
return;
|
|
566
572
|
}
|
|
567
573
|
this.log(`User transcript: "${transcript}"`);
|
|
568
|
-
this.
|
|
574
|
+
this.addUserMessage(transcript);
|
|
569
575
|
if (!this.currentUserStream) {
|
|
570
576
|
this.log("User stopped speaking, answering");
|
|
571
577
|
this.answerUserTurn();
|
|
@@ -577,39 +583,44 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
577
583
|
this.socket.send(audio);
|
|
578
584
|
this.emit("AssistantAudio", audio);
|
|
579
585
|
};
|
|
586
|
+
this.onMessageAgent = (message) => {
|
|
587
|
+
this.sendMessage(message);
|
|
588
|
+
};
|
|
580
589
|
this.socket = socket;
|
|
581
590
|
this.config = config;
|
|
582
591
|
this.log(`Call started`);
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
(
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
)
|
|
608
|
-
);
|
|
592
|
+
config.stt.on("Transcript", this.onTranscriptSTT);
|
|
593
|
+
config.tts?.on("Audio", this.onAudioTTS);
|
|
594
|
+
const agent = config.agent;
|
|
595
|
+
if (agent) {
|
|
596
|
+
agent.on("Message", this.onMessageAgent);
|
|
597
|
+
agent.on(
|
|
598
|
+
"CancelLastUserMessage",
|
|
599
|
+
() => this.socket?.send("CancelLastUserMessage" /* CancelLastUserMessage */)
|
|
600
|
+
);
|
|
601
|
+
agent.on(
|
|
602
|
+
"SkipAnswer",
|
|
603
|
+
() => this.socket?.send("SkipAnswer" /* SkipAnswer */)
|
|
604
|
+
);
|
|
605
|
+
agent.on(
|
|
606
|
+
"EndCall",
|
|
607
|
+
() => this.socket?.send("EndCall" /* EndCall */)
|
|
608
|
+
);
|
|
609
|
+
agent.on(
|
|
610
|
+
"ToolCall",
|
|
611
|
+
(toolCall) => this.socket?.send(
|
|
612
|
+
`${"ToolCall" /* ToolCall */} ${JSON.stringify(toolCall)}`
|
|
613
|
+
)
|
|
614
|
+
);
|
|
615
|
+
}
|
|
609
616
|
queueMicrotask(() => this.sendFirstMessage());
|
|
610
617
|
socket.on("close", this.onClose);
|
|
611
618
|
socket.on("message", this.onMessage);
|
|
612
619
|
}
|
|
620
|
+
/** Everything said so far, whether an agent or the server keeps it */
|
|
621
|
+
get conversation() {
|
|
622
|
+
return this.config?.agent?.conversation ?? this.ownConversation;
|
|
623
|
+
}
|
|
613
624
|
log(...message) {
|
|
614
625
|
this.logger?.log(...message);
|
|
615
626
|
}
|
|
@@ -633,8 +644,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
633
644
|
this.processQueue();
|
|
634
645
|
}
|
|
635
646
|
cancel() {
|
|
636
|
-
this.config?.tts
|
|
637
|
-
this.config?.agent
|
|
647
|
+
this.config?.tts?.cancel();
|
|
648
|
+
this.config?.agent?.cancel();
|
|
638
649
|
this.operationQueue = [];
|
|
639
650
|
}
|
|
640
651
|
onUserAudio(chunk) {
|
|
@@ -671,8 +682,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
671
682
|
return;
|
|
672
683
|
}
|
|
673
684
|
this.turnComplete = this.predictTurnComplete();
|
|
674
|
-
const conversation = this.
|
|
675
|
-
const lastMessage = conversation
|
|
685
|
+
const conversation = this.conversation;
|
|
686
|
+
const lastMessage = conversation[conversation.length - 1];
|
|
676
687
|
if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
|
|
677
688
|
this.log(
|
|
678
689
|
"User stopped speaking and a transcript already exists, answering"
|
|
@@ -725,12 +736,55 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
725
736
|
clearTimeout(this.heldTurnTimer);
|
|
726
737
|
this.heldTurnTimer = void 0;
|
|
727
738
|
}
|
|
739
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
740
|
+
addUserMessage(text, metadata) {
|
|
741
|
+
if (this.config?.agent) {
|
|
742
|
+
this.config.agent.addUserMessage(text, metadata);
|
|
743
|
+
} else {
|
|
744
|
+
this.addOwnMessage("user", text, metadata);
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
748
|
+
addAssistantMessage(text, metadata) {
|
|
749
|
+
if (this.config?.agent) {
|
|
750
|
+
this.config.agent.addAssistantMessage(text, metadata);
|
|
751
|
+
} else {
|
|
752
|
+
this.addOwnMessage("assistant", text, metadata);
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
addOwnMessage(role, text, metadata) {
|
|
756
|
+
if (text.trim() === "") {
|
|
757
|
+
this.log(`Skipping empty ${role} message`);
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
this.log(`Adding ${role} message to conversation: ${text}`);
|
|
761
|
+
const message = {
|
|
762
|
+
role,
|
|
763
|
+
content: text,
|
|
764
|
+
metadata
|
|
765
|
+
};
|
|
766
|
+
this.ownConversation.push(message);
|
|
767
|
+
this.sendMessage(message);
|
|
768
|
+
}
|
|
769
|
+
sendMessage(message) {
|
|
770
|
+
this.socket?.send(
|
|
771
|
+
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
772
|
+
);
|
|
773
|
+
this.emit("Message", message);
|
|
774
|
+
}
|
|
775
|
+
sendPartialAnswer(content) {
|
|
776
|
+
if (!this.socket || content === "") return;
|
|
777
|
+
this.log(`Send partial answer: "${content}"`);
|
|
778
|
+
this.socket.send(
|
|
779
|
+
`${"PartialAssistantMessage" /* PartialAssistantMessage */} ${JSON.stringify(content)}`
|
|
780
|
+
);
|
|
781
|
+
}
|
|
728
782
|
sendFirstMessage() {
|
|
729
783
|
if (!this.config) return;
|
|
730
784
|
if (this.config.firstMessage) {
|
|
731
|
-
this.config.agent
|
|
785
|
+
this.config.agent?.addAssistantMessage(this.config.firstMessage);
|
|
732
786
|
this.speak(this.config.firstMessage);
|
|
733
|
-
} else if (this.config.generateFirstMessage) {
|
|
787
|
+
} else if (this.config.generateFirstMessage && this.config.agent) {
|
|
734
788
|
this.answer();
|
|
735
789
|
} else {
|
|
736
790
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -743,7 +797,13 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
743
797
|
}
|
|
744
798
|
async _answer() {
|
|
745
799
|
if (!this.config) return;
|
|
746
|
-
|
|
800
|
+
if (!this.config.agent) {
|
|
801
|
+
this.log("No agent, skipping answer");
|
|
802
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
803
|
+
return;
|
|
804
|
+
}
|
|
805
|
+
const conversation = this.config.agent.conversation;
|
|
806
|
+
const lastMessage = conversation[conversation.length - 1];
|
|
747
807
|
if (this.lastMessageSpeeched === lastMessage) {
|
|
748
808
|
this.log("Already answered, skipping");
|
|
749
809
|
return;
|
|
@@ -752,13 +812,30 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
752
812
|
try {
|
|
753
813
|
const stream = this.config.agent.answer();
|
|
754
814
|
if (await hasContent(stream)) {
|
|
755
|
-
await this._speak(
|
|
815
|
+
await this._speak(
|
|
816
|
+
this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
|
|
817
|
+
);
|
|
818
|
+
} else {
|
|
819
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
756
820
|
}
|
|
757
821
|
} catch (error) {
|
|
758
822
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
759
823
|
throw error;
|
|
760
824
|
}
|
|
761
825
|
}
|
|
826
|
+
/** Sends the answer to the client as it is written, ahead of the voice */
|
|
827
|
+
forwardPartialAnswer(stream) {
|
|
828
|
+
let content = "";
|
|
829
|
+
const send = (text) => this.sendPartialAnswer(text);
|
|
830
|
+
const forward = new import_stream2.Transform({
|
|
831
|
+
transform(chunk, _encoding, callback) {
|
|
832
|
+
content += chunk.toString();
|
|
833
|
+
send(content);
|
|
834
|
+
callback(null, chunk);
|
|
835
|
+
}
|
|
836
|
+
});
|
|
837
|
+
return stream.pipe(forward);
|
|
838
|
+
}
|
|
762
839
|
// Run text-to-speech and send to client
|
|
763
840
|
speak(message) {
|
|
764
841
|
this.queueOperation(async () => {
|
|
@@ -769,6 +846,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
769
846
|
if (!this.socket || !this.config) return;
|
|
770
847
|
let textStream;
|
|
771
848
|
if (typeof message === "string") {
|
|
849
|
+
if (!this.config.agent) {
|
|
850
|
+
this.addOwnMessage("assistant", message);
|
|
851
|
+
}
|
|
772
852
|
const stream = new import_stream2.PassThrough();
|
|
773
853
|
stream.write(message);
|
|
774
854
|
stream.end();
|
|
@@ -776,7 +856,12 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
776
856
|
} else {
|
|
777
857
|
textStream = message;
|
|
778
858
|
}
|
|
779
|
-
this.config.tts
|
|
859
|
+
if (this.config.tts) {
|
|
860
|
+
this.config.tts.speak(textStream);
|
|
861
|
+
return;
|
|
862
|
+
}
|
|
863
|
+
await drain(textStream);
|
|
864
|
+
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
780
865
|
}
|
|
781
866
|
};
|
|
782
867
|
function hasContent(stream) {
|
|
@@ -797,6 +882,15 @@ function hasContent(stream) {
|
|
|
797
882
|
stream.on("end", onEnd);
|
|
798
883
|
});
|
|
799
884
|
}
|
|
885
|
+
function drain(stream) {
|
|
886
|
+
return new Promise((resolve) => {
|
|
887
|
+
stream.on("data", () => {
|
|
888
|
+
});
|
|
889
|
+
stream.once("end", () => resolve());
|
|
890
|
+
stream.once("error", () => resolve());
|
|
891
|
+
stream.once("close", () => resolve());
|
|
892
|
+
});
|
|
893
|
+
}
|
|
800
894
|
|
|
801
895
|
// src/recorder/MicdropRecorder.ts
|
|
802
896
|
var import_eventemitter33 = require("eventemitter3");
|
|
@@ -834,9 +928,7 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
834
928
|
this.currentAssistantChunks.push(chunk);
|
|
835
929
|
};
|
|
836
930
|
this.onMessage = (message) => {
|
|
837
|
-
const
|
|
838
|
-
if (!conversation) return;
|
|
839
|
-
const messageIndex = conversation.length - 1;
|
|
931
|
+
const messageIndex = this.server.conversation.length - 1;
|
|
840
932
|
if (message.role === "user") {
|
|
841
933
|
this.lastUserMessageIndex = messageIndex;
|
|
842
934
|
if (this.currentUserChunks.length > 0) {
|
|
@@ -862,17 +954,13 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
862
954
|
this.server.on("UserAudio", this.onUserAudio);
|
|
863
955
|
this.server.on("AssistantAudio", this.onAssistantAudio);
|
|
864
956
|
this.server.on("End", this.onEnd);
|
|
865
|
-
|
|
866
|
-
if (agent) {
|
|
867
|
-
agent.on("Message", this.onMessage);
|
|
868
|
-
}
|
|
957
|
+
this.server.on("Message", this.onMessage);
|
|
869
958
|
}
|
|
870
959
|
finalizeUserAudio() {
|
|
871
960
|
if (this.currentUserChunks.length === 0) return;
|
|
872
961
|
if (this.lastUserMessageIndex < 0) return;
|
|
873
|
-
const
|
|
874
|
-
if (!
|
|
875
|
-
const message = conversation[this.lastUserMessageIndex];
|
|
962
|
+
const message = this.server.conversation[this.lastUserMessageIndex];
|
|
963
|
+
if (!message) return;
|
|
876
964
|
const buffer = Buffer.concat(this.currentUserChunks);
|
|
877
965
|
const audioMessage = {
|
|
878
966
|
buffer,
|
|
@@ -891,9 +979,8 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
891
979
|
finalizeAssistantAudio() {
|
|
892
980
|
if (this.currentAssistantChunks.length === 0) return;
|
|
893
981
|
if (this.lastAssistantMessageIndex < 0) return;
|
|
894
|
-
const
|
|
895
|
-
if (!
|
|
896
|
-
const message = conversation[this.lastAssistantMessageIndex];
|
|
982
|
+
const message = this.server.conversation[this.lastAssistantMessageIndex];
|
|
983
|
+
if (!message) return;
|
|
897
984
|
const buffer = Buffer.concat(this.currentAssistantChunks);
|
|
898
985
|
const audioMessage = {
|
|
899
986
|
buffer,
|
|
@@ -917,10 +1004,7 @@ var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
|
|
|
917
1004
|
this.server.off("UserAudio", this.onUserAudio);
|
|
918
1005
|
this.server.off("AssistantAudio", this.onAssistantAudio);
|
|
919
1006
|
this.server.off("End", this.onEnd);
|
|
920
|
-
|
|
921
|
-
if (agent) {
|
|
922
|
-
agent.off("Message", this.onMessage);
|
|
923
|
-
}
|
|
1007
|
+
this.server.off("Message", this.onMessage);
|
|
924
1008
|
this.removeAllListeners();
|
|
925
1009
|
}
|
|
926
1010
|
log(...message) {
|