@micdrop/server 2.9.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/dist/index.d.mts +372 -31
- package/dist/index.d.ts +372 -31
- package/dist/index.js +647 -115
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +639 -115
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -15,32 +15,65 @@ var Agent = class extends EventEmitter {
|
|
|
15
15
|
constructor(options) {
|
|
16
16
|
super();
|
|
17
17
|
this.options = options;
|
|
18
|
+
/**
|
|
19
|
+
* @deprecated Read the signal given to `generateAnswer` instead. Still
|
|
20
|
+
* bumped by every answer and every cancel, for agents written before it.
|
|
21
|
+
*/
|
|
18
22
|
this.answerCount = 0;
|
|
19
|
-
this.answering = false;
|
|
20
23
|
this.conversation = [{ role: "system", content: options.systemPrompt }];
|
|
21
24
|
this.tools = this.getDefaultTools();
|
|
22
25
|
}
|
|
26
|
+
/** Whether an answer is being generated */
|
|
27
|
+
get isAnswering() {
|
|
28
|
+
return !!this.answerController;
|
|
29
|
+
}
|
|
30
|
+
/** @deprecated Read `isAnswering` instead */
|
|
31
|
+
get answering() {
|
|
32
|
+
return this.isAnswering;
|
|
33
|
+
}
|
|
34
|
+
/** @deprecated The answer is over once its stream ends or it is cancelled */
|
|
35
|
+
set answering(_value) {
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Stops the answer being generated, by aborting its signal.
|
|
39
|
+
*
|
|
40
|
+
* An agent overriding it to stop more than the signal does calls
|
|
41
|
+
* `super.cancel()`.
|
|
42
|
+
*/
|
|
43
|
+
cancel() {
|
|
44
|
+
this.answerCount++;
|
|
45
|
+
if (!this.answerController) return;
|
|
46
|
+
this.log("Cancel");
|
|
47
|
+
this.answerController.abort();
|
|
48
|
+
this.answerController = void 0;
|
|
49
|
+
}
|
|
23
50
|
answer() {
|
|
24
51
|
this.log("Start answering");
|
|
25
|
-
|
|
52
|
+
this.answerCount++;
|
|
53
|
+
this.answerController?.abort();
|
|
54
|
+
const controller = new AbortController();
|
|
55
|
+
this.answerController = controller;
|
|
56
|
+
const { signal } = controller;
|
|
26
57
|
const stream = new PassThrough();
|
|
27
|
-
this.answering = true;
|
|
28
58
|
Promise.resolve().then(
|
|
29
59
|
() => this.options.onBeforeAnswer?.bind(this)(stream)
|
|
30
60
|
).then((result) => {
|
|
61
|
+
if (signal.aborted) return;
|
|
31
62
|
if (typeof result === "string") {
|
|
32
63
|
stream.write(result);
|
|
33
64
|
this.addAssistantMessage(result);
|
|
34
65
|
return;
|
|
35
66
|
}
|
|
36
67
|
if (result) return;
|
|
37
|
-
return this.generateAnswer(stream);
|
|
68
|
+
return this.generateAnswer(stream, signal);
|
|
69
|
+
}).catch((error) => {
|
|
70
|
+
if (!signal.aborted) this.log("Error answering:", error);
|
|
38
71
|
}).finally(() => {
|
|
39
72
|
if (stream.writable) {
|
|
40
73
|
stream.end();
|
|
41
74
|
}
|
|
42
|
-
if (
|
|
43
|
-
this.
|
|
75
|
+
if (this.answerController === controller) {
|
|
76
|
+
this.answerController = void 0;
|
|
44
77
|
}
|
|
45
78
|
});
|
|
46
79
|
return stream;
|
|
@@ -163,45 +196,48 @@ var Agent = class extends EventEmitter {
|
|
|
163
196
|
};
|
|
164
197
|
}
|
|
165
198
|
}
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
return
|
|
199
|
+
/**
|
|
200
|
+
* Writes the answer to the stream as it is generated, leaving out the JSON
|
|
201
|
+
* value to extract: the voice stops at its first `{`.
|
|
202
|
+
*
|
|
203
|
+
* The value is cut from the message once the answer is complete, by
|
|
204
|
+
* `extract()`. Only the speech has to be cut while streaming.
|
|
205
|
+
*/
|
|
206
|
+
answerWriter(stream) {
|
|
207
|
+
let extracting = false;
|
|
208
|
+
return (text) => {
|
|
209
|
+
if (extracting || !text) return;
|
|
210
|
+
if (this.options.extract) {
|
|
211
|
+
const start = text.indexOf("{");
|
|
212
|
+
if (start !== -1) {
|
|
213
|
+
extracting = true;
|
|
214
|
+
text = text.slice(0, start).trimEnd();
|
|
215
|
+
if (!text) return;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
stream.write(text);
|
|
219
|
+
};
|
|
176
220
|
}
|
|
221
|
+
/** Cuts the JSON value to extract from a complete answer, and parses it */
|
|
177
222
|
extract(message) {
|
|
178
|
-
const
|
|
223
|
+
const options = this.options.extract;
|
|
179
224
|
let metadata = void 0;
|
|
180
|
-
if (
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
}
|
|
195
|
-
} catch (error) {
|
|
196
|
-
console.error(
|
|
197
|
-
`[OpenaiAgent] Error parsing extracted value (${extractedText}):`,
|
|
198
|
-
error
|
|
199
|
-
);
|
|
200
|
-
}
|
|
201
|
-
message = message.slice(0, startTagIndex).trimEnd();
|
|
202
|
-
}
|
|
225
|
+
if (!options) return { message, metadata };
|
|
226
|
+
const start = message.indexOf("{");
|
|
227
|
+
if (start === -1) return { message, metadata };
|
|
228
|
+
const end = message.lastIndexOf("}");
|
|
229
|
+
const extractedText = message.slice(start, end === -1 ? void 0 : end + 1).trim();
|
|
230
|
+
try {
|
|
231
|
+
const value = JSON.parse(extractedText);
|
|
232
|
+
options.callback?.(value);
|
|
233
|
+
if (options.saveInMetadata) metadata = { extracted: value };
|
|
234
|
+
} catch (error) {
|
|
235
|
+
console.error(
|
|
236
|
+
`[Agent] Error parsing extracted value (${extractedText}):`,
|
|
237
|
+
error
|
|
238
|
+
);
|
|
203
239
|
}
|
|
204
|
-
return { message, metadata };
|
|
240
|
+
return { message: message.slice(0, start).trimEnd(), metadata };
|
|
205
241
|
}
|
|
206
242
|
/**
|
|
207
243
|
* Emits one of the events every agent has.
|
|
@@ -266,10 +302,10 @@ var FallbackAgent = class extends Agent {
|
|
|
266
302
|
extract(message) {
|
|
267
303
|
return this.agent ? this.agent.extract(message) : super.extract(message);
|
|
268
304
|
}
|
|
269
|
-
async generateAnswer(stream) {
|
|
305
|
+
async generateAnswer(stream, signal) {
|
|
270
306
|
for (let attempt = 0; attempt < this.fallbackOptions.factories.length; attempt++) {
|
|
271
307
|
const agent = this.agent;
|
|
272
|
-
if (!agent) return;
|
|
308
|
+
if (!agent || signal.aborted) return;
|
|
273
309
|
let failed = false;
|
|
274
310
|
const onFailed = () => {
|
|
275
311
|
failed = true;
|
|
@@ -280,7 +316,7 @@ var FallbackAgent = class extends Agent {
|
|
|
280
316
|
} finally {
|
|
281
317
|
agent.off("Failed", onFailed);
|
|
282
318
|
}
|
|
283
|
-
if (!failed) return;
|
|
319
|
+
if (!failed || signal.aborted) return;
|
|
284
320
|
this.log("Agent failed, trying next agent");
|
|
285
321
|
this.startNextAgent();
|
|
286
322
|
}
|
|
@@ -288,6 +324,7 @@ var FallbackAgent = class extends Agent {
|
|
|
288
324
|
this.emit("Failed");
|
|
289
325
|
}
|
|
290
326
|
cancel() {
|
|
327
|
+
super.cancel();
|
|
291
328
|
this.agent?.cancel();
|
|
292
329
|
}
|
|
293
330
|
destroy() {
|
|
@@ -350,10 +387,8 @@ var MockAgent = class extends Agent {
|
|
|
350
387
|
}
|
|
351
388
|
async generateAnswer(stream) {
|
|
352
389
|
const message = `Assistant Message ${this.i++}`;
|
|
353
|
-
this.addAssistantMessage(message);
|
|
354
390
|
stream.write(message);
|
|
355
|
-
|
|
356
|
-
cancel() {
|
|
391
|
+
this.addAssistantMessage(message);
|
|
357
392
|
}
|
|
358
393
|
};
|
|
359
394
|
|
|
@@ -436,8 +471,10 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
|
436
471
|
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
437
472
|
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
438
473
|
MicdropServerCommands2["Classification"] = "Classification";
|
|
474
|
+
MicdropServerCommands2["SampleRate"] = "SampleRate";
|
|
439
475
|
return MicdropServerCommands2;
|
|
440
476
|
})(MicdropServerCommands || {});
|
|
477
|
+
var DEFAULT_SAMPLE_RATE = 16e3;
|
|
441
478
|
function currentTurn(conversation) {
|
|
442
479
|
let start = conversation.length;
|
|
443
480
|
while (start > 0) {
|
|
@@ -495,6 +532,20 @@ function turnInput(conversation, history = 1) {
|
|
|
495
532
|
return { input, message: turn.messages[turn.messages.length - 1] };
|
|
496
533
|
}
|
|
497
534
|
|
|
535
|
+
// src/audio/Pcm16Aligner.ts
|
|
536
|
+
var Pcm16Aligner = class {
|
|
537
|
+
process(chunk) {
|
|
538
|
+
const buf = this.leftover ? Buffer.concat([this.leftover, chunk]) : chunk;
|
|
539
|
+
const length = buf.length - buf.length % 2;
|
|
540
|
+
this.leftover = length < buf.length ? buf.subarray(length) : void 0;
|
|
541
|
+
return buf.subarray(0, length);
|
|
542
|
+
}
|
|
543
|
+
/** Forgets the held byte, to start an independent stream */
|
|
544
|
+
reset() {
|
|
545
|
+
this.leftover = void 0;
|
|
546
|
+
}
|
|
547
|
+
};
|
|
548
|
+
|
|
498
549
|
// src/audio/Pcm16Resampler.ts
|
|
499
550
|
var Pcm16Resampler = class {
|
|
500
551
|
// Fractional position into the first sample of the buffer
|
|
@@ -581,10 +632,107 @@ function handleError(socket, error) {
|
|
|
581
632
|
socket.terminate();
|
|
582
633
|
}
|
|
583
634
|
|
|
635
|
+
// src/lists.ts
|
|
636
|
+
import { createHash } from "crypto";
|
|
637
|
+
|
|
638
|
+
// src/SharedInstances.ts
|
|
639
|
+
var SharedInstances = class {
|
|
640
|
+
constructor(options = {}) {
|
|
641
|
+
this.options = options;
|
|
642
|
+
this.entries = /* @__PURE__ */ new Map();
|
|
643
|
+
// When each instance expires, for the ones that do
|
|
644
|
+
this.expiries = /* @__PURE__ */ new Map();
|
|
645
|
+
}
|
|
646
|
+
/**
|
|
647
|
+
* Returns the instance for this key, calling `create` the first time.
|
|
648
|
+
*
|
|
649
|
+
* The key can be any JSON-serializable value, usually an array of the
|
|
650
|
+
* options that define the instance. Concurrent callers get the same promise
|
|
651
|
+
* instead of loading the model twice.
|
|
652
|
+
*/
|
|
653
|
+
load(key, create) {
|
|
654
|
+
const id = typeof key === "string" ? key : JSON.stringify(key);
|
|
655
|
+
this.expire(id);
|
|
656
|
+
const existing = this.entries.get(id);
|
|
657
|
+
if (existing) return existing;
|
|
658
|
+
let loading;
|
|
659
|
+
try {
|
|
660
|
+
loading = create();
|
|
661
|
+
} catch (error) {
|
|
662
|
+
return Promise.reject(error);
|
|
663
|
+
}
|
|
664
|
+
loading.catch(() => {
|
|
665
|
+
if (this.entries.get(id) === loading) this.entries.delete(id);
|
|
666
|
+
});
|
|
667
|
+
this.entries.set(id, loading);
|
|
668
|
+
const { ttl } = this.options;
|
|
669
|
+
if (ttl !== void 0) {
|
|
670
|
+
loading.then(
|
|
671
|
+
() => {
|
|
672
|
+
if (this.entries.get(id) === loading) {
|
|
673
|
+
this.expiries.set(id, Date.now() + ttl);
|
|
674
|
+
}
|
|
675
|
+
},
|
|
676
|
+
() => {
|
|
677
|
+
}
|
|
678
|
+
);
|
|
679
|
+
}
|
|
680
|
+
return loading;
|
|
681
|
+
}
|
|
682
|
+
/** Forgets an instance whose time is up */
|
|
683
|
+
expire(id) {
|
|
684
|
+
const expiry = this.expiries.get(id);
|
|
685
|
+
if (expiry === void 0 || Date.now() < expiry) return;
|
|
686
|
+
this.entries.delete(id);
|
|
687
|
+
this.expiries.delete(id);
|
|
688
|
+
}
|
|
689
|
+
/** The instance for this key, loaded or being loaded, without loading it. */
|
|
690
|
+
get(key) {
|
|
691
|
+
const id = typeof key === "string" ? key : JSON.stringify(key);
|
|
692
|
+
this.expire(id);
|
|
693
|
+
return this.entries.get(id);
|
|
694
|
+
}
|
|
695
|
+
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
696
|
+
has(key) {
|
|
697
|
+
const id = typeof key === "string" ? key : JSON.stringify(key);
|
|
698
|
+
this.expire(id);
|
|
699
|
+
return this.entries.has(id);
|
|
700
|
+
}
|
|
701
|
+
/**
|
|
702
|
+
* Forgets the instance for this key, so the next call loads a new one. For
|
|
703
|
+
* an instance that broke, a crashed worker for instance. Releasing what it
|
|
704
|
+
* holds is left to the caller.
|
|
705
|
+
*/
|
|
706
|
+
delete(key) {
|
|
707
|
+
const id = typeof key === "string" ? key : JSON.stringify(key);
|
|
708
|
+
this.entries.delete(id);
|
|
709
|
+
this.expiries.delete(id);
|
|
710
|
+
}
|
|
711
|
+
/** Drops every instance, so the next call loads again. */
|
|
712
|
+
clear() {
|
|
713
|
+
this.entries.clear();
|
|
714
|
+
this.expiries.clear();
|
|
715
|
+
}
|
|
716
|
+
};
|
|
717
|
+
|
|
718
|
+
// src/lists.ts
|
|
719
|
+
var LIST_TTL = 24 * 60 * 60 * 1e3;
|
|
720
|
+
var lists = new SharedInstances({ ttl: LIST_TTL });
|
|
721
|
+
function cachedList(key, apiKey, load, refresh = false) {
|
|
722
|
+
const account = createHash("sha256").update(apiKey).digest("hex");
|
|
723
|
+
const id = [...key, account];
|
|
724
|
+
if (refresh) lists.delete(id);
|
|
725
|
+
return lists.load(id, load);
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
// src/models.ts
|
|
729
|
+
function cachedModels(key, apiKey, load, refresh = false) {
|
|
730
|
+
return cachedList(["models", ...key], apiKey, load, refresh);
|
|
731
|
+
}
|
|
732
|
+
|
|
584
733
|
// src/MicdropServer.ts
|
|
585
734
|
import { EventEmitter as EventEmitter3 } from "eventemitter3";
|
|
586
735
|
import { PassThrough as PassThrough2, Transform } from "stream";
|
|
587
|
-
var USER_SAMPLE_RATE = 16e3;
|
|
588
736
|
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
589
737
|
var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
|
|
590
738
|
var MicdropServer = class extends EventEmitter3 {
|
|
@@ -593,6 +741,15 @@ var MicdropServer = class extends EventEmitter3 {
|
|
|
593
741
|
this.socket = null;
|
|
594
742
|
this.config = null;
|
|
595
743
|
this.startTime = Date.now();
|
|
744
|
+
// Bumped by every cancel, so an answer cancelled before its first word can
|
|
745
|
+
// tell, and leave its message to the answer that replaces it
|
|
746
|
+
this.cancelRound = 0;
|
|
747
|
+
// Counts the assistant messages sent, so the partial answer of one already
|
|
748
|
+
// settled is not sent after it
|
|
749
|
+
this.assistantMessages = 0;
|
|
750
|
+
// Rate the client plays the assistant voice at, which it takes as the
|
|
751
|
+
// default until told otherwise
|
|
752
|
+
this.assistantSampleRate = DEFAULT_SAMPLE_RATE;
|
|
596
753
|
// Holds the conversation when no agent does, and keeps it readable once the
|
|
597
754
|
// call is over and the config is gone
|
|
598
755
|
this.ownConversation = [];
|
|
@@ -600,10 +757,16 @@ var MicdropServer = class extends EventEmitter3 {
|
|
|
600
757
|
this.operationQueue = [];
|
|
601
758
|
this.isProcessingQueue = false;
|
|
602
759
|
this.userSpeechChunks = 0;
|
|
603
|
-
|
|
760
|
+
// Bumped whenever the user speaks again or mutes, so a decision taken on an
|
|
761
|
+
// earlier stretch of speech is dropped rather than answering over them
|
|
762
|
+
this.turnRound = 0;
|
|
763
|
+
this.onClose = (code, reason) => {
|
|
604
764
|
this.releaseHeldTurn();
|
|
605
765
|
if (!this.config) return;
|
|
606
|
-
this.log("Connection closed"
|
|
766
|
+
this.log("Connection closed", {
|
|
767
|
+
code,
|
|
768
|
+
reason: reason?.toString() || void 0
|
|
769
|
+
});
|
|
607
770
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
608
771
|
const conversation = this.conversation;
|
|
609
772
|
this.agent?.destroy();
|
|
@@ -653,9 +816,16 @@ var MicdropServer = class extends EventEmitter3 {
|
|
|
653
816
|
};
|
|
654
817
|
this.onAssistantAudio = (audio) => {
|
|
655
818
|
if (!this.socket) return;
|
|
819
|
+
const source = this.config?.realtime ?? this.config?.tts;
|
|
820
|
+
const sampleRate = source?.sampleRate ?? DEFAULT_SAMPLE_RATE;
|
|
821
|
+
if (sampleRate !== this.assistantSampleRate) {
|
|
822
|
+
this.log(`Assistant voice now at ${sampleRate} Hz`);
|
|
823
|
+
this.assistantSampleRate = sampleRate;
|
|
824
|
+
this.socket.send(`${"SampleRate" /* SampleRate */} ${sampleRate}`);
|
|
825
|
+
}
|
|
656
826
|
this.log(`Send audio chunk (${audio.byteLength} bytes)`);
|
|
657
827
|
this.socket.send(audio);
|
|
658
|
-
this.emit("AssistantAudio", audio);
|
|
828
|
+
this.emit("AssistantAudio", audio, sampleRate);
|
|
659
829
|
};
|
|
660
830
|
this.onMessageAgent = (message) => {
|
|
661
831
|
this.sendMessage(message);
|
|
@@ -745,6 +915,7 @@ var MicdropServer = class extends EventEmitter3 {
|
|
|
745
915
|
this.processQueue();
|
|
746
916
|
}
|
|
747
917
|
cancel() {
|
|
918
|
+
this.cancelRound++;
|
|
748
919
|
this.config?.classifier?.cancel();
|
|
749
920
|
this.config?.tts?.cancel();
|
|
750
921
|
this.agent?.cancel();
|
|
@@ -754,10 +925,12 @@ var MicdropServer = class extends EventEmitter3 {
|
|
|
754
925
|
this.log(`Received chunk (${chunk.byteLength} bytes)`);
|
|
755
926
|
this.currentUserStream?.write(chunk);
|
|
756
927
|
this.userSpeechChunks++;
|
|
757
|
-
this.config?.turnDetector?.push(pcm16ToFloat32(chunk),
|
|
928
|
+
this.config?.turnDetector?.push(pcm16ToFloat32(chunk), DEFAULT_SAMPLE_RATE);
|
|
758
929
|
this.emit("UserAudio", chunk);
|
|
759
930
|
}
|
|
760
931
|
onMute() {
|
|
932
|
+
this.turnRound++;
|
|
933
|
+
this.releaseHeldTurn();
|
|
761
934
|
this.userSpeechChunks = 0;
|
|
762
935
|
this.currentUserStream?.end();
|
|
763
936
|
this.currentUserStream = void 0;
|
|
@@ -766,6 +939,7 @@ var MicdropServer = class extends EventEmitter3 {
|
|
|
766
939
|
}
|
|
767
940
|
onStartSpeaking() {
|
|
768
941
|
if (!this.config) return;
|
|
942
|
+
this.turnRound++;
|
|
769
943
|
this.userSpeechChunks = 0;
|
|
770
944
|
this.currentUserStream?.end();
|
|
771
945
|
this.currentUserStream = new PassThrough2();
|
|
@@ -817,7 +991,9 @@ var MicdropServer = class extends EventEmitter3 {
|
|
|
817
991
|
}
|
|
818
992
|
/** Answers, unless the sentence sounds like it has more coming */
|
|
819
993
|
async answerUserTurn() {
|
|
994
|
+
const round = this.turnRound;
|
|
820
995
|
const complete = await (this.turnComplete ?? Promise.resolve(true));
|
|
996
|
+
if (round !== this.turnRound || !this.config) return;
|
|
821
997
|
if (!complete) {
|
|
822
998
|
this.log("Waiting for the rest of the sentence");
|
|
823
999
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -893,6 +1069,7 @@ var MicdropServer = class extends EventEmitter3 {
|
|
|
893
1069
|
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
894
1070
|
);
|
|
895
1071
|
this.emit("Message", message);
|
|
1072
|
+
if (message.role === "assistant") this.assistantMessages++;
|
|
896
1073
|
if (message.role === "user" && this.config?.realtime) this.classifyTurn();
|
|
897
1074
|
}
|
|
898
1075
|
sendPartialAnswer(content) {
|
|
@@ -939,12 +1116,18 @@ var MicdropServer = class extends EventEmitter3 {
|
|
|
939
1116
|
return;
|
|
940
1117
|
}
|
|
941
1118
|
this.lastMessageSpeeched = lastMessage;
|
|
1119
|
+
const round = this.cancelRound;
|
|
942
1120
|
try {
|
|
1121
|
+
const sentBefore = this.assistantMessages;
|
|
943
1122
|
const stream = this.config.agent.answer();
|
|
944
1123
|
if (await hasContent(stream)) {
|
|
945
1124
|
await this._speak(
|
|
946
|
-
this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
|
|
1125
|
+
this.config.partialMessages ? this.forwardPartialAnswer(stream, sentBefore) : stream
|
|
947
1126
|
);
|
|
1127
|
+
} else if (round !== this.cancelRound) {
|
|
1128
|
+
if (this.lastMessageSpeeched === lastMessage) {
|
|
1129
|
+
this.lastMessageSpeeched = void 0;
|
|
1130
|
+
}
|
|
948
1131
|
} else {
|
|
949
1132
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
950
1133
|
}
|
|
@@ -953,10 +1136,19 @@ var MicdropServer = class extends EventEmitter3 {
|
|
|
953
1136
|
throw error;
|
|
954
1137
|
}
|
|
955
1138
|
}
|
|
956
|
-
/**
|
|
957
|
-
|
|
1139
|
+
/**
|
|
1140
|
+
* Sends the answer to the client as it is written, ahead of the voice.
|
|
1141
|
+
*
|
|
1142
|
+
* The stream reaches here a moment after the agent wrote it, so the agent
|
|
1143
|
+
* can record the complete message first, which the client then shows. A
|
|
1144
|
+
* partial answer arriving after it would show the answer twice, so the
|
|
1145
|
+
* forwarding stops once the message is sent.
|
|
1146
|
+
*/
|
|
1147
|
+
forwardPartialAnswer(stream, sentBefore) {
|
|
958
1148
|
let content = "";
|
|
959
|
-
const send = (text) =>
|
|
1149
|
+
const send = (text) => {
|
|
1150
|
+
if (this.assistantMessages === sentBefore) this.sendPartialAnswer(text);
|
|
1151
|
+
};
|
|
960
1152
|
const forward = new Transform({
|
|
961
1153
|
transform(chunk, _encoding, callback) {
|
|
962
1154
|
content += chunk.toString();
|
|
@@ -1134,6 +1326,7 @@ var Realtime = class extends Agent {
|
|
|
1134
1326
|
});
|
|
1135
1327
|
}
|
|
1136
1328
|
cancel() {
|
|
1329
|
+
super.cancel();
|
|
1137
1330
|
this.queue = [];
|
|
1138
1331
|
const answer = this.currentAnswer;
|
|
1139
1332
|
if (!answer) return;
|
|
@@ -1195,7 +1388,14 @@ var Realtime = class extends Agent {
|
|
|
1195
1388
|
get isIdle() {
|
|
1196
1389
|
return this.turn === "none" && !this.currentAnswer;
|
|
1197
1390
|
}
|
|
1198
|
-
/**
|
|
1391
|
+
/**
|
|
1392
|
+
* Rate of the audio emitted, in Hz, the one the model speaks at. The server
|
|
1393
|
+
* passes it on to the client, which plays the audio as it comes.
|
|
1394
|
+
*/
|
|
1395
|
+
get sampleRate() {
|
|
1396
|
+
return DEFAULT_SAMPLE_RATE;
|
|
1397
|
+
}
|
|
1398
|
+
/** Voice of the answer, PCM16 at `sampleRate` */
|
|
1199
1399
|
emitAudio(chunk) {
|
|
1200
1400
|
const answer = this.activeAnswer();
|
|
1201
1401
|
if (!answer) return;
|
|
@@ -1322,6 +1522,8 @@ var MicdropRecorder = class extends EventEmitter4 {
|
|
|
1322
1522
|
this.audioMessages = [];
|
|
1323
1523
|
this.currentUserChunks = [];
|
|
1324
1524
|
this.currentAssistantChunks = [];
|
|
1525
|
+
// Rate of the assistant message being recorded, the one of its first chunk
|
|
1526
|
+
this.currentAssistantRate = DEFAULT_SAMPLE_RATE;
|
|
1325
1527
|
this.lastUserMessageIndex = -1;
|
|
1326
1528
|
this.lastAssistantMessageIndex = -1;
|
|
1327
1529
|
this.onUserAudio = (chunk) => {
|
|
@@ -1336,7 +1538,7 @@ var MicdropRecorder = class extends EventEmitter4 {
|
|
|
1336
1538
|
this.log("Recording user audio chunk");
|
|
1337
1539
|
this.currentUserChunks.push(chunk);
|
|
1338
1540
|
};
|
|
1339
|
-
this.onAssistantAudio = (chunk) => {
|
|
1541
|
+
this.onAssistantAudio = (chunk, sampleRate = DEFAULT_SAMPLE_RATE) => {
|
|
1340
1542
|
if (this.currentUserChunks.length > 0) {
|
|
1341
1543
|
if (this.lastUserMessageIndex >= 0) {
|
|
1342
1544
|
this.finalizeUserAudio();
|
|
@@ -1346,7 +1548,7 @@ var MicdropRecorder = class extends EventEmitter4 {
|
|
|
1346
1548
|
}
|
|
1347
1549
|
}
|
|
1348
1550
|
this.log("Recording assistant audio chunk");
|
|
1349
|
-
this.currentAssistantChunks.push(chunk);
|
|
1551
|
+
this.currentAssistantChunks.push(this.toAssistantRate(chunk, sampleRate));
|
|
1350
1552
|
};
|
|
1351
1553
|
this.onMessage = (message) => {
|
|
1352
1554
|
const messageIndex = this.server.conversation.length - 1;
|
|
@@ -1377,6 +1579,21 @@ var MicdropRecorder = class extends EventEmitter4 {
|
|
|
1377
1579
|
this.server.on("End", this.onEnd);
|
|
1378
1580
|
this.server.on("Message", this.onMessage);
|
|
1379
1581
|
}
|
|
1582
|
+
/** A message holds one rate, the one its first chunk came at */
|
|
1583
|
+
toAssistantRate(chunk, sampleRate) {
|
|
1584
|
+
if (this.currentAssistantChunks.length === 0) {
|
|
1585
|
+
this.currentAssistantRate = sampleRate;
|
|
1586
|
+
this.assistantResampler = void 0;
|
|
1587
|
+
}
|
|
1588
|
+
if (sampleRate === this.currentAssistantRate) return chunk;
|
|
1589
|
+
if (this.assistantResampler?.from !== sampleRate) {
|
|
1590
|
+
this.assistantResampler = {
|
|
1591
|
+
from: sampleRate,
|
|
1592
|
+
resampler: new Pcm16Resampler(sampleRate, this.currentAssistantRate)
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
return this.assistantResampler.resampler.process(chunk);
|
|
1596
|
+
}
|
|
1380
1597
|
finalizeUserAudio() {
|
|
1381
1598
|
if (this.currentUserChunks.length === 0) return;
|
|
1382
1599
|
if (this.lastUserMessageIndex < 0) return;
|
|
@@ -1385,6 +1602,7 @@ var MicdropRecorder = class extends EventEmitter4 {
|
|
|
1385
1602
|
const buffer = Buffer.concat(this.currentUserChunks);
|
|
1386
1603
|
const audioMessage = {
|
|
1387
1604
|
buffer,
|
|
1605
|
+
sampleRate: DEFAULT_SAMPLE_RATE,
|
|
1388
1606
|
messageIndex: this.lastUserMessageIndex,
|
|
1389
1607
|
message: "content" in message ? message.content : "",
|
|
1390
1608
|
role: "user"
|
|
@@ -1405,6 +1623,7 @@ var MicdropRecorder = class extends EventEmitter4 {
|
|
|
1405
1623
|
const buffer = Buffer.concat(this.currentAssistantChunks);
|
|
1406
1624
|
const audioMessage = {
|
|
1407
1625
|
buffer,
|
|
1626
|
+
sampleRate: this.currentAssistantRate,
|
|
1408
1627
|
messageIndex: this.lastAssistantMessageIndex,
|
|
1409
1628
|
message: "content" in message ? message.content : "",
|
|
1410
1629
|
role: "assistant"
|
|
@@ -1415,6 +1634,7 @@ var MicdropRecorder = class extends EventEmitter4 {
|
|
|
1415
1634
|
this.audioMessages.push(audioMessage);
|
|
1416
1635
|
this.emit("AudioMessage", audioMessage);
|
|
1417
1636
|
this.currentAssistantChunks = [];
|
|
1637
|
+
this.assistantResampler = void 0;
|
|
1418
1638
|
this.lastAssistantMessageIndex = -1;
|
|
1419
1639
|
}
|
|
1420
1640
|
getAudioMessages() {
|
|
@@ -1433,47 +1653,259 @@ var MicdropRecorder = class extends EventEmitter4 {
|
|
|
1433
1653
|
}
|
|
1434
1654
|
};
|
|
1435
1655
|
|
|
1436
|
-
// src/
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1656
|
+
// src/socket/ReconnectingSocket.ts
|
|
1657
|
+
import { EventEmitter as EventEmitter5 } from "eventemitter3";
|
|
1658
|
+
import WebSocket from "ws";
|
|
1659
|
+
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
1660
|
+
var DEFAULT_RETRY_DELAY = 1e3;
|
|
1661
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
1662
|
+
var ReconnectingSocket = class extends EventEmitter5 {
|
|
1663
|
+
constructor(options) {
|
|
1664
|
+
super();
|
|
1665
|
+
this.options = options;
|
|
1666
|
+
this._status = "idle";
|
|
1667
|
+
this.everOpened = false;
|
|
1668
|
+
this.retryCount = 0;
|
|
1669
|
+
// Numbers the attempts, so one overtaken while it awaited its address stops
|
|
1670
|
+
this.attemptId = 0;
|
|
1671
|
+
this.waiters = [];
|
|
1672
|
+
}
|
|
1673
|
+
get status() {
|
|
1674
|
+
return this._status;
|
|
1675
|
+
}
|
|
1676
|
+
/** Whether a message sent now goes out */
|
|
1677
|
+
get isOpen() {
|
|
1678
|
+
return this._status === "open";
|
|
1440
1679
|
}
|
|
1441
1680
|
/**
|
|
1442
|
-
*
|
|
1443
|
-
*
|
|
1444
|
-
* The key can be any JSON-serializable value, usually an array of the
|
|
1445
|
-
* options that define the instance. Concurrent callers get the same promise
|
|
1446
|
-
* instead of loading the model twice.
|
|
1681
|
+
* Opens the connection. Called again after `failed`, it starts over with a
|
|
1682
|
+
* full retry budget, and does nothing in any other status.
|
|
1447
1683
|
*/
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1684
|
+
connect() {
|
|
1685
|
+
if (this._status !== "idle" && this._status !== "failed") return;
|
|
1686
|
+
this.retryCount = 0;
|
|
1687
|
+
this.attempt();
|
|
1688
|
+
}
|
|
1689
|
+
/**
|
|
1690
|
+
* Waits for the connection to be open.
|
|
1691
|
+
* @returns True once open, false if it failed or was destroyed meanwhile
|
|
1692
|
+
*/
|
|
1693
|
+
ready() {
|
|
1694
|
+
if (this._status === "open") return Promise.resolve(true);
|
|
1695
|
+
if (this._status === "destroyed" || this._status === "failed") {
|
|
1696
|
+
return Promise.resolve(false);
|
|
1697
|
+
}
|
|
1698
|
+
return new Promise((resolve) => this.waiters.push(resolve));
|
|
1699
|
+
}
|
|
1700
|
+
/**
|
|
1701
|
+
* Sends a message if the connection is open
|
|
1702
|
+
* @returns False when it did not go out
|
|
1703
|
+
*/
|
|
1704
|
+
send(data) {
|
|
1705
|
+
if (this._status !== "open" || !this.socket) return false;
|
|
1706
|
+
this.socket.send(data);
|
|
1707
|
+
return true;
|
|
1708
|
+
}
|
|
1709
|
+
/**
|
|
1710
|
+
* Drops the connection and opens a new one right away, for a provider that
|
|
1711
|
+
* can only stop what it is doing by closing. It costs no retry, and the new
|
|
1712
|
+
* connection is not a reconnection.
|
|
1713
|
+
*/
|
|
1714
|
+
restart() {
|
|
1715
|
+
if (this._status === "destroyed") return;
|
|
1716
|
+
this.log("Restarting the connection");
|
|
1717
|
+
this.clearRetryTimer();
|
|
1718
|
+
this.drop();
|
|
1719
|
+
this.everOpened = false;
|
|
1720
|
+
this.retryCount = 0;
|
|
1721
|
+
this.attempt();
|
|
1722
|
+
}
|
|
1723
|
+
/** Closes the connection for good */
|
|
1724
|
+
destroy() {
|
|
1725
|
+
if (this._status === "destroyed") return;
|
|
1726
|
+
this.clearRetryTimer();
|
|
1727
|
+
this.drop();
|
|
1728
|
+
this.setStatus("destroyed");
|
|
1729
|
+
this.settleWaiters(false);
|
|
1730
|
+
this.removeAllListeners();
|
|
1731
|
+
}
|
|
1732
|
+
async attempt() {
|
|
1733
|
+
const id = ++this.attemptId;
|
|
1734
|
+
const isOvertaken = () => id !== this.attemptId || this._status !== "connecting";
|
|
1735
|
+
this.setStatus("connecting");
|
|
1736
|
+
let url;
|
|
1453
1737
|
try {
|
|
1454
|
-
|
|
1738
|
+
const { url: getUrl } = this.options;
|
|
1739
|
+
url = typeof getUrl === "string" ? getUrl : await getUrl();
|
|
1455
1740
|
} catch (error) {
|
|
1456
|
-
|
|
1741
|
+
if (isOvertaken()) return;
|
|
1742
|
+
this.onAttemptFailed(error);
|
|
1743
|
+
return;
|
|
1457
1744
|
}
|
|
1458
|
-
|
|
1459
|
-
|
|
1745
|
+
if (isOvertaken()) return;
|
|
1746
|
+
const socket = new WebSocket(url, this.options.clientOptions);
|
|
1747
|
+
this.socket = socket;
|
|
1748
|
+
let settled = false;
|
|
1749
|
+
const fail = (error) => {
|
|
1750
|
+
if (settled || this.socket !== socket) return;
|
|
1751
|
+
settled = true;
|
|
1752
|
+
clearTimeout(timeout);
|
|
1753
|
+
this.drop();
|
|
1754
|
+
this.onAttemptFailed(error);
|
|
1755
|
+
};
|
|
1756
|
+
const timeout = setTimeout(
|
|
1757
|
+
() => fail(new Error("Connection timeout")),
|
|
1758
|
+
this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
|
|
1759
|
+
);
|
|
1760
|
+
socket.on("error", (error) => {
|
|
1761
|
+
this.log("WebSocket error:", error.message);
|
|
1762
|
+
fail(error);
|
|
1763
|
+
});
|
|
1764
|
+
const { handshake } = this.options;
|
|
1765
|
+
const open = () => {
|
|
1766
|
+
settled = true;
|
|
1767
|
+
clearTimeout(timeout);
|
|
1768
|
+
this.onOpen();
|
|
1769
|
+
};
|
|
1770
|
+
socket.on("open", () => {
|
|
1771
|
+
if (this.socket !== socket) return;
|
|
1772
|
+
try {
|
|
1773
|
+
handshake?.start?.(socket);
|
|
1774
|
+
} catch (error) {
|
|
1775
|
+
fail(error);
|
|
1776
|
+
return;
|
|
1777
|
+
}
|
|
1778
|
+
if (!handshake?.read) open();
|
|
1779
|
+
});
|
|
1780
|
+
socket.on("message", (data, isBinary) => {
|
|
1781
|
+
if (this.socket !== socket) return;
|
|
1782
|
+
if (this._status === "open") {
|
|
1783
|
+
this.prove();
|
|
1784
|
+
this.emit("Message", data, isBinary);
|
|
1785
|
+
return;
|
|
1786
|
+
}
|
|
1787
|
+
if (settled || !handshake?.read) return;
|
|
1788
|
+
try {
|
|
1789
|
+
if (handshake.read(data, isBinary)) open();
|
|
1790
|
+
} catch (error) {
|
|
1791
|
+
fail(error);
|
|
1792
|
+
}
|
|
1793
|
+
});
|
|
1794
|
+
socket.on("close", (code, reasonBuffer) => {
|
|
1795
|
+
const reason = reasonBuffer.toString();
|
|
1796
|
+
if (!settled) {
|
|
1797
|
+
fail(new Error(`Connection closed (${code}) ${reason}`));
|
|
1798
|
+
return;
|
|
1799
|
+
}
|
|
1800
|
+
if (this.socket !== socket) return;
|
|
1801
|
+
this.log("Connection closed", { code, reason });
|
|
1802
|
+
this.drop();
|
|
1803
|
+
this.setStatus("reconnecting");
|
|
1804
|
+
this.emit("Close", code, reason);
|
|
1805
|
+
if (this._status !== "reconnecting") return;
|
|
1806
|
+
this.scheduleRetry(new Error(`Connection closed (${code}) ${reason}`));
|
|
1460
1807
|
});
|
|
1461
|
-
this.entries.set(id, loading);
|
|
1462
|
-
return loading;
|
|
1463
1808
|
}
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1809
|
+
onOpen() {
|
|
1810
|
+
this.log("Connection opened");
|
|
1811
|
+
const reconnected = this.everOpened;
|
|
1812
|
+
this.everOpened = true;
|
|
1813
|
+
if (this.options.handshake?.read) {
|
|
1814
|
+
this.prove();
|
|
1815
|
+
} else {
|
|
1816
|
+
this.proofTimer = setTimeout(
|
|
1817
|
+
() => this.prove(),
|
|
1818
|
+
this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
|
|
1819
|
+
);
|
|
1820
|
+
}
|
|
1821
|
+
this.startKeepAlive();
|
|
1822
|
+
this.setStatus("open");
|
|
1823
|
+
this.emit("Open", { reconnected });
|
|
1824
|
+
this.settleWaiters(true);
|
|
1825
|
+
}
|
|
1826
|
+
/** The connection works, the retry budget is whole again */
|
|
1827
|
+
prove() {
|
|
1828
|
+
this.clearProofTimer();
|
|
1829
|
+
this.retryCount = 0;
|
|
1830
|
+
}
|
|
1831
|
+
clearProofTimer() {
|
|
1832
|
+
if (!this.proofTimer) return;
|
|
1833
|
+
clearTimeout(this.proofTimer);
|
|
1834
|
+
this.proofTimer = void 0;
|
|
1835
|
+
}
|
|
1836
|
+
onAttemptFailed(error) {
|
|
1837
|
+
this.log("Connection failed:", error.message);
|
|
1838
|
+
this.scheduleRetry(error);
|
|
1839
|
+
}
|
|
1840
|
+
scheduleRetry(error) {
|
|
1841
|
+
if (this._status === "destroyed") return;
|
|
1842
|
+
this.retryCount++;
|
|
1843
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
1844
|
+
this.log("Max retries reached, giving up");
|
|
1845
|
+
this.setStatus("failed");
|
|
1846
|
+
this.settleWaiters(false);
|
|
1847
|
+
this.emit("Failed", error);
|
|
1848
|
+
return;
|
|
1849
|
+
}
|
|
1850
|
+
this.setStatus("reconnecting");
|
|
1851
|
+
this.log(`Reconnecting (attempt ${this.retryCount})`);
|
|
1852
|
+
this.clearRetryTimer();
|
|
1853
|
+
this.retryTimer = setTimeout(() => {
|
|
1854
|
+
this.retryTimer = void 0;
|
|
1855
|
+
if (this._status === "reconnecting") this.attempt();
|
|
1856
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
|
|
1857
|
+
}
|
|
1858
|
+
/** Lets go of the current socket, which will never be heard from again */
|
|
1859
|
+
drop() {
|
|
1860
|
+
this.stopKeepAlive();
|
|
1861
|
+
this.clearProofTimer();
|
|
1862
|
+
const socket = this.socket;
|
|
1863
|
+
this.socket = void 0;
|
|
1864
|
+
if (!socket) return;
|
|
1865
|
+
socket.removeAllListeners();
|
|
1866
|
+
socket.on("error", () => {
|
|
1867
|
+
});
|
|
1868
|
+
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
|
|
1869
|
+
socket.close(1e3);
|
|
1870
|
+
}
|
|
1467
1871
|
}
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1872
|
+
startKeepAlive() {
|
|
1873
|
+
const keepAlive = this.options.keepAlive;
|
|
1874
|
+
if (!keepAlive) return;
|
|
1875
|
+
this.stopKeepAlive();
|
|
1876
|
+
this.keepAliveTimer = setInterval(() => {
|
|
1877
|
+
const { message } = keepAlive;
|
|
1878
|
+
this.send(typeof message === "string" ? message : message());
|
|
1879
|
+
}, keepAlive.interval);
|
|
1880
|
+
}
|
|
1881
|
+
stopKeepAlive() {
|
|
1882
|
+
if (!this.keepAliveTimer) return;
|
|
1883
|
+
clearInterval(this.keepAliveTimer);
|
|
1884
|
+
this.keepAliveTimer = void 0;
|
|
1885
|
+
}
|
|
1886
|
+
clearRetryTimer() {
|
|
1887
|
+
if (!this.retryTimer) return;
|
|
1888
|
+
clearTimeout(this.retryTimer);
|
|
1889
|
+
this.retryTimer = void 0;
|
|
1890
|
+
}
|
|
1891
|
+
settleWaiters(open) {
|
|
1892
|
+
const waiters = this.waiters;
|
|
1893
|
+
this.waiters = [];
|
|
1894
|
+
waiters.forEach((resolve) => resolve(open));
|
|
1895
|
+
}
|
|
1896
|
+
setStatus(status) {
|
|
1897
|
+
if (this._status === status) return;
|
|
1898
|
+
this._status = status;
|
|
1899
|
+
this.emit("StatusChange", status);
|
|
1900
|
+
}
|
|
1901
|
+
log(...message) {
|
|
1902
|
+
this.options.log?.(...message);
|
|
1471
1903
|
}
|
|
1472
1904
|
};
|
|
1473
1905
|
|
|
1474
1906
|
// src/stt/STT.ts
|
|
1475
|
-
import { EventEmitter as
|
|
1476
|
-
var STT = class extends
|
|
1907
|
+
import { EventEmitter as EventEmitter6 } from "eventemitter3";
|
|
1908
|
+
var STT = class extends EventEmitter6 {
|
|
1477
1909
|
log(...message) {
|
|
1478
1910
|
this.logger?.log(...message);
|
|
1479
1911
|
}
|
|
@@ -1499,24 +1931,31 @@ var MockSTT = class extends STT {
|
|
|
1499
1931
|
// src/stt/FallbackSTT.ts
|
|
1500
1932
|
import { PassThrough as PassThrough3 } from "stream";
|
|
1501
1933
|
var FallbackSTT = class extends STT {
|
|
1502
|
-
// Start at -1 because we need to increment it before using it
|
|
1503
1934
|
constructor(options) {
|
|
1504
1935
|
super();
|
|
1505
1936
|
this.options = options;
|
|
1506
1937
|
this.stt = null;
|
|
1507
1938
|
this.sttIndex = -1;
|
|
1939
|
+
// Start at -1 because we need to increment it before using it
|
|
1940
|
+
// In the order spoken, each STT giving one transcript per utterance in turn
|
|
1941
|
+
this.pending = [];
|
|
1508
1942
|
this.onTranscript = (transcript) => {
|
|
1943
|
+
this.pending.shift();
|
|
1509
1944
|
this.emit("Transcript", transcript);
|
|
1510
1945
|
};
|
|
1511
|
-
|
|
1946
|
+
/**
|
|
1947
|
+
* Hands every utterance still waiting to the next STT, one stream each so
|
|
1948
|
+
* each gets its own transcript. One still being spoken keeps going there.
|
|
1949
|
+
*/
|
|
1950
|
+
this.onFailed = () => {
|
|
1512
1951
|
this.log("STT failed, trying next STT");
|
|
1513
1952
|
this.startNextSTT();
|
|
1514
|
-
if (
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
this.stt?.transcribe(stream);
|
|
1518
|
-
chunks.forEach((chunk) => stream.write(chunk));
|
|
1519
|
-
stream.end();
|
|
1953
|
+
if (this.pending.length > 0) this.log("Sending audio chunks again");
|
|
1954
|
+
for (const utterance of this.pending) {
|
|
1955
|
+
utterance.stream = new PassThrough3();
|
|
1956
|
+
this.stt?.transcribe(utterance.stream);
|
|
1957
|
+
utterance.chunks.forEach((chunk) => utterance.stream.write(chunk));
|
|
1958
|
+
if (utterance.ended) utterance.stream.end();
|
|
1520
1959
|
}
|
|
1521
1960
|
};
|
|
1522
1961
|
if (this.options.factories.length === 0) {
|
|
@@ -1525,13 +1964,28 @@ var FallbackSTT = class extends STT {
|
|
|
1525
1964
|
this.startNextSTT();
|
|
1526
1965
|
}
|
|
1527
1966
|
transcribe(audioStream) {
|
|
1528
|
-
|
|
1967
|
+
const utterance = {
|
|
1968
|
+
chunks: [],
|
|
1969
|
+
ended: false,
|
|
1970
|
+
stream: new PassThrough3()
|
|
1971
|
+
};
|
|
1972
|
+
this.pending.push(utterance);
|
|
1973
|
+
audioStream.on("data", (chunk) => {
|
|
1974
|
+
utterance.chunks.push(chunk);
|
|
1975
|
+
utterance.stream.write(chunk);
|
|
1976
|
+
});
|
|
1977
|
+
audioStream.on("end", () => {
|
|
1978
|
+
utterance.ended = true;
|
|
1979
|
+
utterance.stream.end();
|
|
1980
|
+
});
|
|
1981
|
+
this.stt?.transcribe(utterance.stream);
|
|
1529
1982
|
}
|
|
1530
1983
|
destroy() {
|
|
1531
1984
|
super.destroy();
|
|
1532
1985
|
this.stt?.destroy();
|
|
1533
1986
|
this.stt = null;
|
|
1534
1987
|
this.sttIndex = -1;
|
|
1988
|
+
this.pending = [];
|
|
1535
1989
|
}
|
|
1536
1990
|
startNextSTT() {
|
|
1537
1991
|
this.sttIndex++;
|
|
@@ -1554,8 +2008,18 @@ var FallbackSTT = class extends STT {
|
|
|
1554
2008
|
import { PassThrough as PassThrough4 } from "stream";
|
|
1555
2009
|
|
|
1556
2010
|
// src/tts/TTS.ts
|
|
1557
|
-
import { EventEmitter as
|
|
1558
|
-
var TTS = class extends
|
|
2011
|
+
import { EventEmitter as EventEmitter7 } from "eventemitter3";
|
|
2012
|
+
var TTS = class extends EventEmitter7 {
|
|
2013
|
+
/**
|
|
2014
|
+
* Rate of the audio emitted, in Hz.
|
|
2015
|
+
*
|
|
2016
|
+
* The server reads it with each chunk and tells the client when it changes,
|
|
2017
|
+
* so the audio reaches the client at the rate the provider produced it. A
|
|
2018
|
+
* provider that offers several rates takes the one chosen in its options.
|
|
2019
|
+
*/
|
|
2020
|
+
get sampleRate() {
|
|
2021
|
+
return DEFAULT_SAMPLE_RATE;
|
|
2022
|
+
}
|
|
1559
2023
|
log(...message) {
|
|
1560
2024
|
this.logger?.log(...message);
|
|
1561
2025
|
}
|
|
@@ -1567,7 +2031,6 @@ var TTS = class extends EventEmitter6 {
|
|
|
1567
2031
|
|
|
1568
2032
|
// src/tts/FallbackTTS.ts
|
|
1569
2033
|
var FallbackTTS = class extends TTS {
|
|
1570
|
-
// Start at -1 because we need to increment it before using it
|
|
1571
2034
|
constructor(options) {
|
|
1572
2035
|
super();
|
|
1573
2036
|
this.options = options;
|
|
@@ -1576,26 +2039,47 @@ var FallbackTTS = class extends TTS {
|
|
|
1576
2039
|
this.onAudio = (audio) => {
|
|
1577
2040
|
this.emit("Audio", audio);
|
|
1578
2041
|
};
|
|
2042
|
+
/**
|
|
2043
|
+
* Hands the unspoken text to the next TTS. An utterance whose text is still
|
|
2044
|
+
* being written goes on there, so its end is spoken too.
|
|
2045
|
+
*/
|
|
1579
2046
|
this.onFailed = (chunks) => {
|
|
1580
2047
|
this.log("TTS failed, trying next TTS");
|
|
1581
2048
|
this.startNextTTS();
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
2049
|
+
const utterance = this.current;
|
|
2050
|
+
const continues = !!utterance && !utterance.ended;
|
|
2051
|
+
if (chunks.length === 0 && !continues) return;
|
|
2052
|
+
this.log("Sending text chunks again");
|
|
2053
|
+
const stream = new PassThrough4();
|
|
2054
|
+
this.tts?.speak(stream);
|
|
2055
|
+
chunks.forEach((chunk) => stream.write(chunk));
|
|
2056
|
+
if (continues) utterance.stream = stream;
|
|
2057
|
+
else stream.end();
|
|
1589
2058
|
};
|
|
1590
2059
|
if (this.options.factories.length === 0) {
|
|
1591
2060
|
throw new Error("FallbackTTS: No factories provided");
|
|
1592
2061
|
}
|
|
1593
2062
|
this.startNextTTS();
|
|
1594
2063
|
}
|
|
2064
|
+
/** Rate of the TTS in use, which changes when it falls back to the next */
|
|
2065
|
+
get sampleRate() {
|
|
2066
|
+
return this.tts?.sampleRate ?? super.sampleRate;
|
|
2067
|
+
}
|
|
1595
2068
|
speak(textStream) {
|
|
1596
|
-
|
|
2069
|
+
const utterance = {
|
|
2070
|
+
ended: false,
|
|
2071
|
+
stream: new PassThrough4()
|
|
2072
|
+
};
|
|
2073
|
+
this.current = utterance;
|
|
2074
|
+
textStream.on("data", (chunk) => utterance.stream.write(chunk));
|
|
2075
|
+
textStream.on("end", () => {
|
|
2076
|
+
utterance.ended = true;
|
|
2077
|
+
utterance.stream.end();
|
|
2078
|
+
});
|
|
2079
|
+
this.tts?.speak(utterance.stream);
|
|
1597
2080
|
}
|
|
1598
2081
|
cancel() {
|
|
2082
|
+
this.current = void 0;
|
|
1599
2083
|
this.tts?.cancel();
|
|
1600
2084
|
}
|
|
1601
2085
|
destroy() {
|
|
@@ -1623,26 +2107,32 @@ var FallbackTTS = class extends TTS {
|
|
|
1623
2107
|
|
|
1624
2108
|
// src/tts/MockTTS.ts
|
|
1625
2109
|
import * as fs from "fs";
|
|
1626
|
-
import { PassThrough as PassThrough5 } from "stream";
|
|
1627
2110
|
var MockTTS = class extends TTS {
|
|
1628
|
-
constructor(audioFilePaths) {
|
|
2111
|
+
constructor(audioFilePaths, rate = DEFAULT_SAMPLE_RATE) {
|
|
1629
2112
|
super();
|
|
1630
2113
|
this.audioFilePaths = audioFilePaths;
|
|
2114
|
+
this.rate = rate;
|
|
2115
|
+
// Bumped by every answer and every cancel, so the chunks of an answer that
|
|
2116
|
+
// is no longer wanted stop there
|
|
2117
|
+
this.generation = 0;
|
|
2118
|
+
}
|
|
2119
|
+
get sampleRate() {
|
|
2120
|
+
return this.rate;
|
|
1631
2121
|
}
|
|
1632
2122
|
speak(textStream) {
|
|
1633
|
-
const
|
|
2123
|
+
const generation = ++this.generation;
|
|
1634
2124
|
textStream.once("data", async () => {
|
|
1635
2125
|
for (const filePath of this.audioFilePaths) {
|
|
1636
2126
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
2127
|
+
if (generation !== this.generation) return;
|
|
2128
|
+
const audio = fs.readFileSync(filePath);
|
|
2129
|
+
this.log(`Loaded chunk (${audio.length} bytes)`);
|
|
2130
|
+
this.emit("Audio", audio);
|
|
1640
2131
|
}
|
|
1641
|
-
audioStream.end();
|
|
1642
2132
|
});
|
|
1643
|
-
return audioStream;
|
|
1644
2133
|
}
|
|
1645
2134
|
cancel() {
|
|
2135
|
+
this.generation++;
|
|
1646
2136
|
}
|
|
1647
2137
|
};
|
|
1648
2138
|
|
|
@@ -1721,6 +2211,9 @@ var SentenceTTS = class extends TTS {
|
|
|
1721
2211
|
this.counter++;
|
|
1722
2212
|
counter = this.counter;
|
|
1723
2213
|
this.splitter.reset();
|
|
2214
|
+
this.queue = [];
|
|
2215
|
+
this.controller?.abort();
|
|
2216
|
+
this.controller = void 0;
|
|
1724
2217
|
return true;
|
|
1725
2218
|
};
|
|
1726
2219
|
textStream.on("data", (chunk) => {
|
|
@@ -1779,6 +2272,29 @@ var SentenceTTS = class extends TTS {
|
|
|
1779
2272
|
}
|
|
1780
2273
|
};
|
|
1781
2274
|
|
|
2275
|
+
// src/tts/voices.ts
|
|
2276
|
+
function voiceGender(value) {
|
|
2277
|
+
const word = value?.toLowerCase();
|
|
2278
|
+
if (!word) return void 0;
|
|
2279
|
+
if (word.startsWith("fem") || word === "woman" || word === "f")
|
|
2280
|
+
return "female";
|
|
2281
|
+
if (word.startsWith("masc") || word === "male" || word === "man" || word === "m") {
|
|
2282
|
+
return "male";
|
|
2283
|
+
}
|
|
2284
|
+
if (word.includes("neutral") || word === "androgynous") return "neutral";
|
|
2285
|
+
return void 0;
|
|
2286
|
+
}
|
|
2287
|
+
function cachedVoices(key, apiKey, load, refresh = false) {
|
|
2288
|
+
return cachedList(["voices", ...key], apiKey, load, refresh);
|
|
2289
|
+
}
|
|
2290
|
+
function voicesForLanguage(voices, language) {
|
|
2291
|
+
if (!language) return voices;
|
|
2292
|
+
const base = language.split("-")[0].toLowerCase();
|
|
2293
|
+
return voices.filter(
|
|
2294
|
+
(voice) => !voice.languages?.length || voice.languages.some((code) => code.split("-")[0].toLowerCase() === base)
|
|
2295
|
+
);
|
|
2296
|
+
}
|
|
2297
|
+
|
|
1782
2298
|
// src/waitForParams.ts
|
|
1783
2299
|
async function waitForParams(socket, validate) {
|
|
1784
2300
|
return new Promise((resolve, reject) => {
|
|
@@ -1807,6 +2323,7 @@ export {
|
|
|
1807
2323
|
AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
1808
2324
|
Agent,
|
|
1809
2325
|
Classifier,
|
|
2326
|
+
DEFAULT_SAMPLE_RATE,
|
|
1810
2327
|
FallbackAgent,
|
|
1811
2328
|
FallbackSTT,
|
|
1812
2329
|
FallbackTTS,
|
|
@@ -1820,19 +2337,26 @@ export {
|
|
|
1820
2337
|
MockAgent,
|
|
1821
2338
|
MockSTT,
|
|
1822
2339
|
MockTTS,
|
|
2340
|
+
Pcm16Aligner,
|
|
1823
2341
|
Pcm16Resampler,
|
|
1824
2342
|
Realtime,
|
|
2343
|
+
ReconnectingSocket,
|
|
1825
2344
|
STT,
|
|
1826
2345
|
SentenceSplitter,
|
|
1827
2346
|
SentenceTTS,
|
|
1828
2347
|
SharedInstances,
|
|
1829
2348
|
TTS,
|
|
2349
|
+
cachedList,
|
|
2350
|
+
cachedModels,
|
|
2351
|
+
cachedVoices,
|
|
1830
2352
|
currentTurn,
|
|
1831
2353
|
float32ToPcm16,
|
|
1832
2354
|
getTurnClassification,
|
|
1833
2355
|
handleError,
|
|
1834
2356
|
pcm16ToFloat32,
|
|
1835
2357
|
turnInput,
|
|
2358
|
+
voiceGender,
|
|
2359
|
+
voicesForLanguage,
|
|
1836
2360
|
waitForParams
|
|
1837
2361
|
};
|
|
1838
2362
|
//# sourceMappingURL=index.mjs.map
|