@micdrop/server 2.9.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/dist/index.d.mts +372 -31
- package/dist/index.d.ts +372 -31
- package/dist/index.js +647 -115
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +639 -115
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -38,6 +38,7 @@ __export(index_exports, {
|
|
|
38
38
|
AUTO_SEMANTIC_TURN_TOOL_NAME: () => AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
39
39
|
Agent: () => Agent,
|
|
40
40
|
Classifier: () => Classifier,
|
|
41
|
+
DEFAULT_SAMPLE_RATE: () => DEFAULT_SAMPLE_RATE,
|
|
41
42
|
FallbackAgent: () => FallbackAgent,
|
|
42
43
|
FallbackSTT: () => FallbackSTT,
|
|
43
44
|
FallbackTTS: () => FallbackTTS,
|
|
@@ -51,19 +52,26 @@ __export(index_exports, {
|
|
|
51
52
|
MockAgent: () => MockAgent,
|
|
52
53
|
MockSTT: () => MockSTT,
|
|
53
54
|
MockTTS: () => MockTTS,
|
|
55
|
+
Pcm16Aligner: () => Pcm16Aligner,
|
|
54
56
|
Pcm16Resampler: () => Pcm16Resampler,
|
|
55
57
|
Realtime: () => Realtime,
|
|
58
|
+
ReconnectingSocket: () => ReconnectingSocket,
|
|
56
59
|
STT: () => STT,
|
|
57
60
|
SentenceSplitter: () => SentenceSplitter,
|
|
58
61
|
SentenceTTS: () => SentenceTTS,
|
|
59
62
|
SharedInstances: () => SharedInstances,
|
|
60
63
|
TTS: () => TTS,
|
|
64
|
+
cachedList: () => cachedList,
|
|
65
|
+
cachedModels: () => cachedModels,
|
|
66
|
+
cachedVoices: () => cachedVoices,
|
|
61
67
|
currentTurn: () => currentTurn,
|
|
62
68
|
float32ToPcm16: () => float32ToPcm16,
|
|
63
69
|
getTurnClassification: () => getTurnClassification,
|
|
64
70
|
handleError: () => handleError,
|
|
65
71
|
pcm16ToFloat32: () => pcm16ToFloat32,
|
|
66
72
|
turnInput: () => turnInput,
|
|
73
|
+
voiceGender: () => voiceGender,
|
|
74
|
+
voicesForLanguage: () => voicesForLanguage,
|
|
67
75
|
waitForParams: () => waitForParams
|
|
68
76
|
});
|
|
69
77
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -85,32 +93,65 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
85
93
|
constructor(options) {
|
|
86
94
|
super();
|
|
87
95
|
this.options = options;
|
|
96
|
+
/**
|
|
97
|
+
* @deprecated Read the signal given to `generateAnswer` instead. Still
|
|
98
|
+
* bumped by every answer and every cancel, for agents written before it.
|
|
99
|
+
*/
|
|
88
100
|
this.answerCount = 0;
|
|
89
|
-
this.answering = false;
|
|
90
101
|
this.conversation = [{ role: "system", content: options.systemPrompt }];
|
|
91
102
|
this.tools = this.getDefaultTools();
|
|
92
103
|
}
|
|
104
|
+
/** Whether an answer is being generated */
|
|
105
|
+
get isAnswering() {
|
|
106
|
+
return !!this.answerController;
|
|
107
|
+
}
|
|
108
|
+
/** @deprecated Read `isAnswering` instead */
|
|
109
|
+
get answering() {
|
|
110
|
+
return this.isAnswering;
|
|
111
|
+
}
|
|
112
|
+
/** @deprecated The answer is over once its stream ends or it is cancelled */
|
|
113
|
+
set answering(_value) {
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Stops the answer being generated, by aborting its signal.
|
|
117
|
+
*
|
|
118
|
+
* An agent overriding it to stop more than the signal does calls
|
|
119
|
+
* `super.cancel()`.
|
|
120
|
+
*/
|
|
121
|
+
cancel() {
|
|
122
|
+
this.answerCount++;
|
|
123
|
+
if (!this.answerController) return;
|
|
124
|
+
this.log("Cancel");
|
|
125
|
+
this.answerController.abort();
|
|
126
|
+
this.answerController = void 0;
|
|
127
|
+
}
|
|
93
128
|
answer() {
|
|
94
129
|
this.log("Start answering");
|
|
95
|
-
|
|
130
|
+
this.answerCount++;
|
|
131
|
+
this.answerController?.abort();
|
|
132
|
+
const controller = new AbortController();
|
|
133
|
+
this.answerController = controller;
|
|
134
|
+
const { signal } = controller;
|
|
96
135
|
const stream = new import_stream.PassThrough();
|
|
97
|
-
this.answering = true;
|
|
98
136
|
Promise.resolve().then(
|
|
99
137
|
() => this.options.onBeforeAnswer?.bind(this)(stream)
|
|
100
138
|
).then((result) => {
|
|
139
|
+
if (signal.aborted) return;
|
|
101
140
|
if (typeof result === "string") {
|
|
102
141
|
stream.write(result);
|
|
103
142
|
this.addAssistantMessage(result);
|
|
104
143
|
return;
|
|
105
144
|
}
|
|
106
145
|
if (result) return;
|
|
107
|
-
return this.generateAnswer(stream);
|
|
146
|
+
return this.generateAnswer(stream, signal);
|
|
147
|
+
}).catch((error) => {
|
|
148
|
+
if (!signal.aborted) this.log("Error answering:", error);
|
|
108
149
|
}).finally(() => {
|
|
109
150
|
if (stream.writable) {
|
|
110
151
|
stream.end();
|
|
111
152
|
}
|
|
112
|
-
if (
|
|
113
|
-
this.
|
|
153
|
+
if (this.answerController === controller) {
|
|
154
|
+
this.answerController = void 0;
|
|
114
155
|
}
|
|
115
156
|
});
|
|
116
157
|
return stream;
|
|
@@ -233,45 +274,48 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
233
274
|
};
|
|
234
275
|
}
|
|
235
276
|
}
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
return
|
|
277
|
+
/**
|
|
278
|
+
* Writes the answer to the stream as it is generated, leaving out the JSON
|
|
279
|
+
* value to extract: the voice stops at its first `{`.
|
|
280
|
+
*
|
|
281
|
+
* The value is cut from the message once the answer is complete, by
|
|
282
|
+
* `extract()`. Only the speech has to be cut while streaming.
|
|
283
|
+
*/
|
|
284
|
+
answerWriter(stream) {
|
|
285
|
+
let extracting = false;
|
|
286
|
+
return (text) => {
|
|
287
|
+
if (extracting || !text) return;
|
|
288
|
+
if (this.options.extract) {
|
|
289
|
+
const start = text.indexOf("{");
|
|
290
|
+
if (start !== -1) {
|
|
291
|
+
extracting = true;
|
|
292
|
+
text = text.slice(0, start).trimEnd();
|
|
293
|
+
if (!text) return;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
stream.write(text);
|
|
297
|
+
};
|
|
246
298
|
}
|
|
299
|
+
/** Cuts the JSON value to extract from a complete answer, and parses it */
|
|
247
300
|
extract(message) {
|
|
248
|
-
const
|
|
301
|
+
const options = this.options.extract;
|
|
249
302
|
let metadata = void 0;
|
|
250
|
-
if (
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
}
|
|
265
|
-
} catch (error) {
|
|
266
|
-
console.error(
|
|
267
|
-
`[OpenaiAgent] Error parsing extracted value (${extractedText}):`,
|
|
268
|
-
error
|
|
269
|
-
);
|
|
270
|
-
}
|
|
271
|
-
message = message.slice(0, startTagIndex).trimEnd();
|
|
272
|
-
}
|
|
303
|
+
if (!options) return { message, metadata };
|
|
304
|
+
const start = message.indexOf("{");
|
|
305
|
+
if (start === -1) return { message, metadata };
|
|
306
|
+
const end = message.lastIndexOf("}");
|
|
307
|
+
const extractedText = message.slice(start, end === -1 ? void 0 : end + 1).trim();
|
|
308
|
+
try {
|
|
309
|
+
const value = JSON.parse(extractedText);
|
|
310
|
+
options.callback?.(value);
|
|
311
|
+
if (options.saveInMetadata) metadata = { extracted: value };
|
|
312
|
+
} catch (error) {
|
|
313
|
+
console.error(
|
|
314
|
+
`[Agent] Error parsing extracted value (${extractedText}):`,
|
|
315
|
+
error
|
|
316
|
+
);
|
|
273
317
|
}
|
|
274
|
-
return { message, metadata };
|
|
318
|
+
return { message: message.slice(0, start).trimEnd(), metadata };
|
|
275
319
|
}
|
|
276
320
|
/**
|
|
277
321
|
* Emits one of the events every agent has.
|
|
@@ -336,10 +380,10 @@ var FallbackAgent = class extends Agent {
|
|
|
336
380
|
extract(message) {
|
|
337
381
|
return this.agent ? this.agent.extract(message) : super.extract(message);
|
|
338
382
|
}
|
|
339
|
-
async generateAnswer(stream) {
|
|
383
|
+
async generateAnswer(stream, signal) {
|
|
340
384
|
for (let attempt = 0; attempt < this.fallbackOptions.factories.length; attempt++) {
|
|
341
385
|
const agent = this.agent;
|
|
342
|
-
if (!agent) return;
|
|
386
|
+
if (!agent || signal.aborted) return;
|
|
343
387
|
let failed = false;
|
|
344
388
|
const onFailed = () => {
|
|
345
389
|
failed = true;
|
|
@@ -350,7 +394,7 @@ var FallbackAgent = class extends Agent {
|
|
|
350
394
|
} finally {
|
|
351
395
|
agent.off("Failed", onFailed);
|
|
352
396
|
}
|
|
353
|
-
if (!failed) return;
|
|
397
|
+
if (!failed || signal.aborted) return;
|
|
354
398
|
this.log("Agent failed, trying next agent");
|
|
355
399
|
this.startNextAgent();
|
|
356
400
|
}
|
|
@@ -358,6 +402,7 @@ var FallbackAgent = class extends Agent {
|
|
|
358
402
|
this.emit("Failed");
|
|
359
403
|
}
|
|
360
404
|
cancel() {
|
|
405
|
+
super.cancel();
|
|
361
406
|
this.agent?.cancel();
|
|
362
407
|
}
|
|
363
408
|
destroy() {
|
|
@@ -420,10 +465,8 @@ var MockAgent = class extends Agent {
|
|
|
420
465
|
}
|
|
421
466
|
async generateAnswer(stream) {
|
|
422
467
|
const message = `Assistant Message ${this.i++}`;
|
|
423
|
-
this.addAssistantMessage(message);
|
|
424
468
|
stream.write(message);
|
|
425
|
-
|
|
426
|
-
cancel() {
|
|
469
|
+
this.addAssistantMessage(message);
|
|
427
470
|
}
|
|
428
471
|
};
|
|
429
472
|
|
|
@@ -506,8 +549,10 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
|
506
549
|
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
507
550
|
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
508
551
|
MicdropServerCommands2["Classification"] = "Classification";
|
|
552
|
+
MicdropServerCommands2["SampleRate"] = "SampleRate";
|
|
509
553
|
return MicdropServerCommands2;
|
|
510
554
|
})(MicdropServerCommands || {});
|
|
555
|
+
var DEFAULT_SAMPLE_RATE = 16e3;
|
|
511
556
|
function currentTurn(conversation) {
|
|
512
557
|
let start = conversation.length;
|
|
513
558
|
while (start > 0) {
|
|
@@ -565,6 +610,20 @@ function turnInput(conversation, history = 1) {
|
|
|
565
610
|
return { input, message: turn.messages[turn.messages.length - 1] };
|
|
566
611
|
}
|
|
567
612
|
|
|
613
|
+
// src/audio/Pcm16Aligner.ts
|
|
614
|
+
var Pcm16Aligner = class {
|
|
615
|
+
process(chunk) {
|
|
616
|
+
const buf = this.leftover ? Buffer.concat([this.leftover, chunk]) : chunk;
|
|
617
|
+
const length = buf.length - buf.length % 2;
|
|
618
|
+
this.leftover = length < buf.length ? buf.subarray(length) : void 0;
|
|
619
|
+
return buf.subarray(0, length);
|
|
620
|
+
}
|
|
621
|
+
/** Forgets the held byte, to start an independent stream */
|
|
622
|
+
reset() {
|
|
623
|
+
this.leftover = void 0;
|
|
624
|
+
}
|
|
625
|
+
};
|
|
626
|
+
|
|
568
627
|
// src/audio/Pcm16Resampler.ts
|
|
569
628
|
var Pcm16Resampler = class {
|
|
570
629
|
// Fractional position into the first sample of the buffer
|
|
@@ -651,10 +710,107 @@ function handleError(socket, error) {
|
|
|
651
710
|
socket.terminate();
|
|
652
711
|
}
|
|
653
712
|
|
|
713
|
+
// src/lists.ts
|
|
714
|
+
var import_crypto = require("crypto");
|
|
715
|
+
|
|
716
|
+
// src/SharedInstances.ts
|
|
717
|
+
var SharedInstances = class {
|
|
718
|
+
constructor(options = {}) {
|
|
719
|
+
this.options = options;
|
|
720
|
+
this.entries = /* @__PURE__ */ new Map();
|
|
721
|
+
// When each instance expires, for the ones that do
|
|
722
|
+
this.expiries = /* @__PURE__ */ new Map();
|
|
723
|
+
}
|
|
724
|
+
/**
|
|
725
|
+
* Returns the instance for this key, calling `create` the first time.
|
|
726
|
+
*
|
|
727
|
+
* The key can be any JSON-serializable value, usually an array of the
|
|
728
|
+
* options that define the instance. Concurrent callers get the same promise
|
|
729
|
+
* instead of loading the model twice.
|
|
730
|
+
*/
|
|
731
|
+
load(key, create) {
|
|
732
|
+
const id = typeof key === "string" ? key : JSON.stringify(key);
|
|
733
|
+
this.expire(id);
|
|
734
|
+
const existing = this.entries.get(id);
|
|
735
|
+
if (existing) return existing;
|
|
736
|
+
let loading;
|
|
737
|
+
try {
|
|
738
|
+
loading = create();
|
|
739
|
+
} catch (error) {
|
|
740
|
+
return Promise.reject(error);
|
|
741
|
+
}
|
|
742
|
+
loading.catch(() => {
|
|
743
|
+
if (this.entries.get(id) === loading) this.entries.delete(id);
|
|
744
|
+
});
|
|
745
|
+
this.entries.set(id, loading);
|
|
746
|
+
const { ttl } = this.options;
|
|
747
|
+
if (ttl !== void 0) {
|
|
748
|
+
loading.then(
|
|
749
|
+
() => {
|
|
750
|
+
if (this.entries.get(id) === loading) {
|
|
751
|
+
this.expiries.set(id, Date.now() + ttl);
|
|
752
|
+
}
|
|
753
|
+
},
|
|
754
|
+
() => {
|
|
755
|
+
}
|
|
756
|
+
);
|
|
757
|
+
}
|
|
758
|
+
return loading;
|
|
759
|
+
}
|
|
760
|
+
/** Forgets an instance whose time is up */
|
|
761
|
+
expire(id) {
|
|
762
|
+
const expiry = this.expiries.get(id);
|
|
763
|
+
if (expiry === void 0 || Date.now() < expiry) return;
|
|
764
|
+
this.entries.delete(id);
|
|
765
|
+
this.expiries.delete(id);
|
|
766
|
+
}
|
|
767
|
+
/** The instance for this key, loaded or being loaded, without loading it. */
|
|
768
|
+
get(key) {
|
|
769
|
+
const id = typeof key === "string" ? key : JSON.stringify(key);
|
|
770
|
+
this.expire(id);
|
|
771
|
+
return this.entries.get(id);
|
|
772
|
+
}
|
|
773
|
+
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
774
|
+
has(key) {
|
|
775
|
+
const id = typeof key === "string" ? key : JSON.stringify(key);
|
|
776
|
+
this.expire(id);
|
|
777
|
+
return this.entries.has(id);
|
|
778
|
+
}
|
|
779
|
+
/**
|
|
780
|
+
* Forgets the instance for this key, so the next call loads a new one. For
|
|
781
|
+
* an instance that broke, a crashed worker for instance. Releasing what it
|
|
782
|
+
* holds is left to the caller.
|
|
783
|
+
*/
|
|
784
|
+
delete(key) {
|
|
785
|
+
const id = typeof key === "string" ? key : JSON.stringify(key);
|
|
786
|
+
this.entries.delete(id);
|
|
787
|
+
this.expiries.delete(id);
|
|
788
|
+
}
|
|
789
|
+
/** Drops every instance, so the next call loads again. */
|
|
790
|
+
clear() {
|
|
791
|
+
this.entries.clear();
|
|
792
|
+
this.expiries.clear();
|
|
793
|
+
}
|
|
794
|
+
};
|
|
795
|
+
|
|
796
|
+
// src/lists.ts
|
|
797
|
+
var LIST_TTL = 24 * 60 * 60 * 1e3;
|
|
798
|
+
var lists = new SharedInstances({ ttl: LIST_TTL });
|
|
799
|
+
function cachedList(key, apiKey, load, refresh = false) {
|
|
800
|
+
const account = (0, import_crypto.createHash)("sha256").update(apiKey).digest("hex");
|
|
801
|
+
const id = [...key, account];
|
|
802
|
+
if (refresh) lists.delete(id);
|
|
803
|
+
return lists.load(id, load);
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
// src/models.ts
|
|
807
|
+
function cachedModels(key, apiKey, load, refresh = false) {
|
|
808
|
+
return cachedList(["models", ...key], apiKey, load, refresh);
|
|
809
|
+
}
|
|
810
|
+
|
|
654
811
|
// src/MicdropServer.ts
|
|
655
812
|
var import_eventemitter33 = require("eventemitter3");
|
|
656
813
|
var import_stream2 = require("stream");
|
|
657
|
-
var USER_SAMPLE_RATE = 16e3;
|
|
658
814
|
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
659
815
|
var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
|
|
660
816
|
var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
@@ -663,6 +819,15 @@ var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
|
663
819
|
this.socket = null;
|
|
664
820
|
this.config = null;
|
|
665
821
|
this.startTime = Date.now();
|
|
822
|
+
// Bumped by every cancel, so an answer cancelled before its first word can
|
|
823
|
+
// tell, and leave its message to the answer that replaces it
|
|
824
|
+
this.cancelRound = 0;
|
|
825
|
+
// Counts the assistant messages sent, so the partial answer of one already
|
|
826
|
+
// settled is not sent after it
|
|
827
|
+
this.assistantMessages = 0;
|
|
828
|
+
// Rate the client plays the assistant voice at, which it takes as the
|
|
829
|
+
// default until told otherwise
|
|
830
|
+
this.assistantSampleRate = DEFAULT_SAMPLE_RATE;
|
|
666
831
|
// Holds the conversation when no agent does, and keeps it readable once the
|
|
667
832
|
// call is over and the config is gone
|
|
668
833
|
this.ownConversation = [];
|
|
@@ -670,10 +835,16 @@ var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
|
670
835
|
this.operationQueue = [];
|
|
671
836
|
this.isProcessingQueue = false;
|
|
672
837
|
this.userSpeechChunks = 0;
|
|
673
|
-
|
|
838
|
+
// Bumped whenever the user speaks again or mutes, so a decision taken on an
|
|
839
|
+
// earlier stretch of speech is dropped rather than answering over them
|
|
840
|
+
this.turnRound = 0;
|
|
841
|
+
this.onClose = (code, reason) => {
|
|
674
842
|
this.releaseHeldTurn();
|
|
675
843
|
if (!this.config) return;
|
|
676
|
-
this.log("Connection closed"
|
|
844
|
+
this.log("Connection closed", {
|
|
845
|
+
code,
|
|
846
|
+
reason: reason?.toString() || void 0
|
|
847
|
+
});
|
|
677
848
|
const duration = Math.round((Date.now() - this.startTime) / 1e3);
|
|
678
849
|
const conversation = this.conversation;
|
|
679
850
|
this.agent?.destroy();
|
|
@@ -723,9 +894,16 @@ var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
|
723
894
|
};
|
|
724
895
|
this.onAssistantAudio = (audio) => {
|
|
725
896
|
if (!this.socket) return;
|
|
897
|
+
const source = this.config?.realtime ?? this.config?.tts;
|
|
898
|
+
const sampleRate = source?.sampleRate ?? DEFAULT_SAMPLE_RATE;
|
|
899
|
+
if (sampleRate !== this.assistantSampleRate) {
|
|
900
|
+
this.log(`Assistant voice now at ${sampleRate} Hz`);
|
|
901
|
+
this.assistantSampleRate = sampleRate;
|
|
902
|
+
this.socket.send(`${"SampleRate" /* SampleRate */} ${sampleRate}`);
|
|
903
|
+
}
|
|
726
904
|
this.log(`Send audio chunk (${audio.byteLength} bytes)`);
|
|
727
905
|
this.socket.send(audio);
|
|
728
|
-
this.emit("AssistantAudio", audio);
|
|
906
|
+
this.emit("AssistantAudio", audio, sampleRate);
|
|
729
907
|
};
|
|
730
908
|
this.onMessageAgent = (message) => {
|
|
731
909
|
this.sendMessage(message);
|
|
@@ -815,6 +993,7 @@ var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
|
815
993
|
this.processQueue();
|
|
816
994
|
}
|
|
817
995
|
cancel() {
|
|
996
|
+
this.cancelRound++;
|
|
818
997
|
this.config?.classifier?.cancel();
|
|
819
998
|
this.config?.tts?.cancel();
|
|
820
999
|
this.agent?.cancel();
|
|
@@ -824,10 +1003,12 @@ var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
|
824
1003
|
this.log(`Received chunk (${chunk.byteLength} bytes)`);
|
|
825
1004
|
this.currentUserStream?.write(chunk);
|
|
826
1005
|
this.userSpeechChunks++;
|
|
827
|
-
this.config?.turnDetector?.push(pcm16ToFloat32(chunk),
|
|
1006
|
+
this.config?.turnDetector?.push(pcm16ToFloat32(chunk), DEFAULT_SAMPLE_RATE);
|
|
828
1007
|
this.emit("UserAudio", chunk);
|
|
829
1008
|
}
|
|
830
1009
|
onMute() {
|
|
1010
|
+
this.turnRound++;
|
|
1011
|
+
this.releaseHeldTurn();
|
|
831
1012
|
this.userSpeechChunks = 0;
|
|
832
1013
|
this.currentUserStream?.end();
|
|
833
1014
|
this.currentUserStream = void 0;
|
|
@@ -836,6 +1017,7 @@ var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
|
836
1017
|
}
|
|
837
1018
|
onStartSpeaking() {
|
|
838
1019
|
if (!this.config) return;
|
|
1020
|
+
this.turnRound++;
|
|
839
1021
|
this.userSpeechChunks = 0;
|
|
840
1022
|
this.currentUserStream?.end();
|
|
841
1023
|
this.currentUserStream = new import_stream2.PassThrough();
|
|
@@ -887,7 +1069,9 @@ var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
|
887
1069
|
}
|
|
888
1070
|
/** Answers, unless the sentence sounds like it has more coming */
|
|
889
1071
|
async answerUserTurn() {
|
|
1072
|
+
const round = this.turnRound;
|
|
890
1073
|
const complete = await (this.turnComplete ?? Promise.resolve(true));
|
|
1074
|
+
if (round !== this.turnRound || !this.config) return;
|
|
891
1075
|
if (!complete) {
|
|
892
1076
|
this.log("Waiting for the rest of the sentence");
|
|
893
1077
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
@@ -963,6 +1147,7 @@ var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
|
963
1147
|
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
964
1148
|
);
|
|
965
1149
|
this.emit("Message", message);
|
|
1150
|
+
if (message.role === "assistant") this.assistantMessages++;
|
|
966
1151
|
if (message.role === "user" && this.config?.realtime) this.classifyTurn();
|
|
967
1152
|
}
|
|
968
1153
|
sendPartialAnswer(content) {
|
|
@@ -1009,12 +1194,18 @@ var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
|
1009
1194
|
return;
|
|
1010
1195
|
}
|
|
1011
1196
|
this.lastMessageSpeeched = lastMessage;
|
|
1197
|
+
const round = this.cancelRound;
|
|
1012
1198
|
try {
|
|
1199
|
+
const sentBefore = this.assistantMessages;
|
|
1013
1200
|
const stream = this.config.agent.answer();
|
|
1014
1201
|
if (await hasContent(stream)) {
|
|
1015
1202
|
await this._speak(
|
|
1016
|
-
this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
|
|
1203
|
+
this.config.partialMessages ? this.forwardPartialAnswer(stream, sentBefore) : stream
|
|
1017
1204
|
);
|
|
1205
|
+
} else if (round !== this.cancelRound) {
|
|
1206
|
+
if (this.lastMessageSpeeched === lastMessage) {
|
|
1207
|
+
this.lastMessageSpeeched = void 0;
|
|
1208
|
+
}
|
|
1018
1209
|
} else {
|
|
1019
1210
|
this.socket?.send("SkipAnswer" /* SkipAnswer */);
|
|
1020
1211
|
}
|
|
@@ -1023,10 +1214,19 @@ var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
|
1023
1214
|
throw error;
|
|
1024
1215
|
}
|
|
1025
1216
|
}
|
|
1026
|
-
/**
|
|
1027
|
-
|
|
1217
|
+
/**
|
|
1218
|
+
* Sends the answer to the client as it is written, ahead of the voice.
|
|
1219
|
+
*
|
|
1220
|
+
* The stream reaches here a moment after the agent wrote it, so the agent
|
|
1221
|
+
* can record the complete message first, which the client then shows. A
|
|
1222
|
+
* partial answer arriving after it would show the answer twice, so the
|
|
1223
|
+
* forwarding stops once the message is sent.
|
|
1224
|
+
*/
|
|
1225
|
+
forwardPartialAnswer(stream, sentBefore) {
|
|
1028
1226
|
let content = "";
|
|
1029
|
-
const send = (text) =>
|
|
1227
|
+
const send = (text) => {
|
|
1228
|
+
if (this.assistantMessages === sentBefore) this.sendPartialAnswer(text);
|
|
1229
|
+
};
|
|
1030
1230
|
const forward = new import_stream2.Transform({
|
|
1031
1231
|
transform(chunk, _encoding, callback) {
|
|
1032
1232
|
content += chunk.toString();
|
|
@@ -1204,6 +1404,7 @@ var Realtime = class extends Agent {
|
|
|
1204
1404
|
});
|
|
1205
1405
|
}
|
|
1206
1406
|
cancel() {
|
|
1407
|
+
super.cancel();
|
|
1207
1408
|
this.queue = [];
|
|
1208
1409
|
const answer = this.currentAnswer;
|
|
1209
1410
|
if (!answer) return;
|
|
@@ -1265,7 +1466,14 @@ var Realtime = class extends Agent {
|
|
|
1265
1466
|
get isIdle() {
|
|
1266
1467
|
return this.turn === "none" && !this.currentAnswer;
|
|
1267
1468
|
}
|
|
1268
|
-
/**
|
|
1469
|
+
/**
|
|
1470
|
+
* Rate of the audio emitted, in Hz, the one the model speaks at. The server
|
|
1471
|
+
* passes it on to the client, which plays the audio as it comes.
|
|
1472
|
+
*/
|
|
1473
|
+
get sampleRate() {
|
|
1474
|
+
return DEFAULT_SAMPLE_RATE;
|
|
1475
|
+
}
|
|
1476
|
+
/** Voice of the answer, PCM16 at `sampleRate` */
|
|
1269
1477
|
emitAudio(chunk) {
|
|
1270
1478
|
const answer = this.activeAnswer();
|
|
1271
1479
|
if (!answer) return;
|
|
@@ -1392,6 +1600,8 @@ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1392
1600
|
this.audioMessages = [];
|
|
1393
1601
|
this.currentUserChunks = [];
|
|
1394
1602
|
this.currentAssistantChunks = [];
|
|
1603
|
+
// Rate of the assistant message being recorded, the one of its first chunk
|
|
1604
|
+
this.currentAssistantRate = DEFAULT_SAMPLE_RATE;
|
|
1395
1605
|
this.lastUserMessageIndex = -1;
|
|
1396
1606
|
this.lastAssistantMessageIndex = -1;
|
|
1397
1607
|
this.onUserAudio = (chunk) => {
|
|
@@ -1406,7 +1616,7 @@ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1406
1616
|
this.log("Recording user audio chunk");
|
|
1407
1617
|
this.currentUserChunks.push(chunk);
|
|
1408
1618
|
};
|
|
1409
|
-
this.onAssistantAudio = (chunk) => {
|
|
1619
|
+
this.onAssistantAudio = (chunk, sampleRate = DEFAULT_SAMPLE_RATE) => {
|
|
1410
1620
|
if (this.currentUserChunks.length > 0) {
|
|
1411
1621
|
if (this.lastUserMessageIndex >= 0) {
|
|
1412
1622
|
this.finalizeUserAudio();
|
|
@@ -1416,7 +1626,7 @@ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1416
1626
|
}
|
|
1417
1627
|
}
|
|
1418
1628
|
this.log("Recording assistant audio chunk");
|
|
1419
|
-
this.currentAssistantChunks.push(chunk);
|
|
1629
|
+
this.currentAssistantChunks.push(this.toAssistantRate(chunk, sampleRate));
|
|
1420
1630
|
};
|
|
1421
1631
|
this.onMessage = (message) => {
|
|
1422
1632
|
const messageIndex = this.server.conversation.length - 1;
|
|
@@ -1447,6 +1657,21 @@ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1447
1657
|
this.server.on("End", this.onEnd);
|
|
1448
1658
|
this.server.on("Message", this.onMessage);
|
|
1449
1659
|
}
|
|
1660
|
+
/** A message holds one rate, the one its first chunk came at */
|
|
1661
|
+
toAssistantRate(chunk, sampleRate) {
|
|
1662
|
+
if (this.currentAssistantChunks.length === 0) {
|
|
1663
|
+
this.currentAssistantRate = sampleRate;
|
|
1664
|
+
this.assistantResampler = void 0;
|
|
1665
|
+
}
|
|
1666
|
+
if (sampleRate === this.currentAssistantRate) return chunk;
|
|
1667
|
+
if (this.assistantResampler?.from !== sampleRate) {
|
|
1668
|
+
this.assistantResampler = {
|
|
1669
|
+
from: sampleRate,
|
|
1670
|
+
resampler: new Pcm16Resampler(sampleRate, this.currentAssistantRate)
|
|
1671
|
+
};
|
|
1672
|
+
}
|
|
1673
|
+
return this.assistantResampler.resampler.process(chunk);
|
|
1674
|
+
}
|
|
1450
1675
|
finalizeUserAudio() {
|
|
1451
1676
|
if (this.currentUserChunks.length === 0) return;
|
|
1452
1677
|
if (this.lastUserMessageIndex < 0) return;
|
|
@@ -1455,6 +1680,7 @@ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1455
1680
|
const buffer = Buffer.concat(this.currentUserChunks);
|
|
1456
1681
|
const audioMessage = {
|
|
1457
1682
|
buffer,
|
|
1683
|
+
sampleRate: DEFAULT_SAMPLE_RATE,
|
|
1458
1684
|
messageIndex: this.lastUserMessageIndex,
|
|
1459
1685
|
message: "content" in message ? message.content : "",
|
|
1460
1686
|
role: "user"
|
|
@@ -1475,6 +1701,7 @@ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1475
1701
|
const buffer = Buffer.concat(this.currentAssistantChunks);
|
|
1476
1702
|
const audioMessage = {
|
|
1477
1703
|
buffer,
|
|
1704
|
+
sampleRate: this.currentAssistantRate,
|
|
1478
1705
|
messageIndex: this.lastAssistantMessageIndex,
|
|
1479
1706
|
message: "content" in message ? message.content : "",
|
|
1480
1707
|
role: "assistant"
|
|
@@ -1485,6 +1712,7 @@ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1485
1712
|
this.audioMessages.push(audioMessage);
|
|
1486
1713
|
this.emit("AudioMessage", audioMessage);
|
|
1487
1714
|
this.currentAssistantChunks = [];
|
|
1715
|
+
this.assistantResampler = void 0;
|
|
1488
1716
|
this.lastAssistantMessageIndex = -1;
|
|
1489
1717
|
}
|
|
1490
1718
|
getAudioMessages() {
|
|
@@ -1503,47 +1731,259 @@ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
|
1503
1731
|
}
|
|
1504
1732
|
};
|
|
1505
1733
|
|
|
1506
|
-
// src/
|
|
1507
|
-
var
|
|
1508
|
-
|
|
1509
|
-
|
|
1734
|
+
// src/socket/ReconnectingSocket.ts
|
|
1735
|
+
var import_eventemitter35 = require("eventemitter3");
|
|
1736
|
+
var import_ws = __toESM(require("ws"));
|
|
1737
|
+
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
1738
|
+
var DEFAULT_RETRY_DELAY = 1e3;
|
|
1739
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
1740
|
+
var ReconnectingSocket = class extends import_eventemitter35.EventEmitter {
|
|
1741
|
+
constructor(options) {
|
|
1742
|
+
super();
|
|
1743
|
+
this.options = options;
|
|
1744
|
+
this._status = "idle";
|
|
1745
|
+
this.everOpened = false;
|
|
1746
|
+
this.retryCount = 0;
|
|
1747
|
+
// Numbers the attempts, so one overtaken while it awaited its address stops
|
|
1748
|
+
this.attemptId = 0;
|
|
1749
|
+
this.waiters = [];
|
|
1750
|
+
}
|
|
1751
|
+
get status() {
|
|
1752
|
+
return this._status;
|
|
1753
|
+
}
|
|
1754
|
+
/** Whether a message sent now goes out */
|
|
1755
|
+
get isOpen() {
|
|
1756
|
+
return this._status === "open";
|
|
1510
1757
|
}
|
|
1511
1758
|
/**
|
|
1512
|
-
*
|
|
1513
|
-
*
|
|
1514
|
-
* The key can be any JSON-serializable value, usually an array of the
|
|
1515
|
-
* options that define the instance. Concurrent callers get the same promise
|
|
1516
|
-
* instead of loading the model twice.
|
|
1759
|
+
* Opens the connection. Called again after `failed`, it starts over with a
|
|
1760
|
+
* full retry budget, and does nothing in any other status.
|
|
1517
1761
|
*/
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1762
|
+
connect() {
|
|
1763
|
+
if (this._status !== "idle" && this._status !== "failed") return;
|
|
1764
|
+
this.retryCount = 0;
|
|
1765
|
+
this.attempt();
|
|
1766
|
+
}
|
|
1767
|
+
/**
|
|
1768
|
+
* Waits for the connection to be open.
|
|
1769
|
+
* @returns True once open, false if it failed or was destroyed meanwhile
|
|
1770
|
+
*/
|
|
1771
|
+
ready() {
|
|
1772
|
+
if (this._status === "open") return Promise.resolve(true);
|
|
1773
|
+
if (this._status === "destroyed" || this._status === "failed") {
|
|
1774
|
+
return Promise.resolve(false);
|
|
1775
|
+
}
|
|
1776
|
+
return new Promise((resolve) => this.waiters.push(resolve));
|
|
1777
|
+
}
|
|
1778
|
+
/**
|
|
1779
|
+
* Sends a message if the connection is open
|
|
1780
|
+
* @returns False when it did not go out
|
|
1781
|
+
*/
|
|
1782
|
+
send(data) {
|
|
1783
|
+
if (this._status !== "open" || !this.socket) return false;
|
|
1784
|
+
this.socket.send(data);
|
|
1785
|
+
return true;
|
|
1786
|
+
}
|
|
1787
|
+
/**
|
|
1788
|
+
* Drops the connection and opens a new one right away, for a provider that
|
|
1789
|
+
* can only stop what it is doing by closing. It costs no retry, and the new
|
|
1790
|
+
* connection is not a reconnection.
|
|
1791
|
+
*/
|
|
1792
|
+
restart() {
|
|
1793
|
+
if (this._status === "destroyed") return;
|
|
1794
|
+
this.log("Restarting the connection");
|
|
1795
|
+
this.clearRetryTimer();
|
|
1796
|
+
this.drop();
|
|
1797
|
+
this.everOpened = false;
|
|
1798
|
+
this.retryCount = 0;
|
|
1799
|
+
this.attempt();
|
|
1800
|
+
}
|
|
1801
|
+
/** Closes the connection for good */
|
|
1802
|
+
destroy() {
|
|
1803
|
+
if (this._status === "destroyed") return;
|
|
1804
|
+
this.clearRetryTimer();
|
|
1805
|
+
this.drop();
|
|
1806
|
+
this.setStatus("destroyed");
|
|
1807
|
+
this.settleWaiters(false);
|
|
1808
|
+
this.removeAllListeners();
|
|
1809
|
+
}
|
|
1810
|
+
async attempt() {
|
|
1811
|
+
const id = ++this.attemptId;
|
|
1812
|
+
const isOvertaken = () => id !== this.attemptId || this._status !== "connecting";
|
|
1813
|
+
this.setStatus("connecting");
|
|
1814
|
+
let url;
|
|
1523
1815
|
try {
|
|
1524
|
-
|
|
1816
|
+
const { url: getUrl } = this.options;
|
|
1817
|
+
url = typeof getUrl === "string" ? getUrl : await getUrl();
|
|
1525
1818
|
} catch (error) {
|
|
1526
|
-
|
|
1819
|
+
if (isOvertaken()) return;
|
|
1820
|
+
this.onAttemptFailed(error);
|
|
1821
|
+
return;
|
|
1527
1822
|
}
|
|
1528
|
-
|
|
1529
|
-
|
|
1823
|
+
if (isOvertaken()) return;
|
|
1824
|
+
const socket = new import_ws.default(url, this.options.clientOptions);
|
|
1825
|
+
this.socket = socket;
|
|
1826
|
+
let settled = false;
|
|
1827
|
+
const fail = (error) => {
|
|
1828
|
+
if (settled || this.socket !== socket) return;
|
|
1829
|
+
settled = true;
|
|
1830
|
+
clearTimeout(timeout);
|
|
1831
|
+
this.drop();
|
|
1832
|
+
this.onAttemptFailed(error);
|
|
1833
|
+
};
|
|
1834
|
+
const timeout = setTimeout(
|
|
1835
|
+
() => fail(new Error("Connection timeout")),
|
|
1836
|
+
this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
|
|
1837
|
+
);
|
|
1838
|
+
socket.on("error", (error) => {
|
|
1839
|
+
this.log("WebSocket error:", error.message);
|
|
1840
|
+
fail(error);
|
|
1841
|
+
});
|
|
1842
|
+
const { handshake } = this.options;
|
|
1843
|
+
const open = () => {
|
|
1844
|
+
settled = true;
|
|
1845
|
+
clearTimeout(timeout);
|
|
1846
|
+
this.onOpen();
|
|
1847
|
+
};
|
|
1848
|
+
socket.on("open", () => {
|
|
1849
|
+
if (this.socket !== socket) return;
|
|
1850
|
+
try {
|
|
1851
|
+
handshake?.start?.(socket);
|
|
1852
|
+
} catch (error) {
|
|
1853
|
+
fail(error);
|
|
1854
|
+
return;
|
|
1855
|
+
}
|
|
1856
|
+
if (!handshake?.read) open();
|
|
1857
|
+
});
|
|
1858
|
+
socket.on("message", (data, isBinary) => {
|
|
1859
|
+
if (this.socket !== socket) return;
|
|
1860
|
+
if (this._status === "open") {
|
|
1861
|
+
this.prove();
|
|
1862
|
+
this.emit("Message", data, isBinary);
|
|
1863
|
+
return;
|
|
1864
|
+
}
|
|
1865
|
+
if (settled || !handshake?.read) return;
|
|
1866
|
+
try {
|
|
1867
|
+
if (handshake.read(data, isBinary)) open();
|
|
1868
|
+
} catch (error) {
|
|
1869
|
+
fail(error);
|
|
1870
|
+
}
|
|
1871
|
+
});
|
|
1872
|
+
socket.on("close", (code, reasonBuffer) => {
|
|
1873
|
+
const reason = reasonBuffer.toString();
|
|
1874
|
+
if (!settled) {
|
|
1875
|
+
fail(new Error(`Connection closed (${code}) ${reason}`));
|
|
1876
|
+
return;
|
|
1877
|
+
}
|
|
1878
|
+
if (this.socket !== socket) return;
|
|
1879
|
+
this.log("Connection closed", { code, reason });
|
|
1880
|
+
this.drop();
|
|
1881
|
+
this.setStatus("reconnecting");
|
|
1882
|
+
this.emit("Close", code, reason);
|
|
1883
|
+
if (this._status !== "reconnecting") return;
|
|
1884
|
+
this.scheduleRetry(new Error(`Connection closed (${code}) ${reason}`));
|
|
1530
1885
|
});
|
|
1531
|
-
this.entries.set(id, loading);
|
|
1532
|
-
return loading;
|
|
1533
1886
|
}
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1887
|
+
onOpen() {
|
|
1888
|
+
this.log("Connection opened");
|
|
1889
|
+
const reconnected = this.everOpened;
|
|
1890
|
+
this.everOpened = true;
|
|
1891
|
+
if (this.options.handshake?.read) {
|
|
1892
|
+
this.prove();
|
|
1893
|
+
} else {
|
|
1894
|
+
this.proofTimer = setTimeout(
|
|
1895
|
+
() => this.prove(),
|
|
1896
|
+
this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
|
|
1897
|
+
);
|
|
1898
|
+
}
|
|
1899
|
+
this.startKeepAlive();
|
|
1900
|
+
this.setStatus("open");
|
|
1901
|
+
this.emit("Open", { reconnected });
|
|
1902
|
+
this.settleWaiters(true);
|
|
1903
|
+
}
|
|
1904
|
+
/** The connection works, the retry budget is whole again */
|
|
1905
|
+
prove() {
|
|
1906
|
+
this.clearProofTimer();
|
|
1907
|
+
this.retryCount = 0;
|
|
1908
|
+
}
|
|
1909
|
+
clearProofTimer() {
|
|
1910
|
+
if (!this.proofTimer) return;
|
|
1911
|
+
clearTimeout(this.proofTimer);
|
|
1912
|
+
this.proofTimer = void 0;
|
|
1913
|
+
}
|
|
1914
|
+
onAttemptFailed(error) {
|
|
1915
|
+
this.log("Connection failed:", error.message);
|
|
1916
|
+
this.scheduleRetry(error);
|
|
1917
|
+
}
|
|
1918
|
+
scheduleRetry(error) {
|
|
1919
|
+
if (this._status === "destroyed") return;
|
|
1920
|
+
this.retryCount++;
|
|
1921
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
1922
|
+
this.log("Max retries reached, giving up");
|
|
1923
|
+
this.setStatus("failed");
|
|
1924
|
+
this.settleWaiters(false);
|
|
1925
|
+
this.emit("Failed", error);
|
|
1926
|
+
return;
|
|
1927
|
+
}
|
|
1928
|
+
this.setStatus("reconnecting");
|
|
1929
|
+
this.log(`Reconnecting (attempt ${this.retryCount})`);
|
|
1930
|
+
this.clearRetryTimer();
|
|
1931
|
+
this.retryTimer = setTimeout(() => {
|
|
1932
|
+
this.retryTimer = void 0;
|
|
1933
|
+
if (this._status === "reconnecting") this.attempt();
|
|
1934
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
|
|
1935
|
+
}
|
|
1936
|
+
/** Lets go of the current socket, which will never be heard from again */
|
|
1937
|
+
drop() {
|
|
1938
|
+
this.stopKeepAlive();
|
|
1939
|
+
this.clearProofTimer();
|
|
1940
|
+
const socket = this.socket;
|
|
1941
|
+
this.socket = void 0;
|
|
1942
|
+
if (!socket) return;
|
|
1943
|
+
socket.removeAllListeners();
|
|
1944
|
+
socket.on("error", () => {
|
|
1945
|
+
});
|
|
1946
|
+
if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
|
|
1947
|
+
socket.close(1e3);
|
|
1948
|
+
}
|
|
1537
1949
|
}
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1950
|
+
startKeepAlive() {
|
|
1951
|
+
const keepAlive = this.options.keepAlive;
|
|
1952
|
+
if (!keepAlive) return;
|
|
1953
|
+
this.stopKeepAlive();
|
|
1954
|
+
this.keepAliveTimer = setInterval(() => {
|
|
1955
|
+
const { message } = keepAlive;
|
|
1956
|
+
this.send(typeof message === "string" ? message : message());
|
|
1957
|
+
}, keepAlive.interval);
|
|
1958
|
+
}
|
|
1959
|
+
stopKeepAlive() {
|
|
1960
|
+
if (!this.keepAliveTimer) return;
|
|
1961
|
+
clearInterval(this.keepAliveTimer);
|
|
1962
|
+
this.keepAliveTimer = void 0;
|
|
1963
|
+
}
|
|
1964
|
+
clearRetryTimer() {
|
|
1965
|
+
if (!this.retryTimer) return;
|
|
1966
|
+
clearTimeout(this.retryTimer);
|
|
1967
|
+
this.retryTimer = void 0;
|
|
1968
|
+
}
|
|
1969
|
+
settleWaiters(open) {
|
|
1970
|
+
const waiters = this.waiters;
|
|
1971
|
+
this.waiters = [];
|
|
1972
|
+
waiters.forEach((resolve) => resolve(open));
|
|
1973
|
+
}
|
|
1974
|
+
setStatus(status) {
|
|
1975
|
+
if (this._status === status) return;
|
|
1976
|
+
this._status = status;
|
|
1977
|
+
this.emit("StatusChange", status);
|
|
1978
|
+
}
|
|
1979
|
+
log(...message) {
|
|
1980
|
+
this.options.log?.(...message);
|
|
1541
1981
|
}
|
|
1542
1982
|
};
|
|
1543
1983
|
|
|
1544
1984
|
// src/stt/STT.ts
|
|
1545
|
-
var
|
|
1546
|
-
var STT = class extends
|
|
1985
|
+
var import_eventemitter36 = require("eventemitter3");
|
|
1986
|
+
var STT = class extends import_eventemitter36.EventEmitter {
|
|
1547
1987
|
log(...message) {
|
|
1548
1988
|
this.logger?.log(...message);
|
|
1549
1989
|
}
|
|
@@ -1569,24 +2009,31 @@ var MockSTT = class extends STT {
|
|
|
1569
2009
|
// src/stt/FallbackSTT.ts
|
|
1570
2010
|
var import_stream3 = require("stream");
|
|
1571
2011
|
var FallbackSTT = class extends STT {
|
|
1572
|
-
// Start at -1 because we need to increment it before using it
|
|
1573
2012
|
constructor(options) {
|
|
1574
2013
|
super();
|
|
1575
2014
|
this.options = options;
|
|
1576
2015
|
this.stt = null;
|
|
1577
2016
|
this.sttIndex = -1;
|
|
2017
|
+
// Start at -1 because we need to increment it before using it
|
|
2018
|
+
// In the order spoken, each STT giving one transcript per utterance in turn
|
|
2019
|
+
this.pending = [];
|
|
1578
2020
|
this.onTranscript = (transcript) => {
|
|
2021
|
+
this.pending.shift();
|
|
1579
2022
|
this.emit("Transcript", transcript);
|
|
1580
2023
|
};
|
|
1581
|
-
|
|
2024
|
+
/**
|
|
2025
|
+
* Hands every utterance still waiting to the next STT, one stream each so
|
|
2026
|
+
* each gets its own transcript. One still being spoken keeps going there.
|
|
2027
|
+
*/
|
|
2028
|
+
this.onFailed = () => {
|
|
1582
2029
|
this.log("STT failed, trying next STT");
|
|
1583
2030
|
this.startNextSTT();
|
|
1584
|
-
if (
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
this.stt?.transcribe(stream);
|
|
1588
|
-
chunks.forEach((chunk) => stream.write(chunk));
|
|
1589
|
-
stream.end();
|
|
2031
|
+
if (this.pending.length > 0) this.log("Sending audio chunks again");
|
|
2032
|
+
for (const utterance of this.pending) {
|
|
2033
|
+
utterance.stream = new import_stream3.PassThrough();
|
|
2034
|
+
this.stt?.transcribe(utterance.stream);
|
|
2035
|
+
utterance.chunks.forEach((chunk) => utterance.stream.write(chunk));
|
|
2036
|
+
if (utterance.ended) utterance.stream.end();
|
|
1590
2037
|
}
|
|
1591
2038
|
};
|
|
1592
2039
|
if (this.options.factories.length === 0) {
|
|
@@ -1595,13 +2042,28 @@ var FallbackSTT = class extends STT {
|
|
|
1595
2042
|
this.startNextSTT();
|
|
1596
2043
|
}
|
|
1597
2044
|
transcribe(audioStream) {
|
|
1598
|
-
|
|
2045
|
+
const utterance = {
|
|
2046
|
+
chunks: [],
|
|
2047
|
+
ended: false,
|
|
2048
|
+
stream: new import_stream3.PassThrough()
|
|
2049
|
+
};
|
|
2050
|
+
this.pending.push(utterance);
|
|
2051
|
+
audioStream.on("data", (chunk) => {
|
|
2052
|
+
utterance.chunks.push(chunk);
|
|
2053
|
+
utterance.stream.write(chunk);
|
|
2054
|
+
});
|
|
2055
|
+
audioStream.on("end", () => {
|
|
2056
|
+
utterance.ended = true;
|
|
2057
|
+
utterance.stream.end();
|
|
2058
|
+
});
|
|
2059
|
+
this.stt?.transcribe(utterance.stream);
|
|
1599
2060
|
}
|
|
1600
2061
|
destroy() {
|
|
1601
2062
|
super.destroy();
|
|
1602
2063
|
this.stt?.destroy();
|
|
1603
2064
|
this.stt = null;
|
|
1604
2065
|
this.sttIndex = -1;
|
|
2066
|
+
this.pending = [];
|
|
1605
2067
|
}
|
|
1606
2068
|
startNextSTT() {
|
|
1607
2069
|
this.sttIndex++;
|
|
@@ -1624,8 +2086,18 @@ var FallbackSTT = class extends STT {
|
|
|
1624
2086
|
var import_stream4 = require("stream");
|
|
1625
2087
|
|
|
1626
2088
|
// src/tts/TTS.ts
|
|
1627
|
-
var
|
|
1628
|
-
var TTS = class extends
|
|
2089
|
+
var import_eventemitter37 = require("eventemitter3");
|
|
2090
|
+
var TTS = class extends import_eventemitter37.EventEmitter {
|
|
2091
|
+
/**
|
|
2092
|
+
* Rate of the audio emitted, in Hz.
|
|
2093
|
+
*
|
|
2094
|
+
* The server reads it with each chunk and tells the client when it changes,
|
|
2095
|
+
* so the audio reaches the client at the rate the provider produced it. A
|
|
2096
|
+
* provider that offers several rates takes the one chosen in its options.
|
|
2097
|
+
*/
|
|
2098
|
+
get sampleRate() {
|
|
2099
|
+
return DEFAULT_SAMPLE_RATE;
|
|
2100
|
+
}
|
|
1629
2101
|
log(...message) {
|
|
1630
2102
|
this.logger?.log(...message);
|
|
1631
2103
|
}
|
|
@@ -1637,7 +2109,6 @@ var TTS = class extends import_eventemitter36.EventEmitter {
|
|
|
1637
2109
|
|
|
1638
2110
|
// src/tts/FallbackTTS.ts
|
|
1639
2111
|
var FallbackTTS = class extends TTS {
|
|
1640
|
-
// Start at -1 because we need to increment it before using it
|
|
1641
2112
|
constructor(options) {
|
|
1642
2113
|
super();
|
|
1643
2114
|
this.options = options;
|
|
@@ -1646,26 +2117,47 @@ var FallbackTTS = class extends TTS {
|
|
|
1646
2117
|
this.onAudio = (audio) => {
|
|
1647
2118
|
this.emit("Audio", audio);
|
|
1648
2119
|
};
|
|
2120
|
+
/**
|
|
2121
|
+
* Hands the unspoken text to the next TTS. An utterance whose text is still
|
|
2122
|
+
* being written goes on there, so its end is spoken too.
|
|
2123
|
+
*/
|
|
1649
2124
|
this.onFailed = (chunks) => {
|
|
1650
2125
|
this.log("TTS failed, trying next TTS");
|
|
1651
2126
|
this.startNextTTS();
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
2127
|
+
const utterance = this.current;
|
|
2128
|
+
const continues = !!utterance && !utterance.ended;
|
|
2129
|
+
if (chunks.length === 0 && !continues) return;
|
|
2130
|
+
this.log("Sending text chunks again");
|
|
2131
|
+
const stream = new import_stream4.PassThrough();
|
|
2132
|
+
this.tts?.speak(stream);
|
|
2133
|
+
chunks.forEach((chunk) => stream.write(chunk));
|
|
2134
|
+
if (continues) utterance.stream = stream;
|
|
2135
|
+
else stream.end();
|
|
1659
2136
|
};
|
|
1660
2137
|
if (this.options.factories.length === 0) {
|
|
1661
2138
|
throw new Error("FallbackTTS: No factories provided");
|
|
1662
2139
|
}
|
|
1663
2140
|
this.startNextTTS();
|
|
1664
2141
|
}
|
|
2142
|
+
/** Rate of the TTS in use, which changes when it falls back to the next */
|
|
2143
|
+
get sampleRate() {
|
|
2144
|
+
return this.tts?.sampleRate ?? super.sampleRate;
|
|
2145
|
+
}
|
|
1665
2146
|
speak(textStream) {
|
|
1666
|
-
|
|
2147
|
+
const utterance = {
|
|
2148
|
+
ended: false,
|
|
2149
|
+
stream: new import_stream4.PassThrough()
|
|
2150
|
+
};
|
|
2151
|
+
this.current = utterance;
|
|
2152
|
+
textStream.on("data", (chunk) => utterance.stream.write(chunk));
|
|
2153
|
+
textStream.on("end", () => {
|
|
2154
|
+
utterance.ended = true;
|
|
2155
|
+
utterance.stream.end();
|
|
2156
|
+
});
|
|
2157
|
+
this.tts?.speak(utterance.stream);
|
|
1667
2158
|
}
|
|
1668
2159
|
cancel() {
|
|
2160
|
+
this.current = void 0;
|
|
1669
2161
|
this.tts?.cancel();
|
|
1670
2162
|
}
|
|
1671
2163
|
destroy() {
|
|
@@ -1693,26 +2185,32 @@ var FallbackTTS = class extends TTS {
|
|
|
1693
2185
|
|
|
1694
2186
|
// src/tts/MockTTS.ts
|
|
1695
2187
|
var fs = __toESM(require("fs"));
|
|
1696
|
-
var import_stream5 = require("stream");
|
|
1697
2188
|
var MockTTS = class extends TTS {
|
|
1698
|
-
constructor(audioFilePaths) {
|
|
2189
|
+
constructor(audioFilePaths, rate = DEFAULT_SAMPLE_RATE) {
|
|
1699
2190
|
super();
|
|
1700
2191
|
this.audioFilePaths = audioFilePaths;
|
|
2192
|
+
this.rate = rate;
|
|
2193
|
+
// Bumped by every answer and every cancel, so the chunks of an answer that
|
|
2194
|
+
// is no longer wanted stop there
|
|
2195
|
+
this.generation = 0;
|
|
2196
|
+
}
|
|
2197
|
+
get sampleRate() {
|
|
2198
|
+
return this.rate;
|
|
1701
2199
|
}
|
|
1702
2200
|
speak(textStream) {
|
|
1703
|
-
const
|
|
2201
|
+
const generation = ++this.generation;
|
|
1704
2202
|
textStream.once("data", async () => {
|
|
1705
2203
|
for (const filePath of this.audioFilePaths) {
|
|
1706
2204
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
2205
|
+
if (generation !== this.generation) return;
|
|
2206
|
+
const audio = fs.readFileSync(filePath);
|
|
2207
|
+
this.log(`Loaded chunk (${audio.length} bytes)`);
|
|
2208
|
+
this.emit("Audio", audio);
|
|
1710
2209
|
}
|
|
1711
|
-
audioStream.end();
|
|
1712
2210
|
});
|
|
1713
|
-
return audioStream;
|
|
1714
2211
|
}
|
|
1715
2212
|
cancel() {
|
|
2213
|
+
this.generation++;
|
|
1716
2214
|
}
|
|
1717
2215
|
};
|
|
1718
2216
|
|
|
@@ -1791,6 +2289,9 @@ var SentenceTTS = class extends TTS {
|
|
|
1791
2289
|
this.counter++;
|
|
1792
2290
|
counter = this.counter;
|
|
1793
2291
|
this.splitter.reset();
|
|
2292
|
+
this.queue = [];
|
|
2293
|
+
this.controller?.abort();
|
|
2294
|
+
this.controller = void 0;
|
|
1794
2295
|
return true;
|
|
1795
2296
|
};
|
|
1796
2297
|
textStream.on("data", (chunk) => {
|
|
@@ -1849,6 +2350,29 @@ var SentenceTTS = class extends TTS {
|
|
|
1849
2350
|
}
|
|
1850
2351
|
};
|
|
1851
2352
|
|
|
2353
|
+
// src/tts/voices.ts
|
|
2354
|
+
function voiceGender(value) {
|
|
2355
|
+
const word = value?.toLowerCase();
|
|
2356
|
+
if (!word) return void 0;
|
|
2357
|
+
if (word.startsWith("fem") || word === "woman" || word === "f")
|
|
2358
|
+
return "female";
|
|
2359
|
+
if (word.startsWith("masc") || word === "male" || word === "man" || word === "m") {
|
|
2360
|
+
return "male";
|
|
2361
|
+
}
|
|
2362
|
+
if (word.includes("neutral") || word === "androgynous") return "neutral";
|
|
2363
|
+
return void 0;
|
|
2364
|
+
}
|
|
2365
|
+
function cachedVoices(key, apiKey, load, refresh = false) {
|
|
2366
|
+
return cachedList(["voices", ...key], apiKey, load, refresh);
|
|
2367
|
+
}
|
|
2368
|
+
function voicesForLanguage(voices, language) {
|
|
2369
|
+
if (!language) return voices;
|
|
2370
|
+
const base = language.split("-")[0].toLowerCase();
|
|
2371
|
+
return voices.filter(
|
|
2372
|
+
(voice) => !voice.languages?.length || voice.languages.some((code) => code.split("-")[0].toLowerCase() === base)
|
|
2373
|
+
);
|
|
2374
|
+
}
|
|
2375
|
+
|
|
1852
2376
|
// src/waitForParams.ts
|
|
1853
2377
|
async function waitForParams(socket, validate) {
|
|
1854
2378
|
return new Promise((resolve, reject) => {
|
|
@@ -1878,6 +2402,7 @@ async function waitForParams(socket, validate) {
|
|
|
1878
2402
|
AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
1879
2403
|
Agent,
|
|
1880
2404
|
Classifier,
|
|
2405
|
+
DEFAULT_SAMPLE_RATE,
|
|
1881
2406
|
FallbackAgent,
|
|
1882
2407
|
FallbackSTT,
|
|
1883
2408
|
FallbackTTS,
|
|
@@ -1891,19 +2416,26 @@ async function waitForParams(socket, validate) {
|
|
|
1891
2416
|
MockAgent,
|
|
1892
2417
|
MockSTT,
|
|
1893
2418
|
MockTTS,
|
|
2419
|
+
Pcm16Aligner,
|
|
1894
2420
|
Pcm16Resampler,
|
|
1895
2421
|
Realtime,
|
|
2422
|
+
ReconnectingSocket,
|
|
1896
2423
|
STT,
|
|
1897
2424
|
SentenceSplitter,
|
|
1898
2425
|
SentenceTTS,
|
|
1899
2426
|
SharedInstances,
|
|
1900
2427
|
TTS,
|
|
2428
|
+
cachedList,
|
|
2429
|
+
cachedModels,
|
|
2430
|
+
cachedVoices,
|
|
1901
2431
|
currentTurn,
|
|
1902
2432
|
float32ToPcm16,
|
|
1903
2433
|
getTurnClassification,
|
|
1904
2434
|
handleError,
|
|
1905
2435
|
pcm16ToFloat32,
|
|
1906
2436
|
turnInput,
|
|
2437
|
+
voiceGender,
|
|
2438
|
+
voicesForLanguage,
|
|
1907
2439
|
waitForParams
|
|
1908
2440
|
});
|
|
1909
2441
|
//# sourceMappingURL=index.js.map
|