@micdrop/server 2.9.0 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -15,32 +15,65 @@ var Agent = class extends EventEmitter {
15
15
  constructor(options) {
16
16
  super();
17
17
  this.options = options;
18
+ /**
19
+ * @deprecated Read the signal given to `generateAnswer` instead. Still
20
+ * bumped by every answer and every cancel, for agents written before it.
21
+ */
18
22
  this.answerCount = 0;
19
- this.answering = false;
20
23
  this.conversation = [{ role: "system", content: options.systemPrompt }];
21
24
  this.tools = this.getDefaultTools();
22
25
  }
26
+ /** Whether an answer is being generated */
27
+ get isAnswering() {
28
+ return !!this.answerController;
29
+ }
30
+ /** @deprecated Read `isAnswering` instead */
31
+ get answering() {
32
+ return this.isAnswering;
33
+ }
34
+ /** @deprecated The answer is over once its stream ends or it is cancelled */
35
+ set answering(_value) {
36
+ }
37
+ /**
38
+ * Stops the answer being generated, by aborting its signal.
39
+ *
40
+ * An agent overriding it to stop more than the signal does calls
41
+ * `super.cancel()`.
42
+ */
43
+ cancel() {
44
+ this.answerCount++;
45
+ if (!this.answerController) return;
46
+ this.log("Cancel");
47
+ this.answerController.abort();
48
+ this.answerController = void 0;
49
+ }
23
50
  answer() {
24
51
  this.log("Start answering");
25
- const answerCount = ++this.answerCount;
52
+ this.answerCount++;
53
+ this.answerController?.abort();
54
+ const controller = new AbortController();
55
+ this.answerController = controller;
56
+ const { signal } = controller;
26
57
  const stream = new PassThrough();
27
- this.answering = true;
28
58
  Promise.resolve().then(
29
59
  () => this.options.onBeforeAnswer?.bind(this)(stream)
30
60
  ).then((result) => {
61
+ if (signal.aborted) return;
31
62
  if (typeof result === "string") {
32
63
  stream.write(result);
33
64
  this.addAssistantMessage(result);
34
65
  return;
35
66
  }
36
67
  if (result) return;
37
- return this.generateAnswer(stream);
68
+ return this.generateAnswer(stream, signal);
69
+ }).catch((error) => {
70
+ if (!signal.aborted) this.log("Error answering:", error);
38
71
  }).finally(() => {
39
72
  if (stream.writable) {
40
73
  stream.end();
41
74
  }
42
- if (answerCount === this.answerCount) {
43
- this.answering = false;
75
+ if (this.answerController === controller) {
76
+ this.answerController = void 0;
44
77
  }
45
78
  });
46
79
  return stream;
@@ -163,45 +196,48 @@ var Agent = class extends EventEmitter {
163
196
  };
164
197
  }
165
198
  }
166
- getExtractOptions() {
167
- const extract = this.options.extract;
168
- if (!extract) return void 0;
169
- if ("json" in extract && extract.json) {
170
- return { ...extract, startTag: "{", endTag: "}" };
171
- }
172
- if ("startTag" in extract && "endTag" in extract) {
173
- return extract;
174
- }
175
- return void 0;
199
+ /**
200
+ * Writes the answer to the stream as it is generated, leaving out the JSON
201
+ * value to extract: the voice stops at its first `{`.
202
+ *
203
+ * The value is cut from the message once the answer is complete, by
204
+ * `extract()`. Only the speech has to be cut while streaming.
205
+ */
206
+ answerWriter(stream) {
207
+ let extracting = false;
208
+ return (text) => {
209
+ if (extracting || !text) return;
210
+ if (this.options.extract) {
211
+ const start = text.indexOf("{");
212
+ if (start !== -1) {
213
+ extracting = true;
214
+ text = text.slice(0, start).trimEnd();
215
+ if (!text) return;
216
+ }
217
+ }
218
+ stream.write(text);
219
+ };
176
220
  }
221
+ /** Cuts the JSON value to extract from a complete answer, and parses it */
177
222
  extract(message) {
178
- const extractOptions = this.getExtractOptions();
223
+ const options = this.options.extract;
179
224
  let metadata = void 0;
180
- if (extractOptions) {
181
- const startTagIndex = message.indexOf(extractOptions.startTag);
182
- if (startTagIndex !== -1) {
183
- let endTagIndex = message.lastIndexOf(extractOptions.endTag);
184
- if (endTagIndex === -1) endTagIndex = message.length + 1;
185
- else endTagIndex += extractOptions.endTag.length;
186
- const extractedText = message.slice(startTagIndex, endTagIndex).trim();
187
- try {
188
- const extractedValue = "json" in extractOptions && extractOptions.json ? JSON.parse(extractedText) : extractedText;
189
- if (extractOptions.callback) {
190
- extractOptions.callback(extractedValue);
191
- }
192
- if (extractOptions.saveInMetadata) {
193
- metadata = { extracted: extractedValue };
194
- }
195
- } catch (error) {
196
- console.error(
197
- `[OpenaiAgent] Error parsing extracted value (${extractedText}):`,
198
- error
199
- );
200
- }
201
- message = message.slice(0, startTagIndex).trimEnd();
202
- }
225
+ if (!options) return { message, metadata };
226
+ const start = message.indexOf("{");
227
+ if (start === -1) return { message, metadata };
228
+ const end = message.lastIndexOf("}");
229
+ const extractedText = message.slice(start, end === -1 ? void 0 : end + 1).trim();
230
+ try {
231
+ const value = JSON.parse(extractedText);
232
+ options.callback?.(value);
233
+ if (options.saveInMetadata) metadata = { extracted: value };
234
+ } catch (error) {
235
+ console.error(
236
+ `[Agent] Error parsing extracted value (${extractedText}):`,
237
+ error
238
+ );
203
239
  }
204
- return { message, metadata };
240
+ return { message: message.slice(0, start).trimEnd(), metadata };
205
241
  }
206
242
  /**
207
243
  * Emits one of the events every agent has.
@@ -266,10 +302,10 @@ var FallbackAgent = class extends Agent {
266
302
  extract(message) {
267
303
  return this.agent ? this.agent.extract(message) : super.extract(message);
268
304
  }
269
- async generateAnswer(stream) {
305
+ async generateAnswer(stream, signal) {
270
306
  for (let attempt = 0; attempt < this.fallbackOptions.factories.length; attempt++) {
271
307
  const agent = this.agent;
272
- if (!agent) return;
308
+ if (!agent || signal.aborted) return;
273
309
  let failed = false;
274
310
  const onFailed = () => {
275
311
  failed = true;
@@ -280,7 +316,7 @@ var FallbackAgent = class extends Agent {
280
316
  } finally {
281
317
  agent.off("Failed", onFailed);
282
318
  }
283
- if (!failed) return;
319
+ if (!failed || signal.aborted) return;
284
320
  this.log("Agent failed, trying next agent");
285
321
  this.startNextAgent();
286
322
  }
@@ -288,6 +324,7 @@ var FallbackAgent = class extends Agent {
288
324
  this.emit("Failed");
289
325
  }
290
326
  cancel() {
327
+ super.cancel();
291
328
  this.agent?.cancel();
292
329
  }
293
330
  destroy() {
@@ -350,10 +387,8 @@ var MockAgent = class extends Agent {
350
387
  }
351
388
  async generateAnswer(stream) {
352
389
  const message = `Assistant Message ${this.i++}`;
353
- this.addAssistantMessage(message);
354
390
  stream.write(message);
355
- }
356
- cancel() {
391
+ this.addAssistantMessage(message);
357
392
  }
358
393
  };
359
394
 
@@ -436,8 +471,10 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
436
471
  MicdropServerCommands2["EndCall"] = "EndCall";
437
472
  MicdropServerCommands2["ToolCall"] = "ToolCall";
438
473
  MicdropServerCommands2["Classification"] = "Classification";
474
+ MicdropServerCommands2["SampleRate"] = "SampleRate";
439
475
  return MicdropServerCommands2;
440
476
  })(MicdropServerCommands || {});
477
+ var DEFAULT_SAMPLE_RATE = 16e3;
441
478
  function currentTurn(conversation) {
442
479
  let start = conversation.length;
443
480
  while (start > 0) {
@@ -495,6 +532,20 @@ function turnInput(conversation, history = 1) {
495
532
  return { input, message: turn.messages[turn.messages.length - 1] };
496
533
  }
497
534
 
535
+ // src/audio/Pcm16Aligner.ts
536
+ var Pcm16Aligner = class {
537
+ process(chunk) {
538
+ const buf = this.leftover ? Buffer.concat([this.leftover, chunk]) : chunk;
539
+ const length = buf.length - buf.length % 2;
540
+ this.leftover = length < buf.length ? buf.subarray(length) : void 0;
541
+ return buf.subarray(0, length);
542
+ }
543
+ /** Forgets the held byte, to start an independent stream */
544
+ reset() {
545
+ this.leftover = void 0;
546
+ }
547
+ };
548
+
498
549
  // src/audio/Pcm16Resampler.ts
499
550
  var Pcm16Resampler = class {
500
551
  // Fractional position into the first sample of the buffer
@@ -581,10 +632,107 @@ function handleError(socket, error) {
581
632
  socket.terminate();
582
633
  }
583
634
 
635
+ // src/lists.ts
636
+ import { createHash } from "crypto";
637
+
638
+ // src/SharedInstances.ts
639
+ var SharedInstances = class {
640
+ constructor(options = {}) {
641
+ this.options = options;
642
+ this.entries = /* @__PURE__ */ new Map();
643
+ // When each instance expires, for the ones that do
644
+ this.expiries = /* @__PURE__ */ new Map();
645
+ }
646
+ /**
647
+ * Returns the instance for this key, calling `create` the first time.
648
+ *
649
+ * The key can be any JSON-serializable value, usually an array of the
650
+ * options that define the instance. Concurrent callers get the same promise
651
+ * instead of loading the model twice.
652
+ */
653
+ load(key, create) {
654
+ const id = typeof key === "string" ? key : JSON.stringify(key);
655
+ this.expire(id);
656
+ const existing = this.entries.get(id);
657
+ if (existing) return existing;
658
+ let loading;
659
+ try {
660
+ loading = create();
661
+ } catch (error) {
662
+ return Promise.reject(error);
663
+ }
664
+ loading.catch(() => {
665
+ if (this.entries.get(id) === loading) this.entries.delete(id);
666
+ });
667
+ this.entries.set(id, loading);
668
+ const { ttl } = this.options;
669
+ if (ttl !== void 0) {
670
+ loading.then(
671
+ () => {
672
+ if (this.entries.get(id) === loading) {
673
+ this.expiries.set(id, Date.now() + ttl);
674
+ }
675
+ },
676
+ () => {
677
+ }
678
+ );
679
+ }
680
+ return loading;
681
+ }
682
+ /** Forgets an instance whose time is up */
683
+ expire(id) {
684
+ const expiry = this.expiries.get(id);
685
+ if (expiry === void 0 || Date.now() < expiry) return;
686
+ this.entries.delete(id);
687
+ this.expiries.delete(id);
688
+ }
689
+ /** The instance for this key, loaded or being loaded, without loading it. */
690
+ get(key) {
691
+ const id = typeof key === "string" ? key : JSON.stringify(key);
692
+ this.expire(id);
693
+ return this.entries.get(id);
694
+ }
695
+ /** Whether an instance is loaded, or being loaded, for this key. */
696
+ has(key) {
697
+ const id = typeof key === "string" ? key : JSON.stringify(key);
698
+ this.expire(id);
699
+ return this.entries.has(id);
700
+ }
701
+ /**
702
+ * Forgets the instance for this key, so the next call loads a new one. For
703
+ * an instance that broke, a crashed worker for instance. Releasing what it
704
+ * holds is left to the caller.
705
+ */
706
+ delete(key) {
707
+ const id = typeof key === "string" ? key : JSON.stringify(key);
708
+ this.entries.delete(id);
709
+ this.expiries.delete(id);
710
+ }
711
+ /** Drops every instance, so the next call loads again. */
712
+ clear() {
713
+ this.entries.clear();
714
+ this.expiries.clear();
715
+ }
716
+ };
717
+
718
+ // src/lists.ts
719
+ var LIST_TTL = 24 * 60 * 60 * 1e3;
720
+ var lists = new SharedInstances({ ttl: LIST_TTL });
721
+ function cachedList(key, apiKey, load, refresh = false) {
722
+ const account = createHash("sha256").update(apiKey).digest("hex");
723
+ const id = [...key, account];
724
+ if (refresh) lists.delete(id);
725
+ return lists.load(id, load);
726
+ }
727
+
728
+ // src/models.ts
729
+ function cachedModels(key, apiKey, load, refresh = false) {
730
+ return cachedList(["models", ...key], apiKey, load, refresh);
731
+ }
732
+
584
733
  // src/MicdropServer.ts
585
734
  import { EventEmitter as EventEmitter3 } from "eventemitter3";
586
735
  import { PassThrough as PassThrough2, Transform } from "stream";
587
- var USER_SAMPLE_RATE = 16e3;
588
736
  var DEFAULT_TURN_MAX_WAIT = 4e3;
589
737
  var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
590
738
  var MicdropServer = class extends EventEmitter3 {
@@ -593,6 +741,15 @@ var MicdropServer = class extends EventEmitter3 {
593
741
  this.socket = null;
594
742
  this.config = null;
595
743
  this.startTime = Date.now();
744
+ // Bumped by every cancel, so an answer cancelled before its first word can
745
+ // tell, and leave its message to the answer that replaces it
746
+ this.cancelRound = 0;
747
+ // Counts the assistant messages sent, so the partial answer of one already
748
+ // settled is not sent after it
749
+ this.assistantMessages = 0;
750
+ // Rate the client plays the assistant voice at, which it takes as the
751
+ // default until told otherwise
752
+ this.assistantSampleRate = DEFAULT_SAMPLE_RATE;
596
753
  // Holds the conversation when no agent does, and keeps it readable once the
597
754
  // call is over and the config is gone
598
755
  this.ownConversation = [];
@@ -600,10 +757,16 @@ var MicdropServer = class extends EventEmitter3 {
600
757
  this.operationQueue = [];
601
758
  this.isProcessingQueue = false;
602
759
  this.userSpeechChunks = 0;
603
- this.onClose = () => {
760
+ // Bumped whenever the user speaks again or mutes, so a decision taken on an
761
+ // earlier stretch of speech is dropped rather than answering over them
762
+ this.turnRound = 0;
763
+ this.onClose = (code, reason) => {
604
764
  this.releaseHeldTurn();
605
765
  if (!this.config) return;
606
- this.log("Connection closed");
766
+ this.log("Connection closed", {
767
+ code,
768
+ reason: reason?.toString() || void 0
769
+ });
607
770
  const duration = Math.round((Date.now() - this.startTime) / 1e3);
608
771
  const conversation = this.conversation;
609
772
  this.agent?.destroy();
@@ -653,9 +816,16 @@ var MicdropServer = class extends EventEmitter3 {
653
816
  };
654
817
  this.onAssistantAudio = (audio) => {
655
818
  if (!this.socket) return;
819
+ const source = this.config?.realtime ?? this.config?.tts;
820
+ const sampleRate = source?.sampleRate ?? DEFAULT_SAMPLE_RATE;
821
+ if (sampleRate !== this.assistantSampleRate) {
822
+ this.log(`Assistant voice now at ${sampleRate} Hz`);
823
+ this.assistantSampleRate = sampleRate;
824
+ this.socket.send(`${"SampleRate" /* SampleRate */} ${sampleRate}`);
825
+ }
656
826
  this.log(`Send audio chunk (${audio.byteLength} bytes)`);
657
827
  this.socket.send(audio);
658
- this.emit("AssistantAudio", audio);
828
+ this.emit("AssistantAudio", audio, sampleRate);
659
829
  };
660
830
  this.onMessageAgent = (message) => {
661
831
  this.sendMessage(message);
@@ -745,6 +915,7 @@ var MicdropServer = class extends EventEmitter3 {
745
915
  this.processQueue();
746
916
  }
747
917
  cancel() {
918
+ this.cancelRound++;
748
919
  this.config?.classifier?.cancel();
749
920
  this.config?.tts?.cancel();
750
921
  this.agent?.cancel();
@@ -754,10 +925,12 @@ var MicdropServer = class extends EventEmitter3 {
754
925
  this.log(`Received chunk (${chunk.byteLength} bytes)`);
755
926
  this.currentUserStream?.write(chunk);
756
927
  this.userSpeechChunks++;
757
- this.config?.turnDetector?.push(pcm16ToFloat32(chunk), USER_SAMPLE_RATE);
928
+ this.config?.turnDetector?.push(pcm16ToFloat32(chunk), DEFAULT_SAMPLE_RATE);
758
929
  this.emit("UserAudio", chunk);
759
930
  }
760
931
  onMute() {
932
+ this.turnRound++;
933
+ this.releaseHeldTurn();
761
934
  this.userSpeechChunks = 0;
762
935
  this.currentUserStream?.end();
763
936
  this.currentUserStream = void 0;
@@ -766,6 +939,7 @@ var MicdropServer = class extends EventEmitter3 {
766
939
  }
767
940
  onStartSpeaking() {
768
941
  if (!this.config) return;
942
+ this.turnRound++;
769
943
  this.userSpeechChunks = 0;
770
944
  this.currentUserStream?.end();
771
945
  this.currentUserStream = new PassThrough2();
@@ -817,7 +991,9 @@ var MicdropServer = class extends EventEmitter3 {
817
991
  }
818
992
  /** Answers, unless the sentence sounds like it has more coming */
819
993
  async answerUserTurn() {
994
+ const round = this.turnRound;
820
995
  const complete = await (this.turnComplete ?? Promise.resolve(true));
996
+ if (round !== this.turnRound || !this.config) return;
821
997
  if (!complete) {
822
998
  this.log("Waiting for the rest of the sentence");
823
999
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -893,6 +1069,7 @@ var MicdropServer = class extends EventEmitter3 {
893
1069
  `${"Message" /* Message */} ${JSON.stringify(message)}`
894
1070
  );
895
1071
  this.emit("Message", message);
1072
+ if (message.role === "assistant") this.assistantMessages++;
896
1073
  if (message.role === "user" && this.config?.realtime) this.classifyTurn();
897
1074
  }
898
1075
  sendPartialAnswer(content) {
@@ -939,12 +1116,18 @@ var MicdropServer = class extends EventEmitter3 {
939
1116
  return;
940
1117
  }
941
1118
  this.lastMessageSpeeched = lastMessage;
1119
+ const round = this.cancelRound;
942
1120
  try {
1121
+ const sentBefore = this.assistantMessages;
943
1122
  const stream = this.config.agent.answer();
944
1123
  if (await hasContent(stream)) {
945
1124
  await this._speak(
946
- this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
1125
+ this.config.partialMessages ? this.forwardPartialAnswer(stream, sentBefore) : stream
947
1126
  );
1127
+ } else if (round !== this.cancelRound) {
1128
+ if (this.lastMessageSpeeched === lastMessage) {
1129
+ this.lastMessageSpeeched = void 0;
1130
+ }
948
1131
  } else {
949
1132
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
950
1133
  }
@@ -953,10 +1136,19 @@ var MicdropServer = class extends EventEmitter3 {
953
1136
  throw error;
954
1137
  }
955
1138
  }
956
- /** Sends the answer to the client as it is written, ahead of the voice */
957
- forwardPartialAnswer(stream) {
1139
+ /**
1140
+ * Sends the answer to the client as it is written, ahead of the voice.
1141
+ *
1142
+ * The stream reaches here a moment after the agent wrote it, so the agent
1143
+ * can record the complete message first, which the client then shows. A
1144
+ * partial answer arriving after it would show the answer twice, so the
1145
+ * forwarding stops once the message is sent.
1146
+ */
1147
+ forwardPartialAnswer(stream, sentBefore) {
958
1148
  let content = "";
959
- const send = (text) => this.sendPartialAnswer(text);
1149
+ const send = (text) => {
1150
+ if (this.assistantMessages === sentBefore) this.sendPartialAnswer(text);
1151
+ };
960
1152
  const forward = new Transform({
961
1153
  transform(chunk, _encoding, callback) {
962
1154
  content += chunk.toString();
@@ -1134,6 +1326,7 @@ var Realtime = class extends Agent {
1134
1326
  });
1135
1327
  }
1136
1328
  cancel() {
1329
+ super.cancel();
1137
1330
  this.queue = [];
1138
1331
  const answer = this.currentAnswer;
1139
1332
  if (!answer) return;
@@ -1195,7 +1388,14 @@ var Realtime = class extends Agent {
1195
1388
  get isIdle() {
1196
1389
  return this.turn === "none" && !this.currentAnswer;
1197
1390
  }
1198
- /** Voice of the answer, PCM16 at 16 kHz */
1391
+ /**
1392
+ * Rate of the audio emitted, in Hz, the one the model speaks at. The server
1393
+ * passes it on to the client, which plays the audio as it comes.
1394
+ */
1395
+ get sampleRate() {
1396
+ return DEFAULT_SAMPLE_RATE;
1397
+ }
1398
+ /** Voice of the answer, PCM16 at `sampleRate` */
1199
1399
  emitAudio(chunk) {
1200
1400
  const answer = this.activeAnswer();
1201
1401
  if (!answer) return;
@@ -1322,6 +1522,8 @@ var MicdropRecorder = class extends EventEmitter4 {
1322
1522
  this.audioMessages = [];
1323
1523
  this.currentUserChunks = [];
1324
1524
  this.currentAssistantChunks = [];
1525
+ // Rate of the assistant message being recorded, the one of its first chunk
1526
+ this.currentAssistantRate = DEFAULT_SAMPLE_RATE;
1325
1527
  this.lastUserMessageIndex = -1;
1326
1528
  this.lastAssistantMessageIndex = -1;
1327
1529
  this.onUserAudio = (chunk) => {
@@ -1336,7 +1538,7 @@ var MicdropRecorder = class extends EventEmitter4 {
1336
1538
  this.log("Recording user audio chunk");
1337
1539
  this.currentUserChunks.push(chunk);
1338
1540
  };
1339
- this.onAssistantAudio = (chunk) => {
1541
+ this.onAssistantAudio = (chunk, sampleRate = DEFAULT_SAMPLE_RATE) => {
1340
1542
  if (this.currentUserChunks.length > 0) {
1341
1543
  if (this.lastUserMessageIndex >= 0) {
1342
1544
  this.finalizeUserAudio();
@@ -1346,7 +1548,7 @@ var MicdropRecorder = class extends EventEmitter4 {
1346
1548
  }
1347
1549
  }
1348
1550
  this.log("Recording assistant audio chunk");
1349
- this.currentAssistantChunks.push(chunk);
1551
+ this.currentAssistantChunks.push(this.toAssistantRate(chunk, sampleRate));
1350
1552
  };
1351
1553
  this.onMessage = (message) => {
1352
1554
  const messageIndex = this.server.conversation.length - 1;
@@ -1377,6 +1579,21 @@ var MicdropRecorder = class extends EventEmitter4 {
1377
1579
  this.server.on("End", this.onEnd);
1378
1580
  this.server.on("Message", this.onMessage);
1379
1581
  }
1582
+ /** A message holds one rate, the one its first chunk came at */
1583
+ toAssistantRate(chunk, sampleRate) {
1584
+ if (this.currentAssistantChunks.length === 0) {
1585
+ this.currentAssistantRate = sampleRate;
1586
+ this.assistantResampler = void 0;
1587
+ }
1588
+ if (sampleRate === this.currentAssistantRate) return chunk;
1589
+ if (this.assistantResampler?.from !== sampleRate) {
1590
+ this.assistantResampler = {
1591
+ from: sampleRate,
1592
+ resampler: new Pcm16Resampler(sampleRate, this.currentAssistantRate)
1593
+ };
1594
+ }
1595
+ return this.assistantResampler.resampler.process(chunk);
1596
+ }
1380
1597
  finalizeUserAudio() {
1381
1598
  if (this.currentUserChunks.length === 0) return;
1382
1599
  if (this.lastUserMessageIndex < 0) return;
@@ -1385,6 +1602,7 @@ var MicdropRecorder = class extends EventEmitter4 {
1385
1602
  const buffer = Buffer.concat(this.currentUserChunks);
1386
1603
  const audioMessage = {
1387
1604
  buffer,
1605
+ sampleRate: DEFAULT_SAMPLE_RATE,
1388
1606
  messageIndex: this.lastUserMessageIndex,
1389
1607
  message: "content" in message ? message.content : "",
1390
1608
  role: "user"
@@ -1405,6 +1623,7 @@ var MicdropRecorder = class extends EventEmitter4 {
1405
1623
  const buffer = Buffer.concat(this.currentAssistantChunks);
1406
1624
  const audioMessage = {
1407
1625
  buffer,
1626
+ sampleRate: this.currentAssistantRate,
1408
1627
  messageIndex: this.lastAssistantMessageIndex,
1409
1628
  message: "content" in message ? message.content : "",
1410
1629
  role: "assistant"
@@ -1415,6 +1634,7 @@ var MicdropRecorder = class extends EventEmitter4 {
1415
1634
  this.audioMessages.push(audioMessage);
1416
1635
  this.emit("AudioMessage", audioMessage);
1417
1636
  this.currentAssistantChunks = [];
1637
+ this.assistantResampler = void 0;
1418
1638
  this.lastAssistantMessageIndex = -1;
1419
1639
  }
1420
1640
  getAudioMessages() {
@@ -1433,47 +1653,259 @@ var MicdropRecorder = class extends EventEmitter4 {
1433
1653
  }
1434
1654
  };
1435
1655
 
1436
- // src/SharedInstances.ts
1437
- var SharedInstances = class {
1438
- constructor() {
1439
- this.entries = /* @__PURE__ */ new Map();
1656
+ // src/socket/ReconnectingSocket.ts
1657
+ import { EventEmitter as EventEmitter5 } from "eventemitter3";
1658
+ import WebSocket from "ws";
1659
+ var DEFAULT_CONNECTION_TIMEOUT = 5e3;
1660
+ var DEFAULT_RETRY_DELAY = 1e3;
1661
+ var DEFAULT_MAX_RETRY = 3;
1662
+ var ReconnectingSocket = class extends EventEmitter5 {
1663
+ constructor(options) {
1664
+ super();
1665
+ this.options = options;
1666
+ this._status = "idle";
1667
+ this.everOpened = false;
1668
+ this.retryCount = 0;
1669
+ // Numbers the attempts, so one overtaken while it awaited its address stops
1670
+ this.attemptId = 0;
1671
+ this.waiters = [];
1672
+ }
1673
+ get status() {
1674
+ return this._status;
1675
+ }
1676
+ /** Whether a message sent now goes out */
1677
+ get isOpen() {
1678
+ return this._status === "open";
1440
1679
  }
1441
1680
  /**
1442
- * Returns the instance for this key, calling `create` the first time.
1443
- *
1444
- * The key can be any JSON-serializable value, usually an array of the
1445
- * options that define the instance. Concurrent callers get the same promise
1446
- * instead of loading the model twice.
1681
+ * Opens the connection. Called again after `failed`, it starts over with a
1682
+ * full retry budget, and does nothing in any other status.
1447
1683
  */
1448
- load(key, create) {
1449
- const id = typeof key === "string" ? key : JSON.stringify(key);
1450
- const existing = this.entries.get(id);
1451
- if (existing) return existing;
1452
- let loading;
1684
+ connect() {
1685
+ if (this._status !== "idle" && this._status !== "failed") return;
1686
+ this.retryCount = 0;
1687
+ this.attempt();
1688
+ }
1689
+ /**
1690
+ * Waits for the connection to be open.
1691
+ * @returns True once open, false if it failed or was destroyed meanwhile
1692
+ */
1693
+ ready() {
1694
+ if (this._status === "open") return Promise.resolve(true);
1695
+ if (this._status === "destroyed" || this._status === "failed") {
1696
+ return Promise.resolve(false);
1697
+ }
1698
+ return new Promise((resolve) => this.waiters.push(resolve));
1699
+ }
1700
+ /**
1701
+ * Sends a message if the connection is open
1702
+ * @returns False when it did not go out
1703
+ */
1704
+ send(data) {
1705
+ if (this._status !== "open" || !this.socket) return false;
1706
+ this.socket.send(data);
1707
+ return true;
1708
+ }
1709
+ /**
1710
+ * Drops the connection and opens a new one right away, for a provider that
1711
+ * can only stop what it is doing by closing. It costs no retry, and the new
1712
+ * connection is not a reconnection.
1713
+ */
1714
+ restart() {
1715
+ if (this._status === "destroyed") return;
1716
+ this.log("Restarting the connection");
1717
+ this.clearRetryTimer();
1718
+ this.drop();
1719
+ this.everOpened = false;
1720
+ this.retryCount = 0;
1721
+ this.attempt();
1722
+ }
1723
+ /** Closes the connection for good */
1724
+ destroy() {
1725
+ if (this._status === "destroyed") return;
1726
+ this.clearRetryTimer();
1727
+ this.drop();
1728
+ this.setStatus("destroyed");
1729
+ this.settleWaiters(false);
1730
+ this.removeAllListeners();
1731
+ }
1732
+ async attempt() {
1733
+ const id = ++this.attemptId;
1734
+ const isOvertaken = () => id !== this.attemptId || this._status !== "connecting";
1735
+ this.setStatus("connecting");
1736
+ let url;
1453
1737
  try {
1454
- loading = create();
1738
+ const { url: getUrl } = this.options;
1739
+ url = typeof getUrl === "string" ? getUrl : await getUrl();
1455
1740
  } catch (error) {
1456
- return Promise.reject(error);
1741
+ if (isOvertaken()) return;
1742
+ this.onAttemptFailed(error);
1743
+ return;
1457
1744
  }
1458
- loading.catch(() => {
1459
- if (this.entries.get(id) === loading) this.entries.delete(id);
1745
+ if (isOvertaken()) return;
1746
+ const socket = new WebSocket(url, this.options.clientOptions);
1747
+ this.socket = socket;
1748
+ let settled = false;
1749
+ const fail = (error) => {
1750
+ if (settled || this.socket !== socket) return;
1751
+ settled = true;
1752
+ clearTimeout(timeout);
1753
+ this.drop();
1754
+ this.onAttemptFailed(error);
1755
+ };
1756
+ const timeout = setTimeout(
1757
+ () => fail(new Error("Connection timeout")),
1758
+ this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
1759
+ );
1760
+ socket.on("error", (error) => {
1761
+ this.log("WebSocket error:", error.message);
1762
+ fail(error);
1763
+ });
1764
+ const { handshake } = this.options;
1765
+ const open = () => {
1766
+ settled = true;
1767
+ clearTimeout(timeout);
1768
+ this.onOpen();
1769
+ };
1770
+ socket.on("open", () => {
1771
+ if (this.socket !== socket) return;
1772
+ try {
1773
+ handshake?.start?.(socket);
1774
+ } catch (error) {
1775
+ fail(error);
1776
+ return;
1777
+ }
1778
+ if (!handshake?.read) open();
1779
+ });
1780
+ socket.on("message", (data, isBinary) => {
1781
+ if (this.socket !== socket) return;
1782
+ if (this._status === "open") {
1783
+ this.prove();
1784
+ this.emit("Message", data, isBinary);
1785
+ return;
1786
+ }
1787
+ if (settled || !handshake?.read) return;
1788
+ try {
1789
+ if (handshake.read(data, isBinary)) open();
1790
+ } catch (error) {
1791
+ fail(error);
1792
+ }
1793
+ });
1794
+ socket.on("close", (code, reasonBuffer) => {
1795
+ const reason = reasonBuffer.toString();
1796
+ if (!settled) {
1797
+ fail(new Error(`Connection closed (${code}) ${reason}`));
1798
+ return;
1799
+ }
1800
+ if (this.socket !== socket) return;
1801
+ this.log("Connection closed", { code, reason });
1802
+ this.drop();
1803
+ this.setStatus("reconnecting");
1804
+ this.emit("Close", code, reason);
1805
+ if (this._status !== "reconnecting") return;
1806
+ this.scheduleRetry(new Error(`Connection closed (${code}) ${reason}`));
1460
1807
  });
1461
- this.entries.set(id, loading);
1462
- return loading;
1463
1808
  }
1464
- /** Whether an instance is loaded, or being loaded, for this key. */
1465
- has(key) {
1466
- return this.entries.has(typeof key === "string" ? key : JSON.stringify(key));
1809
+ onOpen() {
1810
+ this.log("Connection opened");
1811
+ const reconnected = this.everOpened;
1812
+ this.everOpened = true;
1813
+ if (this.options.handshake?.read) {
1814
+ this.prove();
1815
+ } else {
1816
+ this.proofTimer = setTimeout(
1817
+ () => this.prove(),
1818
+ this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
1819
+ );
1820
+ }
1821
+ this.startKeepAlive();
1822
+ this.setStatus("open");
1823
+ this.emit("Open", { reconnected });
1824
+ this.settleWaiters(true);
1825
+ }
1826
+ /** The connection works, the retry budget is whole again */
1827
+ prove() {
1828
+ this.clearProofTimer();
1829
+ this.retryCount = 0;
1830
+ }
1831
+ clearProofTimer() {
1832
+ if (!this.proofTimer) return;
1833
+ clearTimeout(this.proofTimer);
1834
+ this.proofTimer = void 0;
1835
+ }
1836
+ onAttemptFailed(error) {
1837
+ this.log("Connection failed:", error.message);
1838
+ this.scheduleRetry(error);
1839
+ }
1840
+ scheduleRetry(error) {
1841
+ if (this._status === "destroyed") return;
1842
+ this.retryCount++;
1843
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
1844
+ this.log("Max retries reached, giving up");
1845
+ this.setStatus("failed");
1846
+ this.settleWaiters(false);
1847
+ this.emit("Failed", error);
1848
+ return;
1849
+ }
1850
+ this.setStatus("reconnecting");
1851
+ this.log(`Reconnecting (attempt ${this.retryCount})`);
1852
+ this.clearRetryTimer();
1853
+ this.retryTimer = setTimeout(() => {
1854
+ this.retryTimer = void 0;
1855
+ if (this._status === "reconnecting") this.attempt();
1856
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
1857
+ }
1858
+ /** Lets go of the current socket, which will never be heard from again */
1859
+ drop() {
1860
+ this.stopKeepAlive();
1861
+ this.clearProofTimer();
1862
+ const socket = this.socket;
1863
+ this.socket = void 0;
1864
+ if (!socket) return;
1865
+ socket.removeAllListeners();
1866
+ socket.on("error", () => {
1867
+ });
1868
+ if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
1869
+ socket.close(1e3);
1870
+ }
1467
1871
  }
1468
- /** Drops every instance, so the next call loads again. */
1469
- clear() {
1470
- this.entries.clear();
1872
+ startKeepAlive() {
1873
+ const keepAlive = this.options.keepAlive;
1874
+ if (!keepAlive) return;
1875
+ this.stopKeepAlive();
1876
+ this.keepAliveTimer = setInterval(() => {
1877
+ const { message } = keepAlive;
1878
+ this.send(typeof message === "string" ? message : message());
1879
+ }, keepAlive.interval);
1880
+ }
1881
+ stopKeepAlive() {
1882
+ if (!this.keepAliveTimer) return;
1883
+ clearInterval(this.keepAliveTimer);
1884
+ this.keepAliveTimer = void 0;
1885
+ }
1886
+ clearRetryTimer() {
1887
+ if (!this.retryTimer) return;
1888
+ clearTimeout(this.retryTimer);
1889
+ this.retryTimer = void 0;
1890
+ }
1891
+ settleWaiters(open) {
1892
+ const waiters = this.waiters;
1893
+ this.waiters = [];
1894
+ waiters.forEach((resolve) => resolve(open));
1895
+ }
1896
+ setStatus(status) {
1897
+ if (this._status === status) return;
1898
+ this._status = status;
1899
+ this.emit("StatusChange", status);
1900
+ }
1901
+ log(...message) {
1902
+ this.options.log?.(...message);
1471
1903
  }
1472
1904
  };
1473
1905
 
1474
1906
  // src/stt/STT.ts
1475
- import { EventEmitter as EventEmitter5 } from "eventemitter3";
1476
- var STT = class extends EventEmitter5 {
1907
+ import { EventEmitter as EventEmitter6 } from "eventemitter3";
1908
+ var STT = class extends EventEmitter6 {
1477
1909
  log(...message) {
1478
1910
  this.logger?.log(...message);
1479
1911
  }
@@ -1499,24 +1931,31 @@ var MockSTT = class extends STT {
1499
1931
  // src/stt/FallbackSTT.ts
1500
1932
  import { PassThrough as PassThrough3 } from "stream";
1501
1933
  var FallbackSTT = class extends STT {
1502
- // Start at -1 because we need to increment it before using it
1503
1934
  constructor(options) {
1504
1935
  super();
1505
1936
  this.options = options;
1506
1937
  this.stt = null;
1507
1938
  this.sttIndex = -1;
1939
+ // Start at -1 because we need to increment it before using it
1940
+ // In the order spoken, each STT giving one transcript per utterance in turn
1941
+ this.pending = [];
1508
1942
  this.onTranscript = (transcript) => {
1943
+ this.pending.shift();
1509
1944
  this.emit("Transcript", transcript);
1510
1945
  };
1511
- this.onFailed = (chunks) => {
1946
+ /**
1947
+ * Hands every utterance still waiting to the next STT, one stream each so
1948
+ * each gets its own transcript. One still being spoken keeps going there.
1949
+ */
1950
+ this.onFailed = () => {
1512
1951
  this.log("STT failed, trying next STT");
1513
1952
  this.startNextSTT();
1514
- if (chunks.length > 0) {
1515
- this.log("Sending audio chunks again");
1516
- const stream = new PassThrough3();
1517
- this.stt?.transcribe(stream);
1518
- chunks.forEach((chunk) => stream.write(chunk));
1519
- stream.end();
1953
+ if (this.pending.length > 0) this.log("Sending audio chunks again");
1954
+ for (const utterance of this.pending) {
1955
+ utterance.stream = new PassThrough3();
1956
+ this.stt?.transcribe(utterance.stream);
1957
+ utterance.chunks.forEach((chunk) => utterance.stream.write(chunk));
1958
+ if (utterance.ended) utterance.stream.end();
1520
1959
  }
1521
1960
  };
1522
1961
  if (this.options.factories.length === 0) {
@@ -1525,13 +1964,28 @@ var FallbackSTT = class extends STT {
1525
1964
  this.startNextSTT();
1526
1965
  }
1527
1966
  transcribe(audioStream) {
1528
- this.stt?.transcribe(audioStream);
1967
+ const utterance = {
1968
+ chunks: [],
1969
+ ended: false,
1970
+ stream: new PassThrough3()
1971
+ };
1972
+ this.pending.push(utterance);
1973
+ audioStream.on("data", (chunk) => {
1974
+ utterance.chunks.push(chunk);
1975
+ utterance.stream.write(chunk);
1976
+ });
1977
+ audioStream.on("end", () => {
1978
+ utterance.ended = true;
1979
+ utterance.stream.end();
1980
+ });
1981
+ this.stt?.transcribe(utterance.stream);
1529
1982
  }
1530
1983
  destroy() {
1531
1984
  super.destroy();
1532
1985
  this.stt?.destroy();
1533
1986
  this.stt = null;
1534
1987
  this.sttIndex = -1;
1988
+ this.pending = [];
1535
1989
  }
1536
1990
  startNextSTT() {
1537
1991
  this.sttIndex++;
@@ -1554,8 +2008,18 @@ var FallbackSTT = class extends STT {
1554
2008
  import { PassThrough as PassThrough4 } from "stream";
1555
2009
 
1556
2010
  // src/tts/TTS.ts
1557
- import { EventEmitter as EventEmitter6 } from "eventemitter3";
1558
- var TTS = class extends EventEmitter6 {
2011
+ import { EventEmitter as EventEmitter7 } from "eventemitter3";
2012
+ var TTS = class extends EventEmitter7 {
2013
+ /**
2014
+ * Rate of the audio emitted, in Hz.
2015
+ *
2016
+ * The server reads it with each chunk and tells the client when it changes,
2017
+ * so the audio reaches the client at the rate the provider produced it. A
2018
+ * provider that offers several rates takes the one chosen in its options.
2019
+ */
2020
+ get sampleRate() {
2021
+ return DEFAULT_SAMPLE_RATE;
2022
+ }
1559
2023
  log(...message) {
1560
2024
  this.logger?.log(...message);
1561
2025
  }
@@ -1567,7 +2031,6 @@ var TTS = class extends EventEmitter6 {
1567
2031
 
1568
2032
  // src/tts/FallbackTTS.ts
1569
2033
  var FallbackTTS = class extends TTS {
1570
- // Start at -1 because we need to increment it before using it
1571
2034
  constructor(options) {
1572
2035
  super();
1573
2036
  this.options = options;
@@ -1576,26 +2039,47 @@ var FallbackTTS = class extends TTS {
1576
2039
  this.onAudio = (audio) => {
1577
2040
  this.emit("Audio", audio);
1578
2041
  };
2042
+ /**
2043
+ * Hands the unspoken text to the next TTS. An utterance whose text is still
2044
+ * being written goes on there, so its end is spoken too.
2045
+ */
1579
2046
  this.onFailed = (chunks) => {
1580
2047
  this.log("TTS failed, trying next TTS");
1581
2048
  this.startNextTTS();
1582
- if (chunks.length > 0) {
1583
- this.log("Sending text chunks again");
1584
- const stream = new PassThrough4();
1585
- this.tts?.speak(stream);
1586
- chunks.forEach((chunk) => stream.write(chunk));
1587
- stream.end();
1588
- }
2049
+ const utterance = this.current;
2050
+ const continues = !!utterance && !utterance.ended;
2051
+ if (chunks.length === 0 && !continues) return;
2052
+ this.log("Sending text chunks again");
2053
+ const stream = new PassThrough4();
2054
+ this.tts?.speak(stream);
2055
+ chunks.forEach((chunk) => stream.write(chunk));
2056
+ if (continues) utterance.stream = stream;
2057
+ else stream.end();
1589
2058
  };
1590
2059
  if (this.options.factories.length === 0) {
1591
2060
  throw new Error("FallbackTTS: No factories provided");
1592
2061
  }
1593
2062
  this.startNextTTS();
1594
2063
  }
2064
+ /** Rate of the TTS in use, which changes when it falls back to the next */
2065
+ get sampleRate() {
2066
+ return this.tts?.sampleRate ?? super.sampleRate;
2067
+ }
1595
2068
  speak(textStream) {
1596
- this.tts?.speak(textStream);
2069
+ const utterance = {
2070
+ ended: false,
2071
+ stream: new PassThrough4()
2072
+ };
2073
+ this.current = utterance;
2074
+ textStream.on("data", (chunk) => utterance.stream.write(chunk));
2075
+ textStream.on("end", () => {
2076
+ utterance.ended = true;
2077
+ utterance.stream.end();
2078
+ });
2079
+ this.tts?.speak(utterance.stream);
1597
2080
  }
1598
2081
  cancel() {
2082
+ this.current = void 0;
1599
2083
  this.tts?.cancel();
1600
2084
  }
1601
2085
  destroy() {
@@ -1623,26 +2107,32 @@ var FallbackTTS = class extends TTS {
1623
2107
 
1624
2108
  // src/tts/MockTTS.ts
1625
2109
  import * as fs from "fs";
1626
- import { PassThrough as PassThrough5 } from "stream";
1627
2110
  var MockTTS = class extends TTS {
1628
- constructor(audioFilePaths) {
2111
+ constructor(audioFilePaths, rate = DEFAULT_SAMPLE_RATE) {
1629
2112
  super();
1630
2113
  this.audioFilePaths = audioFilePaths;
2114
+ this.rate = rate;
2115
+ // Bumped by every answer and every cancel, so the chunks of an answer that
2116
+ // is no longer wanted stop there
2117
+ this.generation = 0;
2118
+ }
2119
+ get sampleRate() {
2120
+ return this.rate;
1631
2121
  }
1632
2122
  speak(textStream) {
1633
- const audioStream = new PassThrough5();
2123
+ const generation = ++this.generation;
1634
2124
  textStream.once("data", async () => {
1635
2125
  for (const filePath of this.audioFilePaths) {
1636
2126
  await new Promise((resolve) => setTimeout(resolve, 200));
1637
- const audioBuffer = fs.readFileSync(filePath);
1638
- this.log(`Loaded chunk (${audioBuffer.length} bytes)`);
1639
- audioStream.write(audioBuffer);
2127
+ if (generation !== this.generation) return;
2128
+ const audio = fs.readFileSync(filePath);
2129
+ this.log(`Loaded chunk (${audio.length} bytes)`);
2130
+ this.emit("Audio", audio);
1640
2131
  }
1641
- audioStream.end();
1642
2132
  });
1643
- return audioStream;
1644
2133
  }
1645
2134
  cancel() {
2135
+ this.generation++;
1646
2136
  }
1647
2137
  };
1648
2138
 
@@ -1721,6 +2211,9 @@ var SentenceTTS = class extends TTS {
1721
2211
  this.counter++;
1722
2212
  counter = this.counter;
1723
2213
  this.splitter.reset();
2214
+ this.queue = [];
2215
+ this.controller?.abort();
2216
+ this.controller = void 0;
1724
2217
  return true;
1725
2218
  };
1726
2219
  textStream.on("data", (chunk) => {
@@ -1779,6 +2272,29 @@ var SentenceTTS = class extends TTS {
1779
2272
  }
1780
2273
  };
1781
2274
 
2275
+ // src/tts/voices.ts
2276
+ function voiceGender(value) {
2277
+ const word = value?.toLowerCase();
2278
+ if (!word) return void 0;
2279
+ if (word.startsWith("fem") || word === "woman" || word === "f")
2280
+ return "female";
2281
+ if (word.startsWith("masc") || word === "male" || word === "man" || word === "m") {
2282
+ return "male";
2283
+ }
2284
+ if (word.includes("neutral") || word === "androgynous") return "neutral";
2285
+ return void 0;
2286
+ }
2287
+ function cachedVoices(key, apiKey, load, refresh = false) {
2288
+ return cachedList(["voices", ...key], apiKey, load, refresh);
2289
+ }
2290
+ function voicesForLanguage(voices, language) {
2291
+ if (!language) return voices;
2292
+ const base = language.split("-")[0].toLowerCase();
2293
+ return voices.filter(
2294
+ (voice) => !voice.languages?.length || voice.languages.some((code) => code.split("-")[0].toLowerCase() === base)
2295
+ );
2296
+ }
2297
+
1782
2298
  // src/waitForParams.ts
1783
2299
  async function waitForParams(socket, validate) {
1784
2300
  return new Promise((resolve, reject) => {
@@ -1807,6 +2323,7 @@ export {
1807
2323
  AUTO_SEMANTIC_TURN_TOOL_NAME,
1808
2324
  Agent,
1809
2325
  Classifier,
2326
+ DEFAULT_SAMPLE_RATE,
1810
2327
  FallbackAgent,
1811
2328
  FallbackSTT,
1812
2329
  FallbackTTS,
@@ -1820,19 +2337,26 @@ export {
1820
2337
  MockAgent,
1821
2338
  MockSTT,
1822
2339
  MockTTS,
2340
+ Pcm16Aligner,
1823
2341
  Pcm16Resampler,
1824
2342
  Realtime,
2343
+ ReconnectingSocket,
1825
2344
  STT,
1826
2345
  SentenceSplitter,
1827
2346
  SentenceTTS,
1828
2347
  SharedInstances,
1829
2348
  TTS,
2349
+ cachedList,
2350
+ cachedModels,
2351
+ cachedVoices,
1830
2352
  currentTurn,
1831
2353
  float32ToPcm16,
1832
2354
  getTurnClassification,
1833
2355
  handleError,
1834
2356
  pcm16ToFloat32,
1835
2357
  turnInput,
2358
+ voiceGender,
2359
+ voicesForLanguage,
1836
2360
  waitForParams
1837
2361
  };
1838
2362
  //# sourceMappingURL=index.mjs.map