@micdrop/server 2.10.0 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -196,45 +196,48 @@ var Agent = class extends EventEmitter {
196
196
  };
197
197
  }
198
198
  }
199
- getExtractOptions() {
200
- const extract = this.options.extract;
201
- if (!extract) return void 0;
202
- if ("json" in extract && extract.json) {
203
- return { ...extract, startTag: "{", endTag: "}" };
204
- }
205
- if ("startTag" in extract && "endTag" in extract) {
206
- return extract;
207
- }
208
- return void 0;
199
+ /**
200
+ * Writes the answer to the stream as it is generated, leaving out the JSON
201
+ * value to extract: the voice stops at its first `{`.
202
+ *
203
+ * The value is cut from the message once the answer is complete, by
204
+ * `extract()`. Only the speech has to be cut while streaming.
205
+ */
206
+ answerWriter(stream) {
207
+ let extracting = false;
208
+ return (text) => {
209
+ if (extracting || !text) return;
210
+ if (this.options.extract) {
211
+ const start = text.indexOf("{");
212
+ if (start !== -1) {
213
+ extracting = true;
214
+ text = text.slice(0, start).trimEnd();
215
+ if (!text) return;
216
+ }
217
+ }
218
+ stream.write(text);
219
+ };
209
220
  }
221
+ /** Cuts the JSON value to extract from a complete answer, and parses it */
210
222
  extract(message) {
211
- const extractOptions = this.getExtractOptions();
223
+ const options = this.options.extract;
212
224
  let metadata = void 0;
213
- if (extractOptions) {
214
- const startTagIndex = message.indexOf(extractOptions.startTag);
215
- if (startTagIndex !== -1) {
216
- let endTagIndex = message.lastIndexOf(extractOptions.endTag);
217
- if (endTagIndex === -1) endTagIndex = message.length + 1;
218
- else endTagIndex += extractOptions.endTag.length;
219
- const extractedText = message.slice(startTagIndex, endTagIndex).trim();
220
- try {
221
- const extractedValue = "json" in extractOptions && extractOptions.json ? JSON.parse(extractedText) : extractedText;
222
- if (extractOptions.callback) {
223
- extractOptions.callback(extractedValue);
224
- }
225
- if (extractOptions.saveInMetadata) {
226
- metadata = { extracted: extractedValue };
227
- }
228
- } catch (error) {
229
- console.error(
230
- `[OpenaiAgent] Error parsing extracted value (${extractedText}):`,
231
- error
232
- );
233
- }
234
- message = message.slice(0, startTagIndex).trimEnd();
235
- }
225
+ if (!options) return { message, metadata };
226
+ const start = message.indexOf("{");
227
+ if (start === -1) return { message, metadata };
228
+ const end = message.lastIndexOf("}");
229
+ const extractedText = message.slice(start, end === -1 ? void 0 : end + 1).trim();
230
+ try {
231
+ const value = JSON.parse(extractedText);
232
+ options.callback?.(value);
233
+ if (options.saveInMetadata) metadata = { extracted: value };
234
+ } catch (error) {
235
+ console.error(
236
+ `[Agent] Error parsing extracted value (${extractedText}):`,
237
+ error
238
+ );
236
239
  }
237
- return { message, metadata };
240
+ return { message: message.slice(0, start).trimEnd(), metadata };
238
241
  }
239
242
  /**
240
243
  * Emits one of the events every agent has.
@@ -468,8 +471,10 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
468
471
  MicdropServerCommands2["EndCall"] = "EndCall";
469
472
  MicdropServerCommands2["ToolCall"] = "ToolCall";
470
473
  MicdropServerCommands2["Classification"] = "Classification";
474
+ MicdropServerCommands2["SampleRate"] = "SampleRate";
471
475
  return MicdropServerCommands2;
472
476
  })(MicdropServerCommands || {});
477
+ var DEFAULT_SAMPLE_RATE = 16e3;
473
478
  function currentTurn(conversation) {
474
479
  let start = conversation.length;
475
480
  while (start > 0) {
@@ -527,6 +532,20 @@ function turnInput(conversation, history = 1) {
527
532
  return { input, message: turn.messages[turn.messages.length - 1] };
528
533
  }
529
534
 
535
+ // src/audio/Pcm16Aligner.ts
536
+ var Pcm16Aligner = class {
537
+ process(chunk) {
538
+ const buf = this.leftover ? Buffer.concat([this.leftover, chunk]) : chunk;
539
+ const length = buf.length - buf.length % 2;
540
+ this.leftover = length < buf.length ? buf.subarray(length) : void 0;
541
+ return buf.subarray(0, length);
542
+ }
543
+ /** Forgets the held byte, to start an independent stream */
544
+ reset() {
545
+ this.leftover = void 0;
546
+ }
547
+ };
548
+
530
549
  // src/audio/Pcm16Resampler.ts
531
550
  var Pcm16Resampler = class {
532
551
  // Fractional position into the first sample of the buffer
@@ -613,10 +632,107 @@ function handleError(socket, error) {
613
632
  socket.terminate();
614
633
  }
615
634
 
635
+ // src/lists.ts
636
+ import { createHash } from "crypto";
637
+
638
+ // src/SharedInstances.ts
639
+ var SharedInstances = class {
640
+ constructor(options = {}) {
641
+ this.options = options;
642
+ this.entries = /* @__PURE__ */ new Map();
643
+ // When each instance expires, for the ones that do
644
+ this.expiries = /* @__PURE__ */ new Map();
645
+ }
646
+ /**
647
+ * Returns the instance for this key, calling `create` the first time.
648
+ *
649
+ * The key can be any JSON-serializable value, usually an array of the
650
+ * options that define the instance. Concurrent callers get the same promise
651
+ * instead of loading the model twice.
652
+ */
653
+ load(key, create) {
654
+ const id = typeof key === "string" ? key : JSON.stringify(key);
655
+ this.expire(id);
656
+ const existing = this.entries.get(id);
657
+ if (existing) return existing;
658
+ let loading;
659
+ try {
660
+ loading = create();
661
+ } catch (error) {
662
+ return Promise.reject(error);
663
+ }
664
+ loading.catch(() => {
665
+ if (this.entries.get(id) === loading) this.entries.delete(id);
666
+ });
667
+ this.entries.set(id, loading);
668
+ const { ttl } = this.options;
669
+ if (ttl !== void 0) {
670
+ loading.then(
671
+ () => {
672
+ if (this.entries.get(id) === loading) {
673
+ this.expiries.set(id, Date.now() + ttl);
674
+ }
675
+ },
676
+ () => {
677
+ }
678
+ );
679
+ }
680
+ return loading;
681
+ }
682
+ /** Forgets an instance whose time is up */
683
+ expire(id) {
684
+ const expiry = this.expiries.get(id);
685
+ if (expiry === void 0 || Date.now() < expiry) return;
686
+ this.entries.delete(id);
687
+ this.expiries.delete(id);
688
+ }
689
+ /** The instance for this key, loaded or being loaded, without loading it. */
690
+ get(key) {
691
+ const id = typeof key === "string" ? key : JSON.stringify(key);
692
+ this.expire(id);
693
+ return this.entries.get(id);
694
+ }
695
+ /** Whether an instance is loaded, or being loaded, for this key. */
696
+ has(key) {
697
+ const id = typeof key === "string" ? key : JSON.stringify(key);
698
+ this.expire(id);
699
+ return this.entries.has(id);
700
+ }
701
+ /**
702
+ * Forgets the instance for this key, so the next call loads a new one. For
703
+ * an instance that broke, a crashed worker for instance. Releasing what it
704
+ * holds is left to the caller.
705
+ */
706
+ delete(key) {
707
+ const id = typeof key === "string" ? key : JSON.stringify(key);
708
+ this.entries.delete(id);
709
+ this.expiries.delete(id);
710
+ }
711
+ /** Drops every instance, so the next call loads again. */
712
+ clear() {
713
+ this.entries.clear();
714
+ this.expiries.clear();
715
+ }
716
+ };
717
+
718
+ // src/lists.ts
719
+ var LIST_TTL = 24 * 60 * 60 * 1e3;
720
+ var lists = new SharedInstances({ ttl: LIST_TTL });
721
+ function cachedList(key, apiKey, load, refresh = false) {
722
+ const account = createHash("sha256").update(apiKey).digest("hex");
723
+ const id = [...key, account];
724
+ if (refresh) lists.delete(id);
725
+ return lists.load(id, load);
726
+ }
727
+
728
+ // src/models.ts
729
+ function cachedModels(key, apiKey, load, refresh = false) {
730
+ return cachedList(["models", ...key], apiKey, load, refresh);
731
+ }
732
+
616
733
  // src/MicdropServer.ts
617
734
  import { EventEmitter as EventEmitter3 } from "eventemitter3";
618
735
  import { PassThrough as PassThrough2, Transform } from "stream";
619
- var USER_SAMPLE_RATE = 16e3;
620
736
  var DEFAULT_TURN_MAX_WAIT = 4e3;
621
737
  var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
622
738
  var MicdropServer = class extends EventEmitter3 {
@@ -631,6 +747,9 @@ var MicdropServer = class extends EventEmitter3 {
631
747
  // Counts the assistant messages sent, so the partial answer of one already
632
748
  // settled is not sent after it
633
749
  this.assistantMessages = 0;
750
+ // Rate the client plays the assistant voice at, which it takes as the
751
+ // default until told otherwise
752
+ this.assistantSampleRate = DEFAULT_SAMPLE_RATE;
634
753
  // Holds the conversation when no agent does, and keeps it readable once the
635
754
  // call is over and the config is gone
636
755
  this.ownConversation = [];
@@ -697,9 +816,16 @@ var MicdropServer = class extends EventEmitter3 {
697
816
  };
698
817
  this.onAssistantAudio = (audio) => {
699
818
  if (!this.socket) return;
819
+ const source = this.config?.realtime ?? this.config?.tts;
820
+ const sampleRate = source?.sampleRate ?? DEFAULT_SAMPLE_RATE;
821
+ if (sampleRate !== this.assistantSampleRate) {
822
+ this.log(`Assistant voice now at ${sampleRate} Hz`);
823
+ this.assistantSampleRate = sampleRate;
824
+ this.socket.send(`${"SampleRate" /* SampleRate */} ${sampleRate}`);
825
+ }
700
826
  this.log(`Send audio chunk (${audio.byteLength} bytes)`);
701
827
  this.socket.send(audio);
702
- this.emit("AssistantAudio", audio);
828
+ this.emit("AssistantAudio", audio, sampleRate);
703
829
  };
704
830
  this.onMessageAgent = (message) => {
705
831
  this.sendMessage(message);
@@ -799,7 +925,7 @@ var MicdropServer = class extends EventEmitter3 {
799
925
  this.log(`Received chunk (${chunk.byteLength} bytes)`);
800
926
  this.currentUserStream?.write(chunk);
801
927
  this.userSpeechChunks++;
802
- this.config?.turnDetector?.push(pcm16ToFloat32(chunk), USER_SAMPLE_RATE);
928
+ this.config?.turnDetector?.push(pcm16ToFloat32(chunk), DEFAULT_SAMPLE_RATE);
803
929
  this.emit("UserAudio", chunk);
804
930
  }
805
931
  onMute() {
@@ -1262,7 +1388,14 @@ var Realtime = class extends Agent {
1262
1388
  get isIdle() {
1263
1389
  return this.turn === "none" && !this.currentAnswer;
1264
1390
  }
1265
- /** Voice of the answer, PCM16 at 16 kHz */
1391
+ /**
1392
+ * Rate of the audio emitted, in Hz, the one the model speaks at. The server
1393
+ * passes it on to the client, which plays the audio as it comes.
1394
+ */
1395
+ get sampleRate() {
1396
+ return DEFAULT_SAMPLE_RATE;
1397
+ }
1398
+ /** Voice of the answer, PCM16 at `sampleRate` */
1266
1399
  emitAudio(chunk) {
1267
1400
  const answer = this.activeAnswer();
1268
1401
  if (!answer) return;
@@ -1389,6 +1522,8 @@ var MicdropRecorder = class extends EventEmitter4 {
1389
1522
  this.audioMessages = [];
1390
1523
  this.currentUserChunks = [];
1391
1524
  this.currentAssistantChunks = [];
1525
+ // Rate of the assistant message being recorded, the one of its first chunk
1526
+ this.currentAssistantRate = DEFAULT_SAMPLE_RATE;
1392
1527
  this.lastUserMessageIndex = -1;
1393
1528
  this.lastAssistantMessageIndex = -1;
1394
1529
  this.onUserAudio = (chunk) => {
@@ -1403,7 +1538,7 @@ var MicdropRecorder = class extends EventEmitter4 {
1403
1538
  this.log("Recording user audio chunk");
1404
1539
  this.currentUserChunks.push(chunk);
1405
1540
  };
1406
- this.onAssistantAudio = (chunk) => {
1541
+ this.onAssistantAudio = (chunk, sampleRate = DEFAULT_SAMPLE_RATE) => {
1407
1542
  if (this.currentUserChunks.length > 0) {
1408
1543
  if (this.lastUserMessageIndex >= 0) {
1409
1544
  this.finalizeUserAudio();
@@ -1413,7 +1548,7 @@ var MicdropRecorder = class extends EventEmitter4 {
1413
1548
  }
1414
1549
  }
1415
1550
  this.log("Recording assistant audio chunk");
1416
- this.currentAssistantChunks.push(chunk);
1551
+ this.currentAssistantChunks.push(this.toAssistantRate(chunk, sampleRate));
1417
1552
  };
1418
1553
  this.onMessage = (message) => {
1419
1554
  const messageIndex = this.server.conversation.length - 1;
@@ -1444,6 +1579,21 @@ var MicdropRecorder = class extends EventEmitter4 {
1444
1579
  this.server.on("End", this.onEnd);
1445
1580
  this.server.on("Message", this.onMessage);
1446
1581
  }
1582
+ /** A message holds one rate, the one its first chunk came at */
1583
+ toAssistantRate(chunk, sampleRate) {
1584
+ if (this.currentAssistantChunks.length === 0) {
1585
+ this.currentAssistantRate = sampleRate;
1586
+ this.assistantResampler = void 0;
1587
+ }
1588
+ if (sampleRate === this.currentAssistantRate) return chunk;
1589
+ if (this.assistantResampler?.from !== sampleRate) {
1590
+ this.assistantResampler = {
1591
+ from: sampleRate,
1592
+ resampler: new Pcm16Resampler(sampleRate, this.currentAssistantRate)
1593
+ };
1594
+ }
1595
+ return this.assistantResampler.resampler.process(chunk);
1596
+ }
1447
1597
  finalizeUserAudio() {
1448
1598
  if (this.currentUserChunks.length === 0) return;
1449
1599
  if (this.lastUserMessageIndex < 0) return;
@@ -1452,6 +1602,7 @@ var MicdropRecorder = class extends EventEmitter4 {
1452
1602
  const buffer = Buffer.concat(this.currentUserChunks);
1453
1603
  const audioMessage = {
1454
1604
  buffer,
1605
+ sampleRate: DEFAULT_SAMPLE_RATE,
1455
1606
  messageIndex: this.lastUserMessageIndex,
1456
1607
  message: "content" in message ? message.content : "",
1457
1608
  role: "user"
@@ -1472,6 +1623,7 @@ var MicdropRecorder = class extends EventEmitter4 {
1472
1623
  const buffer = Buffer.concat(this.currentAssistantChunks);
1473
1624
  const audioMessage = {
1474
1625
  buffer,
1626
+ sampleRate: this.currentAssistantRate,
1475
1627
  messageIndex: this.lastAssistantMessageIndex,
1476
1628
  message: "content" in message ? message.content : "",
1477
1629
  role: "assistant"
@@ -1482,6 +1634,7 @@ var MicdropRecorder = class extends EventEmitter4 {
1482
1634
  this.audioMessages.push(audioMessage);
1483
1635
  this.emit("AudioMessage", audioMessage);
1484
1636
  this.currentAssistantChunks = [];
1637
+ this.assistantResampler = void 0;
1485
1638
  this.lastAssistantMessageIndex = -1;
1486
1639
  }
1487
1640
  getAudioMessages() {
@@ -1500,56 +1653,6 @@ var MicdropRecorder = class extends EventEmitter4 {
1500
1653
  }
1501
1654
  };
1502
1655
 
1503
- // src/SharedInstances.ts
1504
- var SharedInstances = class {
1505
- constructor() {
1506
- this.entries = /* @__PURE__ */ new Map();
1507
- }
1508
- /**
1509
- * Returns the instance for this key, calling `create` the first time.
1510
- *
1511
- * The key can be any JSON-serializable value, usually an array of the
1512
- * options that define the instance. Concurrent callers get the same promise
1513
- * instead of loading the model twice.
1514
- */
1515
- load(key, create) {
1516
- const id = typeof key === "string" ? key : JSON.stringify(key);
1517
- const existing = this.entries.get(id);
1518
- if (existing) return existing;
1519
- let loading;
1520
- try {
1521
- loading = create();
1522
- } catch (error) {
1523
- return Promise.reject(error);
1524
- }
1525
- loading.catch(() => {
1526
- if (this.entries.get(id) === loading) this.entries.delete(id);
1527
- });
1528
- this.entries.set(id, loading);
1529
- return loading;
1530
- }
1531
- /** The instance for this key, loaded or being loaded, without loading it. */
1532
- get(key) {
1533
- return this.entries.get(typeof key === "string" ? key : JSON.stringify(key));
1534
- }
1535
- /** Whether an instance is loaded, or being loaded, for this key. */
1536
- has(key) {
1537
- return this.entries.has(typeof key === "string" ? key : JSON.stringify(key));
1538
- }
1539
- /**
1540
- * Forgets the instance for this key, so the next call loads a new one. For
1541
- * an instance that broke, a crashed worker for instance. Releasing what it
1542
- * holds is left to the caller.
1543
- */
1544
- delete(key) {
1545
- this.entries.delete(typeof key === "string" ? key : JSON.stringify(key));
1546
- }
1547
- /** Drops every instance, so the next call loads again. */
1548
- clear() {
1549
- this.entries.clear();
1550
- }
1551
- };
1552
-
1553
1656
  // src/socket/ReconnectingSocket.ts
1554
1657
  import { EventEmitter as EventEmitter5 } from "eventemitter3";
1555
1658
  import WebSocket from "ws";
@@ -1907,6 +2010,16 @@ import { PassThrough as PassThrough4 } from "stream";
1907
2010
  // src/tts/TTS.ts
1908
2011
  import { EventEmitter as EventEmitter7 } from "eventemitter3";
1909
2012
  var TTS = class extends EventEmitter7 {
2013
+ /**
2014
+ * Rate of the audio emitted, in Hz.
2015
+ *
2016
+ * The server reads it with each chunk and tells the client when it changes,
2017
+ * so the audio reaches the client at the rate the provider produced it. A
2018
+ * provider that offers several rates takes the one chosen in its options.
2019
+ */
2020
+ get sampleRate() {
2021
+ return DEFAULT_SAMPLE_RATE;
2022
+ }
1910
2023
  log(...message) {
1911
2024
  this.logger?.log(...message);
1912
2025
  }
@@ -1948,6 +2061,10 @@ var FallbackTTS = class extends TTS {
1948
2061
  }
1949
2062
  this.startNextTTS();
1950
2063
  }
2064
+ /** Rate of the TTS in use, which changes when it falls back to the next */
2065
+ get sampleRate() {
2066
+ return this.tts?.sampleRate ?? super.sampleRate;
2067
+ }
1951
2068
  speak(textStream) {
1952
2069
  const utterance = {
1953
2070
  ended: false,
@@ -1991,13 +2108,17 @@ var FallbackTTS = class extends TTS {
1991
2108
  // src/tts/MockTTS.ts
1992
2109
  import * as fs from "fs";
1993
2110
  var MockTTS = class extends TTS {
1994
- constructor(audioFilePaths) {
2111
+ constructor(audioFilePaths, rate = DEFAULT_SAMPLE_RATE) {
1995
2112
  super();
1996
2113
  this.audioFilePaths = audioFilePaths;
2114
+ this.rate = rate;
1997
2115
  // Bumped by every answer and every cancel, so the chunks of an answer that
1998
2116
  // is no longer wanted stop there
1999
2117
  this.generation = 0;
2000
2118
  }
2119
+ get sampleRate() {
2120
+ return this.rate;
2121
+ }
2001
2122
  speak(textStream) {
2002
2123
  const generation = ++this.generation;
2003
2124
  textStream.once("data", async () => {
@@ -2151,6 +2272,29 @@ var SentenceTTS = class extends TTS {
2151
2272
  }
2152
2273
  };
2153
2274
 
2275
+ // src/tts/voices.ts
2276
+ function voiceGender(value) {
2277
+ const word = value?.toLowerCase();
2278
+ if (!word) return void 0;
2279
+ if (word.startsWith("fem") || word === "woman" || word === "f")
2280
+ return "female";
2281
+ if (word.startsWith("masc") || word === "male" || word === "man" || word === "m") {
2282
+ return "male";
2283
+ }
2284
+ if (word.includes("neutral") || word === "androgynous") return "neutral";
2285
+ return void 0;
2286
+ }
2287
+ function cachedVoices(key, apiKey, load, refresh = false) {
2288
+ return cachedList(["voices", ...key], apiKey, load, refresh);
2289
+ }
2290
+ function voicesForLanguage(voices, language) {
2291
+ if (!language) return voices;
2292
+ const base = language.split("-")[0].toLowerCase();
2293
+ return voices.filter(
2294
+ (voice) => !voice.languages?.length || voice.languages.some((code) => code.split("-")[0].toLowerCase() === base)
2295
+ );
2296
+ }
2297
+
2154
2298
  // src/waitForParams.ts
2155
2299
  async function waitForParams(socket, validate) {
2156
2300
  return new Promise((resolve, reject) => {
@@ -2179,6 +2323,7 @@ export {
2179
2323
  AUTO_SEMANTIC_TURN_TOOL_NAME,
2180
2324
  Agent,
2181
2325
  Classifier,
2326
+ DEFAULT_SAMPLE_RATE,
2182
2327
  FallbackAgent,
2183
2328
  FallbackSTT,
2184
2329
  FallbackTTS,
@@ -2192,6 +2337,7 @@ export {
2192
2337
  MockAgent,
2193
2338
  MockSTT,
2194
2339
  MockTTS,
2340
+ Pcm16Aligner,
2195
2341
  Pcm16Resampler,
2196
2342
  Realtime,
2197
2343
  ReconnectingSocket,
@@ -2200,12 +2346,17 @@ export {
2200
2346
  SentenceTTS,
2201
2347
  SharedInstances,
2202
2348
  TTS,
2349
+ cachedList,
2350
+ cachedModels,
2351
+ cachedVoices,
2203
2352
  currentTurn,
2204
2353
  float32ToPcm16,
2205
2354
  getTurnClassification,
2206
2355
  handleError,
2207
2356
  pcm16ToFloat32,
2208
2357
  turnInput,
2358
+ voiceGender,
2359
+ voicesForLanguage,
2209
2360
  waitForParams
2210
2361
  };
2211
2362
  //# sourceMappingURL=index.mjs.map