@micdrop/server 2.8.0 → 2.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -37,6 +37,7 @@ __export(index_exports, {
37
37
  AUTO_SEMANTIC_TURN_PROMPT: () => AUTO_SEMANTIC_TURN_PROMPT,
38
38
  AUTO_SEMANTIC_TURN_TOOL_NAME: () => AUTO_SEMANTIC_TURN_TOOL_NAME,
39
39
  Agent: () => Agent,
40
+ Classifier: () => Classifier,
40
41
  FallbackAgent: () => FallbackAgent,
41
42
  FallbackSTT: () => FallbackSTT,
42
43
  FallbackTTS: () => FallbackTTS,
@@ -52,14 +53,18 @@ __export(index_exports, {
52
53
  MockTTS: () => MockTTS,
53
54
  Pcm16Resampler: () => Pcm16Resampler,
54
55
  Realtime: () => Realtime,
56
+ ReconnectingSocket: () => ReconnectingSocket,
55
57
  STT: () => STT,
56
58
  SentenceSplitter: () => SentenceSplitter,
57
59
  SentenceTTS: () => SentenceTTS,
58
60
  SharedInstances: () => SharedInstances,
59
61
  TTS: () => TTS,
62
+ currentTurn: () => currentTurn,
60
63
  float32ToPcm16: () => float32ToPcm16,
64
+ getTurnClassification: () => getTurnClassification,
61
65
  handleError: () => handleError,
62
66
  pcm16ToFloat32: () => pcm16ToFloat32,
67
+ turnInput: () => turnInput,
63
68
  waitForParams: () => waitForParams
64
69
  });
65
70
  module.exports = __toCommonJS(index_exports);
@@ -81,27 +86,65 @@ var Agent = class extends import_eventemitter3.EventEmitter {
81
86
  constructor(options) {
82
87
  super();
83
88
  this.options = options;
89
+ /**
90
+ * @deprecated Read the signal given to `generateAnswer` instead. Still
91
+ * bumped by every answer and every cancel, for agents written before it.
92
+ */
84
93
  this.answerCount = 0;
85
- this.answering = false;
86
94
  this.conversation = [{ role: "system", content: options.systemPrompt }];
87
95
  this.tools = this.getDefaultTools();
88
96
  }
97
+ /** Whether an answer is being generated */
98
+ get isAnswering() {
99
+ return !!this.answerController;
100
+ }
101
+ /** @deprecated Read `isAnswering` instead */
102
+ get answering() {
103
+ return this.isAnswering;
104
+ }
105
+ /** @deprecated The answer is over once its stream ends or it is cancelled */
106
+ set answering(_value) {
107
+ }
108
+ /**
109
+ * Stops the answer being generated, by aborting its signal.
110
+ *
111
+ * An agent overriding it to stop more than the signal does calls
112
+ * `super.cancel()`.
113
+ */
114
+ cancel() {
115
+ this.answerCount++;
116
+ if (!this.answerController) return;
117
+ this.log("Cancel");
118
+ this.answerController.abort();
119
+ this.answerController = void 0;
120
+ }
89
121
  answer() {
90
122
  this.log("Start answering");
91
- const answerCount = ++this.answerCount;
123
+ this.answerCount++;
124
+ this.answerController?.abort();
125
+ const controller = new AbortController();
126
+ this.answerController = controller;
127
+ const { signal } = controller;
92
128
  const stream = new import_stream.PassThrough();
93
- this.answering = true;
94
129
  Promise.resolve().then(
95
130
  () => this.options.onBeforeAnswer?.bind(this)(stream)
96
- ).then((skip) => {
97
- if (skip) return;
98
- return this.generateAnswer(stream);
131
+ ).then((result) => {
132
+ if (signal.aborted) return;
133
+ if (typeof result === "string") {
134
+ stream.write(result);
135
+ this.addAssistantMessage(result);
136
+ return;
137
+ }
138
+ if (result) return;
139
+ return this.generateAnswer(stream, signal);
140
+ }).catch((error) => {
141
+ if (!signal.aborted) this.log("Error answering:", error);
99
142
  }).finally(() => {
100
143
  if (stream.writable) {
101
144
  stream.end();
102
145
  }
103
- if (answerCount === this.answerCount) {
104
- this.answering = false;
146
+ if (this.answerController === controller) {
147
+ this.answerController = void 0;
105
148
  }
106
149
  });
107
150
  return stream;
@@ -327,10 +370,10 @@ var FallbackAgent = class extends Agent {
327
370
  extract(message) {
328
371
  return this.agent ? this.agent.extract(message) : super.extract(message);
329
372
  }
330
- async generateAnswer(stream) {
373
+ async generateAnswer(stream, signal) {
331
374
  for (let attempt = 0; attempt < this.fallbackOptions.factories.length; attempt++) {
332
375
  const agent = this.agent;
333
- if (!agent) return;
376
+ if (!agent || signal.aborted) return;
334
377
  let failed = false;
335
378
  const onFailed = () => {
336
379
  failed = true;
@@ -341,7 +384,7 @@ var FallbackAgent = class extends Agent {
341
384
  } finally {
342
385
  agent.off("Failed", onFailed);
343
386
  }
344
- if (!failed) return;
387
+ if (!failed || signal.aborted) return;
345
388
  this.log("Agent failed, trying next agent");
346
389
  this.startNextAgent();
347
390
  }
@@ -349,6 +392,7 @@ var FallbackAgent = class extends Agent {
349
392
  this.emit("Failed");
350
393
  }
351
394
  cancel() {
395
+ super.cancel();
352
396
  this.agent?.cancel();
353
397
  }
354
398
  destroy() {
@@ -411,13 +455,149 @@ var MockAgent = class extends Agent {
411
455
  }
412
456
  async generateAnswer(stream) {
413
457
  const message = `Assistant Message ${this.i++}`;
414
- this.addAssistantMessage(message);
415
458
  stream.write(message);
459
+ this.addAssistantMessage(message);
460
+ }
461
+ };
462
+
463
+ // src/classifier/Classifier.ts
464
+ var import_eventemitter32 = require("eventemitter3");
465
+ var Classifier = class extends import_eventemitter32.EventEmitter {
466
+ constructor() {
467
+ super(...arguments);
468
+ this.controllers = /* @__PURE__ */ new Set();
469
+ }
470
+ /**
471
+ * Classifies an input, and emits the result as `Classification`
472
+ * @param input - What to classify: a text, or JSON with named fields
473
+ * @returns The classification, or undefined when cancelled or failed
474
+ */
475
+ classify(input) {
476
+ const controller = new AbortController();
477
+ this.controllers.add(controller);
478
+ const pending = this.run(input, controller.signal).finally(() => {
479
+ this.controllers.delete(controller);
480
+ if (this._pending === pending) this._pending = void 0;
481
+ });
482
+ this._pending = pending;
483
+ return pending;
484
+ }
485
+ /** The last classification started, until it is done */
486
+ get pending() {
487
+ return this._pending;
416
488
  }
489
+ /** Drops every classification in progress, which then emit nothing */
417
490
  cancel() {
491
+ this.controllers.forEach((controller) => controller.abort());
492
+ this.controllers.clear();
493
+ this._pending = void 0;
494
+ }
495
+ log(...message) {
496
+ this.logger?.log(...message);
497
+ }
498
+ destroy() {
499
+ this.log("Destroyed");
500
+ this.cancel();
501
+ this.removeAllListeners();
502
+ }
503
+ async run(input, signal) {
504
+ const start = Date.now();
505
+ try {
506
+ const result = await this.evaluate(input, signal);
507
+ if (signal.aborted) return;
508
+ const classification = {
509
+ input,
510
+ result,
511
+ duration: Date.now() - start
512
+ };
513
+ this.log(
514
+ `Classified in ${classification.duration} ms:`,
515
+ JSON.stringify(input)
516
+ );
517
+ this.lastClassification = classification;
518
+ this.emit("Classification", classification);
519
+ return classification;
520
+ } catch (error) {
521
+ if (!signal.aborted) this.log("Classification failed:", error);
522
+ return;
523
+ }
418
524
  }
419
525
  };
420
526
 
527
+ // src/types.ts
528
+ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
529
+ MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
530
+ MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
531
+ MicdropClientCommands2["Mute"] = "Mute";
532
+ return MicdropClientCommands2;
533
+ })(MicdropClientCommands || {});
534
+ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
535
+ MicdropServerCommands2["Message"] = "Message";
536
+ MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
537
+ MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
538
+ MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
539
+ MicdropServerCommands2["EndCall"] = "EndCall";
540
+ MicdropServerCommands2["ToolCall"] = "ToolCall";
541
+ MicdropServerCommands2["Classification"] = "Classification";
542
+ return MicdropServerCommands2;
543
+ })(MicdropServerCommands || {});
544
+ function currentTurn(conversation) {
545
+ let start = conversation.length;
546
+ while (start > 0) {
547
+ const item = conversation[start - 1];
548
+ if (item.role !== "user" || item.metadata?.classification) break;
549
+ start--;
550
+ }
551
+ const messages = conversation.slice(start);
552
+ return {
553
+ transcript: messages.map((message) => message.content).join(" "),
554
+ messages,
555
+ before: conversation.slice(0, start)
556
+ };
557
+ }
558
+ function getTurnClassification(conversation) {
559
+ for (let index = conversation.length - 1; index >= 0; index--) {
560
+ const item = conversation[index];
561
+ if (item.role === "user") return item.metadata?.classification;
562
+ }
563
+ return void 0;
564
+ }
565
+
566
+ // src/classifier/turnInput.ts
567
+ function turnInput(conversation, history = 1) {
568
+ const turn = currentTurn(conversation);
569
+ const input = { history: [], turn: turn.transcript };
570
+ let rest = turn.before;
571
+ for (let n = 0; n < history && rest.length; n++) {
572
+ let end = rest.length;
573
+ const answers = [];
574
+ while (end > 0 && rest[end - 1].role !== "user") {
575
+ const item = rest[end - 1];
576
+ if (item.role === "assistant" && item.content.trim()) {
577
+ answers.unshift(item.content);
578
+ }
579
+ end--;
580
+ }
581
+ let start = end;
582
+ while (start > 0) {
583
+ const item = rest[start - 1];
584
+ if (item.role !== "user") break;
585
+ if (start < end && item.metadata?.classification)
586
+ break;
587
+ start--;
588
+ }
589
+ if (answers.length) {
590
+ input.history.unshift({ role: "assistant", text: answers.join(" ") });
591
+ }
592
+ if (start < end) {
593
+ const text = rest.slice(start, end).map((item) => "content" in item ? item.content : "").join(" ");
594
+ input.history.unshift({ role: "user", text });
595
+ }
596
+ rest = rest.slice(0, start);
597
+ }
598
+ return { input, message: turn.messages[turn.messages.length - 1] };
599
+ }
600
+
421
601
  // src/audio/Pcm16Resampler.ts
422
602
  var Pcm16Resampler = class {
423
603
  // Fractional position into the first sample of the buffer
@@ -505,35 +685,23 @@ function handleError(socket, error) {
505
685
  }
506
686
 
507
687
  // src/MicdropServer.ts
508
- var import_eventemitter32 = require("eventemitter3");
688
+ var import_eventemitter33 = require("eventemitter3");
509
689
  var import_stream2 = require("stream");
510
-
511
- // src/types.ts
512
- var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
513
- MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
514
- MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
515
- MicdropClientCommands2["Mute"] = "Mute";
516
- return MicdropClientCommands2;
517
- })(MicdropClientCommands || {});
518
- var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
519
- MicdropServerCommands2["Message"] = "Message";
520
- MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
521
- MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
522
- MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
523
- MicdropServerCommands2["EndCall"] = "EndCall";
524
- MicdropServerCommands2["ToolCall"] = "ToolCall";
525
- return MicdropServerCommands2;
526
- })(MicdropServerCommands || {});
527
-
528
- // src/MicdropServer.ts
529
690
  var USER_SAMPLE_RATE = 16e3;
530
691
  var DEFAULT_TURN_MAX_WAIT = 4e3;
531
- var MicdropServer = class extends import_eventemitter32.EventEmitter {
692
+ var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
693
+ var MicdropServer = class extends import_eventemitter33.EventEmitter {
532
694
  constructor(socket, config) {
533
695
  super();
534
696
  this.socket = null;
535
697
  this.config = null;
536
698
  this.startTime = Date.now();
699
+ // Bumped by every cancel, so an answer cancelled before its first word can
700
+ // tell, and leave its message to the answer that replaces it
701
+ this.cancelRound = 0;
702
+ // Counts the assistant messages sent, so the partial answer of one already
703
+ // settled is not sent after it
704
+ this.assistantMessages = 0;
537
705
  // Holds the conversation when no agent does, and keeps it readable once the
538
706
  // call is over and the config is gone
539
707
  this.ownConversation = [];
@@ -541,15 +709,22 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
541
709
  this.operationQueue = [];
542
710
  this.isProcessingQueue = false;
543
711
  this.userSpeechChunks = 0;
544
- this.onClose = () => {
712
+ // Bumped whenever the user speaks again or mutes, so a decision taken on an
713
+ // earlier stretch of speech is dropped rather than answering over them
714
+ this.turnRound = 0;
715
+ this.onClose = (code, reason) => {
545
716
  this.releaseHeldTurn();
546
717
  if (!this.config) return;
547
- this.log("Connection closed");
718
+ this.log("Connection closed", {
719
+ code,
720
+ reason: reason?.toString() || void 0
721
+ });
548
722
  const duration = Math.round((Date.now() - this.startTime) / 1e3);
549
723
  const conversation = this.conversation;
550
724
  this.agent?.destroy();
551
725
  this.config.stt?.destroy();
552
726
  this.config.tts?.destroy();
727
+ this.config.classifier?.destroy();
553
728
  this.emit("End", {
554
729
  conversation,
555
730
  duration
@@ -641,6 +816,14 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
641
816
  )
642
817
  );
643
818
  }
819
+ if (config.classifierOptions?.sendToClient) {
820
+ config.classifier?.on(
821
+ "Classification",
822
+ (classification) => this.socket?.send(
823
+ `${"Classification" /* Classification */} ${JSON.stringify(classification)}`
824
+ )
825
+ );
826
+ }
644
827
  queueMicrotask(() => this.sendFirstMessage());
645
828
  socket.on("close", this.onClose);
646
829
  socket.on("message", this.onMessage);
@@ -677,6 +860,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
677
860
  this.processQueue();
678
861
  }
679
862
  cancel() {
863
+ this.cancelRound++;
864
+ this.config?.classifier?.cancel();
680
865
  this.config?.tts?.cancel();
681
866
  this.agent?.cancel();
682
867
  this.operationQueue = [];
@@ -689,6 +874,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
689
874
  this.emit("UserAudio", chunk);
690
875
  }
691
876
  onMute() {
877
+ this.turnRound++;
878
+ this.releaseHeldTurn();
692
879
  this.userSpeechChunks = 0;
693
880
  this.currentUserStream?.end();
694
881
  this.currentUserStream = void 0;
@@ -697,6 +884,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
697
884
  }
698
885
  onStartSpeaking() {
699
886
  if (!this.config) return;
887
+ this.turnRound++;
700
888
  this.userSpeechChunks = 0;
701
889
  this.currentUserStream?.end();
702
890
  this.currentUserStream = new import_stream2.PassThrough();
@@ -748,7 +936,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
748
936
  }
749
937
  /** Answers, unless the sentence sounds like it has more coming */
750
938
  async answerUserTurn() {
939
+ const round = this.turnRound;
751
940
  const complete = await (this.turnComplete ?? Promise.resolve(true));
941
+ if (round !== this.turnRound || !this.config) return;
752
942
  if (!complete) {
753
943
  this.log("Waiting for the rest of the sentence");
754
944
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -780,6 +970,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
780
970
  if (realtime) {
781
971
  realtime.endTurn();
782
972
  } else {
973
+ this.classifyTurn();
783
974
  this.answer();
784
975
  }
785
976
  }
@@ -823,6 +1014,8 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
823
1014
  `${"Message" /* Message */} ${JSON.stringify(message)}`
824
1015
  );
825
1016
  this.emit("Message", message);
1017
+ if (message.role === "assistant") this.assistantMessages++;
1018
+ if (message.role === "user" && this.config?.realtime) this.classifyTurn();
826
1019
  }
827
1020
  sendPartialAnswer(content) {
828
1021
  if (!this.socket || content === "") return;
@@ -843,6 +1036,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
843
1036
  }
844
1037
  }
845
1038
  answer() {
1039
+ if (this.config?.classifierOptions?.waitBeforeAnswer) {
1040
+ this.queueOperation(() => this.waitForClassification());
1041
+ }
846
1042
  this.queueOperation(async () => {
847
1043
  await this._answer();
848
1044
  });
@@ -865,12 +1061,18 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
865
1061
  return;
866
1062
  }
867
1063
  this.lastMessageSpeeched = lastMessage;
1064
+ const round = this.cancelRound;
868
1065
  try {
1066
+ const sentBefore = this.assistantMessages;
869
1067
  const stream = this.config.agent.answer();
870
1068
  if (await hasContent(stream)) {
871
1069
  await this._speak(
872
- this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
1070
+ this.config.partialMessages ? this.forwardPartialAnswer(stream, sentBefore) : stream
873
1071
  );
1072
+ } else if (round !== this.cancelRound) {
1073
+ if (this.lastMessageSpeeched === lastMessage) {
1074
+ this.lastMessageSpeeched = void 0;
1075
+ }
874
1076
  } else {
875
1077
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
876
1078
  }
@@ -879,10 +1081,19 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
879
1081
  throw error;
880
1082
  }
881
1083
  }
882
- /** Sends the answer to the client as it is written, ahead of the voice */
883
- forwardPartialAnswer(stream) {
1084
+ /**
1085
+ * Sends the answer to the client as it is written, ahead of the voice.
1086
+ *
1087
+ * The stream reaches here a moment after the agent wrote it, so the agent
1088
+ * can record the complete message first, which the client then shows. A
1089
+ * partial answer arriving after it would show the answer twice, so the
1090
+ * forwarding stops once the message is sent.
1091
+ */
1092
+ forwardPartialAnswer(stream, sentBefore) {
884
1093
  let content = "";
885
- const send = (text) => this.sendPartialAnswer(text);
1094
+ const send = (text) => {
1095
+ if (this.assistantMessages === sentBefore) this.sendPartialAnswer(text);
1096
+ };
886
1097
  const forward = new import_stream2.Transform({
887
1098
  transform(chunk, _encoding, callback) {
888
1099
  content += chunk.toString();
@@ -892,6 +1103,38 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
892
1103
  });
893
1104
  return stream.pipe(forward);
894
1105
  }
1106
+ /**
1107
+ * Classifies the turn that just ended, with the one before, and keeps the
1108
+ * result in the metadata of its last message
1109
+ */
1110
+ classifyTurn() {
1111
+ const classifier = this.config?.classifier;
1112
+ if (!classifier) return;
1113
+ const { input, message } = turnInput(
1114
+ this.conversation,
1115
+ this.config?.classifierOptions?.history
1116
+ );
1117
+ if (!message || input.turn.trim() === "") return;
1118
+ classifier.classify(input).then((classification) => {
1119
+ if (!classification) return;
1120
+ message.metadata = { ...message.metadata, classification };
1121
+ });
1122
+ }
1123
+ /** Waits for the classification in progress, never longer than its limit */
1124
+ async waitForClassification() {
1125
+ const pending = this.config?.classifier?.pending;
1126
+ if (!pending) return;
1127
+ const limit = this.config?.classifierOptions?.maxWait ?? DEFAULT_CLASSIFICATION_MAX_WAIT;
1128
+ let timer;
1129
+ const timeout = new Promise((resolve) => {
1130
+ timer = setTimeout(() => {
1131
+ this.log(`Classification took over ${limit} ms, answering without it`);
1132
+ resolve();
1133
+ }, limit);
1134
+ });
1135
+ await Promise.race([pending, timeout]);
1136
+ clearTimeout(timer);
1137
+ }
895
1138
  // Run text-to-speech and send to client
896
1139
  speak(message) {
897
1140
  this.queueOperation(async () => {
@@ -1028,6 +1271,7 @@ var Realtime = class extends Agent {
1028
1271
  });
1029
1272
  }
1030
1273
  cancel() {
1274
+ super.cancel();
1031
1275
  this.queue = [];
1032
1276
  const answer = this.currentAnswer;
1033
1277
  if (!answer) return;
@@ -1208,8 +1452,8 @@ var Realtime = class extends Agent {
1208
1452
  };
1209
1453
 
1210
1454
  // src/recorder/MicdropRecorder.ts
1211
- var import_eventemitter33 = require("eventemitter3");
1212
- var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
1455
+ var import_eventemitter34 = require("eventemitter3");
1456
+ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
1213
1457
  constructor(server) {
1214
1458
  super();
1215
1459
  this.server = server;
@@ -1355,19 +1599,281 @@ var SharedInstances = class {
1355
1599
  this.entries.set(id, loading);
1356
1600
  return loading;
1357
1601
  }
1602
+ /** The instance for this key, loaded or being loaded, without loading it. */
1603
+ get(key) {
1604
+ return this.entries.get(typeof key === "string" ? key : JSON.stringify(key));
1605
+ }
1358
1606
  /** Whether an instance is loaded, or being loaded, for this key. */
1359
1607
  has(key) {
1360
1608
  return this.entries.has(typeof key === "string" ? key : JSON.stringify(key));
1361
1609
  }
1610
+ /**
1611
+ * Forgets the instance for this key, so the next call loads a new one. For
1612
+ * an instance that broke, a crashed worker for instance. Releasing what it
1613
+ * holds is left to the caller.
1614
+ */
1615
+ delete(key) {
1616
+ this.entries.delete(typeof key === "string" ? key : JSON.stringify(key));
1617
+ }
1362
1618
  /** Drops every instance, so the next call loads again. */
1363
1619
  clear() {
1364
1620
  this.entries.clear();
1365
1621
  }
1366
1622
  };
1367
1623
 
1624
+ // src/socket/ReconnectingSocket.ts
1625
+ var import_eventemitter35 = require("eventemitter3");
1626
+ var import_ws = __toESM(require("ws"));
1627
+ var DEFAULT_CONNECTION_TIMEOUT = 5e3;
1628
+ var DEFAULT_RETRY_DELAY = 1e3;
1629
+ var DEFAULT_MAX_RETRY = 3;
1630
+ var ReconnectingSocket = class extends import_eventemitter35.EventEmitter {
1631
+ constructor(options) {
1632
+ super();
1633
+ this.options = options;
1634
+ this._status = "idle";
1635
+ this.everOpened = false;
1636
+ this.retryCount = 0;
1637
+ // Numbers the attempts, so one overtaken while it awaited its address stops
1638
+ this.attemptId = 0;
1639
+ this.waiters = [];
1640
+ }
1641
+ get status() {
1642
+ return this._status;
1643
+ }
1644
+ /** Whether a message sent now goes out */
1645
+ get isOpen() {
1646
+ return this._status === "open";
1647
+ }
1648
+ /**
1649
+ * Opens the connection. Called again after `failed`, it starts over with a
1650
+ * full retry budget, and does nothing in any other status.
1651
+ */
1652
+ connect() {
1653
+ if (this._status !== "idle" && this._status !== "failed") return;
1654
+ this.retryCount = 0;
1655
+ this.attempt();
1656
+ }
1657
+ /**
1658
+ * Waits for the connection to be open.
1659
+ * @returns True once open, false if it failed or was destroyed meanwhile
1660
+ */
1661
+ ready() {
1662
+ if (this._status === "open") return Promise.resolve(true);
1663
+ if (this._status === "destroyed" || this._status === "failed") {
1664
+ return Promise.resolve(false);
1665
+ }
1666
+ return new Promise((resolve) => this.waiters.push(resolve));
1667
+ }
1668
+ /**
1669
+ * Sends a message if the connection is open
1670
+ * @returns False when it did not go out
1671
+ */
1672
+ send(data) {
1673
+ if (this._status !== "open" || !this.socket) return false;
1674
+ this.socket.send(data);
1675
+ return true;
1676
+ }
1677
+ /**
1678
+ * Drops the connection and opens a new one right away, for a provider that
1679
+ * can only stop what it is doing by closing. It costs no retry, and the new
1680
+ * connection is not a reconnection.
1681
+ */
1682
+ restart() {
1683
+ if (this._status === "destroyed") return;
1684
+ this.log("Restarting the connection");
1685
+ this.clearRetryTimer();
1686
+ this.drop();
1687
+ this.everOpened = false;
1688
+ this.retryCount = 0;
1689
+ this.attempt();
1690
+ }
1691
+ /** Closes the connection for good */
1692
+ destroy() {
1693
+ if (this._status === "destroyed") return;
1694
+ this.clearRetryTimer();
1695
+ this.drop();
1696
+ this.setStatus("destroyed");
1697
+ this.settleWaiters(false);
1698
+ this.removeAllListeners();
1699
+ }
1700
+ async attempt() {
1701
+ const id = ++this.attemptId;
1702
+ const isOvertaken = () => id !== this.attemptId || this._status !== "connecting";
1703
+ this.setStatus("connecting");
1704
+ let url;
1705
+ try {
1706
+ const { url: getUrl } = this.options;
1707
+ url = typeof getUrl === "string" ? getUrl : await getUrl();
1708
+ } catch (error) {
1709
+ if (isOvertaken()) return;
1710
+ this.onAttemptFailed(error);
1711
+ return;
1712
+ }
1713
+ if (isOvertaken()) return;
1714
+ const socket = new import_ws.default(url, this.options.clientOptions);
1715
+ this.socket = socket;
1716
+ let settled = false;
1717
+ const fail = (error) => {
1718
+ if (settled || this.socket !== socket) return;
1719
+ settled = true;
1720
+ clearTimeout(timeout);
1721
+ this.drop();
1722
+ this.onAttemptFailed(error);
1723
+ };
1724
+ const timeout = setTimeout(
1725
+ () => fail(new Error("Connection timeout")),
1726
+ this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
1727
+ );
1728
+ socket.on("error", (error) => {
1729
+ this.log("WebSocket error:", error.message);
1730
+ fail(error);
1731
+ });
1732
+ const { handshake } = this.options;
1733
+ const open = () => {
1734
+ settled = true;
1735
+ clearTimeout(timeout);
1736
+ this.onOpen();
1737
+ };
1738
+ socket.on("open", () => {
1739
+ if (this.socket !== socket) return;
1740
+ try {
1741
+ handshake?.start?.(socket);
1742
+ } catch (error) {
1743
+ fail(error);
1744
+ return;
1745
+ }
1746
+ if (!handshake?.read) open();
1747
+ });
1748
+ socket.on("message", (data, isBinary) => {
1749
+ if (this.socket !== socket) return;
1750
+ if (this._status === "open") {
1751
+ this.prove();
1752
+ this.emit("Message", data, isBinary);
1753
+ return;
1754
+ }
1755
+ if (settled || !handshake?.read) return;
1756
+ try {
1757
+ if (handshake.read(data, isBinary)) open();
1758
+ } catch (error) {
1759
+ fail(error);
1760
+ }
1761
+ });
1762
+ socket.on("close", (code, reasonBuffer) => {
1763
+ const reason = reasonBuffer.toString();
1764
+ if (!settled) {
1765
+ fail(new Error(`Connection closed (${code}) ${reason}`));
1766
+ return;
1767
+ }
1768
+ if (this.socket !== socket) return;
1769
+ this.log("Connection closed", { code, reason });
1770
+ this.drop();
1771
+ this.setStatus("reconnecting");
1772
+ this.emit("Close", code, reason);
1773
+ if (this._status !== "reconnecting") return;
1774
+ this.scheduleRetry(new Error(`Connection closed (${code}) ${reason}`));
1775
+ });
1776
+ }
1777
+ onOpen() {
1778
+ this.log("Connection opened");
1779
+ const reconnected = this.everOpened;
1780
+ this.everOpened = true;
1781
+ if (this.options.handshake?.read) {
1782
+ this.prove();
1783
+ } else {
1784
+ this.proofTimer = setTimeout(
1785
+ () => this.prove(),
1786
+ this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
1787
+ );
1788
+ }
1789
+ this.startKeepAlive();
1790
+ this.setStatus("open");
1791
+ this.emit("Open", { reconnected });
1792
+ this.settleWaiters(true);
1793
+ }
1794
+ /** The connection works, the retry budget is whole again */
1795
+ prove() {
1796
+ this.clearProofTimer();
1797
+ this.retryCount = 0;
1798
+ }
1799
+ clearProofTimer() {
1800
+ if (!this.proofTimer) return;
1801
+ clearTimeout(this.proofTimer);
1802
+ this.proofTimer = void 0;
1803
+ }
1804
+ onAttemptFailed(error) {
1805
+ this.log("Connection failed:", error.message);
1806
+ this.scheduleRetry(error);
1807
+ }
1808
+ scheduleRetry(error) {
1809
+ if (this._status === "destroyed") return;
1810
+ this.retryCount++;
1811
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
1812
+ this.log("Max retries reached, giving up");
1813
+ this.setStatus("failed");
1814
+ this.settleWaiters(false);
1815
+ this.emit("Failed", error);
1816
+ return;
1817
+ }
1818
+ this.setStatus("reconnecting");
1819
+ this.log(`Reconnecting (attempt ${this.retryCount})`);
1820
+ this.clearRetryTimer();
1821
+ this.retryTimer = setTimeout(() => {
1822
+ this.retryTimer = void 0;
1823
+ if (this._status === "reconnecting") this.attempt();
1824
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
1825
+ }
1826
+ /** Lets go of the current socket, which will never be heard from again */
1827
+ drop() {
1828
+ this.stopKeepAlive();
1829
+ this.clearProofTimer();
1830
+ const socket = this.socket;
1831
+ this.socket = void 0;
1832
+ if (!socket) return;
1833
+ socket.removeAllListeners();
1834
+ socket.on("error", () => {
1835
+ });
1836
+ if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
1837
+ socket.close(1e3);
1838
+ }
1839
+ }
1840
+ startKeepAlive() {
1841
+ const keepAlive = this.options.keepAlive;
1842
+ if (!keepAlive) return;
1843
+ this.stopKeepAlive();
1844
+ this.keepAliveTimer = setInterval(() => {
1845
+ const { message } = keepAlive;
1846
+ this.send(typeof message === "string" ? message : message());
1847
+ }, keepAlive.interval);
1848
+ }
1849
+ stopKeepAlive() {
1850
+ if (!this.keepAliveTimer) return;
1851
+ clearInterval(this.keepAliveTimer);
1852
+ this.keepAliveTimer = void 0;
1853
+ }
1854
+ clearRetryTimer() {
1855
+ if (!this.retryTimer) return;
1856
+ clearTimeout(this.retryTimer);
1857
+ this.retryTimer = void 0;
1858
+ }
1859
+ settleWaiters(open) {
1860
+ const waiters = this.waiters;
1861
+ this.waiters = [];
1862
+ waiters.forEach((resolve) => resolve(open));
1863
+ }
1864
+ setStatus(status) {
1865
+ if (this._status === status) return;
1866
+ this._status = status;
1867
+ this.emit("StatusChange", status);
1868
+ }
1869
+ log(...message) {
1870
+ this.options.log?.(...message);
1871
+ }
1872
+ };
1873
+
1368
1874
  // src/stt/STT.ts
1369
- var import_eventemitter34 = require("eventemitter3");
1370
- var STT = class extends import_eventemitter34.EventEmitter {
1875
+ var import_eventemitter36 = require("eventemitter3");
1876
+ var STT = class extends import_eventemitter36.EventEmitter {
1371
1877
  log(...message) {
1372
1878
  this.logger?.log(...message);
1373
1879
  }
@@ -1393,24 +1899,31 @@ var MockSTT = class extends STT {
1393
1899
  // src/stt/FallbackSTT.ts
1394
1900
  var import_stream3 = require("stream");
1395
1901
  var FallbackSTT = class extends STT {
1396
- // Start at -1 because we need to increment it before using it
1397
1902
  constructor(options) {
1398
1903
  super();
1399
1904
  this.options = options;
1400
1905
  this.stt = null;
1401
1906
  this.sttIndex = -1;
1907
+ // Start at -1 because we need to increment it before using it
1908
+ // In the order spoken, each STT giving one transcript per utterance in turn
1909
+ this.pending = [];
1402
1910
  this.onTranscript = (transcript) => {
1911
+ this.pending.shift();
1403
1912
  this.emit("Transcript", transcript);
1404
1913
  };
1405
- this.onFailed = (chunks) => {
1914
+ /**
1915
+ * Hands every utterance still waiting to the next STT, one stream each so
1916
+ * each gets its own transcript. One still being spoken keeps going there.
1917
+ */
1918
+ this.onFailed = () => {
1406
1919
  this.log("STT failed, trying next STT");
1407
1920
  this.startNextSTT();
1408
- if (chunks.length > 0) {
1409
- this.log("Sending audio chunks again");
1410
- const stream = new import_stream3.PassThrough();
1411
- this.stt?.transcribe(stream);
1412
- chunks.forEach((chunk) => stream.write(chunk));
1413
- stream.end();
1921
+ if (this.pending.length > 0) this.log("Sending audio chunks again");
1922
+ for (const utterance of this.pending) {
1923
+ utterance.stream = new import_stream3.PassThrough();
1924
+ this.stt?.transcribe(utterance.stream);
1925
+ utterance.chunks.forEach((chunk) => utterance.stream.write(chunk));
1926
+ if (utterance.ended) utterance.stream.end();
1414
1927
  }
1415
1928
  };
1416
1929
  if (this.options.factories.length === 0) {
@@ -1419,13 +1932,28 @@ var FallbackSTT = class extends STT {
1419
1932
  this.startNextSTT();
1420
1933
  }
1421
1934
  transcribe(audioStream) {
1422
- this.stt?.transcribe(audioStream);
1935
+ const utterance = {
1936
+ chunks: [],
1937
+ ended: false,
1938
+ stream: new import_stream3.PassThrough()
1939
+ };
1940
+ this.pending.push(utterance);
1941
+ audioStream.on("data", (chunk) => {
1942
+ utterance.chunks.push(chunk);
1943
+ utterance.stream.write(chunk);
1944
+ });
1945
+ audioStream.on("end", () => {
1946
+ utterance.ended = true;
1947
+ utterance.stream.end();
1948
+ });
1949
+ this.stt?.transcribe(utterance.stream);
1423
1950
  }
1424
1951
  destroy() {
1425
1952
  super.destroy();
1426
1953
  this.stt?.destroy();
1427
1954
  this.stt = null;
1428
1955
  this.sttIndex = -1;
1956
+ this.pending = [];
1429
1957
  }
1430
1958
  startNextSTT() {
1431
1959
  this.sttIndex++;
@@ -1448,8 +1976,8 @@ var FallbackSTT = class extends STT {
1448
1976
  var import_stream4 = require("stream");
1449
1977
 
1450
1978
  // src/tts/TTS.ts
1451
- var import_eventemitter35 = require("eventemitter3");
1452
- var TTS = class extends import_eventemitter35.EventEmitter {
1979
+ var import_eventemitter37 = require("eventemitter3");
1980
+ var TTS = class extends import_eventemitter37.EventEmitter {
1453
1981
  log(...message) {
1454
1982
  this.logger?.log(...message);
1455
1983
  }
@@ -1461,7 +1989,6 @@ var TTS = class extends import_eventemitter35.EventEmitter {
1461
1989
 
1462
1990
  // src/tts/FallbackTTS.ts
1463
1991
  var FallbackTTS = class extends TTS {
1464
- // Start at -1 because we need to increment it before using it
1465
1992
  constructor(options) {
1466
1993
  super();
1467
1994
  this.options = options;
@@ -1470,16 +1997,22 @@ var FallbackTTS = class extends TTS {
1470
1997
  this.onAudio = (audio) => {
1471
1998
  this.emit("Audio", audio);
1472
1999
  };
2000
+ /**
2001
+ * Hands the unspoken text to the next TTS. An utterance whose text is still
2002
+ * being written goes on there, so its end is spoken too.
2003
+ */
1473
2004
  this.onFailed = (chunks) => {
1474
2005
  this.log("TTS failed, trying next TTS");
1475
2006
  this.startNextTTS();
1476
- if (chunks.length > 0) {
1477
- this.log("Sending text chunks again");
1478
- const stream = new import_stream4.PassThrough();
1479
- this.tts?.speak(stream);
1480
- chunks.forEach((chunk) => stream.write(chunk));
1481
- stream.end();
1482
- }
2007
+ const utterance = this.current;
2008
+ const continues = !!utterance && !utterance.ended;
2009
+ if (chunks.length === 0 && !continues) return;
2010
+ this.log("Sending text chunks again");
2011
+ const stream = new import_stream4.PassThrough();
2012
+ this.tts?.speak(stream);
2013
+ chunks.forEach((chunk) => stream.write(chunk));
2014
+ if (continues) utterance.stream = stream;
2015
+ else stream.end();
1483
2016
  };
1484
2017
  if (this.options.factories.length === 0) {
1485
2018
  throw new Error("FallbackTTS: No factories provided");
@@ -1487,9 +2020,20 @@ var FallbackTTS = class extends TTS {
1487
2020
  this.startNextTTS();
1488
2021
  }
1489
2022
  speak(textStream) {
1490
- this.tts?.speak(textStream);
2023
+ const utterance = {
2024
+ ended: false,
2025
+ stream: new import_stream4.PassThrough()
2026
+ };
2027
+ this.current = utterance;
2028
+ textStream.on("data", (chunk) => utterance.stream.write(chunk));
2029
+ textStream.on("end", () => {
2030
+ utterance.ended = true;
2031
+ utterance.stream.end();
2032
+ });
2033
+ this.tts?.speak(utterance.stream);
1491
2034
  }
1492
2035
  cancel() {
2036
+ this.current = void 0;
1493
2037
  this.tts?.cancel();
1494
2038
  }
1495
2039
  destroy() {
@@ -1517,26 +2061,28 @@ var FallbackTTS = class extends TTS {
1517
2061
 
1518
2062
  // src/tts/MockTTS.ts
1519
2063
  var fs = __toESM(require("fs"));
1520
- var import_stream5 = require("stream");
1521
2064
  var MockTTS = class extends TTS {
1522
2065
  constructor(audioFilePaths) {
1523
2066
  super();
1524
2067
  this.audioFilePaths = audioFilePaths;
2068
+ // Bumped by every answer and every cancel, so the chunks of an answer that
2069
+ // is no longer wanted stop there
2070
+ this.generation = 0;
1525
2071
  }
1526
2072
  speak(textStream) {
1527
- const audioStream = new import_stream5.PassThrough();
2073
+ const generation = ++this.generation;
1528
2074
  textStream.once("data", async () => {
1529
2075
  for (const filePath of this.audioFilePaths) {
1530
2076
  await new Promise((resolve) => setTimeout(resolve, 200));
1531
- const audioBuffer = fs.readFileSync(filePath);
1532
- this.log(`Loaded chunk (${audioBuffer.length} bytes)`);
1533
- audioStream.write(audioBuffer);
2077
+ if (generation !== this.generation) return;
2078
+ const audio = fs.readFileSync(filePath);
2079
+ this.log(`Loaded chunk (${audio.length} bytes)`);
2080
+ this.emit("Audio", audio);
1534
2081
  }
1535
- audioStream.end();
1536
2082
  });
1537
- return audioStream;
1538
2083
  }
1539
2084
  cancel() {
2085
+ this.generation++;
1540
2086
  }
1541
2087
  };
1542
2088
 
@@ -1615,6 +2161,9 @@ var SentenceTTS = class extends TTS {
1615
2161
  this.counter++;
1616
2162
  counter = this.counter;
1617
2163
  this.splitter.reset();
2164
+ this.queue = [];
2165
+ this.controller?.abort();
2166
+ this.controller = void 0;
1618
2167
  return true;
1619
2168
  };
1620
2169
  textStream.on("data", (chunk) => {
@@ -1701,6 +2250,7 @@ async function waitForParams(socket, validate) {
1701
2250
  AUTO_SEMANTIC_TURN_PROMPT,
1702
2251
  AUTO_SEMANTIC_TURN_TOOL_NAME,
1703
2252
  Agent,
2253
+ Classifier,
1704
2254
  FallbackAgent,
1705
2255
  FallbackSTT,
1706
2256
  FallbackTTS,
@@ -1716,14 +2266,18 @@ async function waitForParams(socket, validate) {
1716
2266
  MockTTS,
1717
2267
  Pcm16Resampler,
1718
2268
  Realtime,
2269
+ ReconnectingSocket,
1719
2270
  STT,
1720
2271
  SentenceSplitter,
1721
2272
  SentenceTTS,
1722
2273
  SharedInstances,
1723
2274
  TTS,
2275
+ currentTurn,
1724
2276
  float32ToPcm16,
2277
+ getTurnClassification,
1725
2278
  handleError,
1726
2279
  pcm16ToFloat32,
2280
+ turnInput,
1727
2281
  waitForParams
1728
2282
  });
1729
2283
  //# sourceMappingURL=index.js.map