@micdrop/server 2.8.0 → 2.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -15,27 +15,65 @@ var Agent = class extends EventEmitter {
15
15
  constructor(options) {
16
16
  super();
17
17
  this.options = options;
18
+ /**
19
+ * @deprecated Read the signal given to `generateAnswer` instead. Still
20
+ * bumped by every answer and every cancel, for agents written before it.
21
+ */
18
22
  this.answerCount = 0;
19
- this.answering = false;
20
23
  this.conversation = [{ role: "system", content: options.systemPrompt }];
21
24
  this.tools = this.getDefaultTools();
22
25
  }
26
+ /** Whether an answer is being generated */
27
+ get isAnswering() {
28
+ return !!this.answerController;
29
+ }
30
+ /** @deprecated Read `isAnswering` instead */
31
+ get answering() {
32
+ return this.isAnswering;
33
+ }
34
+ /** @deprecated The answer is over once its stream ends or it is cancelled */
35
+ set answering(_value) {
36
+ }
37
+ /**
38
+ * Stops the answer being generated, by aborting its signal.
39
+ *
40
+ * An agent overriding it to stop more than the signal does calls
41
+ * `super.cancel()`.
42
+ */
43
+ cancel() {
44
+ this.answerCount++;
45
+ if (!this.answerController) return;
46
+ this.log("Cancel");
47
+ this.answerController.abort();
48
+ this.answerController = void 0;
49
+ }
23
50
  answer() {
24
51
  this.log("Start answering");
25
- const answerCount = ++this.answerCount;
52
+ this.answerCount++;
53
+ this.answerController?.abort();
54
+ const controller = new AbortController();
55
+ this.answerController = controller;
56
+ const { signal } = controller;
26
57
  const stream = new PassThrough();
27
- this.answering = true;
28
58
  Promise.resolve().then(
29
59
  () => this.options.onBeforeAnswer?.bind(this)(stream)
30
- ).then((skip) => {
31
- if (skip) return;
32
- return this.generateAnswer(stream);
60
+ ).then((result) => {
61
+ if (signal.aborted) return;
62
+ if (typeof result === "string") {
63
+ stream.write(result);
64
+ this.addAssistantMessage(result);
65
+ return;
66
+ }
67
+ if (result) return;
68
+ return this.generateAnswer(stream, signal);
69
+ }).catch((error) => {
70
+ if (!signal.aborted) this.log("Error answering:", error);
33
71
  }).finally(() => {
34
72
  if (stream.writable) {
35
73
  stream.end();
36
74
  }
37
- if (answerCount === this.answerCount) {
38
- this.answering = false;
75
+ if (this.answerController === controller) {
76
+ this.answerController = void 0;
39
77
  }
40
78
  });
41
79
  return stream;
@@ -261,10 +299,10 @@ var FallbackAgent = class extends Agent {
261
299
  extract(message) {
262
300
  return this.agent ? this.agent.extract(message) : super.extract(message);
263
301
  }
264
- async generateAnswer(stream) {
302
+ async generateAnswer(stream, signal) {
265
303
  for (let attempt = 0; attempt < this.fallbackOptions.factories.length; attempt++) {
266
304
  const agent = this.agent;
267
- if (!agent) return;
305
+ if (!agent || signal.aborted) return;
268
306
  let failed = false;
269
307
  const onFailed = () => {
270
308
  failed = true;
@@ -275,7 +313,7 @@ var FallbackAgent = class extends Agent {
275
313
  } finally {
276
314
  agent.off("Failed", onFailed);
277
315
  }
278
- if (!failed) return;
316
+ if (!failed || signal.aborted) return;
279
317
  this.log("Agent failed, trying next agent");
280
318
  this.startNextAgent();
281
319
  }
@@ -283,6 +321,7 @@ var FallbackAgent = class extends Agent {
283
321
  this.emit("Failed");
284
322
  }
285
323
  cancel() {
324
+ super.cancel();
286
325
  this.agent?.cancel();
287
326
  }
288
327
  destroy() {
@@ -345,13 +384,149 @@ var MockAgent = class extends Agent {
345
384
  }
346
385
  async generateAnswer(stream) {
347
386
  const message = `Assistant Message ${this.i++}`;
348
- this.addAssistantMessage(message);
349
387
  stream.write(message);
388
+ this.addAssistantMessage(message);
389
+ }
390
+ };
391
+
392
+ // src/classifier/Classifier.ts
393
+ import { EventEmitter as EventEmitter2 } from "eventemitter3";
394
+ var Classifier = class extends EventEmitter2 {
395
+ constructor() {
396
+ super(...arguments);
397
+ this.controllers = /* @__PURE__ */ new Set();
398
+ }
399
+ /**
400
+ * Classifies an input, and emits the result as `Classification`
401
+ * @param input - What to classify: a text, or JSON with named fields
402
+ * @returns The classification, or undefined when cancelled or failed
403
+ */
404
+ classify(input) {
405
+ const controller = new AbortController();
406
+ this.controllers.add(controller);
407
+ const pending = this.run(input, controller.signal).finally(() => {
408
+ this.controllers.delete(controller);
409
+ if (this._pending === pending) this._pending = void 0;
410
+ });
411
+ this._pending = pending;
412
+ return pending;
413
+ }
414
+ /** The last classification started, until it is done */
415
+ get pending() {
416
+ return this._pending;
350
417
  }
418
+ /** Drops every classification in progress, which then emit nothing */
351
419
  cancel() {
420
+ this.controllers.forEach((controller) => controller.abort());
421
+ this.controllers.clear();
422
+ this._pending = void 0;
423
+ }
424
+ log(...message) {
425
+ this.logger?.log(...message);
426
+ }
427
+ destroy() {
428
+ this.log("Destroyed");
429
+ this.cancel();
430
+ this.removeAllListeners();
431
+ }
432
+ async run(input, signal) {
433
+ const start = Date.now();
434
+ try {
435
+ const result = await this.evaluate(input, signal);
436
+ if (signal.aborted) return;
437
+ const classification = {
438
+ input,
439
+ result,
440
+ duration: Date.now() - start
441
+ };
442
+ this.log(
443
+ `Classified in ${classification.duration} ms:`,
444
+ JSON.stringify(input)
445
+ );
446
+ this.lastClassification = classification;
447
+ this.emit("Classification", classification);
448
+ return classification;
449
+ } catch (error) {
450
+ if (!signal.aborted) this.log("Classification failed:", error);
451
+ return;
452
+ }
352
453
  }
353
454
  };
354
455
 
456
+ // src/types.ts
457
+ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
458
+ MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
459
+ MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
460
+ MicdropClientCommands2["Mute"] = "Mute";
461
+ return MicdropClientCommands2;
462
+ })(MicdropClientCommands || {});
463
+ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
464
+ MicdropServerCommands2["Message"] = "Message";
465
+ MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
466
+ MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
467
+ MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
468
+ MicdropServerCommands2["EndCall"] = "EndCall";
469
+ MicdropServerCommands2["ToolCall"] = "ToolCall";
470
+ MicdropServerCommands2["Classification"] = "Classification";
471
+ return MicdropServerCommands2;
472
+ })(MicdropServerCommands || {});
473
+ function currentTurn(conversation) {
474
+ let start = conversation.length;
475
+ while (start > 0) {
476
+ const item = conversation[start - 1];
477
+ if (item.role !== "user" || item.metadata?.classification) break;
478
+ start--;
479
+ }
480
+ const messages = conversation.slice(start);
481
+ return {
482
+ transcript: messages.map((message) => message.content).join(" "),
483
+ messages,
484
+ before: conversation.slice(0, start)
485
+ };
486
+ }
487
+ function getTurnClassification(conversation) {
488
+ for (let index = conversation.length - 1; index >= 0; index--) {
489
+ const item = conversation[index];
490
+ if (item.role === "user") return item.metadata?.classification;
491
+ }
492
+ return void 0;
493
+ }
494
+
495
+ // src/classifier/turnInput.ts
496
+ function turnInput(conversation, history = 1) {
497
+ const turn = currentTurn(conversation);
498
+ const input = { history: [], turn: turn.transcript };
499
+ let rest = turn.before;
500
+ for (let n = 0; n < history && rest.length; n++) {
501
+ let end = rest.length;
502
+ const answers = [];
503
+ while (end > 0 && rest[end - 1].role !== "user") {
504
+ const item = rest[end - 1];
505
+ if (item.role === "assistant" && item.content.trim()) {
506
+ answers.unshift(item.content);
507
+ }
508
+ end--;
509
+ }
510
+ let start = end;
511
+ while (start > 0) {
512
+ const item = rest[start - 1];
513
+ if (item.role !== "user") break;
514
+ if (start < end && item.metadata?.classification)
515
+ break;
516
+ start--;
517
+ }
518
+ if (answers.length) {
519
+ input.history.unshift({ role: "assistant", text: answers.join(" ") });
520
+ }
521
+ if (start < end) {
522
+ const text = rest.slice(start, end).map((item) => "content" in item ? item.content : "").join(" ");
523
+ input.history.unshift({ role: "user", text });
524
+ }
525
+ rest = rest.slice(0, start);
526
+ }
527
+ return { input, message: turn.messages[turn.messages.length - 1] };
528
+ }
529
+
355
530
  // src/audio/Pcm16Resampler.ts
356
531
  var Pcm16Resampler = class {
357
532
  // Fractional position into the first sample of the buffer
@@ -439,35 +614,23 @@ function handleError(socket, error) {
439
614
  }
440
615
 
441
616
  // src/MicdropServer.ts
442
- import { EventEmitter as EventEmitter2 } from "eventemitter3";
617
+ import { EventEmitter as EventEmitter3 } from "eventemitter3";
443
618
  import { PassThrough as PassThrough2, Transform } from "stream";
444
-
445
- // src/types.ts
446
- var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
447
- MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
448
- MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
449
- MicdropClientCommands2["Mute"] = "Mute";
450
- return MicdropClientCommands2;
451
- })(MicdropClientCommands || {});
452
- var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
453
- MicdropServerCommands2["Message"] = "Message";
454
- MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
455
- MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
456
- MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
457
- MicdropServerCommands2["EndCall"] = "EndCall";
458
- MicdropServerCommands2["ToolCall"] = "ToolCall";
459
- return MicdropServerCommands2;
460
- })(MicdropServerCommands || {});
461
-
462
- // src/MicdropServer.ts
463
619
  var USER_SAMPLE_RATE = 16e3;
464
620
  var DEFAULT_TURN_MAX_WAIT = 4e3;
465
- var MicdropServer = class extends EventEmitter2 {
621
+ var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
622
+ var MicdropServer = class extends EventEmitter3 {
466
623
  constructor(socket, config) {
467
624
  super();
468
625
  this.socket = null;
469
626
  this.config = null;
470
627
  this.startTime = Date.now();
628
+ // Bumped by every cancel, so an answer cancelled before its first word can
629
+ // tell, and leave its message to the answer that replaces it
630
+ this.cancelRound = 0;
631
+ // Counts the assistant messages sent, so the partial answer of one already
632
+ // settled is not sent after it
633
+ this.assistantMessages = 0;
471
634
  // Holds the conversation when no agent does, and keeps it readable once the
472
635
  // call is over and the config is gone
473
636
  this.ownConversation = [];
@@ -475,15 +638,22 @@ var MicdropServer = class extends EventEmitter2 {
475
638
  this.operationQueue = [];
476
639
  this.isProcessingQueue = false;
477
640
  this.userSpeechChunks = 0;
478
- this.onClose = () => {
641
+ // Bumped whenever the user speaks again or mutes, so a decision taken on an
642
+ // earlier stretch of speech is dropped rather than answering over them
643
+ this.turnRound = 0;
644
+ this.onClose = (code, reason) => {
479
645
  this.releaseHeldTurn();
480
646
  if (!this.config) return;
481
- this.log("Connection closed");
647
+ this.log("Connection closed", {
648
+ code,
649
+ reason: reason?.toString() || void 0
650
+ });
482
651
  const duration = Math.round((Date.now() - this.startTime) / 1e3);
483
652
  const conversation = this.conversation;
484
653
  this.agent?.destroy();
485
654
  this.config.stt?.destroy();
486
655
  this.config.tts?.destroy();
656
+ this.config.classifier?.destroy();
487
657
  this.emit("End", {
488
658
  conversation,
489
659
  duration
@@ -575,6 +745,14 @@ var MicdropServer = class extends EventEmitter2 {
575
745
  )
576
746
  );
577
747
  }
748
+ if (config.classifierOptions?.sendToClient) {
749
+ config.classifier?.on(
750
+ "Classification",
751
+ (classification) => this.socket?.send(
752
+ `${"Classification" /* Classification */} ${JSON.stringify(classification)}`
753
+ )
754
+ );
755
+ }
578
756
  queueMicrotask(() => this.sendFirstMessage());
579
757
  socket.on("close", this.onClose);
580
758
  socket.on("message", this.onMessage);
@@ -611,6 +789,8 @@ var MicdropServer = class extends EventEmitter2 {
611
789
  this.processQueue();
612
790
  }
613
791
  cancel() {
792
+ this.cancelRound++;
793
+ this.config?.classifier?.cancel();
614
794
  this.config?.tts?.cancel();
615
795
  this.agent?.cancel();
616
796
  this.operationQueue = [];
@@ -623,6 +803,8 @@ var MicdropServer = class extends EventEmitter2 {
623
803
  this.emit("UserAudio", chunk);
624
804
  }
625
805
  onMute() {
806
+ this.turnRound++;
807
+ this.releaseHeldTurn();
626
808
  this.userSpeechChunks = 0;
627
809
  this.currentUserStream?.end();
628
810
  this.currentUserStream = void 0;
@@ -631,6 +813,7 @@ var MicdropServer = class extends EventEmitter2 {
631
813
  }
632
814
  onStartSpeaking() {
633
815
  if (!this.config) return;
816
+ this.turnRound++;
634
817
  this.userSpeechChunks = 0;
635
818
  this.currentUserStream?.end();
636
819
  this.currentUserStream = new PassThrough2();
@@ -682,7 +865,9 @@ var MicdropServer = class extends EventEmitter2 {
682
865
  }
683
866
  /** Answers, unless the sentence sounds like it has more coming */
684
867
  async answerUserTurn() {
868
+ const round = this.turnRound;
685
869
  const complete = await (this.turnComplete ?? Promise.resolve(true));
870
+ if (round !== this.turnRound || !this.config) return;
686
871
  if (!complete) {
687
872
  this.log("Waiting for the rest of the sentence");
688
873
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
@@ -714,6 +899,7 @@ var MicdropServer = class extends EventEmitter2 {
714
899
  if (realtime) {
715
900
  realtime.endTurn();
716
901
  } else {
902
+ this.classifyTurn();
717
903
  this.answer();
718
904
  }
719
905
  }
@@ -757,6 +943,8 @@ var MicdropServer = class extends EventEmitter2 {
757
943
  `${"Message" /* Message */} ${JSON.stringify(message)}`
758
944
  );
759
945
  this.emit("Message", message);
946
+ if (message.role === "assistant") this.assistantMessages++;
947
+ if (message.role === "user" && this.config?.realtime) this.classifyTurn();
760
948
  }
761
949
  sendPartialAnswer(content) {
762
950
  if (!this.socket || content === "") return;
@@ -777,6 +965,9 @@ var MicdropServer = class extends EventEmitter2 {
777
965
  }
778
966
  }
779
967
  answer() {
968
+ if (this.config?.classifierOptions?.waitBeforeAnswer) {
969
+ this.queueOperation(() => this.waitForClassification());
970
+ }
780
971
  this.queueOperation(async () => {
781
972
  await this._answer();
782
973
  });
@@ -799,12 +990,18 @@ var MicdropServer = class extends EventEmitter2 {
799
990
  return;
800
991
  }
801
992
  this.lastMessageSpeeched = lastMessage;
993
+ const round = this.cancelRound;
802
994
  try {
995
+ const sentBefore = this.assistantMessages;
803
996
  const stream = this.config.agent.answer();
804
997
  if (await hasContent(stream)) {
805
998
  await this._speak(
806
- this.config.partialMessages ? this.forwardPartialAnswer(stream) : stream
999
+ this.config.partialMessages ? this.forwardPartialAnswer(stream, sentBefore) : stream
807
1000
  );
1001
+ } else if (round !== this.cancelRound) {
1002
+ if (this.lastMessageSpeeched === lastMessage) {
1003
+ this.lastMessageSpeeched = void 0;
1004
+ }
808
1005
  } else {
809
1006
  this.socket?.send("SkipAnswer" /* SkipAnswer */);
810
1007
  }
@@ -813,10 +1010,19 @@ var MicdropServer = class extends EventEmitter2 {
813
1010
  throw error;
814
1011
  }
815
1012
  }
816
- /** Sends the answer to the client as it is written, ahead of the voice */
817
- forwardPartialAnswer(stream) {
1013
+ /**
1014
+ * Sends the answer to the client as it is written, ahead of the voice.
1015
+ *
1016
+ * The stream reaches here a moment after the agent wrote it, so the agent
1017
+ * can record the complete message first, which the client then shows. A
1018
+ * partial answer arriving after it would show the answer twice, so the
1019
+ * forwarding stops once the message is sent.
1020
+ */
1021
+ forwardPartialAnswer(stream, sentBefore) {
818
1022
  let content = "";
819
- const send = (text) => this.sendPartialAnswer(text);
1023
+ const send = (text) => {
1024
+ if (this.assistantMessages === sentBefore) this.sendPartialAnswer(text);
1025
+ };
820
1026
  const forward = new Transform({
821
1027
  transform(chunk, _encoding, callback) {
822
1028
  content += chunk.toString();
@@ -826,6 +1032,38 @@ var MicdropServer = class extends EventEmitter2 {
826
1032
  });
827
1033
  return stream.pipe(forward);
828
1034
  }
1035
+ /**
1036
+ * Classifies the turn that just ended, with the one before, and keeps the
1037
+ * result in the metadata of its last message
1038
+ */
1039
+ classifyTurn() {
1040
+ const classifier = this.config?.classifier;
1041
+ if (!classifier) return;
1042
+ const { input, message } = turnInput(
1043
+ this.conversation,
1044
+ this.config?.classifierOptions?.history
1045
+ );
1046
+ if (!message || input.turn.trim() === "") return;
1047
+ classifier.classify(input).then((classification) => {
1048
+ if (!classification) return;
1049
+ message.metadata = { ...message.metadata, classification };
1050
+ });
1051
+ }
1052
+ /** Waits for the classification in progress, never longer than its limit */
1053
+ async waitForClassification() {
1054
+ const pending = this.config?.classifier?.pending;
1055
+ if (!pending) return;
1056
+ const limit = this.config?.classifierOptions?.maxWait ?? DEFAULT_CLASSIFICATION_MAX_WAIT;
1057
+ let timer;
1058
+ const timeout = new Promise((resolve) => {
1059
+ timer = setTimeout(() => {
1060
+ this.log(`Classification took over ${limit} ms, answering without it`);
1061
+ resolve();
1062
+ }, limit);
1063
+ });
1064
+ await Promise.race([pending, timeout]);
1065
+ clearTimeout(timer);
1066
+ }
829
1067
  // Run text-to-speech and send to client
830
1068
  speak(message) {
831
1069
  this.queueOperation(async () => {
@@ -962,6 +1200,7 @@ var Realtime = class extends Agent {
962
1200
  });
963
1201
  }
964
1202
  cancel() {
1203
+ super.cancel();
965
1204
  this.queue = [];
966
1205
  const answer = this.currentAnswer;
967
1206
  if (!answer) return;
@@ -1142,8 +1381,8 @@ var Realtime = class extends Agent {
1142
1381
  };
1143
1382
 
1144
1383
  // src/recorder/MicdropRecorder.ts
1145
- import { EventEmitter as EventEmitter3 } from "eventemitter3";
1146
- var MicdropRecorder = class extends EventEmitter3 {
1384
+ import { EventEmitter as EventEmitter4 } from "eventemitter3";
1385
+ var MicdropRecorder = class extends EventEmitter4 {
1147
1386
  constructor(server) {
1148
1387
  super();
1149
1388
  this.server = server;
@@ -1289,19 +1528,281 @@ var SharedInstances = class {
1289
1528
  this.entries.set(id, loading);
1290
1529
  return loading;
1291
1530
  }
1531
+ /** The instance for this key, loaded or being loaded, without loading it. */
1532
+ get(key) {
1533
+ return this.entries.get(typeof key === "string" ? key : JSON.stringify(key));
1534
+ }
1292
1535
  /** Whether an instance is loaded, or being loaded, for this key. */
1293
1536
  has(key) {
1294
1537
  return this.entries.has(typeof key === "string" ? key : JSON.stringify(key));
1295
1538
  }
1539
+ /**
1540
+ * Forgets the instance for this key, so the next call loads a new one. For
1541
+ * an instance that broke, a crashed worker for instance. Releasing what it
1542
+ * holds is left to the caller.
1543
+ */
1544
+ delete(key) {
1545
+ this.entries.delete(typeof key === "string" ? key : JSON.stringify(key));
1546
+ }
1296
1547
  /** Drops every instance, so the next call loads again. */
1297
1548
  clear() {
1298
1549
  this.entries.clear();
1299
1550
  }
1300
1551
  };
1301
1552
 
1553
+ // src/socket/ReconnectingSocket.ts
1554
+ import { EventEmitter as EventEmitter5 } from "eventemitter3";
1555
+ import WebSocket from "ws";
1556
+ var DEFAULT_CONNECTION_TIMEOUT = 5e3;
1557
+ var DEFAULT_RETRY_DELAY = 1e3;
1558
+ var DEFAULT_MAX_RETRY = 3;
1559
+ var ReconnectingSocket = class extends EventEmitter5 {
1560
+ constructor(options) {
1561
+ super();
1562
+ this.options = options;
1563
+ this._status = "idle";
1564
+ this.everOpened = false;
1565
+ this.retryCount = 0;
1566
+ // Numbers the attempts, so one overtaken while it awaited its address stops
1567
+ this.attemptId = 0;
1568
+ this.waiters = [];
1569
+ }
1570
+ get status() {
1571
+ return this._status;
1572
+ }
1573
+ /** Whether a message sent now goes out */
1574
+ get isOpen() {
1575
+ return this._status === "open";
1576
+ }
1577
+ /**
1578
+ * Opens the connection. Called again after `failed`, it starts over with a
1579
+ * full retry budget, and does nothing in any other status.
1580
+ */
1581
+ connect() {
1582
+ if (this._status !== "idle" && this._status !== "failed") return;
1583
+ this.retryCount = 0;
1584
+ this.attempt();
1585
+ }
1586
+ /**
1587
+ * Waits for the connection to be open.
1588
+ * @returns True once open, false if it failed or was destroyed meanwhile
1589
+ */
1590
+ ready() {
1591
+ if (this._status === "open") return Promise.resolve(true);
1592
+ if (this._status === "destroyed" || this._status === "failed") {
1593
+ return Promise.resolve(false);
1594
+ }
1595
+ return new Promise((resolve) => this.waiters.push(resolve));
1596
+ }
1597
+ /**
1598
+ * Sends a message if the connection is open
1599
+ * @returns False when it did not go out
1600
+ */
1601
+ send(data) {
1602
+ if (this._status !== "open" || !this.socket) return false;
1603
+ this.socket.send(data);
1604
+ return true;
1605
+ }
1606
+ /**
1607
+ * Drops the connection and opens a new one right away, for a provider that
1608
+ * can only stop what it is doing by closing. It costs no retry, and the new
1609
+ * connection is not a reconnection.
1610
+ */
1611
+ restart() {
1612
+ if (this._status === "destroyed") return;
1613
+ this.log("Restarting the connection");
1614
+ this.clearRetryTimer();
1615
+ this.drop();
1616
+ this.everOpened = false;
1617
+ this.retryCount = 0;
1618
+ this.attempt();
1619
+ }
1620
+ /** Closes the connection for good */
1621
+ destroy() {
1622
+ if (this._status === "destroyed") return;
1623
+ this.clearRetryTimer();
1624
+ this.drop();
1625
+ this.setStatus("destroyed");
1626
+ this.settleWaiters(false);
1627
+ this.removeAllListeners();
1628
+ }
1629
+ async attempt() {
1630
+ const id = ++this.attemptId;
1631
+ const isOvertaken = () => id !== this.attemptId || this._status !== "connecting";
1632
+ this.setStatus("connecting");
1633
+ let url;
1634
+ try {
1635
+ const { url: getUrl } = this.options;
1636
+ url = typeof getUrl === "string" ? getUrl : await getUrl();
1637
+ } catch (error) {
1638
+ if (isOvertaken()) return;
1639
+ this.onAttemptFailed(error);
1640
+ return;
1641
+ }
1642
+ if (isOvertaken()) return;
1643
+ const socket = new WebSocket(url, this.options.clientOptions);
1644
+ this.socket = socket;
1645
+ let settled = false;
1646
+ const fail = (error) => {
1647
+ if (settled || this.socket !== socket) return;
1648
+ settled = true;
1649
+ clearTimeout(timeout);
1650
+ this.drop();
1651
+ this.onAttemptFailed(error);
1652
+ };
1653
+ const timeout = setTimeout(
1654
+ () => fail(new Error("Connection timeout")),
1655
+ this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
1656
+ );
1657
+ socket.on("error", (error) => {
1658
+ this.log("WebSocket error:", error.message);
1659
+ fail(error);
1660
+ });
1661
+ const { handshake } = this.options;
1662
+ const open = () => {
1663
+ settled = true;
1664
+ clearTimeout(timeout);
1665
+ this.onOpen();
1666
+ };
1667
+ socket.on("open", () => {
1668
+ if (this.socket !== socket) return;
1669
+ try {
1670
+ handshake?.start?.(socket);
1671
+ } catch (error) {
1672
+ fail(error);
1673
+ return;
1674
+ }
1675
+ if (!handshake?.read) open();
1676
+ });
1677
+ socket.on("message", (data, isBinary) => {
1678
+ if (this.socket !== socket) return;
1679
+ if (this._status === "open") {
1680
+ this.prove();
1681
+ this.emit("Message", data, isBinary);
1682
+ return;
1683
+ }
1684
+ if (settled || !handshake?.read) return;
1685
+ try {
1686
+ if (handshake.read(data, isBinary)) open();
1687
+ } catch (error) {
1688
+ fail(error);
1689
+ }
1690
+ });
1691
+ socket.on("close", (code, reasonBuffer) => {
1692
+ const reason = reasonBuffer.toString();
1693
+ if (!settled) {
1694
+ fail(new Error(`Connection closed (${code}) ${reason}`));
1695
+ return;
1696
+ }
1697
+ if (this.socket !== socket) return;
1698
+ this.log("Connection closed", { code, reason });
1699
+ this.drop();
1700
+ this.setStatus("reconnecting");
1701
+ this.emit("Close", code, reason);
1702
+ if (this._status !== "reconnecting") return;
1703
+ this.scheduleRetry(new Error(`Connection closed (${code}) ${reason}`));
1704
+ });
1705
+ }
1706
+ onOpen() {
1707
+ this.log("Connection opened");
1708
+ const reconnected = this.everOpened;
1709
+ this.everOpened = true;
1710
+ if (this.options.handshake?.read) {
1711
+ this.prove();
1712
+ } else {
1713
+ this.proofTimer = setTimeout(
1714
+ () => this.prove(),
1715
+ this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT
1716
+ );
1717
+ }
1718
+ this.startKeepAlive();
1719
+ this.setStatus("open");
1720
+ this.emit("Open", { reconnected });
1721
+ this.settleWaiters(true);
1722
+ }
1723
+ /** The connection works, the retry budget is whole again */
1724
+ prove() {
1725
+ this.clearProofTimer();
1726
+ this.retryCount = 0;
1727
+ }
1728
+ clearProofTimer() {
1729
+ if (!this.proofTimer) return;
1730
+ clearTimeout(this.proofTimer);
1731
+ this.proofTimer = void 0;
1732
+ }
1733
+ onAttemptFailed(error) {
1734
+ this.log("Connection failed:", error.message);
1735
+ this.scheduleRetry(error);
1736
+ }
1737
+ scheduleRetry(error) {
1738
+ if (this._status === "destroyed") return;
1739
+ this.retryCount++;
1740
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
1741
+ this.log("Max retries reached, giving up");
1742
+ this.setStatus("failed");
1743
+ this.settleWaiters(false);
1744
+ this.emit("Failed", error);
1745
+ return;
1746
+ }
1747
+ this.setStatus("reconnecting");
1748
+ this.log(`Reconnecting (attempt ${this.retryCount})`);
1749
+ this.clearRetryTimer();
1750
+ this.retryTimer = setTimeout(() => {
1751
+ this.retryTimer = void 0;
1752
+ if (this._status === "reconnecting") this.attempt();
1753
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
1754
+ }
1755
+ /** Lets go of the current socket, which will never be heard from again */
1756
+ drop() {
1757
+ this.stopKeepAlive();
1758
+ this.clearProofTimer();
1759
+ const socket = this.socket;
1760
+ this.socket = void 0;
1761
+ if (!socket) return;
1762
+ socket.removeAllListeners();
1763
+ socket.on("error", () => {
1764
+ });
1765
+ if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
1766
+ socket.close(1e3);
1767
+ }
1768
+ }
1769
+ startKeepAlive() {
1770
+ const keepAlive = this.options.keepAlive;
1771
+ if (!keepAlive) return;
1772
+ this.stopKeepAlive();
1773
+ this.keepAliveTimer = setInterval(() => {
1774
+ const { message } = keepAlive;
1775
+ this.send(typeof message === "string" ? message : message());
1776
+ }, keepAlive.interval);
1777
+ }
1778
+ stopKeepAlive() {
1779
+ if (!this.keepAliveTimer) return;
1780
+ clearInterval(this.keepAliveTimer);
1781
+ this.keepAliveTimer = void 0;
1782
+ }
1783
+ clearRetryTimer() {
1784
+ if (!this.retryTimer) return;
1785
+ clearTimeout(this.retryTimer);
1786
+ this.retryTimer = void 0;
1787
+ }
1788
+ settleWaiters(open) {
1789
+ const waiters = this.waiters;
1790
+ this.waiters = [];
1791
+ waiters.forEach((resolve) => resolve(open));
1792
+ }
1793
+ setStatus(status) {
1794
+ if (this._status === status) return;
1795
+ this._status = status;
1796
+ this.emit("StatusChange", status);
1797
+ }
1798
+ log(...message) {
1799
+ this.options.log?.(...message);
1800
+ }
1801
+ };
1802
+
1302
1803
  // src/stt/STT.ts
1303
- import { EventEmitter as EventEmitter4 } from "eventemitter3";
1304
- var STT = class extends EventEmitter4 {
1804
+ import { EventEmitter as EventEmitter6 } from "eventemitter3";
1805
+ var STT = class extends EventEmitter6 {
1305
1806
  log(...message) {
1306
1807
  this.logger?.log(...message);
1307
1808
  }
@@ -1327,24 +1828,31 @@ var MockSTT = class extends STT {
1327
1828
  // src/stt/FallbackSTT.ts
1328
1829
  import { PassThrough as PassThrough3 } from "stream";
1329
1830
  var FallbackSTT = class extends STT {
1330
- // Start at -1 because we need to increment it before using it
1331
1831
  constructor(options) {
1332
1832
  super();
1333
1833
  this.options = options;
1334
1834
  this.stt = null;
1335
1835
  this.sttIndex = -1;
1836
+ // Start at -1 because we need to increment it before using it
1837
+ // In the order spoken, each STT giving one transcript per utterance in turn
1838
+ this.pending = [];
1336
1839
  this.onTranscript = (transcript) => {
1840
+ this.pending.shift();
1337
1841
  this.emit("Transcript", transcript);
1338
1842
  };
1339
- this.onFailed = (chunks) => {
1843
+ /**
1844
+ * Hands every utterance still waiting to the next STT, one stream each so
1845
+ * each gets its own transcript. One still being spoken keeps going there.
1846
+ */
1847
+ this.onFailed = () => {
1340
1848
  this.log("STT failed, trying next STT");
1341
1849
  this.startNextSTT();
1342
- if (chunks.length > 0) {
1343
- this.log("Sending audio chunks again");
1344
- const stream = new PassThrough3();
1345
- this.stt?.transcribe(stream);
1346
- chunks.forEach((chunk) => stream.write(chunk));
1347
- stream.end();
1850
+ if (this.pending.length > 0) this.log("Sending audio chunks again");
1851
+ for (const utterance of this.pending) {
1852
+ utterance.stream = new PassThrough3();
1853
+ this.stt?.transcribe(utterance.stream);
1854
+ utterance.chunks.forEach((chunk) => utterance.stream.write(chunk));
1855
+ if (utterance.ended) utterance.stream.end();
1348
1856
  }
1349
1857
  };
1350
1858
  if (this.options.factories.length === 0) {
@@ -1353,13 +1861,28 @@ var FallbackSTT = class extends STT {
1353
1861
  this.startNextSTT();
1354
1862
  }
1355
1863
  transcribe(audioStream) {
1356
- this.stt?.transcribe(audioStream);
1864
+ const utterance = {
1865
+ chunks: [],
1866
+ ended: false,
1867
+ stream: new PassThrough3()
1868
+ };
1869
+ this.pending.push(utterance);
1870
+ audioStream.on("data", (chunk) => {
1871
+ utterance.chunks.push(chunk);
1872
+ utterance.stream.write(chunk);
1873
+ });
1874
+ audioStream.on("end", () => {
1875
+ utterance.ended = true;
1876
+ utterance.stream.end();
1877
+ });
1878
+ this.stt?.transcribe(utterance.stream);
1357
1879
  }
1358
1880
  destroy() {
1359
1881
  super.destroy();
1360
1882
  this.stt?.destroy();
1361
1883
  this.stt = null;
1362
1884
  this.sttIndex = -1;
1885
+ this.pending = [];
1363
1886
  }
1364
1887
  startNextSTT() {
1365
1888
  this.sttIndex++;
@@ -1382,8 +1905,8 @@ var FallbackSTT = class extends STT {
1382
1905
  import { PassThrough as PassThrough4 } from "stream";
1383
1906
 
1384
1907
  // src/tts/TTS.ts
1385
- import { EventEmitter as EventEmitter5 } from "eventemitter3";
1386
- var TTS = class extends EventEmitter5 {
1908
+ import { EventEmitter as EventEmitter7 } from "eventemitter3";
1909
+ var TTS = class extends EventEmitter7 {
1387
1910
  log(...message) {
1388
1911
  this.logger?.log(...message);
1389
1912
  }
@@ -1395,7 +1918,6 @@ var TTS = class extends EventEmitter5 {
1395
1918
 
1396
1919
  // src/tts/FallbackTTS.ts
1397
1920
  var FallbackTTS = class extends TTS {
1398
- // Start at -1 because we need to increment it before using it
1399
1921
  constructor(options) {
1400
1922
  super();
1401
1923
  this.options = options;
@@ -1404,16 +1926,22 @@ var FallbackTTS = class extends TTS {
1404
1926
  this.onAudio = (audio) => {
1405
1927
  this.emit("Audio", audio);
1406
1928
  };
1929
+ /**
1930
+ * Hands the unspoken text to the next TTS. An utterance whose text is still
1931
+ * being written goes on there, so its end is spoken too.
1932
+ */
1407
1933
  this.onFailed = (chunks) => {
1408
1934
  this.log("TTS failed, trying next TTS");
1409
1935
  this.startNextTTS();
1410
- if (chunks.length > 0) {
1411
- this.log("Sending text chunks again");
1412
- const stream = new PassThrough4();
1413
- this.tts?.speak(stream);
1414
- chunks.forEach((chunk) => stream.write(chunk));
1415
- stream.end();
1416
- }
1936
+ const utterance = this.current;
1937
+ const continues = !!utterance && !utterance.ended;
1938
+ if (chunks.length === 0 && !continues) return;
1939
+ this.log("Sending text chunks again");
1940
+ const stream = new PassThrough4();
1941
+ this.tts?.speak(stream);
1942
+ chunks.forEach((chunk) => stream.write(chunk));
1943
+ if (continues) utterance.stream = stream;
1944
+ else stream.end();
1417
1945
  };
1418
1946
  if (this.options.factories.length === 0) {
1419
1947
  throw new Error("FallbackTTS: No factories provided");
@@ -1421,9 +1949,20 @@ var FallbackTTS = class extends TTS {
1421
1949
  this.startNextTTS();
1422
1950
  }
1423
1951
  speak(textStream) {
1424
- this.tts?.speak(textStream);
1952
+ const utterance = {
1953
+ ended: false,
1954
+ stream: new PassThrough4()
1955
+ };
1956
+ this.current = utterance;
1957
+ textStream.on("data", (chunk) => utterance.stream.write(chunk));
1958
+ textStream.on("end", () => {
1959
+ utterance.ended = true;
1960
+ utterance.stream.end();
1961
+ });
1962
+ this.tts?.speak(utterance.stream);
1425
1963
  }
1426
1964
  cancel() {
1965
+ this.current = void 0;
1427
1966
  this.tts?.cancel();
1428
1967
  }
1429
1968
  destroy() {
@@ -1451,26 +1990,28 @@ var FallbackTTS = class extends TTS {
1451
1990
 
1452
1991
  // src/tts/MockTTS.ts
1453
1992
  import * as fs from "fs";
1454
- import { PassThrough as PassThrough5 } from "stream";
1455
1993
  var MockTTS = class extends TTS {
1456
1994
  constructor(audioFilePaths) {
1457
1995
  super();
1458
1996
  this.audioFilePaths = audioFilePaths;
1997
+ // Bumped by every answer and every cancel, so the chunks of an answer that
1998
+ // is no longer wanted stop there
1999
+ this.generation = 0;
1459
2000
  }
1460
2001
  speak(textStream) {
1461
- const audioStream = new PassThrough5();
2002
+ const generation = ++this.generation;
1462
2003
  textStream.once("data", async () => {
1463
2004
  for (const filePath of this.audioFilePaths) {
1464
2005
  await new Promise((resolve) => setTimeout(resolve, 200));
1465
- const audioBuffer = fs.readFileSync(filePath);
1466
- this.log(`Loaded chunk (${audioBuffer.length} bytes)`);
1467
- audioStream.write(audioBuffer);
2006
+ if (generation !== this.generation) return;
2007
+ const audio = fs.readFileSync(filePath);
2008
+ this.log(`Loaded chunk (${audio.length} bytes)`);
2009
+ this.emit("Audio", audio);
1468
2010
  }
1469
- audioStream.end();
1470
2011
  });
1471
- return audioStream;
1472
2012
  }
1473
2013
  cancel() {
2014
+ this.generation++;
1474
2015
  }
1475
2016
  };
1476
2017
 
@@ -1549,6 +2090,9 @@ var SentenceTTS = class extends TTS {
1549
2090
  this.counter++;
1550
2091
  counter = this.counter;
1551
2092
  this.splitter.reset();
2093
+ this.queue = [];
2094
+ this.controller?.abort();
2095
+ this.controller = void 0;
1552
2096
  return true;
1553
2097
  };
1554
2098
  textStream.on("data", (chunk) => {
@@ -1634,6 +2178,7 @@ export {
1634
2178
  AUTO_SEMANTIC_TURN_PROMPT,
1635
2179
  AUTO_SEMANTIC_TURN_TOOL_NAME,
1636
2180
  Agent,
2181
+ Classifier,
1637
2182
  FallbackAgent,
1638
2183
  FallbackSTT,
1639
2184
  FallbackTTS,
@@ -1649,14 +2194,18 @@ export {
1649
2194
  MockTTS,
1650
2195
  Pcm16Resampler,
1651
2196
  Realtime,
2197
+ ReconnectingSocket,
1652
2198
  STT,
1653
2199
  SentenceSplitter,
1654
2200
  SentenceTTS,
1655
2201
  SharedInstances,
1656
2202
  TTS,
2203
+ currentTurn,
1657
2204
  float32ToPcm16,
2205
+ getTurnClassification,
1658
2206
  handleError,
1659
2207
  pcm16ToFloat32,
2208
+ turnInput,
1660
2209
  waitForParams
1661
2210
  };
1662
2211
  //# sourceMappingURL=index.mjs.map