@micdrop/server 2.8.0 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -37,6 +37,7 @@ __export(index_exports, {
37
37
  AUTO_SEMANTIC_TURN_PROMPT: () => AUTO_SEMANTIC_TURN_PROMPT,
38
38
  AUTO_SEMANTIC_TURN_TOOL_NAME: () => AUTO_SEMANTIC_TURN_TOOL_NAME,
39
39
  Agent: () => Agent,
40
+ Classifier: () => Classifier,
40
41
  FallbackAgent: () => FallbackAgent,
41
42
  FallbackSTT: () => FallbackSTT,
42
43
  FallbackTTS: () => FallbackTTS,
@@ -57,9 +58,12 @@ __export(index_exports, {
57
58
  SentenceTTS: () => SentenceTTS,
58
59
  SharedInstances: () => SharedInstances,
59
60
  TTS: () => TTS,
61
+ currentTurn: () => currentTurn,
60
62
  float32ToPcm16: () => float32ToPcm16,
63
+ getTurnClassification: () => getTurnClassification,
61
64
  handleError: () => handleError,
62
65
  pcm16ToFloat32: () => pcm16ToFloat32,
66
+ turnInput: () => turnInput,
63
67
  waitForParams: () => waitForParams
64
68
  });
65
69
  module.exports = __toCommonJS(index_exports);
@@ -93,8 +97,13 @@ var Agent = class extends import_eventemitter3.EventEmitter {
93
97
  this.answering = true;
94
98
  Promise.resolve().then(
95
99
  () => this.options.onBeforeAnswer?.bind(this)(stream)
96
- ).then((skip) => {
97
- if (skip) return;
100
+ ).then((result) => {
101
+ if (typeof result === "string") {
102
+ stream.write(result);
103
+ this.addAssistantMessage(result);
104
+ return;
105
+ }
106
+ if (result) return;
98
107
  return this.generateAnswer(stream);
99
108
  }).finally(() => {
100
109
  if (stream.writable) {
@@ -418,6 +427,144 @@ var MockAgent = class extends Agent {
418
427
  }
419
428
  };
420
429
 
430
+ // src/classifier/Classifier.ts
431
+ var import_eventemitter32 = require("eventemitter3");
432
+ var Classifier = class extends import_eventemitter32.EventEmitter {
433
+ constructor() {
434
+ super(...arguments);
435
+ this.controllers = /* @__PURE__ */ new Set();
436
+ }
437
+ /**
438
+ * Classifies an input, and emits the result as `Classification`
439
+ * @param input - What to classify: a text, or JSON with named fields
440
+ * @returns The classification, or undefined when cancelled or failed
441
+ */
442
+ classify(input) {
443
+ const controller = new AbortController();
444
+ this.controllers.add(controller);
445
+ const pending = this.run(input, controller.signal).finally(() => {
446
+ this.controllers.delete(controller);
447
+ if (this._pending === pending) this._pending = void 0;
448
+ });
449
+ this._pending = pending;
450
+ return pending;
451
+ }
452
+ /** The last classification started, until it is done */
453
+ get pending() {
454
+ return this._pending;
455
+ }
456
+ /** Drops every classification in progress, which then emit nothing */
457
+ cancel() {
458
+ this.controllers.forEach((controller) => controller.abort());
459
+ this.controllers.clear();
460
+ this._pending = void 0;
461
+ }
462
+ log(...message) {
463
+ this.logger?.log(...message);
464
+ }
465
+ destroy() {
466
+ this.log("Destroyed");
467
+ this.cancel();
468
+ this.removeAllListeners();
469
+ }
470
+ async run(input, signal) {
471
+ const start = Date.now();
472
+ try {
473
+ const result = await this.evaluate(input, signal);
474
+ if (signal.aborted) return;
475
+ const classification = {
476
+ input,
477
+ result,
478
+ duration: Date.now() - start
479
+ };
480
+ this.log(
481
+ `Classified in ${classification.duration} ms:`,
482
+ JSON.stringify(input)
483
+ );
484
+ this.lastClassification = classification;
485
+ this.emit("Classification", classification);
486
+ return classification;
487
+ } catch (error) {
488
+ if (!signal.aborted) this.log("Classification failed:", error);
489
+ return;
490
+ }
491
+ }
492
+ };
493
+
494
+ // src/types.ts
495
+ var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
496
+ MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
497
+ MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
498
+ MicdropClientCommands2["Mute"] = "Mute";
499
+ return MicdropClientCommands2;
500
+ })(MicdropClientCommands || {});
501
+ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
502
+ MicdropServerCommands2["Message"] = "Message";
503
+ MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
504
+ MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
505
+ MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
506
+ MicdropServerCommands2["EndCall"] = "EndCall";
507
+ MicdropServerCommands2["ToolCall"] = "ToolCall";
508
+ MicdropServerCommands2["Classification"] = "Classification";
509
+ return MicdropServerCommands2;
510
+ })(MicdropServerCommands || {});
511
+ function currentTurn(conversation) {
512
+ let start = conversation.length;
513
+ while (start > 0) {
514
+ const item = conversation[start - 1];
515
+ if (item.role !== "user" || item.metadata?.classification) break;
516
+ start--;
517
+ }
518
+ const messages = conversation.slice(start);
519
+ return {
520
+ transcript: messages.map((message) => message.content).join(" "),
521
+ messages,
522
+ before: conversation.slice(0, start)
523
+ };
524
+ }
525
+ function getTurnClassification(conversation) {
526
+ for (let index = conversation.length - 1; index >= 0; index--) {
527
+ const item = conversation[index];
528
+ if (item.role === "user") return item.metadata?.classification;
529
+ }
530
+ return void 0;
531
+ }
532
+
533
+ // src/classifier/turnInput.ts
534
+ function turnInput(conversation, history = 1) {
535
+ const turn = currentTurn(conversation);
536
+ const input = { history: [], turn: turn.transcript };
537
+ let rest = turn.before;
538
+ for (let n = 0; n < history && rest.length; n++) {
539
+ let end = rest.length;
540
+ const answers = [];
541
+ while (end > 0 && rest[end - 1].role !== "user") {
542
+ const item = rest[end - 1];
543
+ if (item.role === "assistant" && item.content.trim()) {
544
+ answers.unshift(item.content);
545
+ }
546
+ end--;
547
+ }
548
+ let start = end;
549
+ while (start > 0) {
550
+ const item = rest[start - 1];
551
+ if (item.role !== "user") break;
552
+ if (start < end && item.metadata?.classification)
553
+ break;
554
+ start--;
555
+ }
556
+ if (answers.length) {
557
+ input.history.unshift({ role: "assistant", text: answers.join(" ") });
558
+ }
559
+ if (start < end) {
560
+ const text = rest.slice(start, end).map((item) => "content" in item ? item.content : "").join(" ");
561
+ input.history.unshift({ role: "user", text });
562
+ }
563
+ rest = rest.slice(0, start);
564
+ }
565
+ return { input, message: turn.messages[turn.messages.length - 1] };
566
+ }
567
+
421
568
  // src/audio/Pcm16Resampler.ts
422
569
  var Pcm16Resampler = class {
423
570
  // Fractional position into the first sample of the buffer
@@ -505,30 +652,12 @@ function handleError(socket, error) {
505
652
  }
506
653
 
507
654
  // src/MicdropServer.ts
508
- var import_eventemitter32 = require("eventemitter3");
655
+ var import_eventemitter33 = require("eventemitter3");
509
656
  var import_stream2 = require("stream");
510
-
511
- // src/types.ts
512
- var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
513
- MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
514
- MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
515
- MicdropClientCommands2["Mute"] = "Mute";
516
- return MicdropClientCommands2;
517
- })(MicdropClientCommands || {});
518
- var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
519
- MicdropServerCommands2["Message"] = "Message";
520
- MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
521
- MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
522
- MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
523
- MicdropServerCommands2["EndCall"] = "EndCall";
524
- MicdropServerCommands2["ToolCall"] = "ToolCall";
525
- return MicdropServerCommands2;
526
- })(MicdropServerCommands || {});
527
-
528
- // src/MicdropServer.ts
529
657
  var USER_SAMPLE_RATE = 16e3;
530
658
  var DEFAULT_TURN_MAX_WAIT = 4e3;
531
- var MicdropServer = class extends import_eventemitter32.EventEmitter {
659
+ var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
660
+ var MicdropServer = class extends import_eventemitter33.EventEmitter {
532
661
  constructor(socket, config) {
533
662
  super();
534
663
  this.socket = null;
@@ -550,6 +679,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
550
679
  this.agent?.destroy();
551
680
  this.config.stt?.destroy();
552
681
  this.config.tts?.destroy();
682
+ this.config.classifier?.destroy();
553
683
  this.emit("End", {
554
684
  conversation,
555
685
  duration
@@ -641,6 +771,14 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
641
771
  )
642
772
  );
643
773
  }
774
+ if (config.classifierOptions?.sendToClient) {
775
+ config.classifier?.on(
776
+ "Classification",
777
+ (classification) => this.socket?.send(
778
+ `${"Classification" /* Classification */} ${JSON.stringify(classification)}`
779
+ )
780
+ );
781
+ }
644
782
  queueMicrotask(() => this.sendFirstMessage());
645
783
  socket.on("close", this.onClose);
646
784
  socket.on("message", this.onMessage);
@@ -677,6 +815,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
677
815
  this.processQueue();
678
816
  }
679
817
  cancel() {
818
+ this.config?.classifier?.cancel();
680
819
  this.config?.tts?.cancel();
681
820
  this.agent?.cancel();
682
821
  this.operationQueue = [];
@@ -780,6 +919,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
780
919
  if (realtime) {
781
920
  realtime.endTurn();
782
921
  } else {
922
+ this.classifyTurn();
783
923
  this.answer();
784
924
  }
785
925
  }
@@ -823,6 +963,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
823
963
  `${"Message" /* Message */} ${JSON.stringify(message)}`
824
964
  );
825
965
  this.emit("Message", message);
966
+ if (message.role === "user" && this.config?.realtime) this.classifyTurn();
826
967
  }
827
968
  sendPartialAnswer(content) {
828
969
  if (!this.socket || content === "") return;
@@ -843,6 +984,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
843
984
  }
844
985
  }
845
986
  answer() {
987
+ if (this.config?.classifierOptions?.waitBeforeAnswer) {
988
+ this.queueOperation(() => this.waitForClassification());
989
+ }
846
990
  this.queueOperation(async () => {
847
991
  await this._answer();
848
992
  });
@@ -892,6 +1036,38 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
892
1036
  });
893
1037
  return stream.pipe(forward);
894
1038
  }
1039
+ /**
1040
+ * Classifies the turn that just ended, with the one before, and keeps the
1041
+ * result in the metadata of its last message
1042
+ */
1043
+ classifyTurn() {
1044
+ const classifier = this.config?.classifier;
1045
+ if (!classifier) return;
1046
+ const { input, message } = turnInput(
1047
+ this.conversation,
1048
+ this.config?.classifierOptions?.history
1049
+ );
1050
+ if (!message || input.turn.trim() === "") return;
1051
+ classifier.classify(input).then((classification) => {
1052
+ if (!classification) return;
1053
+ message.metadata = { ...message.metadata, classification };
1054
+ });
1055
+ }
1056
+ /** Waits for the classification in progress, never longer than its limit */
1057
+ async waitForClassification() {
1058
+ const pending = this.config?.classifier?.pending;
1059
+ if (!pending) return;
1060
+ const limit = this.config?.classifierOptions?.maxWait ?? DEFAULT_CLASSIFICATION_MAX_WAIT;
1061
+ let timer;
1062
+ const timeout = new Promise((resolve) => {
1063
+ timer = setTimeout(() => {
1064
+ this.log(`Classification took over ${limit} ms, answering without it`);
1065
+ resolve();
1066
+ }, limit);
1067
+ });
1068
+ await Promise.race([pending, timeout]);
1069
+ clearTimeout(timer);
1070
+ }
895
1071
  // Run text-to-speech and send to client
896
1072
  speak(message) {
897
1073
  this.queueOperation(async () => {
@@ -1208,8 +1384,8 @@ var Realtime = class extends Agent {
1208
1384
  };
1209
1385
 
1210
1386
  // src/recorder/MicdropRecorder.ts
1211
- var import_eventemitter33 = require("eventemitter3");
1212
- var MicdropRecorder = class extends import_eventemitter33.EventEmitter {
1387
+ var import_eventemitter34 = require("eventemitter3");
1388
+ var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
1213
1389
  constructor(server) {
1214
1390
  super();
1215
1391
  this.server = server;
@@ -1366,8 +1542,8 @@ var SharedInstances = class {
1366
1542
  };
1367
1543
 
1368
1544
  // src/stt/STT.ts
1369
- var import_eventemitter34 = require("eventemitter3");
1370
- var STT = class extends import_eventemitter34.EventEmitter {
1545
+ var import_eventemitter35 = require("eventemitter3");
1546
+ var STT = class extends import_eventemitter35.EventEmitter {
1371
1547
  log(...message) {
1372
1548
  this.logger?.log(...message);
1373
1549
  }
@@ -1448,8 +1624,8 @@ var FallbackSTT = class extends STT {
1448
1624
  var import_stream4 = require("stream");
1449
1625
 
1450
1626
  // src/tts/TTS.ts
1451
- var import_eventemitter35 = require("eventemitter3");
1452
- var TTS = class extends import_eventemitter35.EventEmitter {
1627
+ var import_eventemitter36 = require("eventemitter3");
1628
+ var TTS = class extends import_eventemitter36.EventEmitter {
1453
1629
  log(...message) {
1454
1630
  this.logger?.log(...message);
1455
1631
  }
@@ -1701,6 +1877,7 @@ async function waitForParams(socket, validate) {
1701
1877
  AUTO_SEMANTIC_TURN_PROMPT,
1702
1878
  AUTO_SEMANTIC_TURN_TOOL_NAME,
1703
1879
  Agent,
1880
+ Classifier,
1704
1881
  FallbackAgent,
1705
1882
  FallbackSTT,
1706
1883
  FallbackTTS,
@@ -1721,9 +1898,12 @@ async function waitForParams(socket, validate) {
1721
1898
  SentenceTTS,
1722
1899
  SharedInstances,
1723
1900
  TTS,
1901
+ currentTurn,
1724
1902
  float32ToPcm16,
1903
+ getTurnClassification,
1725
1904
  handleError,
1726
1905
  pcm16ToFloat32,
1906
+ turnInput,
1727
1907
  waitForParams
1728
1908
  });
1729
1909
  //# sourceMappingURL=index.js.map