@micdrop/server 2.8.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +144 -3
- package/dist/index.d.ts +144 -3
- package/dist/index.js +209 -29
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +205 -29
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -37,6 +37,7 @@ __export(index_exports, {
|
|
|
37
37
|
AUTO_SEMANTIC_TURN_PROMPT: () => AUTO_SEMANTIC_TURN_PROMPT,
|
|
38
38
|
AUTO_SEMANTIC_TURN_TOOL_NAME: () => AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
39
39
|
Agent: () => Agent,
|
|
40
|
+
Classifier: () => Classifier,
|
|
40
41
|
FallbackAgent: () => FallbackAgent,
|
|
41
42
|
FallbackSTT: () => FallbackSTT,
|
|
42
43
|
FallbackTTS: () => FallbackTTS,
|
|
@@ -57,9 +58,12 @@ __export(index_exports, {
|
|
|
57
58
|
SentenceTTS: () => SentenceTTS,
|
|
58
59
|
SharedInstances: () => SharedInstances,
|
|
59
60
|
TTS: () => TTS,
|
|
61
|
+
currentTurn: () => currentTurn,
|
|
60
62
|
float32ToPcm16: () => float32ToPcm16,
|
|
63
|
+
getTurnClassification: () => getTurnClassification,
|
|
61
64
|
handleError: () => handleError,
|
|
62
65
|
pcm16ToFloat32: () => pcm16ToFloat32,
|
|
66
|
+
turnInput: () => turnInput,
|
|
63
67
|
waitForParams: () => waitForParams
|
|
64
68
|
});
|
|
65
69
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -93,8 +97,13 @@ var Agent = class extends import_eventemitter3.EventEmitter {
|
|
|
93
97
|
this.answering = true;
|
|
94
98
|
Promise.resolve().then(
|
|
95
99
|
() => this.options.onBeforeAnswer?.bind(this)(stream)
|
|
96
|
-
).then((
|
|
97
|
-
if (
|
|
100
|
+
).then((result) => {
|
|
101
|
+
if (typeof result === "string") {
|
|
102
|
+
stream.write(result);
|
|
103
|
+
this.addAssistantMessage(result);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
if (result) return;
|
|
98
107
|
return this.generateAnswer(stream);
|
|
99
108
|
}).finally(() => {
|
|
100
109
|
if (stream.writable) {
|
|
@@ -418,6 +427,144 @@ var MockAgent = class extends Agent {
|
|
|
418
427
|
}
|
|
419
428
|
};
|
|
420
429
|
|
|
430
|
+
// src/classifier/Classifier.ts
|
|
431
|
+
var import_eventemitter32 = require("eventemitter3");
|
|
432
|
+
var Classifier = class extends import_eventemitter32.EventEmitter {
|
|
433
|
+
constructor() {
|
|
434
|
+
super(...arguments);
|
|
435
|
+
this.controllers = /* @__PURE__ */ new Set();
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Classifies an input, and emits the result as `Classification`
|
|
439
|
+
* @param input - What to classify: a text, or JSON with named fields
|
|
440
|
+
* @returns The classification, or undefined when cancelled or failed
|
|
441
|
+
*/
|
|
442
|
+
classify(input) {
|
|
443
|
+
const controller = new AbortController();
|
|
444
|
+
this.controllers.add(controller);
|
|
445
|
+
const pending = this.run(input, controller.signal).finally(() => {
|
|
446
|
+
this.controllers.delete(controller);
|
|
447
|
+
if (this._pending === pending) this._pending = void 0;
|
|
448
|
+
});
|
|
449
|
+
this._pending = pending;
|
|
450
|
+
return pending;
|
|
451
|
+
}
|
|
452
|
+
/** The last classification started, until it is done */
|
|
453
|
+
get pending() {
|
|
454
|
+
return this._pending;
|
|
455
|
+
}
|
|
456
|
+
/** Drops every classification in progress, which then emit nothing */
|
|
457
|
+
cancel() {
|
|
458
|
+
this.controllers.forEach((controller) => controller.abort());
|
|
459
|
+
this.controllers.clear();
|
|
460
|
+
this._pending = void 0;
|
|
461
|
+
}
|
|
462
|
+
log(...message) {
|
|
463
|
+
this.logger?.log(...message);
|
|
464
|
+
}
|
|
465
|
+
destroy() {
|
|
466
|
+
this.log("Destroyed");
|
|
467
|
+
this.cancel();
|
|
468
|
+
this.removeAllListeners();
|
|
469
|
+
}
|
|
470
|
+
async run(input, signal) {
|
|
471
|
+
const start = Date.now();
|
|
472
|
+
try {
|
|
473
|
+
const result = await this.evaluate(input, signal);
|
|
474
|
+
if (signal.aborted) return;
|
|
475
|
+
const classification = {
|
|
476
|
+
input,
|
|
477
|
+
result,
|
|
478
|
+
duration: Date.now() - start
|
|
479
|
+
};
|
|
480
|
+
this.log(
|
|
481
|
+
`Classified in ${classification.duration} ms:`,
|
|
482
|
+
JSON.stringify(input)
|
|
483
|
+
);
|
|
484
|
+
this.lastClassification = classification;
|
|
485
|
+
this.emit("Classification", classification);
|
|
486
|
+
return classification;
|
|
487
|
+
} catch (error) {
|
|
488
|
+
if (!signal.aborted) this.log("Classification failed:", error);
|
|
489
|
+
return;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
};
|
|
493
|
+
|
|
494
|
+
// src/types.ts
|
|
495
|
+
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
496
|
+
MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
497
|
+
MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
498
|
+
MicdropClientCommands2["Mute"] = "Mute";
|
|
499
|
+
return MicdropClientCommands2;
|
|
500
|
+
})(MicdropClientCommands || {});
|
|
501
|
+
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
502
|
+
MicdropServerCommands2["Message"] = "Message";
|
|
503
|
+
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
504
|
+
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
505
|
+
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
506
|
+
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
507
|
+
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
508
|
+
MicdropServerCommands2["Classification"] = "Classification";
|
|
509
|
+
return MicdropServerCommands2;
|
|
510
|
+
})(MicdropServerCommands || {});
|
|
511
|
+
function currentTurn(conversation) {
|
|
512
|
+
let start = conversation.length;
|
|
513
|
+
while (start > 0) {
|
|
514
|
+
const item = conversation[start - 1];
|
|
515
|
+
if (item.role !== "user" || item.metadata?.classification) break;
|
|
516
|
+
start--;
|
|
517
|
+
}
|
|
518
|
+
const messages = conversation.slice(start);
|
|
519
|
+
return {
|
|
520
|
+
transcript: messages.map((message) => message.content).join(" "),
|
|
521
|
+
messages,
|
|
522
|
+
before: conversation.slice(0, start)
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
function getTurnClassification(conversation) {
|
|
526
|
+
for (let index = conversation.length - 1; index >= 0; index--) {
|
|
527
|
+
const item = conversation[index];
|
|
528
|
+
if (item.role === "user") return item.metadata?.classification;
|
|
529
|
+
}
|
|
530
|
+
return void 0;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
// src/classifier/turnInput.ts
|
|
534
|
+
function turnInput(conversation, history = 1) {
|
|
535
|
+
const turn = currentTurn(conversation);
|
|
536
|
+
const input = { history: [], turn: turn.transcript };
|
|
537
|
+
let rest = turn.before;
|
|
538
|
+
for (let n = 0; n < history && rest.length; n++) {
|
|
539
|
+
let end = rest.length;
|
|
540
|
+
const answers = [];
|
|
541
|
+
while (end > 0 && rest[end - 1].role !== "user") {
|
|
542
|
+
const item = rest[end - 1];
|
|
543
|
+
if (item.role === "assistant" && item.content.trim()) {
|
|
544
|
+
answers.unshift(item.content);
|
|
545
|
+
}
|
|
546
|
+
end--;
|
|
547
|
+
}
|
|
548
|
+
let start = end;
|
|
549
|
+
while (start > 0) {
|
|
550
|
+
const item = rest[start - 1];
|
|
551
|
+
if (item.role !== "user") break;
|
|
552
|
+
if (start < end && item.metadata?.classification)
|
|
553
|
+
break;
|
|
554
|
+
start--;
|
|
555
|
+
}
|
|
556
|
+
if (answers.length) {
|
|
557
|
+
input.history.unshift({ role: "assistant", text: answers.join(" ") });
|
|
558
|
+
}
|
|
559
|
+
if (start < end) {
|
|
560
|
+
const text = rest.slice(start, end).map((item) => "content" in item ? item.content : "").join(" ");
|
|
561
|
+
input.history.unshift({ role: "user", text });
|
|
562
|
+
}
|
|
563
|
+
rest = rest.slice(0, start);
|
|
564
|
+
}
|
|
565
|
+
return { input, message: turn.messages[turn.messages.length - 1] };
|
|
566
|
+
}
|
|
567
|
+
|
|
421
568
|
// src/audio/Pcm16Resampler.ts
|
|
422
569
|
var Pcm16Resampler = class {
|
|
423
570
|
// Fractional position into the first sample of the buffer
|
|
@@ -505,30 +652,12 @@ function handleError(socket, error) {
|
|
|
505
652
|
}
|
|
506
653
|
|
|
507
654
|
// src/MicdropServer.ts
|
|
508
|
-
var
|
|
655
|
+
var import_eventemitter33 = require("eventemitter3");
|
|
509
656
|
var import_stream2 = require("stream");
|
|
510
|
-
|
|
511
|
-
// src/types.ts
|
|
512
|
-
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
513
|
-
MicdropClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
514
|
-
MicdropClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
515
|
-
MicdropClientCommands2["Mute"] = "Mute";
|
|
516
|
-
return MicdropClientCommands2;
|
|
517
|
-
})(MicdropClientCommands || {});
|
|
518
|
-
var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
|
|
519
|
-
MicdropServerCommands2["Message"] = "Message";
|
|
520
|
-
MicdropServerCommands2["PartialAssistantMessage"] = "PartialAssistantMessage";
|
|
521
|
-
MicdropServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
522
|
-
MicdropServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
523
|
-
MicdropServerCommands2["EndCall"] = "EndCall";
|
|
524
|
-
MicdropServerCommands2["ToolCall"] = "ToolCall";
|
|
525
|
-
return MicdropServerCommands2;
|
|
526
|
-
})(MicdropServerCommands || {});
|
|
527
|
-
|
|
528
|
-
// src/MicdropServer.ts
|
|
529
657
|
var USER_SAMPLE_RATE = 16e3;
|
|
530
658
|
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
531
|
-
var
|
|
659
|
+
var DEFAULT_CLASSIFICATION_MAX_WAIT = 1e3;
|
|
660
|
+
var MicdropServer = class extends import_eventemitter33.EventEmitter {
|
|
532
661
|
constructor(socket, config) {
|
|
533
662
|
super();
|
|
534
663
|
this.socket = null;
|
|
@@ -550,6 +679,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
550
679
|
this.agent?.destroy();
|
|
551
680
|
this.config.stt?.destroy();
|
|
552
681
|
this.config.tts?.destroy();
|
|
682
|
+
this.config.classifier?.destroy();
|
|
553
683
|
this.emit("End", {
|
|
554
684
|
conversation,
|
|
555
685
|
duration
|
|
@@ -641,6 +771,14 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
641
771
|
)
|
|
642
772
|
);
|
|
643
773
|
}
|
|
774
|
+
if (config.classifierOptions?.sendToClient) {
|
|
775
|
+
config.classifier?.on(
|
|
776
|
+
"Classification",
|
|
777
|
+
(classification) => this.socket?.send(
|
|
778
|
+
`${"Classification" /* Classification */} ${JSON.stringify(classification)}`
|
|
779
|
+
)
|
|
780
|
+
);
|
|
781
|
+
}
|
|
644
782
|
queueMicrotask(() => this.sendFirstMessage());
|
|
645
783
|
socket.on("close", this.onClose);
|
|
646
784
|
socket.on("message", this.onMessage);
|
|
@@ -677,6 +815,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
677
815
|
this.processQueue();
|
|
678
816
|
}
|
|
679
817
|
cancel() {
|
|
818
|
+
this.config?.classifier?.cancel();
|
|
680
819
|
this.config?.tts?.cancel();
|
|
681
820
|
this.agent?.cancel();
|
|
682
821
|
this.operationQueue = [];
|
|
@@ -780,6 +919,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
780
919
|
if (realtime) {
|
|
781
920
|
realtime.endTurn();
|
|
782
921
|
} else {
|
|
922
|
+
this.classifyTurn();
|
|
783
923
|
this.answer();
|
|
784
924
|
}
|
|
785
925
|
}
|
|
@@ -823,6 +963,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
823
963
|
`${"Message" /* Message */} ${JSON.stringify(message)}`
|
|
824
964
|
);
|
|
825
965
|
this.emit("Message", message);
|
|
966
|
+
if (message.role === "user" && this.config?.realtime) this.classifyTurn();
|
|
826
967
|
}
|
|
827
968
|
sendPartialAnswer(content) {
|
|
828
969
|
if (!this.socket || content === "") return;
|
|
@@ -843,6 +984,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
843
984
|
}
|
|
844
985
|
}
|
|
845
986
|
answer() {
|
|
987
|
+
if (this.config?.classifierOptions?.waitBeforeAnswer) {
|
|
988
|
+
this.queueOperation(() => this.waitForClassification());
|
|
989
|
+
}
|
|
846
990
|
this.queueOperation(async () => {
|
|
847
991
|
await this._answer();
|
|
848
992
|
});
|
|
@@ -892,6 +1036,38 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
|
|
|
892
1036
|
});
|
|
893
1037
|
return stream.pipe(forward);
|
|
894
1038
|
}
|
|
1039
|
+
/**
|
|
1040
|
+
* Classifies the turn that just ended, with the one before, and keeps the
|
|
1041
|
+
* result in the metadata of its last message
|
|
1042
|
+
*/
|
|
1043
|
+
classifyTurn() {
|
|
1044
|
+
const classifier = this.config?.classifier;
|
|
1045
|
+
if (!classifier) return;
|
|
1046
|
+
const { input, message } = turnInput(
|
|
1047
|
+
this.conversation,
|
|
1048
|
+
this.config?.classifierOptions?.history
|
|
1049
|
+
);
|
|
1050
|
+
if (!message || input.turn.trim() === "") return;
|
|
1051
|
+
classifier.classify(input).then((classification) => {
|
|
1052
|
+
if (!classification) return;
|
|
1053
|
+
message.metadata = { ...message.metadata, classification };
|
|
1054
|
+
});
|
|
1055
|
+
}
|
|
1056
|
+
/** Waits for the classification in progress, never longer than its limit */
|
|
1057
|
+
async waitForClassification() {
|
|
1058
|
+
const pending = this.config?.classifier?.pending;
|
|
1059
|
+
if (!pending) return;
|
|
1060
|
+
const limit = this.config?.classifierOptions?.maxWait ?? DEFAULT_CLASSIFICATION_MAX_WAIT;
|
|
1061
|
+
let timer;
|
|
1062
|
+
const timeout = new Promise((resolve) => {
|
|
1063
|
+
timer = setTimeout(() => {
|
|
1064
|
+
this.log(`Classification took over ${limit} ms, answering without it`);
|
|
1065
|
+
resolve();
|
|
1066
|
+
}, limit);
|
|
1067
|
+
});
|
|
1068
|
+
await Promise.race([pending, timeout]);
|
|
1069
|
+
clearTimeout(timer);
|
|
1070
|
+
}
|
|
895
1071
|
// Run text-to-speech and send to client
|
|
896
1072
|
speak(message) {
|
|
897
1073
|
this.queueOperation(async () => {
|
|
@@ -1208,8 +1384,8 @@ var Realtime = class extends Agent {
|
|
|
1208
1384
|
};
|
|
1209
1385
|
|
|
1210
1386
|
// src/recorder/MicdropRecorder.ts
|
|
1211
|
-
var
|
|
1212
|
-
var MicdropRecorder = class extends
|
|
1387
|
+
var import_eventemitter34 = require("eventemitter3");
|
|
1388
|
+
var MicdropRecorder = class extends import_eventemitter34.EventEmitter {
|
|
1213
1389
|
constructor(server) {
|
|
1214
1390
|
super();
|
|
1215
1391
|
this.server = server;
|
|
@@ -1366,8 +1542,8 @@ var SharedInstances = class {
|
|
|
1366
1542
|
};
|
|
1367
1543
|
|
|
1368
1544
|
// src/stt/STT.ts
|
|
1369
|
-
var
|
|
1370
|
-
var STT = class extends
|
|
1545
|
+
var import_eventemitter35 = require("eventemitter3");
|
|
1546
|
+
var STT = class extends import_eventemitter35.EventEmitter {
|
|
1371
1547
|
log(...message) {
|
|
1372
1548
|
this.logger?.log(...message);
|
|
1373
1549
|
}
|
|
@@ -1448,8 +1624,8 @@ var FallbackSTT = class extends STT {
|
|
|
1448
1624
|
var import_stream4 = require("stream");
|
|
1449
1625
|
|
|
1450
1626
|
// src/tts/TTS.ts
|
|
1451
|
-
var
|
|
1452
|
-
var TTS = class extends
|
|
1627
|
+
var import_eventemitter36 = require("eventemitter3");
|
|
1628
|
+
var TTS = class extends import_eventemitter36.EventEmitter {
|
|
1453
1629
|
log(...message) {
|
|
1454
1630
|
this.logger?.log(...message);
|
|
1455
1631
|
}
|
|
@@ -1701,6 +1877,7 @@ async function waitForParams(socket, validate) {
|
|
|
1701
1877
|
AUTO_SEMANTIC_TURN_PROMPT,
|
|
1702
1878
|
AUTO_SEMANTIC_TURN_TOOL_NAME,
|
|
1703
1879
|
Agent,
|
|
1880
|
+
Classifier,
|
|
1704
1881
|
FallbackAgent,
|
|
1705
1882
|
FallbackSTT,
|
|
1706
1883
|
FallbackTTS,
|
|
@@ -1721,9 +1898,12 @@ async function waitForParams(socket, validate) {
|
|
|
1721
1898
|
SentenceTTS,
|
|
1722
1899
|
SharedInstances,
|
|
1723
1900
|
TTS,
|
|
1901
|
+
currentTurn,
|
|
1724
1902
|
float32ToPcm16,
|
|
1903
|
+
getTurnClassification,
|
|
1725
1904
|
handleError,
|
|
1726
1905
|
pcm16ToFloat32,
|
|
1906
|
+
turnInput,
|
|
1727
1907
|
waitForParams
|
|
1728
1908
|
});
|
|
1729
1909
|
//# sourceMappingURL=index.js.map
|