@teamlearners/clawops 0.1.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +6 -0
- package/dist/agent/index.cjs +863 -413
- package/dist/agent/index.cjs.map +1 -1
- package/dist/agent/index.d.cts +255 -196
- package/dist/agent/index.d.ts +255 -196
- package/dist/agent/index.js +863 -413
- package/dist/agent/index.js.map +1 -1
- package/dist/index.cjs +4 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +7 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/agent/index.cjs
CHANGED
|
@@ -347,13 +347,16 @@ function resamplePcm16(pcm, fromRate, toRate) {
|
|
|
347
347
|
}
|
|
348
348
|
|
|
349
349
|
// src/agent/control-ws.ts
|
|
350
|
-
var DEFAULT_PATH = "/v1/agent/control";
|
|
351
350
|
var INITIAL_RECONNECT_DELAY = 1e3;
|
|
352
351
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
353
352
|
function buildControlWsUrl(options) {
|
|
354
|
-
const
|
|
355
|
-
const
|
|
356
|
-
|
|
353
|
+
const scheme = options.baseUrl.startsWith("https") ? "wss" : "ws";
|
|
354
|
+
const host = options.baseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "");
|
|
355
|
+
let url = `${scheme}://${host}/v1/accounts/${encodeURIComponent(options.accountId)}/agent/listen`;
|
|
356
|
+
if (options.number) {
|
|
357
|
+
url += `?number=${encodeURIComponent(options.number)}`;
|
|
358
|
+
}
|
|
359
|
+
return url;
|
|
357
360
|
}
|
|
358
361
|
var ControlWebSocket = class {
|
|
359
362
|
constructor(_options) {
|
|
@@ -404,7 +407,12 @@ var ControlWebSocket = class {
|
|
|
404
407
|
}
|
|
405
408
|
async _doConnect() {
|
|
406
409
|
const { WebSocket } = await import('ws');
|
|
407
|
-
const ws = new WebSocket(this._url
|
|
410
|
+
const ws = new WebSocket(this._url, {
|
|
411
|
+
followRedirects: true,
|
|
412
|
+
headers: {
|
|
413
|
+
Authorization: `Bearer ${this._options.apiKey}`
|
|
414
|
+
}
|
|
415
|
+
});
|
|
408
416
|
this._ws = ws;
|
|
409
417
|
ws.on("open", () => {
|
|
410
418
|
this._reconnectDelay = INITIAL_RECONNECT_DELAY;
|
|
@@ -547,47 +555,37 @@ var MCPClient = class {
|
|
|
547
555
|
// src/agent/media-ws.ts
|
|
548
556
|
function parseStartEvent(data) {
|
|
549
557
|
const start = data["start"];
|
|
558
|
+
const fmt = start["mediaFormat"] ?? {};
|
|
550
559
|
return {
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
customParameters: start["customParameters"] ?? {},
|
|
556
|
-
mediaFormat: start["mediaFormat"] ?? {
|
|
557
|
-
encoding: "audio/x-mulaw",
|
|
558
|
-
sampleRate: 8e3,
|
|
559
|
-
channels: 1
|
|
560
|
-
}
|
|
560
|
+
streamId: start["streamId"] ?? "",
|
|
561
|
+
callId: start["callId"] ?? "",
|
|
562
|
+
accountId: start["accountId"] ?? "",
|
|
563
|
+
sampleRate: fmt["sampleRate"] ?? 8e3
|
|
561
564
|
};
|
|
562
565
|
}
|
|
563
566
|
function parseMediaEvent(data) {
|
|
564
567
|
const media = data["media"];
|
|
565
568
|
return {
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
timestamp: media["timestamp"] ?? ""
|
|
569
|
+
audio: Buffer.from(media["payload"] ?? "", "base64"),
|
|
570
|
+
timestamp: parseInt(media["timestamp"] ?? "0", 10) || 0
|
|
569
571
|
};
|
|
570
572
|
}
|
|
571
|
-
function buildMediaResponse(
|
|
573
|
+
function buildMediaResponse(audioBase64) {
|
|
572
574
|
return JSON.stringify({
|
|
573
575
|
event: "media",
|
|
574
|
-
streamSid,
|
|
575
576
|
media: {
|
|
576
|
-
payload
|
|
577
|
+
payload: audioBase64
|
|
577
578
|
}
|
|
578
579
|
});
|
|
579
580
|
}
|
|
580
581
|
var MediaWebSocket = class {
|
|
581
582
|
_ws = null;
|
|
582
|
-
_streamSid = null;
|
|
583
583
|
_audioQueue = [];
|
|
584
584
|
_sendLoopRunning = false;
|
|
585
585
|
_closed = false;
|
|
586
586
|
_onAudio = null;
|
|
587
587
|
_onStart = null;
|
|
588
588
|
_onClose = null;
|
|
589
|
-
_markSeq = 0;
|
|
590
|
-
_markResolves = /* @__PURE__ */ new Map();
|
|
591
589
|
/** Set the handler for inbound audio data. */
|
|
592
590
|
onAudio(handler) {
|
|
593
591
|
this._onAudio = handler;
|
|
@@ -600,11 +598,16 @@ var MediaWebSocket = class {
|
|
|
600
598
|
onClose(handler) {
|
|
601
599
|
this._onClose = handler;
|
|
602
600
|
}
|
|
603
|
-
/** Connect to a media WebSocket URL. */
|
|
604
|
-
async connect(url) {
|
|
601
|
+
/** Connect to a media WebSocket URL with Bearer authentication. */
|
|
602
|
+
async connect(url, apiKey) {
|
|
605
603
|
const { WebSocket } = await import('ws');
|
|
606
604
|
return new Promise((resolve, reject) => {
|
|
607
|
-
const ws = new WebSocket(url
|
|
605
|
+
const ws = new WebSocket(url, {
|
|
606
|
+
followRedirects: true,
|
|
607
|
+
headers: {
|
|
608
|
+
Authorization: `Bearer ${apiKey}`
|
|
609
|
+
}
|
|
610
|
+
});
|
|
608
611
|
this._ws = ws;
|
|
609
612
|
this._closed = false;
|
|
610
613
|
ws.on("open", () => {
|
|
@@ -639,31 +642,21 @@ var MediaWebSocket = class {
|
|
|
639
642
|
/** Clear all queued outbound audio. */
|
|
640
643
|
sendClear() {
|
|
641
644
|
this._audioQueue.length = 0;
|
|
642
|
-
if (this._ws && this.
|
|
645
|
+
if (this._ws && this._ws.readyState === 1) {
|
|
646
|
+
this._ws.send(JSON.stringify({ event: "clear" }));
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
/** Send a mark event. */
|
|
650
|
+
sendMark(name) {
|
|
651
|
+
if (this._ws && this._ws.readyState === 1) {
|
|
643
652
|
this._ws.send(
|
|
644
653
|
JSON.stringify({
|
|
645
|
-
event: "
|
|
646
|
-
|
|
654
|
+
event: "mark",
|
|
655
|
+
mark: { name }
|
|
647
656
|
})
|
|
648
657
|
);
|
|
649
658
|
}
|
|
650
659
|
}
|
|
651
|
-
/** Send a mark event and return a promise that resolves when the mark is acknowledged. */
|
|
652
|
-
sendMark() {
|
|
653
|
-
const label = `mark_${++this._markSeq}`;
|
|
654
|
-
return new Promise((resolve) => {
|
|
655
|
-
this._markResolves.set(label, resolve);
|
|
656
|
-
if (this._ws && this._streamSid && this._ws.readyState === 1) {
|
|
657
|
-
this._ws.send(
|
|
658
|
-
JSON.stringify({
|
|
659
|
-
event: "mark",
|
|
660
|
-
streamSid: this._streamSid,
|
|
661
|
-
mark: { name: label }
|
|
662
|
-
})
|
|
663
|
-
);
|
|
664
|
-
}
|
|
665
|
-
});
|
|
666
|
-
}
|
|
667
660
|
/** Close the media WebSocket. */
|
|
668
661
|
close() {
|
|
669
662
|
this._closed = true;
|
|
@@ -671,17 +664,12 @@ var MediaWebSocket = class {
|
|
|
671
664
|
this._ws.close();
|
|
672
665
|
this._ws = null;
|
|
673
666
|
}
|
|
674
|
-
for (const resolve of this._markResolves.values()) {
|
|
675
|
-
resolve();
|
|
676
|
-
}
|
|
677
|
-
this._markResolves.clear();
|
|
678
667
|
}
|
|
679
668
|
_handleMessage(msg) {
|
|
680
669
|
const event = msg["event"];
|
|
681
670
|
switch (event) {
|
|
682
671
|
case "start": {
|
|
683
672
|
const startEvt = parseStartEvent(msg);
|
|
684
|
-
this._streamSid = startEvt.streamSid;
|
|
685
673
|
if (this._onStart) {
|
|
686
674
|
this._onStart(startEvt);
|
|
687
675
|
}
|
|
@@ -689,18 +677,8 @@ var MediaWebSocket = class {
|
|
|
689
677
|
}
|
|
690
678
|
case "media": {
|
|
691
679
|
const mediaEvt = parseMediaEvent(msg);
|
|
692
|
-
if (
|
|
693
|
-
|
|
694
|
-
this._onAudio(audioBuf);
|
|
695
|
-
}
|
|
696
|
-
break;
|
|
697
|
-
}
|
|
698
|
-
case "mark": {
|
|
699
|
-
const mark = msg["mark"];
|
|
700
|
-
const name = mark?.["name"];
|
|
701
|
-
if (name && this._markResolves.has(name)) {
|
|
702
|
-
this._markResolves.get(name)();
|
|
703
|
-
this._markResolves.delete(name);
|
|
680
|
+
if (this._onAudio) {
|
|
681
|
+
this._onAudio(mediaEvt.audio, mediaEvt.timestamp);
|
|
704
682
|
}
|
|
705
683
|
break;
|
|
706
684
|
}
|
|
@@ -720,98 +698,162 @@ var MediaWebSocket = class {
|
|
|
720
698
|
}
|
|
721
699
|
while (this._audioQueue.length > 0 && this._ws && this._ws.readyState === 1) {
|
|
722
700
|
const payload = this._audioQueue.shift();
|
|
723
|
-
|
|
724
|
-
this._ws.send(buildMediaResponse(this._streamSid, payload));
|
|
725
|
-
}
|
|
701
|
+
this._ws.send(buildMediaResponse(payload));
|
|
726
702
|
}
|
|
727
703
|
setTimeout(loop, 20);
|
|
728
704
|
};
|
|
729
705
|
loop();
|
|
730
706
|
}
|
|
731
707
|
};
|
|
732
|
-
var
|
|
733
|
-
var
|
|
734
|
-
var
|
|
735
|
-
|
|
708
|
+
var SAMPLE_RATE = 8e3;
|
|
709
|
+
var CHANNELS = 1;
|
|
710
|
+
var BITS_PER_SAMPLE = 16;
|
|
711
|
+
var BYTES_PER_SECOND = SAMPLE_RATE * CHANNELS * (BITS_PER_SAMPLE / 8);
|
|
712
|
+
function makeWavHeader(dataSize = 0) {
|
|
736
713
|
const header = Buffer.alloc(44);
|
|
737
|
-
|
|
738
|
-
const blockAlign = WAV_CHANNELS * (WAV_BITS_PER_SAMPLE / 8);
|
|
739
|
-
header.write("RIFF", 0);
|
|
714
|
+
header.write("RIFF", 0, "ascii");
|
|
740
715
|
header.writeUInt32LE(36 + dataSize, 4);
|
|
741
|
-
header.write("WAVE", 8);
|
|
742
|
-
header.write("fmt ", 12);
|
|
716
|
+
header.write("WAVE", 8, "ascii");
|
|
717
|
+
header.write("fmt ", 12, "ascii");
|
|
743
718
|
header.writeUInt32LE(16, 16);
|
|
744
719
|
header.writeUInt16LE(1, 20);
|
|
745
|
-
header.writeUInt16LE(
|
|
746
|
-
header.writeUInt32LE(
|
|
747
|
-
header.writeUInt32LE(
|
|
748
|
-
header.writeUInt16LE(
|
|
749
|
-
header.writeUInt16LE(
|
|
750
|
-
header.write("data", 36);
|
|
720
|
+
header.writeUInt16LE(CHANNELS, 22);
|
|
721
|
+
header.writeUInt32LE(SAMPLE_RATE, 24);
|
|
722
|
+
header.writeUInt32LE(SAMPLE_RATE * CHANNELS * (BITS_PER_SAMPLE / 8), 28);
|
|
723
|
+
header.writeUInt16LE(CHANNELS * (BITS_PER_SAMPLE / 8), 32);
|
|
724
|
+
header.writeUInt16LE(BITS_PER_SAMPLE, 34);
|
|
725
|
+
header.write("data", 36, "ascii");
|
|
751
726
|
header.writeUInt32LE(dataSize, 40);
|
|
752
727
|
return header;
|
|
753
728
|
}
|
|
729
|
+
function mixSamples(a, b) {
|
|
730
|
+
const n = Math.min(a.length, b.length) >> 1;
|
|
731
|
+
const result = Buffer.alloc(n * 2);
|
|
732
|
+
for (let i = 0; i < n; i++) {
|
|
733
|
+
const sa = a.readInt16LE(i * 2);
|
|
734
|
+
const sb = b.readInt16LE(i * 2);
|
|
735
|
+
result.writeInt16LE(Math.max(-32768, Math.min(32767, sa + sb)), i * 2);
|
|
736
|
+
}
|
|
737
|
+
return result;
|
|
738
|
+
}
|
|
754
739
|
var AudioRecorder = class {
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
740
|
+
_dir;
|
|
741
|
+
_fdIn = null;
|
|
742
|
+
_fdOut = null;
|
|
743
|
+
_fdMix = null;
|
|
744
|
+
_inWritten = 0;
|
|
745
|
+
_outWritten = 0;
|
|
746
|
+
_mixWritten = 0;
|
|
747
|
+
_startTime = 0;
|
|
760
748
|
_started = false;
|
|
761
|
-
constructor(
|
|
762
|
-
this.
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
this.
|
|
768
|
-
this.
|
|
769
|
-
this.
|
|
749
|
+
constructor(recordingPath, callId) {
|
|
750
|
+
this._dir = path__namespace.join(recordingPath, callId);
|
|
751
|
+
}
|
|
752
|
+
start() {
|
|
753
|
+
fs__namespace.mkdirSync(this._dir, { recursive: true });
|
|
754
|
+
const header = makeWavHeader();
|
|
755
|
+
this._fdIn = fs__namespace.openSync(path__namespace.join(this._dir, "in.wav"), "w");
|
|
756
|
+
this._fdOut = fs__namespace.openSync(path__namespace.join(this._dir, "out.wav"), "w");
|
|
757
|
+
this._fdMix = fs__namespace.openSync(path__namespace.join(this._dir, "mix.wav"), "w+");
|
|
758
|
+
fs__namespace.writeSync(this._fdIn, header);
|
|
759
|
+
fs__namespace.writeSync(this._fdOut, header);
|
|
760
|
+
fs__namespace.writeSync(this._fdMix, header);
|
|
761
|
+
this._startTime = performance.now();
|
|
770
762
|
this._started = true;
|
|
771
|
-
|
|
772
|
-
|
|
763
|
+
}
|
|
764
|
+
_expectedBytes() {
|
|
765
|
+
const elapsed = (performance.now() - this._startTime) / 1e3;
|
|
766
|
+
return Math.floor(elapsed * BYTES_PER_SECOND);
|
|
767
|
+
}
|
|
768
|
+
_padSilence(fd, written) {
|
|
769
|
+
const expected = this._expectedBytes();
|
|
770
|
+
let gap = expected - written;
|
|
771
|
+
if (gap <= 0) return 0;
|
|
772
|
+
gap = gap - gap % 2;
|
|
773
|
+
if (gap > 0) {
|
|
774
|
+
fs__namespace.writeSync(fd, Buffer.alloc(gap));
|
|
775
|
+
}
|
|
776
|
+
return gap;
|
|
777
|
+
}
|
|
778
|
+
_writeToMix(data, trackPos) {
|
|
779
|
+
if (this._fdMix === null) return;
|
|
780
|
+
const filePos = 44 + trackPos;
|
|
781
|
+
if (trackPos < this._mixWritten) {
|
|
782
|
+
const overlap = Math.min(data.length, this._mixWritten - trackPos);
|
|
783
|
+
const existing = Buffer.alloc(overlap);
|
|
784
|
+
fs__namespace.readSync(this._fdMix, existing, 0, overlap, filePos);
|
|
785
|
+
const mixed = mixSamples(existing, data.subarray(0, overlap));
|
|
786
|
+
fs__namespace.writeSync(this._fdMix, mixed, 0, mixed.length, filePos);
|
|
787
|
+
if (data.length > overlap) {
|
|
788
|
+
fs__namespace.writeSync(this._fdMix, data, overlap, data.length - overlap, filePos + overlap);
|
|
789
|
+
this._mixWritten = trackPos + data.length;
|
|
790
|
+
}
|
|
791
|
+
} else {
|
|
792
|
+
if (trackPos > this._mixWritten) {
|
|
793
|
+
let gap = trackPos - this._mixWritten;
|
|
794
|
+
gap = gap - gap % 2;
|
|
795
|
+
if (gap > 0) {
|
|
796
|
+
const silence = Buffer.alloc(gap);
|
|
797
|
+
fs__namespace.writeSync(this._fdMix, silence, 0, silence.length, 44 + this._mixWritten);
|
|
798
|
+
this._mixWritten += gap;
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
fs__namespace.writeSync(this._fdMix, data, 0, data.length, 44 + this._mixWritten);
|
|
802
|
+
this._mixWritten += data.length;
|
|
773
803
|
}
|
|
774
804
|
}
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
this.
|
|
805
|
+
writeInbound(pcm16_8k) {
|
|
806
|
+
if (!this._started || this._fdIn === null) return;
|
|
807
|
+
try {
|
|
808
|
+
const gap = this._padSilence(this._fdIn, this._inWritten);
|
|
809
|
+
this._inWritten += gap;
|
|
810
|
+
const posBefore = this._inWritten;
|
|
811
|
+
fs__namespace.writeSync(this._fdIn, pcm16_8k);
|
|
812
|
+
this._inWritten += pcm16_8k.length;
|
|
813
|
+
this._writeToMix(pcm16_8k, posBefore);
|
|
814
|
+
} catch (err) {
|
|
815
|
+
console.error("Error writing inbound audio:", err);
|
|
779
816
|
}
|
|
780
817
|
}
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
this.
|
|
818
|
+
writeOutbound(pcm16_8k) {
|
|
819
|
+
if (!this._started || this._fdOut === null) return;
|
|
820
|
+
try {
|
|
821
|
+
const gap = this._padSilence(this._fdOut, this._outWritten);
|
|
822
|
+
this._outWritten += gap;
|
|
823
|
+
const posBefore = this._outWritten;
|
|
824
|
+
fs__namespace.writeSync(this._fdOut, pcm16_8k);
|
|
825
|
+
this._outWritten += pcm16_8k.length;
|
|
826
|
+
this._writeToMix(pcm16_8k, posBefore);
|
|
827
|
+
} catch (err) {
|
|
828
|
+
console.error("Error writing outbound audio:", err);
|
|
785
829
|
}
|
|
786
830
|
}
|
|
787
|
-
/** Stop recording and write WAV files to disk. Returns file paths. */
|
|
788
831
|
stop() {
|
|
789
|
-
if (!this._started
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
832
|
+
if (!this._started) return;
|
|
833
|
+
try {
|
|
834
|
+
let maxWritten = Math.max(this._inWritten, this._outWritten, this._mixWritten);
|
|
835
|
+
maxWritten = maxWritten & ~1;
|
|
836
|
+
for (const [fd, written] of [
|
|
837
|
+
[this._fdIn, this._inWritten],
|
|
838
|
+
[this._fdOut, this._outWritten],
|
|
839
|
+
[this._fdMix, this._mixWritten]
|
|
840
|
+
]) {
|
|
841
|
+
if (fd === null) continue;
|
|
842
|
+
const pad = maxWritten - written;
|
|
843
|
+
if (pad > 0) {
|
|
844
|
+
fs__namespace.writeSync(fd, Buffer.alloc(pad), 0, pad, 44 + written);
|
|
845
|
+
}
|
|
846
|
+
fs__namespace.writeSync(fd, makeWavHeader(maxWritten), 0, 44, 0);
|
|
847
|
+
fs__namespace.closeSync(fd);
|
|
848
|
+
}
|
|
849
|
+
} catch (err) {
|
|
850
|
+
console.error("Error stopping recorder:", err);
|
|
851
|
+
} finally {
|
|
852
|
+
this._fdIn = null;
|
|
853
|
+
this._fdOut = null;
|
|
854
|
+
this._fdMix = null;
|
|
855
|
+
this._started = false;
|
|
803
856
|
}
|
|
804
|
-
this._inboundChunks = [];
|
|
805
|
-
this._outboundChunks = [];
|
|
806
|
-
return result;
|
|
807
|
-
}
|
|
808
|
-
_writeWav(filePath, chunks) {
|
|
809
|
-
const pcmData = Buffer.concat(chunks);
|
|
810
|
-
const header = makeWavHeader(pcmData.length);
|
|
811
|
-
const fd = fs__namespace.openSync(filePath, "w");
|
|
812
|
-
fs__namespace.writeSync(fd, header);
|
|
813
|
-
fs__namespace.writeSync(fd, pcmData);
|
|
814
|
-
fs__namespace.closeSync(fd);
|
|
815
857
|
}
|
|
816
858
|
};
|
|
817
859
|
|
|
@@ -891,23 +933,23 @@ var CallSession = class {
|
|
|
891
933
|
/** Mark the session as ended (called internally). */
|
|
892
934
|
_markEnded() {
|
|
893
935
|
this._status = "ended";
|
|
894
|
-
this._emit({ type: "ended" });
|
|
895
936
|
this._resolveEnded();
|
|
896
937
|
}
|
|
897
|
-
/** Emit an event to registered handlers. */
|
|
898
|
-
|
|
899
|
-
|
|
938
|
+
/** Emit an event to registered handlers. Matches Python SDK: _emit(event, ...args) */
|
|
939
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
940
|
+
_emit(event, ...args) {
|
|
941
|
+
const handlers = this._handlers.get(event);
|
|
900
942
|
if (handlers) {
|
|
901
943
|
for (const handler of handlers) {
|
|
902
944
|
try {
|
|
903
|
-
const result = handler(
|
|
945
|
+
const result = handler(this, ...args);
|
|
904
946
|
if (result && typeof result.catch === "function") {
|
|
905
947
|
result.catch((err) => {
|
|
906
|
-
console.error(`[CallSession] Error in ${event
|
|
948
|
+
console.error(`[CallSession] Error in ${event} handler:`, err);
|
|
907
949
|
});
|
|
908
950
|
}
|
|
909
951
|
} catch (err) {
|
|
910
|
-
console.error(`[CallSession] Error in ${event
|
|
952
|
+
console.error(`[CallSession] Error in ${event} handler:`, err);
|
|
911
953
|
}
|
|
912
954
|
}
|
|
913
955
|
}
|
|
@@ -1111,44 +1153,61 @@ var ATTR_AGENT_ID = "clawops.agent.id";
|
|
|
1111
1153
|
// src/agent/agent.ts
|
|
1112
1154
|
var ClawOpsAgent = class {
|
|
1113
1155
|
_apiKey;
|
|
1114
|
-
|
|
1156
|
+
_accountId;
|
|
1115
1157
|
_baseUrl;
|
|
1116
|
-
|
|
1158
|
+
_fromNumber;
|
|
1159
|
+
_session;
|
|
1117
1160
|
_tools = new ToolRegistry();
|
|
1118
1161
|
_handlers = /* @__PURE__ */ new Map();
|
|
1119
1162
|
_controlWs = null;
|
|
1120
|
-
|
|
1121
|
-
|
|
1163
|
+
_mcpServers;
|
|
1164
|
+
_recording;
|
|
1165
|
+
_recordingPath;
|
|
1122
1166
|
_activeSessions = /* @__PURE__ */ new Map();
|
|
1123
|
-
constructor(options
|
|
1167
|
+
constructor(options) {
|
|
1124
1168
|
this._apiKey = options.apiKey ?? process.env["CLAWOPS_API_KEY"] ?? "";
|
|
1125
|
-
this.
|
|
1169
|
+
this._accountId = options.accountId ?? process.env["CLAWOPS_ACCOUNT_ID"] ?? "";
|
|
1126
1170
|
this._baseUrl = options.baseUrl ?? chunkJNSHMKDI_cjs.DEFAULT_BASE_URL;
|
|
1127
|
-
this.
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
const sessionInstance = options.session;
|
|
1133
|
-
this._sessionFactory = () => sessionInstance;
|
|
1134
|
-
}
|
|
1135
|
-
}
|
|
1136
|
-
if (options.mcpServers) {
|
|
1137
|
-
this._mcpClient = new MCPClient();
|
|
1138
|
-
for (const [name, config] of Object.entries(options.mcpServers)) {
|
|
1139
|
-
this._mcpClient.addServer(name, config);
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1171
|
+
this._fromNumber = options.from;
|
|
1172
|
+
this._session = options.session;
|
|
1173
|
+
this._recording = options.recording ?? false;
|
|
1174
|
+
this._recordingPath = options.recordingPath ?? "./recordings";
|
|
1175
|
+
this._mcpServers = options.mcpServers ?? [];
|
|
1142
1176
|
if (options.tracing) {
|
|
1143
1177
|
setTracingConfig(options.tracing);
|
|
1144
1178
|
}
|
|
1145
1179
|
}
|
|
1146
|
-
/**
|
|
1147
|
-
|
|
1148
|
-
|
|
1180
|
+
/**
|
|
1181
|
+
* Register a function tool.
|
|
1182
|
+
*
|
|
1183
|
+
* Supports two signatures (matching Python SDK):
|
|
1184
|
+
* agent.tool(name, description, parameters, handler)
|
|
1185
|
+
* agent.tool(functionToolObject)
|
|
1186
|
+
*/
|
|
1187
|
+
tool(nameOrTool, description, parameters, handler) {
|
|
1188
|
+
if (typeof nameOrTool === "string") {
|
|
1189
|
+
if (!description || !parameters || !handler) {
|
|
1190
|
+
throw new chunkJNSHMKDI_cjs.AgentError("tool(name, description, parameters, handler) requires all arguments.");
|
|
1191
|
+
}
|
|
1192
|
+
this._tools.register({
|
|
1193
|
+
name: nameOrTool,
|
|
1194
|
+
description,
|
|
1195
|
+
parameters,
|
|
1196
|
+
required: Object.keys(parameters),
|
|
1197
|
+
handler
|
|
1198
|
+
});
|
|
1199
|
+
} else {
|
|
1200
|
+
this._tools.register(nameOrTool);
|
|
1201
|
+
}
|
|
1149
1202
|
return this;
|
|
1150
1203
|
}
|
|
1151
|
-
/**
|
|
1204
|
+
/**
|
|
1205
|
+
* Register an event handler.
|
|
1206
|
+
*
|
|
1207
|
+
* Matches Python SDK decorator style:
|
|
1208
|
+
* agent.on("call_start", (call) => { ... })
|
|
1209
|
+
* agent.on("transcript", (call, role, text) => { ... })
|
|
1210
|
+
*/
|
|
1152
1211
|
on(event, handler) {
|
|
1153
1212
|
let list = this._handlers.get(event);
|
|
1154
1213
|
if (!list) {
|
|
@@ -1163,21 +1222,14 @@ var ClawOpsAgent = class {
|
|
|
1163
1222
|
if (!this._apiKey) {
|
|
1164
1223
|
throw new chunkJNSHMKDI_cjs.AgentError("API key is required. Set CLAWOPS_API_KEY or pass apiKey option.");
|
|
1165
1224
|
}
|
|
1166
|
-
if (!this.
|
|
1167
|
-
throw new chunkJNSHMKDI_cjs.AgentError("
|
|
1168
|
-
}
|
|
1169
|
-
if (this._mcpClient) {
|
|
1170
|
-
try {
|
|
1171
|
-
const mcpTools = await this._mcpClient.connect();
|
|
1172
|
-
this._tools.registerMcpTools(mcpTools);
|
|
1173
|
-
} catch (err) {
|
|
1174
|
-
console.error("[ClawOpsAgent] MCP connection error:", err);
|
|
1175
|
-
}
|
|
1225
|
+
if (!this._accountId) {
|
|
1226
|
+
throw new chunkJNSHMKDI_cjs.AgentError("Account ID is required. Set CLAWOPS_ACCOUNT_ID or pass accountId option.");
|
|
1176
1227
|
}
|
|
1177
1228
|
this._controlWs = new ControlWebSocket({
|
|
1178
1229
|
baseUrl: this._baseUrl,
|
|
1179
1230
|
apiKey: this._apiKey,
|
|
1180
|
-
|
|
1231
|
+
accountId: this._accountId,
|
|
1232
|
+
number: this._fromNumber
|
|
1181
1233
|
});
|
|
1182
1234
|
this._controlWs.on("call.incoming", (event) => this._handleIncoming(event));
|
|
1183
1235
|
this._controlWs.on("call.ended", (event) => this._handleEnded(event));
|
|
@@ -1192,6 +1244,7 @@ var ClawOpsAgent = class {
|
|
|
1192
1244
|
`Failed to connect to ClawOps: ${err instanceof Error ? err.message : String(err)}`
|
|
1193
1245
|
);
|
|
1194
1246
|
}
|
|
1247
|
+
console.log(`[ClawOpsAgent] Connected on ${this._fromNumber}`);
|
|
1195
1248
|
}
|
|
1196
1249
|
/**
|
|
1197
1250
|
* Connect and block until disconnected.
|
|
@@ -1217,83 +1270,118 @@ var ClawOpsAgent = class {
|
|
|
1217
1270
|
session._markEnded();
|
|
1218
1271
|
}
|
|
1219
1272
|
this._activeSessions.clear();
|
|
1220
|
-
|
|
1221
|
-
await this._mcpClient.disconnect();
|
|
1222
|
-
this._tools.clearMcpTools();
|
|
1223
|
-
}
|
|
1273
|
+
console.log("[ClawOpsAgent] Disconnected");
|
|
1224
1274
|
}
|
|
1225
1275
|
/**
|
|
1226
1276
|
* Initiate an outbound call.
|
|
1277
|
+
* Matches Python SDK: agent.call(to, { timeout })
|
|
1227
1278
|
*/
|
|
1228
|
-
call(options) {
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
}
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1279
|
+
async call(to, options) {
|
|
1280
|
+
await this.connect();
|
|
1281
|
+
const url = `${this._baseUrl}/v1/accounts/${this._accountId}/calls`;
|
|
1282
|
+
const body = { To: to, From: this._fromNumber, Timeout: options?.timeout ?? 60 };
|
|
1283
|
+
const resp = await fetch(url, {
|
|
1284
|
+
method: "POST",
|
|
1285
|
+
headers: {
|
|
1286
|
+
Authorization: `Bearer ${this._apiKey}`,
|
|
1287
|
+
"Content-Type": "application/json"
|
|
1288
|
+
},
|
|
1289
|
+
body: JSON.stringify(body)
|
|
1290
|
+
});
|
|
1291
|
+
if (resp.status !== 201) {
|
|
1292
|
+
const error = await resp.json();
|
|
1293
|
+
throw new chunkJNSHMKDI_cjs.AgentError(`\uBC1C\uC2E0 \uC2E4\uD328 (${resp.status}): ${error["error"] ?? ""}`);
|
|
1294
|
+
}
|
|
1295
|
+
const data = await resp.json();
|
|
1296
|
+
const callSession = new CallSession({
|
|
1297
|
+
callId: data["callId"],
|
|
1298
|
+
fromNumber: this._fromNumber,
|
|
1299
|
+
toNumber: to,
|
|
1300
|
+
accountId: this._accountId,
|
|
1301
|
+
direction: "outbound"
|
|
1239
1302
|
});
|
|
1303
|
+
for (const [evt, handlers] of this._handlers) {
|
|
1304
|
+
for (const handler of handlers) {
|
|
1305
|
+
callSession.on(evt, handler);
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
this._activeSessions.set(callSession.callId, callSession);
|
|
1309
|
+
console.log(`[ClawOpsAgent] Outbound call initiated: ${this._fromNumber} -> ${to} (${callSession.callId})`);
|
|
1310
|
+
return callSession;
|
|
1240
1311
|
}
|
|
1241
1312
|
_handleIncoming(event) {
|
|
1242
|
-
const
|
|
1313
|
+
const callId = event["callId"];
|
|
1314
|
+
const fromNumber = event["from"] ?? "";
|
|
1315
|
+
const mediaUrl = event["mediaUrl"] ?? "";
|
|
1243
1316
|
const session = new CallSession({
|
|
1244
|
-
callId
|
|
1245
|
-
fromNumber
|
|
1246
|
-
toNumber:
|
|
1247
|
-
accountId:
|
|
1248
|
-
direction:
|
|
1249
|
-
metadata: data.metadata
|
|
1317
|
+
callId,
|
|
1318
|
+
fromNumber,
|
|
1319
|
+
toNumber: this._fromNumber,
|
|
1320
|
+
accountId: this._accountId,
|
|
1321
|
+
direction: "inbound"
|
|
1250
1322
|
});
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1323
|
+
for (const [evt, handlers] of this._handlers) {
|
|
1324
|
+
for (const handler of handlers) {
|
|
1325
|
+
session.on(evt, handler);
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
this._activeSessions.set(callId, session);
|
|
1329
|
+
if (this._controlWs) {
|
|
1330
|
+
this._controlWs.send({ event: "call.accept", callId });
|
|
1331
|
+
}
|
|
1332
|
+
if (mediaUrl) {
|
|
1333
|
+
this._startCallSession(session, mediaUrl).catch((err) => {
|
|
1334
|
+
console.error(`[ClawOpsAgent] Error in call session ${callId}:`, err);
|
|
1256
1335
|
});
|
|
1257
1336
|
}
|
|
1258
1337
|
}
|
|
1259
1338
|
_handleEnded(event) {
|
|
1260
|
-
const
|
|
1339
|
+
const callId = event["callId"];
|
|
1340
|
+
const session = this._activeSessions.get(callId);
|
|
1261
1341
|
if (session) {
|
|
1262
1342
|
session._markEnded();
|
|
1263
|
-
this._activeSessions.delete(
|
|
1264
|
-
this._emitEvent("call.ended", session);
|
|
1343
|
+
this._activeSessions.delete(callId);
|
|
1265
1344
|
}
|
|
1266
1345
|
}
|
|
1267
1346
|
_handleOutboundReady(event) {
|
|
1268
|
-
const
|
|
1269
|
-
const
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1347
|
+
const callId = event["callId"];
|
|
1348
|
+
const mediaUrl = event["mediaUrl"] ?? "";
|
|
1349
|
+
let session = this._activeSessions.get(callId);
|
|
1350
|
+
if (!session) {
|
|
1351
|
+
session = new CallSession({
|
|
1352
|
+
callId,
|
|
1353
|
+
fromNumber: this._fromNumber,
|
|
1354
|
+
toNumber: event["to"] ?? "",
|
|
1355
|
+
accountId: this._accountId,
|
|
1356
|
+
direction: "outbound"
|
|
1357
|
+
});
|
|
1358
|
+
for (const [evt, handlers] of this._handlers) {
|
|
1359
|
+
for (const handler of handlers) {
|
|
1360
|
+
session.on(evt, handler);
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
this._activeSessions.set(callId, session);
|
|
1364
|
+
}
|
|
1365
|
+
if (mediaUrl) {
|
|
1366
|
+
this._startCallSession(session, mediaUrl).catch((err) => {
|
|
1367
|
+
console.error(`[ClawOpsAgent] Error in call session ${callId}:`, err);
|
|
1282
1368
|
});
|
|
1283
1369
|
}
|
|
1284
1370
|
}
|
|
1285
1371
|
_handleRinging(event) {
|
|
1286
|
-
const
|
|
1372
|
+
const callId = event["callId"];
|
|
1373
|
+
const session = this._activeSessions.get(callId);
|
|
1287
1374
|
if (session) {
|
|
1288
|
-
|
|
1375
|
+
console.log(`[ClawOpsAgent] Outbound call ringing: ${callId}`);
|
|
1289
1376
|
}
|
|
1290
1377
|
}
|
|
1291
1378
|
_handleFailed(event) {
|
|
1292
|
-
const
|
|
1379
|
+
const callId = event["callId"];
|
|
1380
|
+
const session = this._activeSessions.get(callId);
|
|
1293
1381
|
if (session) {
|
|
1382
|
+
session._emit("call_failed", event["reason"] ?? "failed");
|
|
1294
1383
|
session._markEnded();
|
|
1295
|
-
this._activeSessions.delete(
|
|
1296
|
-
this._emitEvent("call.failed", session);
|
|
1384
|
+
this._activeSessions.delete(callId);
|
|
1297
1385
|
}
|
|
1298
1386
|
}
|
|
1299
1387
|
async _startCallSession(session, mediaWsUrl) {
|
|
@@ -1302,23 +1390,33 @@ var ClawOpsAgent = class {
|
|
|
1302
1390
|
{
|
|
1303
1391
|
[ATTR_CALL_ID]: session.callId,
|
|
1304
1392
|
[ATTR_CALL_DIRECTION]: session.direction,
|
|
1305
|
-
[ATTR_AGENT_ID]: this.
|
|
1393
|
+
[ATTR_AGENT_ID]: this._accountId
|
|
1306
1394
|
},
|
|
1307
1395
|
async () => {
|
|
1308
1396
|
const sessionTools = this._tools.fork();
|
|
1397
|
+
const mcpClients = [];
|
|
1398
|
+
if (this._mcpServers.length > 0) {
|
|
1399
|
+
for (const serverConfig of this._mcpServers) {
|
|
1400
|
+
const client = new MCPClient();
|
|
1401
|
+
client.addServer("mcp", serverConfig);
|
|
1402
|
+
try {
|
|
1403
|
+
const tools = await client.connect();
|
|
1404
|
+
sessionTools.registerMcpTools(tools);
|
|
1405
|
+
mcpClients.push(client);
|
|
1406
|
+
} catch (err) {
|
|
1407
|
+
console.error("[ClawOpsAgent] MCP connection error:", err);
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1309
1411
|
let recorder = null;
|
|
1310
|
-
if (this.
|
|
1311
|
-
recorder = new AudioRecorder(
|
|
1312
|
-
recorder.start(
|
|
1412
|
+
if (this._recording) {
|
|
1413
|
+
recorder = new AudioRecorder(this._recordingPath, session.callId);
|
|
1414
|
+
recorder.start();
|
|
1313
1415
|
}
|
|
1314
1416
|
const mediaWs = new MediaWebSocket();
|
|
1315
1417
|
session._bindTransport(
|
|
1316
1418
|
(audio) => {
|
|
1317
|
-
|
|
1318
|
-
mediaWs.sendAudio(ulaw.toString("base64"));
|
|
1319
|
-
if (recorder) {
|
|
1320
|
-
recorder.writeRawOutbound(audio);
|
|
1321
|
-
}
|
|
1419
|
+
mediaWs.sendAudio(audio.toString("base64"));
|
|
1322
1420
|
},
|
|
1323
1421
|
() => {
|
|
1324
1422
|
mediaWs.sendClear();
|
|
@@ -1327,15 +1425,19 @@ var ClawOpsAgent = class {
|
|
|
1327
1425
|
mediaWs.close();
|
|
1328
1426
|
}
|
|
1329
1427
|
);
|
|
1330
|
-
const sessionHandler = this.
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1428
|
+
const sessionHandler = this._session;
|
|
1429
|
+
if ("setToolRegistry" in sessionHandler && typeof sessionHandler.setToolRegistry === "function") {
|
|
1430
|
+
sessionHandler.setToolRegistry(sessionTools);
|
|
1431
|
+
}
|
|
1432
|
+
if (recorder && "setRecorder" in sessionHandler && typeof sessionHandler.setRecorder === "function") {
|
|
1433
|
+
sessionHandler.setRecorder(recorder);
|
|
1434
|
+
}
|
|
1435
|
+
mediaWs.onAudio((ulawAudio, _timestamp) => {
|
|
1336
1436
|
if (sessionHandler) {
|
|
1337
|
-
|
|
1338
|
-
|
|
1437
|
+
sessionHandler.feedAudio(ulawAudio);
|
|
1438
|
+
}
|
|
1439
|
+
if (recorder) {
|
|
1440
|
+
recorder.writeInbound(ulawToPcm16(ulawAudio));
|
|
1339
1441
|
}
|
|
1340
1442
|
});
|
|
1341
1443
|
mediaWs.onClose(() => {
|
|
@@ -1344,74 +1446,93 @@ var ClawOpsAgent = class {
|
|
|
1344
1446
|
}
|
|
1345
1447
|
session._markEnded();
|
|
1346
1448
|
});
|
|
1449
|
+
session._emit("call_start");
|
|
1347
1450
|
try {
|
|
1348
|
-
await mediaWs.connect(mediaWsUrl);
|
|
1349
|
-
|
|
1350
|
-
await sessionHandler.start(session, sessionTools);
|
|
1351
|
-
}
|
|
1451
|
+
await mediaWs.connect(mediaWsUrl, this._apiKey);
|
|
1452
|
+
await sessionHandler.start(session, sessionTools);
|
|
1352
1453
|
await session.wait();
|
|
1353
|
-
|
|
1354
|
-
await sessionHandler.stop();
|
|
1355
|
-
}
|
|
1454
|
+
await sessionHandler.stop();
|
|
1356
1455
|
} catch (err) {
|
|
1357
1456
|
console.error(`[ClawOpsAgent] Call session error:`, err);
|
|
1358
1457
|
} finally {
|
|
1458
|
+
if (mcpClients.length > 0) {
|
|
1459
|
+
sessionTools.clearMcpTools();
|
|
1460
|
+
for (const c of mcpClients) {
|
|
1461
|
+
await c.disconnect();
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1359
1464
|
mediaWs.close();
|
|
1360
1465
|
if (recorder) {
|
|
1361
1466
|
recorder.stop();
|
|
1362
1467
|
}
|
|
1468
|
+
session._emit("call_end");
|
|
1469
|
+
session._markEnded();
|
|
1470
|
+
this._activeSessions.delete(session.callId);
|
|
1363
1471
|
}
|
|
1364
1472
|
}
|
|
1365
1473
|
);
|
|
1366
1474
|
}
|
|
1367
|
-
_emitEvent(event, session) {
|
|
1368
|
-
const handlers = this._handlers.get(event);
|
|
1369
|
-
if (handlers) {
|
|
1370
|
-
for (const handler of handlers) {
|
|
1371
|
-
try {
|
|
1372
|
-
const result = handler(session);
|
|
1373
|
-
if (result && typeof result.catch === "function") {
|
|
1374
|
-
result.catch((err) => {
|
|
1375
|
-
console.error(`[ClawOpsAgent] Error in ${event} handler:`, err);
|
|
1376
|
-
});
|
|
1377
|
-
}
|
|
1378
|
-
} catch (err) {
|
|
1379
|
-
console.error(`[ClawOpsAgent] Error in ${event} handler:`, err);
|
|
1380
|
-
}
|
|
1381
|
-
}
|
|
1382
|
-
}
|
|
1383
|
-
}
|
|
1384
1475
|
};
|
|
1385
1476
|
|
|
1386
1477
|
// src/agent/pipeline/openai-realtime.ts
|
|
1478
|
+
var OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime?model=";
|
|
1479
|
+
var HANG_UP_TOOL = {
|
|
1480
|
+
type: "function",
|
|
1481
|
+
name: "hang_up",
|
|
1482
|
+
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1483
|
+
parameters: { type: "object", properties: {}, required: [] }
|
|
1484
|
+
};
|
|
1387
1485
|
var OpenAIRealtime = class {
|
|
1388
|
-
|
|
1486
|
+
_apiKey;
|
|
1487
|
+
_systemPrompt;
|
|
1488
|
+
_model;
|
|
1489
|
+
_voice;
|
|
1490
|
+
_language;
|
|
1491
|
+
_eagerness;
|
|
1492
|
+
_greeting;
|
|
1389
1493
|
_ws = null;
|
|
1390
|
-
|
|
1494
|
+
_call = null;
|
|
1391
1495
|
_tools = null;
|
|
1496
|
+
_recorder = null;
|
|
1392
1497
|
_closed = false;
|
|
1498
|
+
// Truncation / barge-in tracking (matching Python SDK)
|
|
1499
|
+
_lastAssistantItem = null;
|
|
1500
|
+
_responseStartTs = null;
|
|
1501
|
+
_sentAudioChunks = 0;
|
|
1502
|
+
_audioRemainder = Buffer.alloc(0);
|
|
1393
1503
|
constructor(options = {}) {
|
|
1394
|
-
this.
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1504
|
+
this._apiKey = options.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
|
|
1505
|
+
this._systemPrompt = options.systemPrompt ?? "";
|
|
1506
|
+
this._model = options.model ?? "gpt-realtime-1.5";
|
|
1507
|
+
this._voice = options.voice ?? "marin";
|
|
1508
|
+
this._language = options.language ?? "ko";
|
|
1509
|
+
this._eagerness = options.eagerness ?? "high";
|
|
1510
|
+
this._greeting = options.greeting ?? true;
|
|
1511
|
+
}
|
|
1512
|
+
/** Inject per-call ToolRegistry. */
|
|
1513
|
+
setToolRegistry(registry) {
|
|
1514
|
+
this._tools = registry;
|
|
1515
|
+
}
|
|
1516
|
+
/** Inject per-call AudioRecorder. */
|
|
1517
|
+
setRecorder(recorder) {
|
|
1518
|
+
this._recorder = recorder;
|
|
1401
1519
|
}
|
|
1402
1520
|
async start(callSession, tools) {
|
|
1403
|
-
this.
|
|
1404
|
-
this._tools = tools
|
|
1521
|
+
this._call = callSession;
|
|
1522
|
+
if (tools) this._tools = tools;
|
|
1405
1523
|
this._closed = false;
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1524
|
+
this._lastAssistantItem = null;
|
|
1525
|
+
this._responseStartTs = null;
|
|
1526
|
+
this._sentAudioChunks = 0;
|
|
1527
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1528
|
+
if (!this._apiKey) {
|
|
1529
|
+
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey option.");
|
|
1409
1530
|
}
|
|
1410
1531
|
const { WebSocket } = await import('ws');
|
|
1411
|
-
const url =
|
|
1532
|
+
const url = `${OPENAI_REALTIME_URL}${this._model}`;
|
|
1412
1533
|
this._ws = new WebSocket(url, {
|
|
1413
1534
|
headers: {
|
|
1414
|
-
Authorization: `Bearer ${
|
|
1535
|
+
Authorization: `Bearer ${this._apiKey}`,
|
|
1415
1536
|
"OpenAI-Beta": "realtime=v1"
|
|
1416
1537
|
}
|
|
1417
1538
|
});
|
|
@@ -1419,6 +1540,9 @@ var OpenAIRealtime = class {
|
|
|
1419
1540
|
const ws = this._ws;
|
|
1420
1541
|
ws.on("open", () => {
|
|
1421
1542
|
this._sendSessionUpdate();
|
|
1543
|
+
if (this._greeting) {
|
|
1544
|
+
this._send({ type: "response.create" });
|
|
1545
|
+
}
|
|
1422
1546
|
resolve();
|
|
1423
1547
|
});
|
|
1424
1548
|
ws.on("message", (data) => {
|
|
@@ -1441,12 +1565,10 @@ var OpenAIRealtime = class {
|
|
|
1441
1565
|
}
|
|
1442
1566
|
feedAudio(audio) {
|
|
1443
1567
|
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1444
|
-
this.
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
})
|
|
1449
|
-
);
|
|
1568
|
+
this._send({
|
|
1569
|
+
type: "input_audio_buffer.append",
|
|
1570
|
+
audio: audio.toString("base64")
|
|
1571
|
+
});
|
|
1450
1572
|
}
|
|
1451
1573
|
}
|
|
1452
1574
|
async stop() {
|
|
@@ -1458,52 +1580,71 @@ var OpenAIRealtime = class {
|
|
|
1458
1580
|
}
|
|
1459
1581
|
_sendSessionUpdate() {
|
|
1460
1582
|
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1461
|
-
const
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
);
|
|
1583
|
+
const toolSchemas = this._tools ? this._tools.toOpenAITools().map((t) => ({ type: "function", ...t.function })) : [];
|
|
1584
|
+
toolSchemas.push(HANG_UP_TOOL);
|
|
1585
|
+
this._send({
|
|
1586
|
+
type: "session.update",
|
|
1587
|
+
session: {
|
|
1588
|
+
modalities: ["text", "audio"],
|
|
1589
|
+
voice: this._voice,
|
|
1590
|
+
instructions: this._systemPrompt,
|
|
1591
|
+
input_audio_format: "g711_ulaw",
|
|
1592
|
+
output_audio_format: "g711_ulaw",
|
|
1593
|
+
input_audio_transcription: {
|
|
1594
|
+
model: "gpt-4o-mini-transcribe",
|
|
1595
|
+
language: this._language
|
|
1596
|
+
},
|
|
1597
|
+
input_audio_noise_reduction: { type: "far_field" },
|
|
1598
|
+
turn_detection: {
|
|
1599
|
+
type: "semantic_vad",
|
|
1600
|
+
interrupt_response: true,
|
|
1601
|
+
eagerness: this._eagerness
|
|
1602
|
+
},
|
|
1603
|
+
tools: toolSchemas
|
|
1604
|
+
}
|
|
1605
|
+
});
|
|
1485
1606
|
}
|
|
1486
1607
|
_handleMessage(msg) {
|
|
1487
1608
|
const type = msg["type"];
|
|
1488
1609
|
switch (type) {
|
|
1489
1610
|
case "response.audio.delta": {
|
|
1490
|
-
|
|
1491
|
-
if (delta && this._callSession) {
|
|
1492
|
-
const audio = Buffer.from(delta, "base64");
|
|
1493
|
-
this._callSession.sendAudio(audio);
|
|
1494
|
-
}
|
|
1611
|
+
this._handleAudioDelta(msg);
|
|
1495
1612
|
break;
|
|
1496
1613
|
}
|
|
1497
1614
|
case "response.audio.done": {
|
|
1615
|
+
if (this._audioRemainder.length > 0) {
|
|
1616
|
+
const padded = Buffer.concat([
|
|
1617
|
+
this._audioRemainder,
|
|
1618
|
+
Buffer.alloc(160 - this._audioRemainder.length, 255)
|
|
1619
|
+
]);
|
|
1620
|
+
if (this._call) {
|
|
1621
|
+
this._call.sendAudio(padded);
|
|
1622
|
+
}
|
|
1623
|
+
this._sentAudioChunks++;
|
|
1624
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1625
|
+
}
|
|
1626
|
+
break;
|
|
1627
|
+
}
|
|
1628
|
+
case "input_audio_buffer.speech_started": {
|
|
1629
|
+
this._handleTruncation();
|
|
1498
1630
|
break;
|
|
1499
1631
|
}
|
|
1500
|
-
case "response.
|
|
1501
|
-
|
|
1632
|
+
case "response.output_item.done": {
|
|
1633
|
+
const item = msg["item"];
|
|
1634
|
+
if (item && item["type"] === "function_call") {
|
|
1635
|
+
this._handleToolCall(item);
|
|
1636
|
+
}
|
|
1502
1637
|
break;
|
|
1503
1638
|
}
|
|
1504
|
-
case "
|
|
1505
|
-
if (this.
|
|
1506
|
-
this.
|
|
1639
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
1640
|
+
if (this._call) {
|
|
1641
|
+
this._call._emit("transcript", "user", msg["transcript"] ?? "");
|
|
1642
|
+
}
|
|
1643
|
+
break;
|
|
1644
|
+
}
|
|
1645
|
+
case "response.audio_transcript.done": {
|
|
1646
|
+
if (this._call) {
|
|
1647
|
+
this._call._emit("transcript", "assistant", msg["transcript"] ?? "");
|
|
1507
1648
|
}
|
|
1508
1649
|
break;
|
|
1509
1650
|
}
|
|
@@ -1513,73 +1654,215 @@ var OpenAIRealtime = class {
|
|
|
1513
1654
|
}
|
|
1514
1655
|
}
|
|
1515
1656
|
}
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1657
|
+
_handleAudioDelta(msg) {
|
|
1658
|
+
if (this._responseStartTs === null) {
|
|
1659
|
+
this._responseStartTs = Date.now();
|
|
1660
|
+
this._sentAudioChunks = 0;
|
|
1661
|
+
}
|
|
1662
|
+
if (msg["item_id"]) {
|
|
1663
|
+
this._lastAssistantItem = msg["item_id"];
|
|
1664
|
+
}
|
|
1665
|
+
const ulaw = Buffer.from(msg["delta"], "base64");
|
|
1666
|
+
if (this._recorder) {
|
|
1667
|
+
this._recorder.writeOutbound(ulawToPcm16(ulaw));
|
|
1668
|
+
}
|
|
1669
|
+
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
1670
|
+
const chunkSize = 160;
|
|
1671
|
+
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
1672
|
+
for (let off = 0; off < fullEnd; off += chunkSize) {
|
|
1673
|
+
if (this._call) {
|
|
1674
|
+
this._call.sendAudio(combined.subarray(off, off + chunkSize));
|
|
1675
|
+
}
|
|
1676
|
+
this._sentAudioChunks++;
|
|
1677
|
+
}
|
|
1678
|
+
this._audioRemainder = combined.subarray(fullEnd);
|
|
1679
|
+
}
|
|
1680
|
+
_handleTruncation() {
|
|
1681
|
+
if (!this._lastAssistantItem || this._responseStartTs === null) {
|
|
1522
1682
|
return;
|
|
1523
1683
|
}
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1684
|
+
const audioEndMs = Math.max(0, this._sentAudioChunks * 20);
|
|
1685
|
+
this._send({
|
|
1686
|
+
type: "conversation.item.truncate",
|
|
1687
|
+
item_id: this._lastAssistantItem,
|
|
1688
|
+
content_index: 0,
|
|
1689
|
+
audio_end_ms: audioEndMs
|
|
1690
|
+
});
|
|
1691
|
+
if (this._call) {
|
|
1692
|
+
this._call.clearAudio();
|
|
1693
|
+
}
|
|
1694
|
+
this._lastAssistantItem = null;
|
|
1695
|
+
this._responseStartTs = null;
|
|
1696
|
+
this._sentAudioChunks = 0;
|
|
1697
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1698
|
+
}
|
|
1699
|
+
async _handleToolCall(item) {
|
|
1700
|
+
const funcName = item["name"];
|
|
1701
|
+
const callId = item["call_id"];
|
|
1702
|
+
if (funcName === "hang_up") {
|
|
1703
|
+
if (this._call) {
|
|
1704
|
+
this._call.hangup();
|
|
1539
1705
|
}
|
|
1706
|
+
return;
|
|
1707
|
+
}
|
|
1708
|
+
if (!this._tools || !this._tools.has(funcName)) {
|
|
1709
|
+
console.error(`[OpenAIRealtime] Unknown tool: ${funcName}`);
|
|
1710
|
+
return;
|
|
1711
|
+
}
|
|
1712
|
+
let result;
|
|
1713
|
+
try {
|
|
1714
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
1715
|
+
result = await this._tools.call(funcName, args);
|
|
1540
1716
|
} catch (err) {
|
|
1541
|
-
console.error(`[OpenAIRealtime] Tool call
|
|
1717
|
+
console.error(`[OpenAIRealtime] Tool call failed: ${funcName}:`, err);
|
|
1718
|
+
result = `Error: ${err}`;
|
|
1719
|
+
}
|
|
1720
|
+
this._send({
|
|
1721
|
+
type: "conversation.item.create",
|
|
1722
|
+
item: {
|
|
1723
|
+
type: "function_call_output",
|
|
1724
|
+
call_id: callId,
|
|
1725
|
+
output: typeof result === "string" ? result : JSON.stringify(result)
|
|
1726
|
+
}
|
|
1727
|
+
});
|
|
1728
|
+
this._send({ type: "response.create" });
|
|
1729
|
+
}
|
|
1730
|
+
_send(data) {
|
|
1731
|
+
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1732
|
+
this._ws.send(JSON.stringify(data));
|
|
1542
1733
|
}
|
|
1543
1734
|
}
|
|
1544
1735
|
};
|
|
1545
1736
|
|
|
1546
1737
|
// src/agent/pipeline/gemini-realtime.ts
|
|
1738
|
+
var HANG_UP_TOOL2 = {
|
|
1739
|
+
name: "hang_up",
|
|
1740
|
+
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1741
|
+
parameters: { type: "object", properties: {} }
|
|
1742
|
+
};
|
|
1743
|
+
function resolveRef(ref, defs) {
|
|
1744
|
+
const parts = ref.replace(/^#\//, "").split("/");
|
|
1745
|
+
let result = defs;
|
|
1746
|
+
for (const part of parts) {
|
|
1747
|
+
if (result && typeof result === "object" && !Array.isArray(result)) {
|
|
1748
|
+
result = result[part];
|
|
1749
|
+
} else {
|
|
1750
|
+
return {};
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
return typeof result === "object" && result !== null && !Array.isArray(result) ? result : {};
|
|
1754
|
+
}
|
|
1755
|
+
function sanitizeSchemaForGemini(schema, defs, depth = 0) {
|
|
1756
|
+
if (depth > 15) return { type: "object", properties: {} };
|
|
1757
|
+
if (!schema || typeof schema !== "object") return { type: "object", properties: {} };
|
|
1758
|
+
if (defs === void 0) {
|
|
1759
|
+
defs = schema["$defs"] ?? schema["definitions"] ?? {};
|
|
1760
|
+
}
|
|
1761
|
+
if (typeof schema["$ref"] === "string") {
|
|
1762
|
+
const resolved = resolveRef(schema["$ref"], { $defs: defs, definitions: defs });
|
|
1763
|
+
if (resolved && Object.keys(resolved).length > 0) {
|
|
1764
|
+
return sanitizeSchemaForGemini(resolved, defs, depth + 1);
|
|
1765
|
+
}
|
|
1766
|
+
return { type: "object", properties: {} };
|
|
1767
|
+
}
|
|
1768
|
+
for (const comboKey of ["oneOf", "anyOf", "allOf"]) {
|
|
1769
|
+
const variants = schema[comboKey];
|
|
1770
|
+
if (Array.isArray(variants) && variants.length > 0) {
|
|
1771
|
+
for (const v of variants) {
|
|
1772
|
+
if (v && typeof v === "object") {
|
|
1773
|
+
const resolved = sanitizeSchemaForGemini(v, defs, depth + 1);
|
|
1774
|
+
if (resolved["type"] === "object" && resolved["properties"]) {
|
|
1775
|
+
return resolved;
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1779
|
+
const first = variants[0];
|
|
1780
|
+
if (first && typeof first === "object") {
|
|
1781
|
+
return sanitizeSchemaForGemini(first, defs, depth + 1);
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
const result = {};
|
|
1786
|
+
let schemaType = schema["type"];
|
|
1787
|
+
if (Array.isArray(schemaType)) {
|
|
1788
|
+
const nonNull = schemaType.filter((t) => t !== "null");
|
|
1789
|
+
schemaType = nonNull[0] ?? "string";
|
|
1790
|
+
}
|
|
1791
|
+
if (schemaType) result["type"] = schemaType;
|
|
1792
|
+
if (schema["description"]) result["description"] = schema["description"];
|
|
1793
|
+
if (schema["enum"]) result["enum"] = schema["enum"];
|
|
1794
|
+
if (schema["required"]) result["required"] = schema["required"];
|
|
1795
|
+
if (schema["properties"] && typeof schema["properties"] === "object") {
|
|
1796
|
+
const props = {};
|
|
1797
|
+
for (const [key, val] of Object.entries(schema["properties"])) {
|
|
1798
|
+
if (val && typeof val === "object") {
|
|
1799
|
+
props[key] = sanitizeSchemaForGemini(val, defs, depth + 1);
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
result["properties"] = props;
|
|
1803
|
+
}
|
|
1804
|
+
if (schema["items"] && typeof schema["items"] === "object" && !Array.isArray(schema["items"])) {
|
|
1805
|
+
result["items"] = sanitizeSchemaForGemini(schema["items"], defs, depth + 1);
|
|
1806
|
+
}
|
|
1807
|
+
if (!result["type"] && result["properties"]) result["type"] = "object";
|
|
1808
|
+
if (result["type"] === "object" && !result["properties"]) result["properties"] = {};
|
|
1809
|
+
return result;
|
|
1810
|
+
}
|
|
1547
1811
|
var GeminiRealtime = class {
|
|
1548
|
-
|
|
1812
|
+
_apiKey;
|
|
1813
|
+
_systemPrompt;
|
|
1814
|
+
_model;
|
|
1815
|
+
_voice;
|
|
1816
|
+
_language;
|
|
1817
|
+
_greeting;
|
|
1818
|
+
_generationConfig;
|
|
1549
1819
|
_ws = null;
|
|
1550
|
-
|
|
1820
|
+
_call = null;
|
|
1551
1821
|
_tools = null;
|
|
1822
|
+
_recorder = null;
|
|
1552
1823
|
_closed = false;
|
|
1824
|
+
_sentAudioChunks = 0;
|
|
1825
|
+
_audioRemainder = Buffer.alloc(0);
|
|
1553
1826
|
constructor(options = {}) {
|
|
1554
|
-
this.
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1827
|
+
this._apiKey = options.apiKey ?? process.env["GOOGLE_API_KEY"] ?? "";
|
|
1828
|
+
this._systemPrompt = options.systemPrompt ?? "";
|
|
1829
|
+
this._model = options.model ?? "gemini-2.5-flash-native-audio-preview-12-2025";
|
|
1830
|
+
this._voice = options.voice ?? "Kore";
|
|
1831
|
+
this._language = options.language ?? "ko";
|
|
1832
|
+
this._greeting = options.greeting ?? true;
|
|
1833
|
+
this._generationConfig = options.generationConfig;
|
|
1834
|
+
}
|
|
1835
|
+
/** Inject per-call ToolRegistry. */
|
|
1836
|
+
setToolRegistry(registry) {
|
|
1837
|
+
this._tools = registry;
|
|
1838
|
+
}
|
|
1839
|
+
/** Inject per-call AudioRecorder. */
|
|
1840
|
+
setRecorder(recorder) {
|
|
1841
|
+
this._recorder = recorder;
|
|
1558
1842
|
}
|
|
1559
1843
|
async start(callSession, tools) {
|
|
1560
|
-
this.
|
|
1561
|
-
this._tools = tools
|
|
1844
|
+
this._call = callSession;
|
|
1845
|
+
if (tools) this._tools = tools;
|
|
1562
1846
|
this._closed = false;
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1847
|
+
this._sentAudioChunks = 0;
|
|
1848
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1849
|
+
if (!this._apiKey) {
|
|
1850
|
+
throw new Error("Google API key is required. Set GOOGLE_API_KEY or pass apiKey option.");
|
|
1566
1851
|
}
|
|
1567
1852
|
const { WebSocket } = await import('ws');
|
|
1568
|
-
this.
|
|
1569
|
-
const url = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=${apiKey}`;
|
|
1853
|
+
const url = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=${this._apiKey}`;
|
|
1570
1854
|
this._ws = new WebSocket(url);
|
|
1571
1855
|
return new Promise((resolve, reject) => {
|
|
1572
1856
|
const ws = this._ws;
|
|
1573
1857
|
ws.on("open", () => {
|
|
1574
1858
|
this._sendSetup();
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
}
|
|
1582
|
-
}
|
|
1859
|
+
this._waitSetupComplete().then(() => {
|
|
1860
|
+
if (this._greeting) {
|
|
1861
|
+
this._sendGreeting();
|
|
1862
|
+
}
|
|
1863
|
+
this._receiveLoop();
|
|
1864
|
+
resolve();
|
|
1865
|
+
}).catch(reject);
|
|
1583
1866
|
});
|
|
1584
1867
|
ws.on("close", () => {
|
|
1585
1868
|
this._closed = true;
|
|
@@ -1594,13 +1877,15 @@ var GeminiRealtime = class {
|
|
|
1594
1877
|
}
|
|
1595
1878
|
feedAudio(audio) {
|
|
1596
1879
|
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1880
|
+
const pcm8k = ulawToPcm16(audio);
|
|
1881
|
+
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
1597
1882
|
this._ws.send(
|
|
1598
1883
|
JSON.stringify({
|
|
1599
1884
|
realtimeInput: {
|
|
1600
1885
|
mediaChunks: [
|
|
1601
1886
|
{
|
|
1602
1887
|
mimeType: "audio/pcm;rate=16000",
|
|
1603
|
-
data:
|
|
1888
|
+
data: pcm16k.toString("base64")
|
|
1604
1889
|
}
|
|
1605
1890
|
]
|
|
1606
1891
|
}
|
|
@@ -1618,35 +1903,87 @@ var GeminiRealtime = class {
|
|
|
1618
1903
|
_sendSetup() {
|
|
1619
1904
|
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1620
1905
|
const setupConfig = {
|
|
1621
|
-
model: `models/${this.
|
|
1906
|
+
model: `models/${this._model}`,
|
|
1622
1907
|
generationConfig: {
|
|
1623
1908
|
responseModalities: ["AUDIO"],
|
|
1624
1909
|
speechConfig: {
|
|
1625
1910
|
voiceConfig: {
|
|
1626
1911
|
prebuiltVoiceConfig: {
|
|
1627
|
-
voiceName: this.
|
|
1912
|
+
voiceName: this._voice
|
|
1628
1913
|
}
|
|
1629
1914
|
}
|
|
1630
1915
|
},
|
|
1631
|
-
...this.
|
|
1916
|
+
...this._generationConfig
|
|
1917
|
+
},
|
|
1918
|
+
realtimeInputConfig: {
|
|
1919
|
+
automaticActivityDetection: {
|
|
1920
|
+
disabled: false
|
|
1921
|
+
}
|
|
1632
1922
|
}
|
|
1633
1923
|
};
|
|
1634
|
-
if (this.
|
|
1924
|
+
if (this._systemPrompt) {
|
|
1635
1925
|
setupConfig["systemInstruction"] = {
|
|
1636
|
-
parts: [{ text: this.
|
|
1926
|
+
parts: [{ text: this._systemPrompt }]
|
|
1637
1927
|
};
|
|
1638
1928
|
}
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1929
|
+
const toolDefs = this._tools ? this._tools.toOpenAITools().map((t) => ({
|
|
1930
|
+
name: t.function.name,
|
|
1931
|
+
description: t.function.description,
|
|
1932
|
+
parameters: sanitizeSchemaForGemini(
|
|
1933
|
+
t.function.parameters ?? { type: "object", properties: {} }
|
|
1934
|
+
)
|
|
1935
|
+
})) : [];
|
|
1936
|
+
toolDefs.push(HANG_UP_TOOL2);
|
|
1937
|
+
setupConfig["tools"] = [{ functionDeclarations: toolDefs }];
|
|
1647
1938
|
this._ws.send(JSON.stringify({ setup: setupConfig }));
|
|
1648
1939
|
}
|
|
1940
|
+
_waitSetupComplete() {
|
|
1941
|
+
return new Promise((resolve, reject) => {
|
|
1942
|
+
if (!this._ws) {
|
|
1943
|
+
reject(new Error("WebSocket not connected"));
|
|
1944
|
+
return;
|
|
1945
|
+
}
|
|
1946
|
+
const onMessage = (data) => {
|
|
1947
|
+
try {
|
|
1948
|
+
const msg = JSON.parse(data.toString());
|
|
1949
|
+
if ("setupComplete" in msg) {
|
|
1950
|
+
this._ws?.removeListener("message", onMessage);
|
|
1951
|
+
resolve();
|
|
1952
|
+
}
|
|
1953
|
+
} catch {
|
|
1954
|
+
}
|
|
1955
|
+
};
|
|
1956
|
+
this._ws.on("message", onMessage);
|
|
1957
|
+
});
|
|
1958
|
+
}
|
|
1959
|
+
_sendGreeting() {
|
|
1960
|
+
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1961
|
+
this._ws.send(
|
|
1962
|
+
JSON.stringify({
|
|
1963
|
+
clientContent: {
|
|
1964
|
+
turns: [
|
|
1965
|
+
{
|
|
1966
|
+
role: "user",
|
|
1967
|
+
parts: [{ text: "\uC778\uC0AC\uD574 \uC8FC\uC138\uC694." }]
|
|
1968
|
+
}
|
|
1969
|
+
],
|
|
1970
|
+
turnComplete: true
|
|
1971
|
+
}
|
|
1972
|
+
})
|
|
1973
|
+
);
|
|
1974
|
+
}
|
|
1975
|
+
_receiveLoop() {
|
|
1976
|
+
if (!this._ws) return;
|
|
1977
|
+
this._ws.on("message", (data) => {
|
|
1978
|
+
try {
|
|
1979
|
+
const msg = JSON.parse(data.toString());
|
|
1980
|
+
this._handleMessage(msg);
|
|
1981
|
+
} catch {
|
|
1982
|
+
}
|
|
1983
|
+
});
|
|
1984
|
+
}
|
|
1649
1985
|
_handleMessage(msg) {
|
|
1986
|
+
if (!this._call) return;
|
|
1650
1987
|
const serverContent = msg["serverContent"];
|
|
1651
1988
|
if (serverContent) {
|
|
1652
1989
|
const modelTurn = serverContent["modelTurn"];
|
|
@@ -1655,35 +1992,108 @@ var GeminiRealtime = class {
|
|
|
1655
1992
|
if (parts) {
|
|
1656
1993
|
for (const part of parts) {
|
|
1657
1994
|
const inlineData = part["inlineData"];
|
|
1658
|
-
if (inlineData && inlineData["data"]
|
|
1659
|
-
const
|
|
1660
|
-
|
|
1995
|
+
if (inlineData && inlineData["data"]) {
|
|
1996
|
+
const mimeType = inlineData["mimeType"] ?? "";
|
|
1997
|
+
if (mimeType.includes("audio")) {
|
|
1998
|
+
this._handleAudioData(inlineData["data"]);
|
|
1999
|
+
}
|
|
2000
|
+
}
|
|
2001
|
+
const text = part["text"];
|
|
2002
|
+
if (text && this._call) {
|
|
2003
|
+
this._call._emit("transcript", "assistant", text);
|
|
1661
2004
|
}
|
|
1662
2005
|
}
|
|
1663
2006
|
}
|
|
1664
2007
|
}
|
|
2008
|
+
if (serverContent["turnComplete"]) {
|
|
2009
|
+
this._flushAudioRemainder();
|
|
2010
|
+
}
|
|
2011
|
+
if (serverContent["interrupted"]) {
|
|
2012
|
+
if (this._call) {
|
|
2013
|
+
this._call.clearAudio();
|
|
2014
|
+
}
|
|
2015
|
+
this._sentAudioChunks = 0;
|
|
2016
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
2017
|
+
}
|
|
2018
|
+
}
|
|
2019
|
+
const inputTranscription = msg["inputTranscription"];
|
|
2020
|
+
if (inputTranscription) {
|
|
2021
|
+
const text = inputTranscription["text"];
|
|
2022
|
+
if (text && this._call) {
|
|
2023
|
+
this._call._emit("transcript", "user", text);
|
|
2024
|
+
}
|
|
2025
|
+
}
|
|
2026
|
+
const outputTranscription = msg["outputTranscription"];
|
|
2027
|
+
if (outputTranscription) {
|
|
2028
|
+
const text = outputTranscription["text"];
|
|
2029
|
+
if (text && this._call) {
|
|
2030
|
+
this._call._emit("transcript", "assistant", text);
|
|
2031
|
+
}
|
|
1665
2032
|
}
|
|
1666
2033
|
const toolCall = msg["toolCall"];
|
|
1667
2034
|
if (toolCall) {
|
|
1668
2035
|
this._handleToolCall(toolCall);
|
|
1669
2036
|
}
|
|
2037
|
+
if (msg["toolCallCancellation"]) ;
|
|
2038
|
+
}
|
|
2039
|
+
_handleAudioData(b64Data) {
|
|
2040
|
+
if (!this._call) return;
|
|
2041
|
+
const pcm24k = Buffer.from(b64Data, "base64");
|
|
2042
|
+
if (this._recorder) {
|
|
2043
|
+
this._recorder.writeOutbound(resamplePcm16(pcm24k, 24e3, 8e3));
|
|
2044
|
+
}
|
|
2045
|
+
const pcm8k = resamplePcm16(pcm24k, 24e3, 8e3);
|
|
2046
|
+
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2047
|
+
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
2048
|
+
const chunkSize = 160;
|
|
2049
|
+
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
2050
|
+
for (let off = 0; off < fullEnd; off += chunkSize) {
|
|
2051
|
+
this._call.sendAudio(combined.subarray(off, off + chunkSize));
|
|
2052
|
+
this._sentAudioChunks++;
|
|
2053
|
+
}
|
|
2054
|
+
this._audioRemainder = combined.subarray(fullEnd);
|
|
2055
|
+
}
|
|
2056
|
+
_flushAudioRemainder() {
|
|
2057
|
+
if (this._audioRemainder.length > 0 && this._call) {
|
|
2058
|
+
const padded = Buffer.concat([
|
|
2059
|
+
this._audioRemainder,
|
|
2060
|
+
Buffer.alloc(160 - this._audioRemainder.length, 255)
|
|
2061
|
+
]);
|
|
2062
|
+
this._call.sendAudio(padded);
|
|
2063
|
+
this._sentAudioChunks++;
|
|
2064
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
2065
|
+
}
|
|
1670
2066
|
}
|
|
1671
2067
|
async _handleToolCall(toolCall) {
|
|
1672
2068
|
const functionCalls = toolCall["functionCalls"];
|
|
1673
|
-
if (!functionCalls
|
|
2069
|
+
if (!functionCalls) return;
|
|
1674
2070
|
const responses = [];
|
|
1675
2071
|
for (const fc of functionCalls) {
|
|
1676
2072
|
const name = fc["name"];
|
|
2073
|
+
const fcId = fc["id"] ?? "";
|
|
1677
2074
|
const args = fc["args"] ?? {};
|
|
2075
|
+
if (name === "hang_up") {
|
|
2076
|
+
if (this._call) {
|
|
2077
|
+
this._call.hangup();
|
|
2078
|
+
}
|
|
2079
|
+
return;
|
|
2080
|
+
}
|
|
2081
|
+
if (!this._tools || !this._tools.has(name)) {
|
|
2082
|
+
console.error(`[GeminiRealtime] Unknown tool: ${name}`);
|
|
2083
|
+
responses.push({ id: fcId, name, response: { error: `Unknown tool: ${name}` } });
|
|
2084
|
+
continue;
|
|
2085
|
+
}
|
|
1678
2086
|
try {
|
|
1679
2087
|
const result = await this._tools.call(name, args);
|
|
1680
2088
|
responses.push({
|
|
2089
|
+
id: fcId,
|
|
1681
2090
|
name,
|
|
1682
2091
|
response: { result: typeof result === "string" ? result : JSON.stringify(result) }
|
|
1683
2092
|
});
|
|
1684
2093
|
} catch (err) {
|
|
1685
2094
|
console.error(`[GeminiRealtime] Tool call error for ${name}:`, err);
|
|
1686
2095
|
responses.push({
|
|
2096
|
+
id: fcId,
|
|
1687
2097
|
name,
|
|
1688
2098
|
response: { error: String(err) }
|
|
1689
2099
|
});
|
|
@@ -1707,12 +2117,15 @@ var PipelineSession = class {
|
|
|
1707
2117
|
_llm;
|
|
1708
2118
|
_tts;
|
|
1709
2119
|
_systemPrompt;
|
|
2120
|
+
_greeting;
|
|
2121
|
+
_language;
|
|
1710
2122
|
_temperature;
|
|
1711
2123
|
_maxTokens;
|
|
1712
2124
|
_sampleRate;
|
|
1713
2125
|
_interruptOnSpeech;
|
|
1714
2126
|
_callSession = null;
|
|
1715
2127
|
_tools = null;
|
|
2128
|
+
_recorder = null;
|
|
1716
2129
|
_conversation = [];
|
|
1717
2130
|
_audioBuffer = [];
|
|
1718
2131
|
_running = false;
|
|
@@ -1722,10 +2135,20 @@ var PipelineSession = class {
|
|
|
1722
2135
|
this._llm = options.llm;
|
|
1723
2136
|
this._tts = options.tts;
|
|
1724
2137
|
this._systemPrompt = options.systemPrompt;
|
|
2138
|
+
this._greeting = options.greeting ?? true;
|
|
2139
|
+
this._language = options.language ?? "ko";
|
|
1725
2140
|
this._temperature = options.temperature;
|
|
1726
2141
|
this._maxTokens = options.maxTokens;
|
|
1727
2142
|
this._sampleRate = options.sampleRate ?? 8e3;
|
|
1728
2143
|
this._interruptOnSpeech = options.interruptOnSpeech ?? true;
|
|
2144
|
+
if (options.toolRegistry) this._tools = options.toolRegistry;
|
|
2145
|
+
if (options.recorder) this._recorder = options.recorder;
|
|
2146
|
+
}
|
|
2147
|
+
setToolRegistry(registry) {
|
|
2148
|
+
this._tools = registry;
|
|
2149
|
+
}
|
|
2150
|
+
setRecorder(recorder) {
|
|
2151
|
+
this._recorder = recorder;
|
|
1729
2152
|
}
|
|
1730
2153
|
async start(callSession, tools) {
|
|
1731
2154
|
this._callSession = callSession;
|
|
@@ -1738,6 +2161,11 @@ var PipelineSession = class {
|
|
|
1738
2161
|
content: this._systemPrompt
|
|
1739
2162
|
});
|
|
1740
2163
|
}
|
|
2164
|
+
if (this._greeting) {
|
|
2165
|
+
this._generateGreeting().catch((err) => {
|
|
2166
|
+
console.error("[PipelineSession] Greeting error:", err);
|
|
2167
|
+
});
|
|
2168
|
+
}
|
|
1741
2169
|
this._runSttLoop().catch((err) => {
|
|
1742
2170
|
console.error("[PipelineSession] STT loop error:", err);
|
|
1743
2171
|
});
|
|
@@ -1771,14 +2199,24 @@ var PipelineSession = class {
|
|
|
1771
2199
|
async *_createAudioStream() {
|
|
1772
2200
|
while (this._running) {
|
|
1773
2201
|
if (this._audioBuffer.length > 0) {
|
|
1774
|
-
|
|
2202
|
+
const ulaw = this._audioBuffer.shift();
|
|
2203
|
+
const pcm8k = ulawToPcm16(ulaw);
|
|
2204
|
+
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
2205
|
+
yield pcm16k;
|
|
1775
2206
|
} else {
|
|
1776
2207
|
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1777
2208
|
}
|
|
1778
2209
|
}
|
|
1779
2210
|
}
|
|
2211
|
+
async _generateGreeting() {
|
|
2212
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
2213
|
+
await this._respond();
|
|
2214
|
+
}
|
|
1780
2215
|
async _handleUserSpeech(transcript) {
|
|
1781
2216
|
this._conversation.push({ role: "user", content: transcript });
|
|
2217
|
+
await this._respond();
|
|
2218
|
+
}
|
|
2219
|
+
async _respond() {
|
|
1782
2220
|
let fullResponse = "";
|
|
1783
2221
|
const textChunks = [];
|
|
1784
2222
|
const llmStream = this._llm.generate(this._conversation, {
|
|
@@ -1845,7 +2283,19 @@ var PipelineSession = class {
|
|
|
1845
2283
|
sampleRate: this._sampleRate
|
|
1846
2284
|
})) {
|
|
1847
2285
|
if (!this._running || !this._speaking) break;
|
|
1848
|
-
this.
|
|
2286
|
+
if (this._recorder) {
|
|
2287
|
+
const pcm8k2 = this._sampleRate !== 8e3 ? resamplePcm16(audioChunk, this._sampleRate, 8e3) : audioChunk;
|
|
2288
|
+
this._recorder.writeOutbound(pcm8k2);
|
|
2289
|
+
}
|
|
2290
|
+
const pcm8k = resamplePcm16(audioChunk, this._sampleRate, 8e3);
|
|
2291
|
+
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2292
|
+
for (let off = 0; off < ulaw.length; off += 160) {
|
|
2293
|
+
let chunk = ulaw.subarray(off, off + 160);
|
|
2294
|
+
if (chunk.length < 160) {
|
|
2295
|
+
chunk = Buffer.concat([chunk, Buffer.alloc(160 - chunk.length, 255)]);
|
|
2296
|
+
}
|
|
2297
|
+
this._callSession.sendAudio(chunk);
|
|
2298
|
+
}
|
|
1849
2299
|
}
|
|
1850
2300
|
} catch (err) {
|
|
1851
2301
|
console.error("[PipelineSession] TTS error:", err);
|
|
@@ -1860,14 +2310,14 @@ var DeepgramSTT = class {
|
|
|
1860
2310
|
_options;
|
|
1861
2311
|
constructor(options = {}) {
|
|
1862
2312
|
this._options = {
|
|
1863
|
-
model: "nova-
|
|
2313
|
+
model: "nova-3",
|
|
1864
2314
|
language: "ko",
|
|
1865
|
-
|
|
1866
|
-
punctuate: true,
|
|
1867
|
-
smartFormat: true,
|
|
2315
|
+
sampleRate: 16e3,
|
|
1868
2316
|
encoding: "linear16",
|
|
1869
|
-
|
|
1870
|
-
|
|
2317
|
+
punctuate: true,
|
|
2318
|
+
interimResults: true,
|
|
2319
|
+
endpointing: 300,
|
|
2320
|
+
utteranceEndMs: 1e3,
|
|
1871
2321
|
...options
|
|
1872
2322
|
};
|
|
1873
2323
|
}
|
|
@@ -1884,10 +2334,10 @@ var DeepgramSTT = class {
|
|
|
1884
2334
|
language,
|
|
1885
2335
|
punctuate: String(this._options.punctuate),
|
|
1886
2336
|
interim_results: String(this._options.interimResults),
|
|
1887
|
-
smart_format: String(this._options.smartFormat),
|
|
1888
2337
|
encoding: this._options.encoding,
|
|
1889
2338
|
sample_rate: String(sampleRate),
|
|
1890
|
-
|
|
2339
|
+
endpointing: String(this._options.endpointing),
|
|
2340
|
+
utterance_end_ms: String(this._options.utteranceEndMs)
|
|
1891
2341
|
});
|
|
1892
2342
|
const url = `wss://api.deepgram.com/v1/listen?${params.toString()}`;
|
|
1893
2343
|
const ws = new WebSocket(url, {
|
|
@@ -1976,13 +2426,12 @@ var ElevenLabsTTS = class {
|
|
|
1976
2426
|
_options;
|
|
1977
2427
|
constructor(options = {}) {
|
|
1978
2428
|
this._options = {
|
|
1979
|
-
voiceId: "
|
|
1980
|
-
|
|
1981
|
-
outputFormat: "
|
|
2429
|
+
voiceId: "EXAVITQu4vr4xnSDxMaL",
|
|
2430
|
+
model: "eleven_flash_v2_5",
|
|
2431
|
+
outputFormat: "pcm_24000",
|
|
1982
2432
|
stability: 0.5,
|
|
1983
2433
|
similarityBoost: 0.75,
|
|
1984
|
-
|
|
1985
|
-
useSpeakerBoost: true,
|
|
2434
|
+
languageCode: "ko",
|
|
1986
2435
|
...options
|
|
1987
2436
|
};
|
|
1988
2437
|
}
|
|
@@ -2008,12 +2457,10 @@ var ElevenLabsTTS = class {
|
|
|
2008
2457
|
},
|
|
2009
2458
|
body: JSON.stringify({
|
|
2010
2459
|
text,
|
|
2011
|
-
model_id: this._options.
|
|
2460
|
+
model_id: this._options.model,
|
|
2012
2461
|
voice_settings: {
|
|
2013
2462
|
stability: this._options.stability,
|
|
2014
|
-
similarity_boost: this._options.similarityBoost
|
|
2015
|
-
style: this._options.style,
|
|
2016
|
-
use_speaker_boost: this._options.useSpeakerBoost
|
|
2463
|
+
similarity_boost: this._options.similarityBoost
|
|
2017
2464
|
}
|
|
2018
2465
|
})
|
|
2019
2466
|
});
|
|
@@ -2037,7 +2484,7 @@ var ElevenLabsTTS = class {
|
|
|
2037
2484
|
}
|
|
2038
2485
|
async *_synthesizeStreaming(apiKey, voiceId, textStream) {
|
|
2039
2486
|
const { WebSocket } = await import('ws');
|
|
2040
|
-
const url = `wss://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream-input?model_id=${this._options.
|
|
2487
|
+
const url = `wss://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream-input?model_id=${this._options.model}&output_format=${this._options.outputFormat}`;
|
|
2041
2488
|
const ws = new WebSocket(url);
|
|
2042
2489
|
const audioQueue = [];
|
|
2043
2490
|
let resolveWait = null;
|
|
@@ -2048,9 +2495,7 @@ var ElevenLabsTTS = class {
|
|
|
2048
2495
|
text: " ",
|
|
2049
2496
|
voice_settings: {
|
|
2050
2497
|
stability: this._options.stability,
|
|
2051
|
-
similarity_boost: this._options.similarityBoost
|
|
2052
|
-
style: this._options.style,
|
|
2053
|
-
use_speaker_boost: this._options.useSpeakerBoost
|
|
2498
|
+
similarity_boost: this._options.similarityBoost
|
|
2054
2499
|
},
|
|
2055
2500
|
xi_api_key: apiKey
|
|
2056
2501
|
})
|
|
@@ -2127,7 +2572,9 @@ var OpenAILLM = class {
|
|
|
2127
2572
|
_options;
|
|
2128
2573
|
constructor(options = {}) {
|
|
2129
2574
|
this._options = {
|
|
2130
|
-
model: "gpt-4o",
|
|
2575
|
+
model: "gpt-4o-mini",
|
|
2576
|
+
temperature: 0.8,
|
|
2577
|
+
maxTokens: 4096,
|
|
2131
2578
|
...options
|
|
2132
2579
|
};
|
|
2133
2580
|
}
|
|
@@ -2211,8 +2658,9 @@ var AnthropicLLM = class {
|
|
|
2211
2658
|
_options;
|
|
2212
2659
|
constructor(options = {}) {
|
|
2213
2660
|
this._options = {
|
|
2214
|
-
model: "claude-sonnet-4-
|
|
2215
|
-
|
|
2661
|
+
model: "claude-sonnet-4-6",
|
|
2662
|
+
temperature: 0.8,
|
|
2663
|
+
maxTokens: 4096,
|
|
2216
2664
|
...options
|
|
2217
2665
|
};
|
|
2218
2666
|
}
|
|
@@ -2315,7 +2763,9 @@ var GeminiLLM = class {
|
|
|
2315
2763
|
_options;
|
|
2316
2764
|
constructor(options = {}) {
|
|
2317
2765
|
this._options = {
|
|
2318
|
-
model: "gemini-2.
|
|
2766
|
+
model: "gemini-2.5-flash",
|
|
2767
|
+
temperature: 0.8,
|
|
2768
|
+
maxTokens: 4096,
|
|
2319
2769
|
...options
|
|
2320
2770
|
};
|
|
2321
2771
|
}
|
|
@@ -2479,13 +2929,13 @@ var OpenAICompatLLM = class {
|
|
|
2479
2929
|
var OllamaLLM = class {
|
|
2480
2930
|
_inner;
|
|
2481
2931
|
constructor(options = {}) {
|
|
2482
|
-
const baseUrl =
|
|
2932
|
+
const baseUrl = options.baseUrl ?? process.env["OLLAMA_BASE_URL"] ?? "http://localhost:11434/v1";
|
|
2483
2933
|
this._inner = new OpenAICompatLLM({
|
|
2484
|
-
baseUrl:
|
|
2485
|
-
model: options.model ?? "llama3.
|
|
2934
|
+
baseUrl: baseUrl.replace(/\/$/, ""),
|
|
2935
|
+
model: options.model ?? "llama3.2",
|
|
2486
2936
|
apiKey: "ollama",
|
|
2487
|
-
temperature: options.temperature,
|
|
2488
|
-
maxTokens: options.maxTokens
|
|
2937
|
+
temperature: options.temperature ?? 0.8,
|
|
2938
|
+
maxTokens: options.maxTokens ?? 4096
|
|
2489
2939
|
});
|
|
2490
2940
|
}
|
|
2491
2941
|
async *generate(messages, options) {
|