@teamlearners/clawops 0.1.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +6 -0
- package/dist/agent/index.cjs +863 -413
- package/dist/agent/index.cjs.map +1 -1
- package/dist/agent/index.d.cts +255 -196
- package/dist/agent/index.d.ts +255 -196
- package/dist/agent/index.js +863 -413
- package/dist/agent/index.js.map +1 -1
- package/dist/index.cjs +4 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +7 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/agent/index.js
CHANGED
|
@@ -324,13 +324,16 @@ function resamplePcm16(pcm, fromRate, toRate) {
|
|
|
324
324
|
}
|
|
325
325
|
|
|
326
326
|
// src/agent/control-ws.ts
|
|
327
|
-
var DEFAULT_PATH = "/v1/agent/control";
|
|
328
327
|
var INITIAL_RECONNECT_DELAY = 1e3;
|
|
329
328
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
330
329
|
function buildControlWsUrl(options) {
|
|
331
|
-
const
|
|
332
|
-
const
|
|
333
|
-
|
|
330
|
+
const scheme = options.baseUrl.startsWith("https") ? "wss" : "ws";
|
|
331
|
+
const host = options.baseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "");
|
|
332
|
+
let url = `${scheme}://${host}/v1/accounts/${encodeURIComponent(options.accountId)}/agent/listen`;
|
|
333
|
+
if (options.number) {
|
|
334
|
+
url += `?number=${encodeURIComponent(options.number)}`;
|
|
335
|
+
}
|
|
336
|
+
return url;
|
|
334
337
|
}
|
|
335
338
|
var ControlWebSocket = class {
|
|
336
339
|
constructor(_options) {
|
|
@@ -381,7 +384,12 @@ var ControlWebSocket = class {
|
|
|
381
384
|
}
|
|
382
385
|
async _doConnect() {
|
|
383
386
|
const { WebSocket } = await import('ws');
|
|
384
|
-
const ws = new WebSocket(this._url
|
|
387
|
+
const ws = new WebSocket(this._url, {
|
|
388
|
+
followRedirects: true,
|
|
389
|
+
headers: {
|
|
390
|
+
Authorization: `Bearer ${this._options.apiKey}`
|
|
391
|
+
}
|
|
392
|
+
});
|
|
385
393
|
this._ws = ws;
|
|
386
394
|
ws.on("open", () => {
|
|
387
395
|
this._reconnectDelay = INITIAL_RECONNECT_DELAY;
|
|
@@ -524,47 +532,37 @@ var MCPClient = class {
|
|
|
524
532
|
// src/agent/media-ws.ts
|
|
525
533
|
function parseStartEvent(data) {
|
|
526
534
|
const start = data["start"];
|
|
535
|
+
const fmt = start["mediaFormat"] ?? {};
|
|
527
536
|
return {
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
customParameters: start["customParameters"] ?? {},
|
|
533
|
-
mediaFormat: start["mediaFormat"] ?? {
|
|
534
|
-
encoding: "audio/x-mulaw",
|
|
535
|
-
sampleRate: 8e3,
|
|
536
|
-
channels: 1
|
|
537
|
-
}
|
|
537
|
+
streamId: start["streamId"] ?? "",
|
|
538
|
+
callId: start["callId"] ?? "",
|
|
539
|
+
accountId: start["accountId"] ?? "",
|
|
540
|
+
sampleRate: fmt["sampleRate"] ?? 8e3
|
|
538
541
|
};
|
|
539
542
|
}
|
|
540
543
|
function parseMediaEvent(data) {
|
|
541
544
|
const media = data["media"];
|
|
542
545
|
return {
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
timestamp: media["timestamp"] ?? ""
|
|
546
|
+
audio: Buffer.from(media["payload"] ?? "", "base64"),
|
|
547
|
+
timestamp: parseInt(media["timestamp"] ?? "0", 10) || 0
|
|
546
548
|
};
|
|
547
549
|
}
|
|
548
|
-
function buildMediaResponse(
|
|
550
|
+
function buildMediaResponse(audioBase64) {
|
|
549
551
|
return JSON.stringify({
|
|
550
552
|
event: "media",
|
|
551
|
-
streamSid,
|
|
552
553
|
media: {
|
|
553
|
-
payload
|
|
554
|
+
payload: audioBase64
|
|
554
555
|
}
|
|
555
556
|
});
|
|
556
557
|
}
|
|
557
558
|
var MediaWebSocket = class {
|
|
558
559
|
_ws = null;
|
|
559
|
-
_streamSid = null;
|
|
560
560
|
_audioQueue = [];
|
|
561
561
|
_sendLoopRunning = false;
|
|
562
562
|
_closed = false;
|
|
563
563
|
_onAudio = null;
|
|
564
564
|
_onStart = null;
|
|
565
565
|
_onClose = null;
|
|
566
|
-
_markSeq = 0;
|
|
567
|
-
_markResolves = /* @__PURE__ */ new Map();
|
|
568
566
|
/** Set the handler for inbound audio data. */
|
|
569
567
|
onAudio(handler) {
|
|
570
568
|
this._onAudio = handler;
|
|
@@ -577,11 +575,16 @@ var MediaWebSocket = class {
|
|
|
577
575
|
onClose(handler) {
|
|
578
576
|
this._onClose = handler;
|
|
579
577
|
}
|
|
580
|
-
/** Connect to a media WebSocket URL. */
|
|
581
|
-
async connect(url) {
|
|
578
|
+
/** Connect to a media WebSocket URL with Bearer authentication. */
|
|
579
|
+
async connect(url, apiKey) {
|
|
582
580
|
const { WebSocket } = await import('ws');
|
|
583
581
|
return new Promise((resolve, reject) => {
|
|
584
|
-
const ws = new WebSocket(url
|
|
582
|
+
const ws = new WebSocket(url, {
|
|
583
|
+
followRedirects: true,
|
|
584
|
+
headers: {
|
|
585
|
+
Authorization: `Bearer ${apiKey}`
|
|
586
|
+
}
|
|
587
|
+
});
|
|
585
588
|
this._ws = ws;
|
|
586
589
|
this._closed = false;
|
|
587
590
|
ws.on("open", () => {
|
|
@@ -616,31 +619,21 @@ var MediaWebSocket = class {
|
|
|
616
619
|
/** Clear all queued outbound audio. */
|
|
617
620
|
sendClear() {
|
|
618
621
|
this._audioQueue.length = 0;
|
|
619
|
-
if (this._ws && this.
|
|
622
|
+
if (this._ws && this._ws.readyState === 1) {
|
|
623
|
+
this._ws.send(JSON.stringify({ event: "clear" }));
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
/** Send a mark event. */
|
|
627
|
+
sendMark(name) {
|
|
628
|
+
if (this._ws && this._ws.readyState === 1) {
|
|
620
629
|
this._ws.send(
|
|
621
630
|
JSON.stringify({
|
|
622
|
-
event: "
|
|
623
|
-
|
|
631
|
+
event: "mark",
|
|
632
|
+
mark: { name }
|
|
624
633
|
})
|
|
625
634
|
);
|
|
626
635
|
}
|
|
627
636
|
}
|
|
628
|
-
/** Send a mark event and return a promise that resolves when the mark is acknowledged. */
|
|
629
|
-
sendMark() {
|
|
630
|
-
const label = `mark_${++this._markSeq}`;
|
|
631
|
-
return new Promise((resolve) => {
|
|
632
|
-
this._markResolves.set(label, resolve);
|
|
633
|
-
if (this._ws && this._streamSid && this._ws.readyState === 1) {
|
|
634
|
-
this._ws.send(
|
|
635
|
-
JSON.stringify({
|
|
636
|
-
event: "mark",
|
|
637
|
-
streamSid: this._streamSid,
|
|
638
|
-
mark: { name: label }
|
|
639
|
-
})
|
|
640
|
-
);
|
|
641
|
-
}
|
|
642
|
-
});
|
|
643
|
-
}
|
|
644
637
|
/** Close the media WebSocket. */
|
|
645
638
|
close() {
|
|
646
639
|
this._closed = true;
|
|
@@ -648,17 +641,12 @@ var MediaWebSocket = class {
|
|
|
648
641
|
this._ws.close();
|
|
649
642
|
this._ws = null;
|
|
650
643
|
}
|
|
651
|
-
for (const resolve of this._markResolves.values()) {
|
|
652
|
-
resolve();
|
|
653
|
-
}
|
|
654
|
-
this._markResolves.clear();
|
|
655
644
|
}
|
|
656
645
|
_handleMessage(msg) {
|
|
657
646
|
const event = msg["event"];
|
|
658
647
|
switch (event) {
|
|
659
648
|
case "start": {
|
|
660
649
|
const startEvt = parseStartEvent(msg);
|
|
661
|
-
this._streamSid = startEvt.streamSid;
|
|
662
650
|
if (this._onStart) {
|
|
663
651
|
this._onStart(startEvt);
|
|
664
652
|
}
|
|
@@ -666,18 +654,8 @@ var MediaWebSocket = class {
|
|
|
666
654
|
}
|
|
667
655
|
case "media": {
|
|
668
656
|
const mediaEvt = parseMediaEvent(msg);
|
|
669
|
-
if (
|
|
670
|
-
|
|
671
|
-
this._onAudio(audioBuf);
|
|
672
|
-
}
|
|
673
|
-
break;
|
|
674
|
-
}
|
|
675
|
-
case "mark": {
|
|
676
|
-
const mark = msg["mark"];
|
|
677
|
-
const name = mark?.["name"];
|
|
678
|
-
if (name && this._markResolves.has(name)) {
|
|
679
|
-
this._markResolves.get(name)();
|
|
680
|
-
this._markResolves.delete(name);
|
|
657
|
+
if (this._onAudio) {
|
|
658
|
+
this._onAudio(mediaEvt.audio, mediaEvt.timestamp);
|
|
681
659
|
}
|
|
682
660
|
break;
|
|
683
661
|
}
|
|
@@ -697,98 +675,162 @@ var MediaWebSocket = class {
|
|
|
697
675
|
}
|
|
698
676
|
while (this._audioQueue.length > 0 && this._ws && this._ws.readyState === 1) {
|
|
699
677
|
const payload = this._audioQueue.shift();
|
|
700
|
-
|
|
701
|
-
this._ws.send(buildMediaResponse(this._streamSid, payload));
|
|
702
|
-
}
|
|
678
|
+
this._ws.send(buildMediaResponse(payload));
|
|
703
679
|
}
|
|
704
680
|
setTimeout(loop, 20);
|
|
705
681
|
};
|
|
706
682
|
loop();
|
|
707
683
|
}
|
|
708
684
|
};
|
|
709
|
-
var
|
|
710
|
-
var
|
|
711
|
-
var
|
|
712
|
-
|
|
685
|
+
var SAMPLE_RATE = 8e3;
|
|
686
|
+
var CHANNELS = 1;
|
|
687
|
+
var BITS_PER_SAMPLE = 16;
|
|
688
|
+
var BYTES_PER_SECOND = SAMPLE_RATE * CHANNELS * (BITS_PER_SAMPLE / 8);
|
|
689
|
+
function makeWavHeader(dataSize = 0) {
|
|
713
690
|
const header = Buffer.alloc(44);
|
|
714
|
-
|
|
715
|
-
const blockAlign = WAV_CHANNELS * (WAV_BITS_PER_SAMPLE / 8);
|
|
716
|
-
header.write("RIFF", 0);
|
|
691
|
+
header.write("RIFF", 0, "ascii");
|
|
717
692
|
header.writeUInt32LE(36 + dataSize, 4);
|
|
718
|
-
header.write("WAVE", 8);
|
|
719
|
-
header.write("fmt ", 12);
|
|
693
|
+
header.write("WAVE", 8, "ascii");
|
|
694
|
+
header.write("fmt ", 12, "ascii");
|
|
720
695
|
header.writeUInt32LE(16, 16);
|
|
721
696
|
header.writeUInt16LE(1, 20);
|
|
722
|
-
header.writeUInt16LE(
|
|
723
|
-
header.writeUInt32LE(
|
|
724
|
-
header.writeUInt32LE(
|
|
725
|
-
header.writeUInt16LE(
|
|
726
|
-
header.writeUInt16LE(
|
|
727
|
-
header.write("data", 36);
|
|
697
|
+
header.writeUInt16LE(CHANNELS, 22);
|
|
698
|
+
header.writeUInt32LE(SAMPLE_RATE, 24);
|
|
699
|
+
header.writeUInt32LE(SAMPLE_RATE * CHANNELS * (BITS_PER_SAMPLE / 8), 28);
|
|
700
|
+
header.writeUInt16LE(CHANNELS * (BITS_PER_SAMPLE / 8), 32);
|
|
701
|
+
header.writeUInt16LE(BITS_PER_SAMPLE, 34);
|
|
702
|
+
header.write("data", 36, "ascii");
|
|
728
703
|
header.writeUInt32LE(dataSize, 40);
|
|
729
704
|
return header;
|
|
730
705
|
}
|
|
706
|
+
function mixSamples(a, b) {
|
|
707
|
+
const n = Math.min(a.length, b.length) >> 1;
|
|
708
|
+
const result = Buffer.alloc(n * 2);
|
|
709
|
+
for (let i = 0; i < n; i++) {
|
|
710
|
+
const sa = a.readInt16LE(i * 2);
|
|
711
|
+
const sb = b.readInt16LE(i * 2);
|
|
712
|
+
result.writeInt16LE(Math.max(-32768, Math.min(32767, sa + sb)), i * 2);
|
|
713
|
+
}
|
|
714
|
+
return result;
|
|
715
|
+
}
|
|
731
716
|
var AudioRecorder = class {
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
717
|
+
_dir;
|
|
718
|
+
_fdIn = null;
|
|
719
|
+
_fdOut = null;
|
|
720
|
+
_fdMix = null;
|
|
721
|
+
_inWritten = 0;
|
|
722
|
+
_outWritten = 0;
|
|
723
|
+
_mixWritten = 0;
|
|
724
|
+
_startTime = 0;
|
|
737
725
|
_started = false;
|
|
738
|
-
constructor(
|
|
739
|
-
this.
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
this.
|
|
745
|
-
this.
|
|
746
|
-
this.
|
|
726
|
+
constructor(recordingPath, callId) {
|
|
727
|
+
this._dir = path.join(recordingPath, callId);
|
|
728
|
+
}
|
|
729
|
+
start() {
|
|
730
|
+
fs.mkdirSync(this._dir, { recursive: true });
|
|
731
|
+
const header = makeWavHeader();
|
|
732
|
+
this._fdIn = fs.openSync(path.join(this._dir, "in.wav"), "w");
|
|
733
|
+
this._fdOut = fs.openSync(path.join(this._dir, "out.wav"), "w");
|
|
734
|
+
this._fdMix = fs.openSync(path.join(this._dir, "mix.wav"), "w+");
|
|
735
|
+
fs.writeSync(this._fdIn, header);
|
|
736
|
+
fs.writeSync(this._fdOut, header);
|
|
737
|
+
fs.writeSync(this._fdMix, header);
|
|
738
|
+
this._startTime = performance.now();
|
|
747
739
|
this._started = true;
|
|
748
|
-
|
|
749
|
-
|
|
740
|
+
}
|
|
741
|
+
_expectedBytes() {
|
|
742
|
+
const elapsed = (performance.now() - this._startTime) / 1e3;
|
|
743
|
+
return Math.floor(elapsed * BYTES_PER_SECOND);
|
|
744
|
+
}
|
|
745
|
+
_padSilence(fd, written) {
|
|
746
|
+
const expected = this._expectedBytes();
|
|
747
|
+
let gap = expected - written;
|
|
748
|
+
if (gap <= 0) return 0;
|
|
749
|
+
gap = gap - gap % 2;
|
|
750
|
+
if (gap > 0) {
|
|
751
|
+
fs.writeSync(fd, Buffer.alloc(gap));
|
|
752
|
+
}
|
|
753
|
+
return gap;
|
|
754
|
+
}
|
|
755
|
+
_writeToMix(data, trackPos) {
|
|
756
|
+
if (this._fdMix === null) return;
|
|
757
|
+
const filePos = 44 + trackPos;
|
|
758
|
+
if (trackPos < this._mixWritten) {
|
|
759
|
+
const overlap = Math.min(data.length, this._mixWritten - trackPos);
|
|
760
|
+
const existing = Buffer.alloc(overlap);
|
|
761
|
+
fs.readSync(this._fdMix, existing, 0, overlap, filePos);
|
|
762
|
+
const mixed = mixSamples(existing, data.subarray(0, overlap));
|
|
763
|
+
fs.writeSync(this._fdMix, mixed, 0, mixed.length, filePos);
|
|
764
|
+
if (data.length > overlap) {
|
|
765
|
+
fs.writeSync(this._fdMix, data, overlap, data.length - overlap, filePos + overlap);
|
|
766
|
+
this._mixWritten = trackPos + data.length;
|
|
767
|
+
}
|
|
768
|
+
} else {
|
|
769
|
+
if (trackPos > this._mixWritten) {
|
|
770
|
+
let gap = trackPos - this._mixWritten;
|
|
771
|
+
gap = gap - gap % 2;
|
|
772
|
+
if (gap > 0) {
|
|
773
|
+
const silence = Buffer.alloc(gap);
|
|
774
|
+
fs.writeSync(this._fdMix, silence, 0, silence.length, 44 + this._mixWritten);
|
|
775
|
+
this._mixWritten += gap;
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
fs.writeSync(this._fdMix, data, 0, data.length, 44 + this._mixWritten);
|
|
779
|
+
this._mixWritten += data.length;
|
|
750
780
|
}
|
|
751
781
|
}
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
this.
|
|
782
|
+
writeInbound(pcm16_8k) {
|
|
783
|
+
if (!this._started || this._fdIn === null) return;
|
|
784
|
+
try {
|
|
785
|
+
const gap = this._padSilence(this._fdIn, this._inWritten);
|
|
786
|
+
this._inWritten += gap;
|
|
787
|
+
const posBefore = this._inWritten;
|
|
788
|
+
fs.writeSync(this._fdIn, pcm16_8k);
|
|
789
|
+
this._inWritten += pcm16_8k.length;
|
|
790
|
+
this._writeToMix(pcm16_8k, posBefore);
|
|
791
|
+
} catch (err) {
|
|
792
|
+
console.error("Error writing inbound audio:", err);
|
|
756
793
|
}
|
|
757
794
|
}
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
this.
|
|
795
|
+
writeOutbound(pcm16_8k) {
|
|
796
|
+
if (!this._started || this._fdOut === null) return;
|
|
797
|
+
try {
|
|
798
|
+
const gap = this._padSilence(this._fdOut, this._outWritten);
|
|
799
|
+
this._outWritten += gap;
|
|
800
|
+
const posBefore = this._outWritten;
|
|
801
|
+
fs.writeSync(this._fdOut, pcm16_8k);
|
|
802
|
+
this._outWritten += pcm16_8k.length;
|
|
803
|
+
this._writeToMix(pcm16_8k, posBefore);
|
|
804
|
+
} catch (err) {
|
|
805
|
+
console.error("Error writing outbound audio:", err);
|
|
762
806
|
}
|
|
763
807
|
}
|
|
764
|
-
/** Stop recording and write WAV files to disk. Returns file paths. */
|
|
765
808
|
stop() {
|
|
766
|
-
if (!this._started
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
809
|
+
if (!this._started) return;
|
|
810
|
+
try {
|
|
811
|
+
let maxWritten = Math.max(this._inWritten, this._outWritten, this._mixWritten);
|
|
812
|
+
maxWritten = maxWritten & ~1;
|
|
813
|
+
for (const [fd, written] of [
|
|
814
|
+
[this._fdIn, this._inWritten],
|
|
815
|
+
[this._fdOut, this._outWritten],
|
|
816
|
+
[this._fdMix, this._mixWritten]
|
|
817
|
+
]) {
|
|
818
|
+
if (fd === null) continue;
|
|
819
|
+
const pad = maxWritten - written;
|
|
820
|
+
if (pad > 0) {
|
|
821
|
+
fs.writeSync(fd, Buffer.alloc(pad), 0, pad, 44 + written);
|
|
822
|
+
}
|
|
823
|
+
fs.writeSync(fd, makeWavHeader(maxWritten), 0, 44, 0);
|
|
824
|
+
fs.closeSync(fd);
|
|
825
|
+
}
|
|
826
|
+
} catch (err) {
|
|
827
|
+
console.error("Error stopping recorder:", err);
|
|
828
|
+
} finally {
|
|
829
|
+
this._fdIn = null;
|
|
830
|
+
this._fdOut = null;
|
|
831
|
+
this._fdMix = null;
|
|
832
|
+
this._started = false;
|
|
780
833
|
}
|
|
781
|
-
this._inboundChunks = [];
|
|
782
|
-
this._outboundChunks = [];
|
|
783
|
-
return result;
|
|
784
|
-
}
|
|
785
|
-
_writeWav(filePath, chunks) {
|
|
786
|
-
const pcmData = Buffer.concat(chunks);
|
|
787
|
-
const header = makeWavHeader(pcmData.length);
|
|
788
|
-
const fd = fs.openSync(filePath, "w");
|
|
789
|
-
fs.writeSync(fd, header);
|
|
790
|
-
fs.writeSync(fd, pcmData);
|
|
791
|
-
fs.closeSync(fd);
|
|
792
834
|
}
|
|
793
835
|
};
|
|
794
836
|
|
|
@@ -868,23 +910,23 @@ var CallSession = class {
|
|
|
868
910
|
/** Mark the session as ended (called internally). */
|
|
869
911
|
_markEnded() {
|
|
870
912
|
this._status = "ended";
|
|
871
|
-
this._emit({ type: "ended" });
|
|
872
913
|
this._resolveEnded();
|
|
873
914
|
}
|
|
874
|
-
/** Emit an event to registered handlers. */
|
|
875
|
-
|
|
876
|
-
|
|
915
|
+
/** Emit an event to registered handlers. Matches Python SDK: _emit(event, ...args) */
|
|
916
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
917
|
+
_emit(event, ...args) {
|
|
918
|
+
const handlers = this._handlers.get(event);
|
|
877
919
|
if (handlers) {
|
|
878
920
|
for (const handler of handlers) {
|
|
879
921
|
try {
|
|
880
|
-
const result = handler(
|
|
922
|
+
const result = handler(this, ...args);
|
|
881
923
|
if (result && typeof result.catch === "function") {
|
|
882
924
|
result.catch((err) => {
|
|
883
|
-
console.error(`[CallSession] Error in ${event
|
|
925
|
+
console.error(`[CallSession] Error in ${event} handler:`, err);
|
|
884
926
|
});
|
|
885
927
|
}
|
|
886
928
|
} catch (err) {
|
|
887
|
-
console.error(`[CallSession] Error in ${event
|
|
929
|
+
console.error(`[CallSession] Error in ${event} handler:`, err);
|
|
888
930
|
}
|
|
889
931
|
}
|
|
890
932
|
}
|
|
@@ -1088,44 +1130,61 @@ var ATTR_AGENT_ID = "clawops.agent.id";
|
|
|
1088
1130
|
// src/agent/agent.ts
|
|
1089
1131
|
var ClawOpsAgent = class {
|
|
1090
1132
|
_apiKey;
|
|
1091
|
-
|
|
1133
|
+
_accountId;
|
|
1092
1134
|
_baseUrl;
|
|
1093
|
-
|
|
1135
|
+
_fromNumber;
|
|
1136
|
+
_session;
|
|
1094
1137
|
_tools = new ToolRegistry();
|
|
1095
1138
|
_handlers = /* @__PURE__ */ new Map();
|
|
1096
1139
|
_controlWs = null;
|
|
1097
|
-
|
|
1098
|
-
|
|
1140
|
+
_mcpServers;
|
|
1141
|
+
_recording;
|
|
1142
|
+
_recordingPath;
|
|
1099
1143
|
_activeSessions = /* @__PURE__ */ new Map();
|
|
1100
|
-
constructor(options
|
|
1144
|
+
constructor(options) {
|
|
1101
1145
|
this._apiKey = options.apiKey ?? process.env["CLAWOPS_API_KEY"] ?? "";
|
|
1102
|
-
this.
|
|
1146
|
+
this._accountId = options.accountId ?? process.env["CLAWOPS_ACCOUNT_ID"] ?? "";
|
|
1103
1147
|
this._baseUrl = options.baseUrl ?? DEFAULT_BASE_URL;
|
|
1104
|
-
this.
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
const sessionInstance = options.session;
|
|
1110
|
-
this._sessionFactory = () => sessionInstance;
|
|
1111
|
-
}
|
|
1112
|
-
}
|
|
1113
|
-
if (options.mcpServers) {
|
|
1114
|
-
this._mcpClient = new MCPClient();
|
|
1115
|
-
for (const [name, config] of Object.entries(options.mcpServers)) {
|
|
1116
|
-
this._mcpClient.addServer(name, config);
|
|
1117
|
-
}
|
|
1118
|
-
}
|
|
1148
|
+
this._fromNumber = options.from;
|
|
1149
|
+
this._session = options.session;
|
|
1150
|
+
this._recording = options.recording ?? false;
|
|
1151
|
+
this._recordingPath = options.recordingPath ?? "./recordings";
|
|
1152
|
+
this._mcpServers = options.mcpServers ?? [];
|
|
1119
1153
|
if (options.tracing) {
|
|
1120
1154
|
setTracingConfig(options.tracing);
|
|
1121
1155
|
}
|
|
1122
1156
|
}
|
|
1123
|
-
/**
|
|
1124
|
-
|
|
1125
|
-
|
|
1157
|
+
/**
|
|
1158
|
+
* Register a function tool.
|
|
1159
|
+
*
|
|
1160
|
+
* Supports two signatures (matching Python SDK):
|
|
1161
|
+
* agent.tool(name, description, parameters, handler)
|
|
1162
|
+
* agent.tool(functionToolObject)
|
|
1163
|
+
*/
|
|
1164
|
+
tool(nameOrTool, description, parameters, handler) {
|
|
1165
|
+
if (typeof nameOrTool === "string") {
|
|
1166
|
+
if (!description || !parameters || !handler) {
|
|
1167
|
+
throw new AgentError("tool(name, description, parameters, handler) requires all arguments.");
|
|
1168
|
+
}
|
|
1169
|
+
this._tools.register({
|
|
1170
|
+
name: nameOrTool,
|
|
1171
|
+
description,
|
|
1172
|
+
parameters,
|
|
1173
|
+
required: Object.keys(parameters),
|
|
1174
|
+
handler
|
|
1175
|
+
});
|
|
1176
|
+
} else {
|
|
1177
|
+
this._tools.register(nameOrTool);
|
|
1178
|
+
}
|
|
1126
1179
|
return this;
|
|
1127
1180
|
}
|
|
1128
|
-
/**
|
|
1181
|
+
/**
|
|
1182
|
+
* Register an event handler.
|
|
1183
|
+
*
|
|
1184
|
+
* Matches Python SDK decorator style:
|
|
1185
|
+
* agent.on("call_start", (call) => { ... })
|
|
1186
|
+
* agent.on("transcript", (call, role, text) => { ... })
|
|
1187
|
+
*/
|
|
1129
1188
|
on(event, handler) {
|
|
1130
1189
|
let list = this._handlers.get(event);
|
|
1131
1190
|
if (!list) {
|
|
@@ -1140,21 +1199,14 @@ var ClawOpsAgent = class {
|
|
|
1140
1199
|
if (!this._apiKey) {
|
|
1141
1200
|
throw new AgentError("API key is required. Set CLAWOPS_API_KEY or pass apiKey option.");
|
|
1142
1201
|
}
|
|
1143
|
-
if (!this.
|
|
1144
|
-
throw new AgentError("
|
|
1145
|
-
}
|
|
1146
|
-
if (this._mcpClient) {
|
|
1147
|
-
try {
|
|
1148
|
-
const mcpTools = await this._mcpClient.connect();
|
|
1149
|
-
this._tools.registerMcpTools(mcpTools);
|
|
1150
|
-
} catch (err) {
|
|
1151
|
-
console.error("[ClawOpsAgent] MCP connection error:", err);
|
|
1152
|
-
}
|
|
1202
|
+
if (!this._accountId) {
|
|
1203
|
+
throw new AgentError("Account ID is required. Set CLAWOPS_ACCOUNT_ID or pass accountId option.");
|
|
1153
1204
|
}
|
|
1154
1205
|
this._controlWs = new ControlWebSocket({
|
|
1155
1206
|
baseUrl: this._baseUrl,
|
|
1156
1207
|
apiKey: this._apiKey,
|
|
1157
|
-
|
|
1208
|
+
accountId: this._accountId,
|
|
1209
|
+
number: this._fromNumber
|
|
1158
1210
|
});
|
|
1159
1211
|
this._controlWs.on("call.incoming", (event) => this._handleIncoming(event));
|
|
1160
1212
|
this._controlWs.on("call.ended", (event) => this._handleEnded(event));
|
|
@@ -1169,6 +1221,7 @@ var ClawOpsAgent = class {
|
|
|
1169
1221
|
`Failed to connect to ClawOps: ${err instanceof Error ? err.message : String(err)}`
|
|
1170
1222
|
);
|
|
1171
1223
|
}
|
|
1224
|
+
console.log(`[ClawOpsAgent] Connected on ${this._fromNumber}`);
|
|
1172
1225
|
}
|
|
1173
1226
|
/**
|
|
1174
1227
|
* Connect and block until disconnected.
|
|
@@ -1194,83 +1247,118 @@ var ClawOpsAgent = class {
|
|
|
1194
1247
|
session._markEnded();
|
|
1195
1248
|
}
|
|
1196
1249
|
this._activeSessions.clear();
|
|
1197
|
-
|
|
1198
|
-
await this._mcpClient.disconnect();
|
|
1199
|
-
this._tools.clearMcpTools();
|
|
1200
|
-
}
|
|
1250
|
+
console.log("[ClawOpsAgent] Disconnected");
|
|
1201
1251
|
}
|
|
1202
1252
|
/**
|
|
1203
1253
|
* Initiate an outbound call.
|
|
1254
|
+
* Matches Python SDK: agent.call(to, { timeout })
|
|
1204
1255
|
*/
|
|
1205
|
-
call(options) {
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
}
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1256
|
+
async call(to, options) {
|
|
1257
|
+
await this.connect();
|
|
1258
|
+
const url = `${this._baseUrl}/v1/accounts/${this._accountId}/calls`;
|
|
1259
|
+
const body = { To: to, From: this._fromNumber, Timeout: options?.timeout ?? 60 };
|
|
1260
|
+
const resp = await fetch(url, {
|
|
1261
|
+
method: "POST",
|
|
1262
|
+
headers: {
|
|
1263
|
+
Authorization: `Bearer ${this._apiKey}`,
|
|
1264
|
+
"Content-Type": "application/json"
|
|
1265
|
+
},
|
|
1266
|
+
body: JSON.stringify(body)
|
|
1267
|
+
});
|
|
1268
|
+
if (resp.status !== 201) {
|
|
1269
|
+
const error = await resp.json();
|
|
1270
|
+
throw new AgentError(`\uBC1C\uC2E0 \uC2E4\uD328 (${resp.status}): ${error["error"] ?? ""}`);
|
|
1271
|
+
}
|
|
1272
|
+
const data = await resp.json();
|
|
1273
|
+
const callSession = new CallSession({
|
|
1274
|
+
callId: data["callId"],
|
|
1275
|
+
fromNumber: this._fromNumber,
|
|
1276
|
+
toNumber: to,
|
|
1277
|
+
accountId: this._accountId,
|
|
1278
|
+
direction: "outbound"
|
|
1216
1279
|
});
|
|
1280
|
+
for (const [evt, handlers] of this._handlers) {
|
|
1281
|
+
for (const handler of handlers) {
|
|
1282
|
+
callSession.on(evt, handler);
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
this._activeSessions.set(callSession.callId, callSession);
|
|
1286
|
+
console.log(`[ClawOpsAgent] Outbound call initiated: ${this._fromNumber} -> ${to} (${callSession.callId})`);
|
|
1287
|
+
return callSession;
|
|
1217
1288
|
}
|
|
1218
1289
|
_handleIncoming(event) {
|
|
1219
|
-
const
|
|
1290
|
+
const callId = event["callId"];
|
|
1291
|
+
const fromNumber = event["from"] ?? "";
|
|
1292
|
+
const mediaUrl = event["mediaUrl"] ?? "";
|
|
1220
1293
|
const session = new CallSession({
|
|
1221
|
-
callId
|
|
1222
|
-
fromNumber
|
|
1223
|
-
toNumber:
|
|
1224
|
-
accountId:
|
|
1225
|
-
direction:
|
|
1226
|
-
metadata: data.metadata
|
|
1294
|
+
callId,
|
|
1295
|
+
fromNumber,
|
|
1296
|
+
toNumber: this._fromNumber,
|
|
1297
|
+
accountId: this._accountId,
|
|
1298
|
+
direction: "inbound"
|
|
1227
1299
|
});
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1300
|
+
for (const [evt, handlers] of this._handlers) {
|
|
1301
|
+
for (const handler of handlers) {
|
|
1302
|
+
session.on(evt, handler);
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
this._activeSessions.set(callId, session);
|
|
1306
|
+
if (this._controlWs) {
|
|
1307
|
+
this._controlWs.send({ event: "call.accept", callId });
|
|
1308
|
+
}
|
|
1309
|
+
if (mediaUrl) {
|
|
1310
|
+
this._startCallSession(session, mediaUrl).catch((err) => {
|
|
1311
|
+
console.error(`[ClawOpsAgent] Error in call session ${callId}:`, err);
|
|
1233
1312
|
});
|
|
1234
1313
|
}
|
|
1235
1314
|
}
|
|
1236
1315
|
_handleEnded(event) {
|
|
1237
|
-
const
|
|
1316
|
+
const callId = event["callId"];
|
|
1317
|
+
const session = this._activeSessions.get(callId);
|
|
1238
1318
|
if (session) {
|
|
1239
1319
|
session._markEnded();
|
|
1240
|
-
this._activeSessions.delete(
|
|
1241
|
-
this._emitEvent("call.ended", session);
|
|
1320
|
+
this._activeSessions.delete(callId);
|
|
1242
1321
|
}
|
|
1243
1322
|
}
|
|
1244
1323
|
_handleOutboundReady(event) {
|
|
1245
|
-
const
|
|
1246
|
-
const
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1324
|
+
const callId = event["callId"];
|
|
1325
|
+
const mediaUrl = event["mediaUrl"] ?? "";
|
|
1326
|
+
let session = this._activeSessions.get(callId);
|
|
1327
|
+
if (!session) {
|
|
1328
|
+
session = new CallSession({
|
|
1329
|
+
callId,
|
|
1330
|
+
fromNumber: this._fromNumber,
|
|
1331
|
+
toNumber: event["to"] ?? "",
|
|
1332
|
+
accountId: this._accountId,
|
|
1333
|
+
direction: "outbound"
|
|
1334
|
+
});
|
|
1335
|
+
for (const [evt, handlers] of this._handlers) {
|
|
1336
|
+
for (const handler of handlers) {
|
|
1337
|
+
session.on(evt, handler);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
this._activeSessions.set(callId, session);
|
|
1341
|
+
}
|
|
1342
|
+
if (mediaUrl) {
|
|
1343
|
+
this._startCallSession(session, mediaUrl).catch((err) => {
|
|
1344
|
+
console.error(`[ClawOpsAgent] Error in call session ${callId}:`, err);
|
|
1259
1345
|
});
|
|
1260
1346
|
}
|
|
1261
1347
|
}
|
|
1262
1348
|
_handleRinging(event) {
|
|
1263
|
-
const
|
|
1349
|
+
const callId = event["callId"];
|
|
1350
|
+
const session = this._activeSessions.get(callId);
|
|
1264
1351
|
if (session) {
|
|
1265
|
-
|
|
1352
|
+
console.log(`[ClawOpsAgent] Outbound call ringing: ${callId}`);
|
|
1266
1353
|
}
|
|
1267
1354
|
}
|
|
1268
1355
|
_handleFailed(event) {
|
|
1269
|
-
const
|
|
1356
|
+
const callId = event["callId"];
|
|
1357
|
+
const session = this._activeSessions.get(callId);
|
|
1270
1358
|
if (session) {
|
|
1359
|
+
session._emit("call_failed", event["reason"] ?? "failed");
|
|
1271
1360
|
session._markEnded();
|
|
1272
|
-
this._activeSessions.delete(
|
|
1273
|
-
this._emitEvent("call.failed", session);
|
|
1361
|
+
this._activeSessions.delete(callId);
|
|
1274
1362
|
}
|
|
1275
1363
|
}
|
|
1276
1364
|
async _startCallSession(session, mediaWsUrl) {
|
|
@@ -1279,23 +1367,33 @@ var ClawOpsAgent = class {
|
|
|
1279
1367
|
{
|
|
1280
1368
|
[ATTR_CALL_ID]: session.callId,
|
|
1281
1369
|
[ATTR_CALL_DIRECTION]: session.direction,
|
|
1282
|
-
[ATTR_AGENT_ID]: this.
|
|
1370
|
+
[ATTR_AGENT_ID]: this._accountId
|
|
1283
1371
|
},
|
|
1284
1372
|
async () => {
|
|
1285
1373
|
const sessionTools = this._tools.fork();
|
|
1374
|
+
const mcpClients = [];
|
|
1375
|
+
if (this._mcpServers.length > 0) {
|
|
1376
|
+
for (const serverConfig of this._mcpServers) {
|
|
1377
|
+
const client = new MCPClient();
|
|
1378
|
+
client.addServer("mcp", serverConfig);
|
|
1379
|
+
try {
|
|
1380
|
+
const tools = await client.connect();
|
|
1381
|
+
sessionTools.registerMcpTools(tools);
|
|
1382
|
+
mcpClients.push(client);
|
|
1383
|
+
} catch (err) {
|
|
1384
|
+
console.error("[ClawOpsAgent] MCP connection error:", err);
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1286
1388
|
let recorder = null;
|
|
1287
|
-
if (this.
|
|
1288
|
-
recorder = new AudioRecorder(
|
|
1289
|
-
recorder.start(
|
|
1389
|
+
if (this._recording) {
|
|
1390
|
+
recorder = new AudioRecorder(this._recordingPath, session.callId);
|
|
1391
|
+
recorder.start();
|
|
1290
1392
|
}
|
|
1291
1393
|
const mediaWs = new MediaWebSocket();
|
|
1292
1394
|
session._bindTransport(
|
|
1293
1395
|
(audio) => {
|
|
1294
|
-
|
|
1295
|
-
mediaWs.sendAudio(ulaw.toString("base64"));
|
|
1296
|
-
if (recorder) {
|
|
1297
|
-
recorder.writeRawOutbound(audio);
|
|
1298
|
-
}
|
|
1396
|
+
mediaWs.sendAudio(audio.toString("base64"));
|
|
1299
1397
|
},
|
|
1300
1398
|
() => {
|
|
1301
1399
|
mediaWs.sendClear();
|
|
@@ -1304,15 +1402,19 @@ var ClawOpsAgent = class {
|
|
|
1304
1402
|
mediaWs.close();
|
|
1305
1403
|
}
|
|
1306
1404
|
);
|
|
1307
|
-
const sessionHandler = this.
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1405
|
+
const sessionHandler = this._session;
|
|
1406
|
+
if ("setToolRegistry" in sessionHandler && typeof sessionHandler.setToolRegistry === "function") {
|
|
1407
|
+
sessionHandler.setToolRegistry(sessionTools);
|
|
1408
|
+
}
|
|
1409
|
+
if (recorder && "setRecorder" in sessionHandler && typeof sessionHandler.setRecorder === "function") {
|
|
1410
|
+
sessionHandler.setRecorder(recorder);
|
|
1411
|
+
}
|
|
1412
|
+
mediaWs.onAudio((ulawAudio, _timestamp) => {
|
|
1313
1413
|
if (sessionHandler) {
|
|
1314
|
-
|
|
1315
|
-
|
|
1414
|
+
sessionHandler.feedAudio(ulawAudio);
|
|
1415
|
+
}
|
|
1416
|
+
if (recorder) {
|
|
1417
|
+
recorder.writeInbound(ulawToPcm16(ulawAudio));
|
|
1316
1418
|
}
|
|
1317
1419
|
});
|
|
1318
1420
|
mediaWs.onClose(() => {
|
|
@@ -1321,74 +1423,93 @@ var ClawOpsAgent = class {
|
|
|
1321
1423
|
}
|
|
1322
1424
|
session._markEnded();
|
|
1323
1425
|
});
|
|
1426
|
+
session._emit("call_start");
|
|
1324
1427
|
try {
|
|
1325
|
-
await mediaWs.connect(mediaWsUrl);
|
|
1326
|
-
|
|
1327
|
-
await sessionHandler.start(session, sessionTools);
|
|
1328
|
-
}
|
|
1428
|
+
await mediaWs.connect(mediaWsUrl, this._apiKey);
|
|
1429
|
+
await sessionHandler.start(session, sessionTools);
|
|
1329
1430
|
await session.wait();
|
|
1330
|
-
|
|
1331
|
-
await sessionHandler.stop();
|
|
1332
|
-
}
|
|
1431
|
+
await sessionHandler.stop();
|
|
1333
1432
|
} catch (err) {
|
|
1334
1433
|
console.error(`[ClawOpsAgent] Call session error:`, err);
|
|
1335
1434
|
} finally {
|
|
1435
|
+
if (mcpClients.length > 0) {
|
|
1436
|
+
sessionTools.clearMcpTools();
|
|
1437
|
+
for (const c of mcpClients) {
|
|
1438
|
+
await c.disconnect();
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1336
1441
|
mediaWs.close();
|
|
1337
1442
|
if (recorder) {
|
|
1338
1443
|
recorder.stop();
|
|
1339
1444
|
}
|
|
1445
|
+
session._emit("call_end");
|
|
1446
|
+
session._markEnded();
|
|
1447
|
+
this._activeSessions.delete(session.callId);
|
|
1340
1448
|
}
|
|
1341
1449
|
}
|
|
1342
1450
|
);
|
|
1343
1451
|
}
|
|
1344
|
-
_emitEvent(event, session) {
|
|
1345
|
-
const handlers = this._handlers.get(event);
|
|
1346
|
-
if (handlers) {
|
|
1347
|
-
for (const handler of handlers) {
|
|
1348
|
-
try {
|
|
1349
|
-
const result = handler(session);
|
|
1350
|
-
if (result && typeof result.catch === "function") {
|
|
1351
|
-
result.catch((err) => {
|
|
1352
|
-
console.error(`[ClawOpsAgent] Error in ${event} handler:`, err);
|
|
1353
|
-
});
|
|
1354
|
-
}
|
|
1355
|
-
} catch (err) {
|
|
1356
|
-
console.error(`[ClawOpsAgent] Error in ${event} handler:`, err);
|
|
1357
|
-
}
|
|
1358
|
-
}
|
|
1359
|
-
}
|
|
1360
|
-
}
|
|
1361
1452
|
};
|
|
1362
1453
|
|
|
1363
1454
|
// src/agent/pipeline/openai-realtime.ts
|
|
1455
|
+
var OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime?model=";
|
|
1456
|
+
var HANG_UP_TOOL = {
|
|
1457
|
+
type: "function",
|
|
1458
|
+
name: "hang_up",
|
|
1459
|
+
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1460
|
+
parameters: { type: "object", properties: {}, required: [] }
|
|
1461
|
+
};
|
|
1364
1462
|
var OpenAIRealtime = class {
|
|
1365
|
-
|
|
1463
|
+
_apiKey;
|
|
1464
|
+
_systemPrompt;
|
|
1465
|
+
_model;
|
|
1466
|
+
_voice;
|
|
1467
|
+
_language;
|
|
1468
|
+
_eagerness;
|
|
1469
|
+
_greeting;
|
|
1366
1470
|
_ws = null;
|
|
1367
|
-
|
|
1471
|
+
_call = null;
|
|
1368
1472
|
_tools = null;
|
|
1473
|
+
_recorder = null;
|
|
1369
1474
|
_closed = false;
|
|
1475
|
+
// Truncation / barge-in tracking (matching Python SDK)
|
|
1476
|
+
_lastAssistantItem = null;
|
|
1477
|
+
_responseStartTs = null;
|
|
1478
|
+
_sentAudioChunks = 0;
|
|
1479
|
+
_audioRemainder = Buffer.alloc(0);
|
|
1370
1480
|
constructor(options = {}) {
|
|
1371
|
-
this.
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1481
|
+
this._apiKey = options.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
|
|
1482
|
+
this._systemPrompt = options.systemPrompt ?? "";
|
|
1483
|
+
this._model = options.model ?? "gpt-realtime-1.5";
|
|
1484
|
+
this._voice = options.voice ?? "marin";
|
|
1485
|
+
this._language = options.language ?? "ko";
|
|
1486
|
+
this._eagerness = options.eagerness ?? "high";
|
|
1487
|
+
this._greeting = options.greeting ?? true;
|
|
1488
|
+
}
|
|
1489
|
+
/** Inject per-call ToolRegistry. */
|
|
1490
|
+
setToolRegistry(registry) {
|
|
1491
|
+
this._tools = registry;
|
|
1492
|
+
}
|
|
1493
|
+
/** Inject per-call AudioRecorder. */
|
|
1494
|
+
setRecorder(recorder) {
|
|
1495
|
+
this._recorder = recorder;
|
|
1378
1496
|
}
|
|
1379
1497
|
async start(callSession, tools) {
|
|
1380
|
-
this.
|
|
1381
|
-
this._tools = tools
|
|
1498
|
+
this._call = callSession;
|
|
1499
|
+
if (tools) this._tools = tools;
|
|
1382
1500
|
this._closed = false;
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1501
|
+
this._lastAssistantItem = null;
|
|
1502
|
+
this._responseStartTs = null;
|
|
1503
|
+
this._sentAudioChunks = 0;
|
|
1504
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1505
|
+
if (!this._apiKey) {
|
|
1506
|
+
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey option.");
|
|
1386
1507
|
}
|
|
1387
1508
|
const { WebSocket } = await import('ws');
|
|
1388
|
-
const url =
|
|
1509
|
+
const url = `${OPENAI_REALTIME_URL}${this._model}`;
|
|
1389
1510
|
this._ws = new WebSocket(url, {
|
|
1390
1511
|
headers: {
|
|
1391
|
-
Authorization: `Bearer ${
|
|
1512
|
+
Authorization: `Bearer ${this._apiKey}`,
|
|
1392
1513
|
"OpenAI-Beta": "realtime=v1"
|
|
1393
1514
|
}
|
|
1394
1515
|
});
|
|
@@ -1396,6 +1517,9 @@ var OpenAIRealtime = class {
|
|
|
1396
1517
|
const ws = this._ws;
|
|
1397
1518
|
ws.on("open", () => {
|
|
1398
1519
|
this._sendSessionUpdate();
|
|
1520
|
+
if (this._greeting) {
|
|
1521
|
+
this._send({ type: "response.create" });
|
|
1522
|
+
}
|
|
1399
1523
|
resolve();
|
|
1400
1524
|
});
|
|
1401
1525
|
ws.on("message", (data) => {
|
|
@@ -1418,12 +1542,10 @@ var OpenAIRealtime = class {
|
|
|
1418
1542
|
}
|
|
1419
1543
|
feedAudio(audio) {
|
|
1420
1544
|
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1421
|
-
this.
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
})
|
|
1426
|
-
);
|
|
1545
|
+
this._send({
|
|
1546
|
+
type: "input_audio_buffer.append",
|
|
1547
|
+
audio: audio.toString("base64")
|
|
1548
|
+
});
|
|
1427
1549
|
}
|
|
1428
1550
|
}
|
|
1429
1551
|
async stop() {
|
|
@@ -1435,52 +1557,71 @@ var OpenAIRealtime = class {
|
|
|
1435
1557
|
}
|
|
1436
1558
|
_sendSessionUpdate() {
|
|
1437
1559
|
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1438
|
-
const
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
);
|
|
1560
|
+
const toolSchemas = this._tools ? this._tools.toOpenAITools().map((t) => ({ type: "function", ...t.function })) : [];
|
|
1561
|
+
toolSchemas.push(HANG_UP_TOOL);
|
|
1562
|
+
this._send({
|
|
1563
|
+
type: "session.update",
|
|
1564
|
+
session: {
|
|
1565
|
+
modalities: ["text", "audio"],
|
|
1566
|
+
voice: this._voice,
|
|
1567
|
+
instructions: this._systemPrompt,
|
|
1568
|
+
input_audio_format: "g711_ulaw",
|
|
1569
|
+
output_audio_format: "g711_ulaw",
|
|
1570
|
+
input_audio_transcription: {
|
|
1571
|
+
model: "gpt-4o-mini-transcribe",
|
|
1572
|
+
language: this._language
|
|
1573
|
+
},
|
|
1574
|
+
input_audio_noise_reduction: { type: "far_field" },
|
|
1575
|
+
turn_detection: {
|
|
1576
|
+
type: "semantic_vad",
|
|
1577
|
+
interrupt_response: true,
|
|
1578
|
+
eagerness: this._eagerness
|
|
1579
|
+
},
|
|
1580
|
+
tools: toolSchemas
|
|
1581
|
+
}
|
|
1582
|
+
});
|
|
1462
1583
|
}
|
|
1463
1584
|
_handleMessage(msg) {
|
|
1464
1585
|
const type = msg["type"];
|
|
1465
1586
|
switch (type) {
|
|
1466
1587
|
case "response.audio.delta": {
|
|
1467
|
-
|
|
1468
|
-
if (delta && this._callSession) {
|
|
1469
|
-
const audio = Buffer.from(delta, "base64");
|
|
1470
|
-
this._callSession.sendAudio(audio);
|
|
1471
|
-
}
|
|
1588
|
+
this._handleAudioDelta(msg);
|
|
1472
1589
|
break;
|
|
1473
1590
|
}
|
|
1474
1591
|
case "response.audio.done": {
|
|
1592
|
+
if (this._audioRemainder.length > 0) {
|
|
1593
|
+
const padded = Buffer.concat([
|
|
1594
|
+
this._audioRemainder,
|
|
1595
|
+
Buffer.alloc(160 - this._audioRemainder.length, 255)
|
|
1596
|
+
]);
|
|
1597
|
+
if (this._call) {
|
|
1598
|
+
this._call.sendAudio(padded);
|
|
1599
|
+
}
|
|
1600
|
+
this._sentAudioChunks++;
|
|
1601
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1602
|
+
}
|
|
1603
|
+
break;
|
|
1604
|
+
}
|
|
1605
|
+
case "input_audio_buffer.speech_started": {
|
|
1606
|
+
this._handleTruncation();
|
|
1475
1607
|
break;
|
|
1476
1608
|
}
|
|
1477
|
-
case "response.
|
|
1478
|
-
|
|
1609
|
+
case "response.output_item.done": {
|
|
1610
|
+
const item = msg["item"];
|
|
1611
|
+
if (item && item["type"] === "function_call") {
|
|
1612
|
+
this._handleToolCall(item);
|
|
1613
|
+
}
|
|
1479
1614
|
break;
|
|
1480
1615
|
}
|
|
1481
|
-
case "
|
|
1482
|
-
if (this.
|
|
1483
|
-
this.
|
|
1616
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
1617
|
+
if (this._call) {
|
|
1618
|
+
this._call._emit("transcript", "user", msg["transcript"] ?? "");
|
|
1619
|
+
}
|
|
1620
|
+
break;
|
|
1621
|
+
}
|
|
1622
|
+
case "response.audio_transcript.done": {
|
|
1623
|
+
if (this._call) {
|
|
1624
|
+
this._call._emit("transcript", "assistant", msg["transcript"] ?? "");
|
|
1484
1625
|
}
|
|
1485
1626
|
break;
|
|
1486
1627
|
}
|
|
@@ -1490,73 +1631,215 @@ var OpenAIRealtime = class {
|
|
|
1490
1631
|
}
|
|
1491
1632
|
}
|
|
1492
1633
|
}
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1634
|
+
_handleAudioDelta(msg) {
|
|
1635
|
+
if (this._responseStartTs === null) {
|
|
1636
|
+
this._responseStartTs = Date.now();
|
|
1637
|
+
this._sentAudioChunks = 0;
|
|
1638
|
+
}
|
|
1639
|
+
if (msg["item_id"]) {
|
|
1640
|
+
this._lastAssistantItem = msg["item_id"];
|
|
1641
|
+
}
|
|
1642
|
+
const ulaw = Buffer.from(msg["delta"], "base64");
|
|
1643
|
+
if (this._recorder) {
|
|
1644
|
+
this._recorder.writeOutbound(ulawToPcm16(ulaw));
|
|
1645
|
+
}
|
|
1646
|
+
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
1647
|
+
const chunkSize = 160;
|
|
1648
|
+
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
1649
|
+
for (let off = 0; off < fullEnd; off += chunkSize) {
|
|
1650
|
+
if (this._call) {
|
|
1651
|
+
this._call.sendAudio(combined.subarray(off, off + chunkSize));
|
|
1652
|
+
}
|
|
1653
|
+
this._sentAudioChunks++;
|
|
1654
|
+
}
|
|
1655
|
+
this._audioRemainder = combined.subarray(fullEnd);
|
|
1656
|
+
}
|
|
1657
|
+
_handleTruncation() {
|
|
1658
|
+
if (!this._lastAssistantItem || this._responseStartTs === null) {
|
|
1499
1659
|
return;
|
|
1500
1660
|
}
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1661
|
+
const audioEndMs = Math.max(0, this._sentAudioChunks * 20);
|
|
1662
|
+
this._send({
|
|
1663
|
+
type: "conversation.item.truncate",
|
|
1664
|
+
item_id: this._lastAssistantItem,
|
|
1665
|
+
content_index: 0,
|
|
1666
|
+
audio_end_ms: audioEndMs
|
|
1667
|
+
});
|
|
1668
|
+
if (this._call) {
|
|
1669
|
+
this._call.clearAudio();
|
|
1670
|
+
}
|
|
1671
|
+
this._lastAssistantItem = null;
|
|
1672
|
+
this._responseStartTs = null;
|
|
1673
|
+
this._sentAudioChunks = 0;
|
|
1674
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1675
|
+
}
|
|
1676
|
+
async _handleToolCall(item) {
|
|
1677
|
+
const funcName = item["name"];
|
|
1678
|
+
const callId = item["call_id"];
|
|
1679
|
+
if (funcName === "hang_up") {
|
|
1680
|
+
if (this._call) {
|
|
1681
|
+
this._call.hangup();
|
|
1516
1682
|
}
|
|
1683
|
+
return;
|
|
1684
|
+
}
|
|
1685
|
+
if (!this._tools || !this._tools.has(funcName)) {
|
|
1686
|
+
console.error(`[OpenAIRealtime] Unknown tool: ${funcName}`);
|
|
1687
|
+
return;
|
|
1688
|
+
}
|
|
1689
|
+
let result;
|
|
1690
|
+
try {
|
|
1691
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
1692
|
+
result = await this._tools.call(funcName, args);
|
|
1517
1693
|
} catch (err) {
|
|
1518
|
-
console.error(`[OpenAIRealtime] Tool call
|
|
1694
|
+
console.error(`[OpenAIRealtime] Tool call failed: ${funcName}:`, err);
|
|
1695
|
+
result = `Error: ${err}`;
|
|
1696
|
+
}
|
|
1697
|
+
this._send({
|
|
1698
|
+
type: "conversation.item.create",
|
|
1699
|
+
item: {
|
|
1700
|
+
type: "function_call_output",
|
|
1701
|
+
call_id: callId,
|
|
1702
|
+
output: typeof result === "string" ? result : JSON.stringify(result)
|
|
1703
|
+
}
|
|
1704
|
+
});
|
|
1705
|
+
this._send({ type: "response.create" });
|
|
1706
|
+
}
|
|
1707
|
+
_send(data) {
|
|
1708
|
+
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1709
|
+
this._ws.send(JSON.stringify(data));
|
|
1519
1710
|
}
|
|
1520
1711
|
}
|
|
1521
1712
|
};
|
|
1522
1713
|
|
|
1523
1714
|
// src/agent/pipeline/gemini-realtime.ts
|
|
1715
|
+
var HANG_UP_TOOL2 = {
|
|
1716
|
+
name: "hang_up",
|
|
1717
|
+
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1718
|
+
parameters: { type: "object", properties: {} }
|
|
1719
|
+
};
|
|
1720
|
+
function resolveRef(ref, defs) {
|
|
1721
|
+
const parts = ref.replace(/^#\//, "").split("/");
|
|
1722
|
+
let result = defs;
|
|
1723
|
+
for (const part of parts) {
|
|
1724
|
+
if (result && typeof result === "object" && !Array.isArray(result)) {
|
|
1725
|
+
result = result[part];
|
|
1726
|
+
} else {
|
|
1727
|
+
return {};
|
|
1728
|
+
}
|
|
1729
|
+
}
|
|
1730
|
+
return typeof result === "object" && result !== null && !Array.isArray(result) ? result : {};
|
|
1731
|
+
}
|
|
1732
|
+
function sanitizeSchemaForGemini(schema, defs, depth = 0) {
|
|
1733
|
+
if (depth > 15) return { type: "object", properties: {} };
|
|
1734
|
+
if (!schema || typeof schema !== "object") return { type: "object", properties: {} };
|
|
1735
|
+
if (defs === void 0) {
|
|
1736
|
+
defs = schema["$defs"] ?? schema["definitions"] ?? {};
|
|
1737
|
+
}
|
|
1738
|
+
if (typeof schema["$ref"] === "string") {
|
|
1739
|
+
const resolved = resolveRef(schema["$ref"], { $defs: defs, definitions: defs });
|
|
1740
|
+
if (resolved && Object.keys(resolved).length > 0) {
|
|
1741
|
+
return sanitizeSchemaForGemini(resolved, defs, depth + 1);
|
|
1742
|
+
}
|
|
1743
|
+
return { type: "object", properties: {} };
|
|
1744
|
+
}
|
|
1745
|
+
for (const comboKey of ["oneOf", "anyOf", "allOf"]) {
|
|
1746
|
+
const variants = schema[comboKey];
|
|
1747
|
+
if (Array.isArray(variants) && variants.length > 0) {
|
|
1748
|
+
for (const v of variants) {
|
|
1749
|
+
if (v && typeof v === "object") {
|
|
1750
|
+
const resolved = sanitizeSchemaForGemini(v, defs, depth + 1);
|
|
1751
|
+
if (resolved["type"] === "object" && resolved["properties"]) {
|
|
1752
|
+
return resolved;
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
}
|
|
1756
|
+
const first = variants[0];
|
|
1757
|
+
if (first && typeof first === "object") {
|
|
1758
|
+
return sanitizeSchemaForGemini(first, defs, depth + 1);
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
const result = {};
|
|
1763
|
+
let schemaType = schema["type"];
|
|
1764
|
+
if (Array.isArray(schemaType)) {
|
|
1765
|
+
const nonNull = schemaType.filter((t) => t !== "null");
|
|
1766
|
+
schemaType = nonNull[0] ?? "string";
|
|
1767
|
+
}
|
|
1768
|
+
if (schemaType) result["type"] = schemaType;
|
|
1769
|
+
if (schema["description"]) result["description"] = schema["description"];
|
|
1770
|
+
if (schema["enum"]) result["enum"] = schema["enum"];
|
|
1771
|
+
if (schema["required"]) result["required"] = schema["required"];
|
|
1772
|
+
if (schema["properties"] && typeof schema["properties"] === "object") {
|
|
1773
|
+
const props = {};
|
|
1774
|
+
for (const [key, val] of Object.entries(schema["properties"])) {
|
|
1775
|
+
if (val && typeof val === "object") {
|
|
1776
|
+
props[key] = sanitizeSchemaForGemini(val, defs, depth + 1);
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1779
|
+
result["properties"] = props;
|
|
1780
|
+
}
|
|
1781
|
+
if (schema["items"] && typeof schema["items"] === "object" && !Array.isArray(schema["items"])) {
|
|
1782
|
+
result["items"] = sanitizeSchemaForGemini(schema["items"], defs, depth + 1);
|
|
1783
|
+
}
|
|
1784
|
+
if (!result["type"] && result["properties"]) result["type"] = "object";
|
|
1785
|
+
if (result["type"] === "object" && !result["properties"]) result["properties"] = {};
|
|
1786
|
+
return result;
|
|
1787
|
+
}
|
|
1524
1788
|
var GeminiRealtime = class {
|
|
1525
|
-
|
|
1789
|
+
_apiKey;
|
|
1790
|
+
_systemPrompt;
|
|
1791
|
+
_model;
|
|
1792
|
+
_voice;
|
|
1793
|
+
_language;
|
|
1794
|
+
_greeting;
|
|
1795
|
+
_generationConfig;
|
|
1526
1796
|
_ws = null;
|
|
1527
|
-
|
|
1797
|
+
_call = null;
|
|
1528
1798
|
_tools = null;
|
|
1799
|
+
_recorder = null;
|
|
1529
1800
|
_closed = false;
|
|
1801
|
+
_sentAudioChunks = 0;
|
|
1802
|
+
_audioRemainder = Buffer.alloc(0);
|
|
1530
1803
|
constructor(options = {}) {
|
|
1531
|
-
this.
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1804
|
+
this._apiKey = options.apiKey ?? process.env["GOOGLE_API_KEY"] ?? "";
|
|
1805
|
+
this._systemPrompt = options.systemPrompt ?? "";
|
|
1806
|
+
this._model = options.model ?? "gemini-2.5-flash-native-audio-preview-12-2025";
|
|
1807
|
+
this._voice = options.voice ?? "Kore";
|
|
1808
|
+
this._language = options.language ?? "ko";
|
|
1809
|
+
this._greeting = options.greeting ?? true;
|
|
1810
|
+
this._generationConfig = options.generationConfig;
|
|
1811
|
+
}
|
|
1812
|
+
/** Inject per-call ToolRegistry. */
|
|
1813
|
+
setToolRegistry(registry) {
|
|
1814
|
+
this._tools = registry;
|
|
1815
|
+
}
|
|
1816
|
+
/** Inject per-call AudioRecorder. */
|
|
1817
|
+
setRecorder(recorder) {
|
|
1818
|
+
this._recorder = recorder;
|
|
1535
1819
|
}
|
|
1536
1820
|
async start(callSession, tools) {
|
|
1537
|
-
this.
|
|
1538
|
-
this._tools = tools
|
|
1821
|
+
this._call = callSession;
|
|
1822
|
+
if (tools) this._tools = tools;
|
|
1539
1823
|
this._closed = false;
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1824
|
+
this._sentAudioChunks = 0;
|
|
1825
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1826
|
+
if (!this._apiKey) {
|
|
1827
|
+
throw new Error("Google API key is required. Set GOOGLE_API_KEY or pass apiKey option.");
|
|
1543
1828
|
}
|
|
1544
1829
|
const { WebSocket } = await import('ws');
|
|
1545
|
-
this.
|
|
1546
|
-
const url = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=${apiKey}`;
|
|
1830
|
+
const url = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=${this._apiKey}`;
|
|
1547
1831
|
this._ws = new WebSocket(url);
|
|
1548
1832
|
return new Promise((resolve, reject) => {
|
|
1549
1833
|
const ws = this._ws;
|
|
1550
1834
|
ws.on("open", () => {
|
|
1551
1835
|
this._sendSetup();
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
}
|
|
1559
|
-
}
|
|
1836
|
+
this._waitSetupComplete().then(() => {
|
|
1837
|
+
if (this._greeting) {
|
|
1838
|
+
this._sendGreeting();
|
|
1839
|
+
}
|
|
1840
|
+
this._receiveLoop();
|
|
1841
|
+
resolve();
|
|
1842
|
+
}).catch(reject);
|
|
1560
1843
|
});
|
|
1561
1844
|
ws.on("close", () => {
|
|
1562
1845
|
this._closed = true;
|
|
@@ -1571,13 +1854,15 @@ var GeminiRealtime = class {
|
|
|
1571
1854
|
}
|
|
1572
1855
|
feedAudio(audio) {
|
|
1573
1856
|
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1857
|
+
const pcm8k = ulawToPcm16(audio);
|
|
1858
|
+
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
1574
1859
|
this._ws.send(
|
|
1575
1860
|
JSON.stringify({
|
|
1576
1861
|
realtimeInput: {
|
|
1577
1862
|
mediaChunks: [
|
|
1578
1863
|
{
|
|
1579
1864
|
mimeType: "audio/pcm;rate=16000",
|
|
1580
|
-
data:
|
|
1865
|
+
data: pcm16k.toString("base64")
|
|
1581
1866
|
}
|
|
1582
1867
|
]
|
|
1583
1868
|
}
|
|
@@ -1595,35 +1880,87 @@ var GeminiRealtime = class {
|
|
|
1595
1880
|
_sendSetup() {
|
|
1596
1881
|
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1597
1882
|
const setupConfig = {
|
|
1598
|
-
model: `models/${this.
|
|
1883
|
+
model: `models/${this._model}`,
|
|
1599
1884
|
generationConfig: {
|
|
1600
1885
|
responseModalities: ["AUDIO"],
|
|
1601
1886
|
speechConfig: {
|
|
1602
1887
|
voiceConfig: {
|
|
1603
1888
|
prebuiltVoiceConfig: {
|
|
1604
|
-
voiceName: this.
|
|
1889
|
+
voiceName: this._voice
|
|
1605
1890
|
}
|
|
1606
1891
|
}
|
|
1607
1892
|
},
|
|
1608
|
-
...this.
|
|
1893
|
+
...this._generationConfig
|
|
1894
|
+
},
|
|
1895
|
+
realtimeInputConfig: {
|
|
1896
|
+
automaticActivityDetection: {
|
|
1897
|
+
disabled: false
|
|
1898
|
+
}
|
|
1609
1899
|
}
|
|
1610
1900
|
};
|
|
1611
|
-
if (this.
|
|
1901
|
+
if (this._systemPrompt) {
|
|
1612
1902
|
setupConfig["systemInstruction"] = {
|
|
1613
|
-
parts: [{ text: this.
|
|
1903
|
+
parts: [{ text: this._systemPrompt }]
|
|
1614
1904
|
};
|
|
1615
1905
|
}
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1906
|
+
const toolDefs = this._tools ? this._tools.toOpenAITools().map((t) => ({
|
|
1907
|
+
name: t.function.name,
|
|
1908
|
+
description: t.function.description,
|
|
1909
|
+
parameters: sanitizeSchemaForGemini(
|
|
1910
|
+
t.function.parameters ?? { type: "object", properties: {} }
|
|
1911
|
+
)
|
|
1912
|
+
})) : [];
|
|
1913
|
+
toolDefs.push(HANG_UP_TOOL2);
|
|
1914
|
+
setupConfig["tools"] = [{ functionDeclarations: toolDefs }];
|
|
1624
1915
|
this._ws.send(JSON.stringify({ setup: setupConfig }));
|
|
1625
1916
|
}
|
|
1917
|
+
_waitSetupComplete() {
|
|
1918
|
+
return new Promise((resolve, reject) => {
|
|
1919
|
+
if (!this._ws) {
|
|
1920
|
+
reject(new Error("WebSocket not connected"));
|
|
1921
|
+
return;
|
|
1922
|
+
}
|
|
1923
|
+
const onMessage = (data) => {
|
|
1924
|
+
try {
|
|
1925
|
+
const msg = JSON.parse(data.toString());
|
|
1926
|
+
if ("setupComplete" in msg) {
|
|
1927
|
+
this._ws?.removeListener("message", onMessage);
|
|
1928
|
+
resolve();
|
|
1929
|
+
}
|
|
1930
|
+
} catch {
|
|
1931
|
+
}
|
|
1932
|
+
};
|
|
1933
|
+
this._ws.on("message", onMessage);
|
|
1934
|
+
});
|
|
1935
|
+
}
|
|
1936
|
+
_sendGreeting() {
|
|
1937
|
+
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1938
|
+
this._ws.send(
|
|
1939
|
+
JSON.stringify({
|
|
1940
|
+
clientContent: {
|
|
1941
|
+
turns: [
|
|
1942
|
+
{
|
|
1943
|
+
role: "user",
|
|
1944
|
+
parts: [{ text: "\uC778\uC0AC\uD574 \uC8FC\uC138\uC694." }]
|
|
1945
|
+
}
|
|
1946
|
+
],
|
|
1947
|
+
turnComplete: true
|
|
1948
|
+
}
|
|
1949
|
+
})
|
|
1950
|
+
);
|
|
1951
|
+
}
|
|
1952
|
+
_receiveLoop() {
|
|
1953
|
+
if (!this._ws) return;
|
|
1954
|
+
this._ws.on("message", (data) => {
|
|
1955
|
+
try {
|
|
1956
|
+
const msg = JSON.parse(data.toString());
|
|
1957
|
+
this._handleMessage(msg);
|
|
1958
|
+
} catch {
|
|
1959
|
+
}
|
|
1960
|
+
});
|
|
1961
|
+
}
|
|
1626
1962
|
_handleMessage(msg) {
|
|
1963
|
+
if (!this._call) return;
|
|
1627
1964
|
const serverContent = msg["serverContent"];
|
|
1628
1965
|
if (serverContent) {
|
|
1629
1966
|
const modelTurn = serverContent["modelTurn"];
|
|
@@ -1632,35 +1969,108 @@ var GeminiRealtime = class {
|
|
|
1632
1969
|
if (parts) {
|
|
1633
1970
|
for (const part of parts) {
|
|
1634
1971
|
const inlineData = part["inlineData"];
|
|
1635
|
-
if (inlineData && inlineData["data"]
|
|
1636
|
-
const
|
|
1637
|
-
|
|
1972
|
+
if (inlineData && inlineData["data"]) {
|
|
1973
|
+
const mimeType = inlineData["mimeType"] ?? "";
|
|
1974
|
+
if (mimeType.includes("audio")) {
|
|
1975
|
+
this._handleAudioData(inlineData["data"]);
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
const text = part["text"];
|
|
1979
|
+
if (text && this._call) {
|
|
1980
|
+
this._call._emit("transcript", "assistant", text);
|
|
1638
1981
|
}
|
|
1639
1982
|
}
|
|
1640
1983
|
}
|
|
1641
1984
|
}
|
|
1985
|
+
if (serverContent["turnComplete"]) {
|
|
1986
|
+
this._flushAudioRemainder();
|
|
1987
|
+
}
|
|
1988
|
+
if (serverContent["interrupted"]) {
|
|
1989
|
+
if (this._call) {
|
|
1990
|
+
this._call.clearAudio();
|
|
1991
|
+
}
|
|
1992
|
+
this._sentAudioChunks = 0;
|
|
1993
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1994
|
+
}
|
|
1995
|
+
}
|
|
1996
|
+
const inputTranscription = msg["inputTranscription"];
|
|
1997
|
+
if (inputTranscription) {
|
|
1998
|
+
const text = inputTranscription["text"];
|
|
1999
|
+
if (text && this._call) {
|
|
2000
|
+
this._call._emit("transcript", "user", text);
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
const outputTranscription = msg["outputTranscription"];
|
|
2004
|
+
if (outputTranscription) {
|
|
2005
|
+
const text = outputTranscription["text"];
|
|
2006
|
+
if (text && this._call) {
|
|
2007
|
+
this._call._emit("transcript", "assistant", text);
|
|
2008
|
+
}
|
|
1642
2009
|
}
|
|
1643
2010
|
const toolCall = msg["toolCall"];
|
|
1644
2011
|
if (toolCall) {
|
|
1645
2012
|
this._handleToolCall(toolCall);
|
|
1646
2013
|
}
|
|
2014
|
+
if (msg["toolCallCancellation"]) ;
|
|
2015
|
+
}
|
|
2016
|
+
_handleAudioData(b64Data) {
|
|
2017
|
+
if (!this._call) return;
|
|
2018
|
+
const pcm24k = Buffer.from(b64Data, "base64");
|
|
2019
|
+
if (this._recorder) {
|
|
2020
|
+
this._recorder.writeOutbound(resamplePcm16(pcm24k, 24e3, 8e3));
|
|
2021
|
+
}
|
|
2022
|
+
const pcm8k = resamplePcm16(pcm24k, 24e3, 8e3);
|
|
2023
|
+
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2024
|
+
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
2025
|
+
const chunkSize = 160;
|
|
2026
|
+
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
2027
|
+
for (let off = 0; off < fullEnd; off += chunkSize) {
|
|
2028
|
+
this._call.sendAudio(combined.subarray(off, off + chunkSize));
|
|
2029
|
+
this._sentAudioChunks++;
|
|
2030
|
+
}
|
|
2031
|
+
this._audioRemainder = combined.subarray(fullEnd);
|
|
2032
|
+
}
|
|
2033
|
+
_flushAudioRemainder() {
|
|
2034
|
+
if (this._audioRemainder.length > 0 && this._call) {
|
|
2035
|
+
const padded = Buffer.concat([
|
|
2036
|
+
this._audioRemainder,
|
|
2037
|
+
Buffer.alloc(160 - this._audioRemainder.length, 255)
|
|
2038
|
+
]);
|
|
2039
|
+
this._call.sendAudio(padded);
|
|
2040
|
+
this._sentAudioChunks++;
|
|
2041
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
2042
|
+
}
|
|
1647
2043
|
}
|
|
1648
2044
|
async _handleToolCall(toolCall) {
|
|
1649
2045
|
const functionCalls = toolCall["functionCalls"];
|
|
1650
|
-
if (!functionCalls
|
|
2046
|
+
if (!functionCalls) return;
|
|
1651
2047
|
const responses = [];
|
|
1652
2048
|
for (const fc of functionCalls) {
|
|
1653
2049
|
const name = fc["name"];
|
|
2050
|
+
const fcId = fc["id"] ?? "";
|
|
1654
2051
|
const args = fc["args"] ?? {};
|
|
2052
|
+
if (name === "hang_up") {
|
|
2053
|
+
if (this._call) {
|
|
2054
|
+
this._call.hangup();
|
|
2055
|
+
}
|
|
2056
|
+
return;
|
|
2057
|
+
}
|
|
2058
|
+
if (!this._tools || !this._tools.has(name)) {
|
|
2059
|
+
console.error(`[GeminiRealtime] Unknown tool: ${name}`);
|
|
2060
|
+
responses.push({ id: fcId, name, response: { error: `Unknown tool: ${name}` } });
|
|
2061
|
+
continue;
|
|
2062
|
+
}
|
|
1655
2063
|
try {
|
|
1656
2064
|
const result = await this._tools.call(name, args);
|
|
1657
2065
|
responses.push({
|
|
2066
|
+
id: fcId,
|
|
1658
2067
|
name,
|
|
1659
2068
|
response: { result: typeof result === "string" ? result : JSON.stringify(result) }
|
|
1660
2069
|
});
|
|
1661
2070
|
} catch (err) {
|
|
1662
2071
|
console.error(`[GeminiRealtime] Tool call error for ${name}:`, err);
|
|
1663
2072
|
responses.push({
|
|
2073
|
+
id: fcId,
|
|
1664
2074
|
name,
|
|
1665
2075
|
response: { error: String(err) }
|
|
1666
2076
|
});
|
|
@@ -1684,12 +2094,15 @@ var PipelineSession = class {
|
|
|
1684
2094
|
_llm;
|
|
1685
2095
|
_tts;
|
|
1686
2096
|
_systemPrompt;
|
|
2097
|
+
_greeting;
|
|
2098
|
+
_language;
|
|
1687
2099
|
_temperature;
|
|
1688
2100
|
_maxTokens;
|
|
1689
2101
|
_sampleRate;
|
|
1690
2102
|
_interruptOnSpeech;
|
|
1691
2103
|
_callSession = null;
|
|
1692
2104
|
_tools = null;
|
|
2105
|
+
_recorder = null;
|
|
1693
2106
|
_conversation = [];
|
|
1694
2107
|
_audioBuffer = [];
|
|
1695
2108
|
_running = false;
|
|
@@ -1699,10 +2112,20 @@ var PipelineSession = class {
|
|
|
1699
2112
|
this._llm = options.llm;
|
|
1700
2113
|
this._tts = options.tts;
|
|
1701
2114
|
this._systemPrompt = options.systemPrompt;
|
|
2115
|
+
this._greeting = options.greeting ?? true;
|
|
2116
|
+
this._language = options.language ?? "ko";
|
|
1702
2117
|
this._temperature = options.temperature;
|
|
1703
2118
|
this._maxTokens = options.maxTokens;
|
|
1704
2119
|
this._sampleRate = options.sampleRate ?? 8e3;
|
|
1705
2120
|
this._interruptOnSpeech = options.interruptOnSpeech ?? true;
|
|
2121
|
+
if (options.toolRegistry) this._tools = options.toolRegistry;
|
|
2122
|
+
if (options.recorder) this._recorder = options.recorder;
|
|
2123
|
+
}
|
|
2124
|
+
setToolRegistry(registry) {
|
|
2125
|
+
this._tools = registry;
|
|
2126
|
+
}
|
|
2127
|
+
setRecorder(recorder) {
|
|
2128
|
+
this._recorder = recorder;
|
|
1706
2129
|
}
|
|
1707
2130
|
async start(callSession, tools) {
|
|
1708
2131
|
this._callSession = callSession;
|
|
@@ -1715,6 +2138,11 @@ var PipelineSession = class {
|
|
|
1715
2138
|
content: this._systemPrompt
|
|
1716
2139
|
});
|
|
1717
2140
|
}
|
|
2141
|
+
if (this._greeting) {
|
|
2142
|
+
this._generateGreeting().catch((err) => {
|
|
2143
|
+
console.error("[PipelineSession] Greeting error:", err);
|
|
2144
|
+
});
|
|
2145
|
+
}
|
|
1718
2146
|
this._runSttLoop().catch((err) => {
|
|
1719
2147
|
console.error("[PipelineSession] STT loop error:", err);
|
|
1720
2148
|
});
|
|
@@ -1748,14 +2176,24 @@ var PipelineSession = class {
|
|
|
1748
2176
|
async *_createAudioStream() {
|
|
1749
2177
|
while (this._running) {
|
|
1750
2178
|
if (this._audioBuffer.length > 0) {
|
|
1751
|
-
|
|
2179
|
+
const ulaw = this._audioBuffer.shift();
|
|
2180
|
+
const pcm8k = ulawToPcm16(ulaw);
|
|
2181
|
+
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
2182
|
+
yield pcm16k;
|
|
1752
2183
|
} else {
|
|
1753
2184
|
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1754
2185
|
}
|
|
1755
2186
|
}
|
|
1756
2187
|
}
|
|
2188
|
+
async _generateGreeting() {
|
|
2189
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
2190
|
+
await this._respond();
|
|
2191
|
+
}
|
|
1757
2192
|
async _handleUserSpeech(transcript) {
|
|
1758
2193
|
this._conversation.push({ role: "user", content: transcript });
|
|
2194
|
+
await this._respond();
|
|
2195
|
+
}
|
|
2196
|
+
async _respond() {
|
|
1759
2197
|
let fullResponse = "";
|
|
1760
2198
|
const textChunks = [];
|
|
1761
2199
|
const llmStream = this._llm.generate(this._conversation, {
|
|
@@ -1822,7 +2260,19 @@ var PipelineSession = class {
|
|
|
1822
2260
|
sampleRate: this._sampleRate
|
|
1823
2261
|
})) {
|
|
1824
2262
|
if (!this._running || !this._speaking) break;
|
|
1825
|
-
this.
|
|
2263
|
+
if (this._recorder) {
|
|
2264
|
+
const pcm8k2 = this._sampleRate !== 8e3 ? resamplePcm16(audioChunk, this._sampleRate, 8e3) : audioChunk;
|
|
2265
|
+
this._recorder.writeOutbound(pcm8k2);
|
|
2266
|
+
}
|
|
2267
|
+
const pcm8k = resamplePcm16(audioChunk, this._sampleRate, 8e3);
|
|
2268
|
+
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2269
|
+
for (let off = 0; off < ulaw.length; off += 160) {
|
|
2270
|
+
let chunk = ulaw.subarray(off, off + 160);
|
|
2271
|
+
if (chunk.length < 160) {
|
|
2272
|
+
chunk = Buffer.concat([chunk, Buffer.alloc(160 - chunk.length, 255)]);
|
|
2273
|
+
}
|
|
2274
|
+
this._callSession.sendAudio(chunk);
|
|
2275
|
+
}
|
|
1826
2276
|
}
|
|
1827
2277
|
} catch (err) {
|
|
1828
2278
|
console.error("[PipelineSession] TTS error:", err);
|
|
@@ -1837,14 +2287,14 @@ var DeepgramSTT = class {
|
|
|
1837
2287
|
_options;
|
|
1838
2288
|
constructor(options = {}) {
|
|
1839
2289
|
this._options = {
|
|
1840
|
-
model: "nova-
|
|
2290
|
+
model: "nova-3",
|
|
1841
2291
|
language: "ko",
|
|
1842
|
-
|
|
1843
|
-
punctuate: true,
|
|
1844
|
-
smartFormat: true,
|
|
2292
|
+
sampleRate: 16e3,
|
|
1845
2293
|
encoding: "linear16",
|
|
1846
|
-
|
|
1847
|
-
|
|
2294
|
+
punctuate: true,
|
|
2295
|
+
interimResults: true,
|
|
2296
|
+
endpointing: 300,
|
|
2297
|
+
utteranceEndMs: 1e3,
|
|
1848
2298
|
...options
|
|
1849
2299
|
};
|
|
1850
2300
|
}
|
|
@@ -1861,10 +2311,10 @@ var DeepgramSTT = class {
|
|
|
1861
2311
|
language,
|
|
1862
2312
|
punctuate: String(this._options.punctuate),
|
|
1863
2313
|
interim_results: String(this._options.interimResults),
|
|
1864
|
-
smart_format: String(this._options.smartFormat),
|
|
1865
2314
|
encoding: this._options.encoding,
|
|
1866
2315
|
sample_rate: String(sampleRate),
|
|
1867
|
-
|
|
2316
|
+
endpointing: String(this._options.endpointing),
|
|
2317
|
+
utterance_end_ms: String(this._options.utteranceEndMs)
|
|
1868
2318
|
});
|
|
1869
2319
|
const url = `wss://api.deepgram.com/v1/listen?${params.toString()}`;
|
|
1870
2320
|
const ws = new WebSocket(url, {
|
|
@@ -1953,13 +2403,12 @@ var ElevenLabsTTS = class {
|
|
|
1953
2403
|
_options;
|
|
1954
2404
|
constructor(options = {}) {
|
|
1955
2405
|
this._options = {
|
|
1956
|
-
voiceId: "
|
|
1957
|
-
|
|
1958
|
-
outputFormat: "
|
|
2406
|
+
voiceId: "EXAVITQu4vr4xnSDxMaL",
|
|
2407
|
+
model: "eleven_flash_v2_5",
|
|
2408
|
+
outputFormat: "pcm_24000",
|
|
1959
2409
|
stability: 0.5,
|
|
1960
2410
|
similarityBoost: 0.75,
|
|
1961
|
-
|
|
1962
|
-
useSpeakerBoost: true,
|
|
2411
|
+
languageCode: "ko",
|
|
1963
2412
|
...options
|
|
1964
2413
|
};
|
|
1965
2414
|
}
|
|
@@ -1985,12 +2434,10 @@ var ElevenLabsTTS = class {
|
|
|
1985
2434
|
},
|
|
1986
2435
|
body: JSON.stringify({
|
|
1987
2436
|
text,
|
|
1988
|
-
model_id: this._options.
|
|
2437
|
+
model_id: this._options.model,
|
|
1989
2438
|
voice_settings: {
|
|
1990
2439
|
stability: this._options.stability,
|
|
1991
|
-
similarity_boost: this._options.similarityBoost
|
|
1992
|
-
style: this._options.style,
|
|
1993
|
-
use_speaker_boost: this._options.useSpeakerBoost
|
|
2440
|
+
similarity_boost: this._options.similarityBoost
|
|
1994
2441
|
}
|
|
1995
2442
|
})
|
|
1996
2443
|
});
|
|
@@ -2014,7 +2461,7 @@ var ElevenLabsTTS = class {
|
|
|
2014
2461
|
}
|
|
2015
2462
|
async *_synthesizeStreaming(apiKey, voiceId, textStream) {
|
|
2016
2463
|
const { WebSocket } = await import('ws');
|
|
2017
|
-
const url = `wss://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream-input?model_id=${this._options.
|
|
2464
|
+
const url = `wss://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream-input?model_id=${this._options.model}&output_format=${this._options.outputFormat}`;
|
|
2018
2465
|
const ws = new WebSocket(url);
|
|
2019
2466
|
const audioQueue = [];
|
|
2020
2467
|
let resolveWait = null;
|
|
@@ -2025,9 +2472,7 @@ var ElevenLabsTTS = class {
|
|
|
2025
2472
|
text: " ",
|
|
2026
2473
|
voice_settings: {
|
|
2027
2474
|
stability: this._options.stability,
|
|
2028
|
-
similarity_boost: this._options.similarityBoost
|
|
2029
|
-
style: this._options.style,
|
|
2030
|
-
use_speaker_boost: this._options.useSpeakerBoost
|
|
2475
|
+
similarity_boost: this._options.similarityBoost
|
|
2031
2476
|
},
|
|
2032
2477
|
xi_api_key: apiKey
|
|
2033
2478
|
})
|
|
@@ -2104,7 +2549,9 @@ var OpenAILLM = class {
|
|
|
2104
2549
|
_options;
|
|
2105
2550
|
constructor(options = {}) {
|
|
2106
2551
|
this._options = {
|
|
2107
|
-
model: "gpt-4o",
|
|
2552
|
+
model: "gpt-4o-mini",
|
|
2553
|
+
temperature: 0.8,
|
|
2554
|
+
maxTokens: 4096,
|
|
2108
2555
|
...options
|
|
2109
2556
|
};
|
|
2110
2557
|
}
|
|
@@ -2188,8 +2635,9 @@ var AnthropicLLM = class {
|
|
|
2188
2635
|
_options;
|
|
2189
2636
|
constructor(options = {}) {
|
|
2190
2637
|
this._options = {
|
|
2191
|
-
model: "claude-sonnet-4-
|
|
2192
|
-
|
|
2638
|
+
model: "claude-sonnet-4-6",
|
|
2639
|
+
temperature: 0.8,
|
|
2640
|
+
maxTokens: 4096,
|
|
2193
2641
|
...options
|
|
2194
2642
|
};
|
|
2195
2643
|
}
|
|
@@ -2292,7 +2740,9 @@ var GeminiLLM = class {
|
|
|
2292
2740
|
_options;
|
|
2293
2741
|
constructor(options = {}) {
|
|
2294
2742
|
this._options = {
|
|
2295
|
-
model: "gemini-2.
|
|
2743
|
+
model: "gemini-2.5-flash",
|
|
2744
|
+
temperature: 0.8,
|
|
2745
|
+
maxTokens: 4096,
|
|
2296
2746
|
...options
|
|
2297
2747
|
};
|
|
2298
2748
|
}
|
|
@@ -2456,13 +2906,13 @@ var OpenAICompatLLM = class {
|
|
|
2456
2906
|
var OllamaLLM = class {
|
|
2457
2907
|
_inner;
|
|
2458
2908
|
constructor(options = {}) {
|
|
2459
|
-
const baseUrl =
|
|
2909
|
+
const baseUrl = options.baseUrl ?? process.env["OLLAMA_BASE_URL"] ?? "http://localhost:11434/v1";
|
|
2460
2910
|
this._inner = new OpenAICompatLLM({
|
|
2461
|
-
baseUrl:
|
|
2462
|
-
model: options.model ?? "llama3.
|
|
2911
|
+
baseUrl: baseUrl.replace(/\/$/, ""),
|
|
2912
|
+
model: options.model ?? "llama3.2",
|
|
2463
2913
|
apiKey: "ollama",
|
|
2464
|
-
temperature: options.temperature,
|
|
2465
|
-
maxTokens: options.maxTokens
|
|
2914
|
+
temperature: options.temperature ?? 0.8,
|
|
2915
|
+
maxTokens: options.maxTokens ?? 4096
|
|
2466
2916
|
});
|
|
2467
2917
|
}
|
|
2468
2918
|
async *generate(messages, options) {
|