@teamlearners/clawops 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/agent/index.cjs +794 -414
- package/dist/agent/index.cjs.map +1 -1
- package/dist/agent/index.d.cts +255 -196
- package/dist/agent/index.d.ts +255 -196
- package/dist/agent/index.js +794 -414
- package/dist/agent/index.js.map +1 -1
- package/dist/index.cjs +4 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +7 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/agent/index.cjs
CHANGED
|
@@ -347,13 +347,16 @@ function resamplePcm16(pcm, fromRate, toRate) {
|
|
|
347
347
|
}
|
|
348
348
|
|
|
349
349
|
// src/agent/control-ws.ts
|
|
350
|
-
var DEFAULT_PATH = "/v1/agent/control";
|
|
351
350
|
var INITIAL_RECONNECT_DELAY = 1e3;
|
|
352
351
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
353
352
|
function buildControlWsUrl(options) {
|
|
354
|
-
const
|
|
355
|
-
const
|
|
356
|
-
|
|
353
|
+
const scheme = options.baseUrl.startsWith("https") ? "wss" : "ws";
|
|
354
|
+
const host = options.baseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "");
|
|
355
|
+
let url = `${scheme}://${host}/v1/accounts/${encodeURIComponent(options.accountId)}/agent/listen`;
|
|
356
|
+
if (options.number) {
|
|
357
|
+
url += `?number=${encodeURIComponent(options.number)}`;
|
|
358
|
+
}
|
|
359
|
+
return url;
|
|
357
360
|
}
|
|
358
361
|
var ControlWebSocket = class {
|
|
359
362
|
constructor(_options) {
|
|
@@ -404,7 +407,12 @@ var ControlWebSocket = class {
|
|
|
404
407
|
}
|
|
405
408
|
async _doConnect() {
|
|
406
409
|
const { WebSocket } = await import('ws');
|
|
407
|
-
const ws = new WebSocket(this._url
|
|
410
|
+
const ws = new WebSocket(this._url, {
|
|
411
|
+
followRedirects: true,
|
|
412
|
+
headers: {
|
|
413
|
+
Authorization: `Bearer ${this._options.apiKey}`
|
|
414
|
+
}
|
|
415
|
+
});
|
|
408
416
|
this._ws = ws;
|
|
409
417
|
ws.on("open", () => {
|
|
410
418
|
this._reconnectDelay = INITIAL_RECONNECT_DELAY;
|
|
@@ -547,47 +555,37 @@ var MCPClient = class {
|
|
|
547
555
|
// src/agent/media-ws.ts
|
|
548
556
|
function parseStartEvent(data) {
|
|
549
557
|
const start = data["start"];
|
|
558
|
+
const fmt = start["mediaFormat"] ?? {};
|
|
550
559
|
return {
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
customParameters: start["customParameters"] ?? {},
|
|
556
|
-
mediaFormat: start["mediaFormat"] ?? {
|
|
557
|
-
encoding: "audio/x-mulaw",
|
|
558
|
-
sampleRate: 8e3,
|
|
559
|
-
channels: 1
|
|
560
|
-
}
|
|
560
|
+
streamId: start["streamId"] ?? "",
|
|
561
|
+
callId: start["callId"] ?? "",
|
|
562
|
+
accountId: start["accountId"] ?? "",
|
|
563
|
+
sampleRate: fmt["sampleRate"] ?? 8e3
|
|
561
564
|
};
|
|
562
565
|
}
|
|
563
566
|
function parseMediaEvent(data) {
|
|
564
567
|
const media = data["media"];
|
|
565
568
|
return {
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
timestamp: media["timestamp"] ?? ""
|
|
569
|
+
audio: Buffer.from(media["payload"] ?? "", "base64"),
|
|
570
|
+
timestamp: parseInt(media["timestamp"] ?? "0", 10) || 0
|
|
569
571
|
};
|
|
570
572
|
}
|
|
571
|
-
function buildMediaResponse(
|
|
573
|
+
function buildMediaResponse(audioBase64) {
|
|
572
574
|
return JSON.stringify({
|
|
573
575
|
event: "media",
|
|
574
|
-
streamSid,
|
|
575
576
|
media: {
|
|
576
|
-
payload
|
|
577
|
+
payload: audioBase64
|
|
577
578
|
}
|
|
578
579
|
});
|
|
579
580
|
}
|
|
580
581
|
var MediaWebSocket = class {
|
|
581
582
|
_ws = null;
|
|
582
|
-
_streamSid = null;
|
|
583
583
|
_audioQueue = [];
|
|
584
584
|
_sendLoopRunning = false;
|
|
585
585
|
_closed = false;
|
|
586
586
|
_onAudio = null;
|
|
587
587
|
_onStart = null;
|
|
588
588
|
_onClose = null;
|
|
589
|
-
_markSeq = 0;
|
|
590
|
-
_markResolves = /* @__PURE__ */ new Map();
|
|
591
589
|
/** Set the handler for inbound audio data. */
|
|
592
590
|
onAudio(handler) {
|
|
593
591
|
this._onAudio = handler;
|
|
@@ -600,11 +598,16 @@ var MediaWebSocket = class {
|
|
|
600
598
|
onClose(handler) {
|
|
601
599
|
this._onClose = handler;
|
|
602
600
|
}
|
|
603
|
-
/** Connect to a media WebSocket URL. */
|
|
604
|
-
async connect(url) {
|
|
601
|
+
/** Connect to a media WebSocket URL with Bearer authentication. */
|
|
602
|
+
async connect(url, apiKey) {
|
|
605
603
|
const { WebSocket } = await import('ws');
|
|
606
604
|
return new Promise((resolve, reject) => {
|
|
607
|
-
const ws = new WebSocket(url
|
|
605
|
+
const ws = new WebSocket(url, {
|
|
606
|
+
followRedirects: true,
|
|
607
|
+
headers: {
|
|
608
|
+
Authorization: `Bearer ${apiKey}`
|
|
609
|
+
}
|
|
610
|
+
});
|
|
608
611
|
this._ws = ws;
|
|
609
612
|
this._closed = false;
|
|
610
613
|
ws.on("open", () => {
|
|
@@ -639,31 +642,21 @@ var MediaWebSocket = class {
|
|
|
639
642
|
/** Clear all queued outbound audio. */
|
|
640
643
|
sendClear() {
|
|
641
644
|
this._audioQueue.length = 0;
|
|
642
|
-
if (this._ws && this.
|
|
645
|
+
if (this._ws && this._ws.readyState === 1) {
|
|
646
|
+
this._ws.send(JSON.stringify({ event: "clear" }));
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
/** Send a mark event. */
|
|
650
|
+
sendMark(name) {
|
|
651
|
+
if (this._ws && this._ws.readyState === 1) {
|
|
643
652
|
this._ws.send(
|
|
644
653
|
JSON.stringify({
|
|
645
|
-
event: "
|
|
646
|
-
|
|
654
|
+
event: "mark",
|
|
655
|
+
mark: { name }
|
|
647
656
|
})
|
|
648
657
|
);
|
|
649
658
|
}
|
|
650
659
|
}
|
|
651
|
-
/** Send a mark event and return a promise that resolves when the mark is acknowledged. */
|
|
652
|
-
sendMark() {
|
|
653
|
-
const label = `mark_${++this._markSeq}`;
|
|
654
|
-
return new Promise((resolve) => {
|
|
655
|
-
this._markResolves.set(label, resolve);
|
|
656
|
-
if (this._ws && this._streamSid && this._ws.readyState === 1) {
|
|
657
|
-
this._ws.send(
|
|
658
|
-
JSON.stringify({
|
|
659
|
-
event: "mark",
|
|
660
|
-
streamSid: this._streamSid,
|
|
661
|
-
mark: { name: label }
|
|
662
|
-
})
|
|
663
|
-
);
|
|
664
|
-
}
|
|
665
|
-
});
|
|
666
|
-
}
|
|
667
660
|
/** Close the media WebSocket. */
|
|
668
661
|
close() {
|
|
669
662
|
this._closed = true;
|
|
@@ -671,17 +664,12 @@ var MediaWebSocket = class {
|
|
|
671
664
|
this._ws.close();
|
|
672
665
|
this._ws = null;
|
|
673
666
|
}
|
|
674
|
-
for (const resolve of this._markResolves.values()) {
|
|
675
|
-
resolve();
|
|
676
|
-
}
|
|
677
|
-
this._markResolves.clear();
|
|
678
667
|
}
|
|
679
668
|
_handleMessage(msg) {
|
|
680
669
|
const event = msg["event"];
|
|
681
670
|
switch (event) {
|
|
682
671
|
case "start": {
|
|
683
672
|
const startEvt = parseStartEvent(msg);
|
|
684
|
-
this._streamSid = startEvt.streamSid;
|
|
685
673
|
if (this._onStart) {
|
|
686
674
|
this._onStart(startEvt);
|
|
687
675
|
}
|
|
@@ -689,18 +677,8 @@ var MediaWebSocket = class {
|
|
|
689
677
|
}
|
|
690
678
|
case "media": {
|
|
691
679
|
const mediaEvt = parseMediaEvent(msg);
|
|
692
|
-
if (
|
|
693
|
-
|
|
694
|
-
this._onAudio(audioBuf);
|
|
695
|
-
}
|
|
696
|
-
break;
|
|
697
|
-
}
|
|
698
|
-
case "mark": {
|
|
699
|
-
const mark = msg["mark"];
|
|
700
|
-
const name = mark?.["name"];
|
|
701
|
-
if (name && this._markResolves.has(name)) {
|
|
702
|
-
this._markResolves.get(name)();
|
|
703
|
-
this._markResolves.delete(name);
|
|
680
|
+
if (this._onAudio) {
|
|
681
|
+
this._onAudio(mediaEvt.audio, mediaEvt.timestamp);
|
|
704
682
|
}
|
|
705
683
|
break;
|
|
706
684
|
}
|
|
@@ -720,98 +698,162 @@ var MediaWebSocket = class {
|
|
|
720
698
|
}
|
|
721
699
|
while (this._audioQueue.length > 0 && this._ws && this._ws.readyState === 1) {
|
|
722
700
|
const payload = this._audioQueue.shift();
|
|
723
|
-
|
|
724
|
-
this._ws.send(buildMediaResponse(this._streamSid, payload));
|
|
725
|
-
}
|
|
701
|
+
this._ws.send(buildMediaResponse(payload));
|
|
726
702
|
}
|
|
727
703
|
setTimeout(loop, 20);
|
|
728
704
|
};
|
|
729
705
|
loop();
|
|
730
706
|
}
|
|
731
707
|
};
|
|
732
|
-
var
|
|
733
|
-
var
|
|
734
|
-
var
|
|
735
|
-
|
|
708
|
+
var SAMPLE_RATE = 8e3;
|
|
709
|
+
var CHANNELS = 1;
|
|
710
|
+
var BITS_PER_SAMPLE = 16;
|
|
711
|
+
var BYTES_PER_SECOND = SAMPLE_RATE * CHANNELS * (BITS_PER_SAMPLE / 8);
|
|
712
|
+
function makeWavHeader(dataSize = 0) {
|
|
736
713
|
const header = Buffer.alloc(44);
|
|
737
|
-
|
|
738
|
-
const blockAlign = WAV_CHANNELS * (WAV_BITS_PER_SAMPLE / 8);
|
|
739
|
-
header.write("RIFF", 0);
|
|
714
|
+
header.write("RIFF", 0, "ascii");
|
|
740
715
|
header.writeUInt32LE(36 + dataSize, 4);
|
|
741
|
-
header.write("WAVE", 8);
|
|
742
|
-
header.write("fmt ", 12);
|
|
716
|
+
header.write("WAVE", 8, "ascii");
|
|
717
|
+
header.write("fmt ", 12, "ascii");
|
|
743
718
|
header.writeUInt32LE(16, 16);
|
|
744
719
|
header.writeUInt16LE(1, 20);
|
|
745
|
-
header.writeUInt16LE(
|
|
746
|
-
header.writeUInt32LE(
|
|
747
|
-
header.writeUInt32LE(
|
|
748
|
-
header.writeUInt16LE(
|
|
749
|
-
header.writeUInt16LE(
|
|
750
|
-
header.write("data", 36);
|
|
720
|
+
header.writeUInt16LE(CHANNELS, 22);
|
|
721
|
+
header.writeUInt32LE(SAMPLE_RATE, 24);
|
|
722
|
+
header.writeUInt32LE(SAMPLE_RATE * CHANNELS * (BITS_PER_SAMPLE / 8), 28);
|
|
723
|
+
header.writeUInt16LE(CHANNELS * (BITS_PER_SAMPLE / 8), 32);
|
|
724
|
+
header.writeUInt16LE(BITS_PER_SAMPLE, 34);
|
|
725
|
+
header.write("data", 36, "ascii");
|
|
751
726
|
header.writeUInt32LE(dataSize, 40);
|
|
752
727
|
return header;
|
|
753
728
|
}
|
|
729
|
+
function mixSamples(a, b) {
|
|
730
|
+
const n = Math.min(a.length, b.length) >> 1;
|
|
731
|
+
const result = Buffer.alloc(n * 2);
|
|
732
|
+
for (let i = 0; i < n; i++) {
|
|
733
|
+
const sa = a.readInt16LE(i * 2);
|
|
734
|
+
const sb = b.readInt16LE(i * 2);
|
|
735
|
+
result.writeInt16LE(Math.max(-32768, Math.min(32767, sa + sb)), i * 2);
|
|
736
|
+
}
|
|
737
|
+
return result;
|
|
738
|
+
}
|
|
754
739
|
var AudioRecorder = class {
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
740
|
+
_dir;
|
|
741
|
+
_fdIn = null;
|
|
742
|
+
_fdOut = null;
|
|
743
|
+
_fdMix = null;
|
|
744
|
+
_inWritten = 0;
|
|
745
|
+
_outWritten = 0;
|
|
746
|
+
_mixWritten = 0;
|
|
747
|
+
_startTime = 0;
|
|
760
748
|
_started = false;
|
|
761
|
-
constructor(
|
|
762
|
-
this.
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
this.
|
|
768
|
-
this.
|
|
769
|
-
this.
|
|
749
|
+
constructor(recordingPath, callId) {
|
|
750
|
+
this._dir = path__namespace.join(recordingPath, callId);
|
|
751
|
+
}
|
|
752
|
+
start() {
|
|
753
|
+
fs__namespace.mkdirSync(this._dir, { recursive: true });
|
|
754
|
+
const header = makeWavHeader();
|
|
755
|
+
this._fdIn = fs__namespace.openSync(path__namespace.join(this._dir, "in.wav"), "w");
|
|
756
|
+
this._fdOut = fs__namespace.openSync(path__namespace.join(this._dir, "out.wav"), "w");
|
|
757
|
+
this._fdMix = fs__namespace.openSync(path__namespace.join(this._dir, "mix.wav"), "w+");
|
|
758
|
+
fs__namespace.writeSync(this._fdIn, header);
|
|
759
|
+
fs__namespace.writeSync(this._fdOut, header);
|
|
760
|
+
fs__namespace.writeSync(this._fdMix, header);
|
|
761
|
+
this._startTime = performance.now();
|
|
770
762
|
this._started = true;
|
|
771
|
-
|
|
772
|
-
|
|
763
|
+
}
|
|
764
|
+
_expectedBytes() {
|
|
765
|
+
const elapsed = (performance.now() - this._startTime) / 1e3;
|
|
766
|
+
return Math.floor(elapsed * BYTES_PER_SECOND);
|
|
767
|
+
}
|
|
768
|
+
_padSilence(fd, written) {
|
|
769
|
+
const expected = this._expectedBytes();
|
|
770
|
+
let gap = expected - written;
|
|
771
|
+
if (gap <= 0) return 0;
|
|
772
|
+
gap = gap - gap % 2;
|
|
773
|
+
if (gap > 0) {
|
|
774
|
+
fs__namespace.writeSync(fd, Buffer.alloc(gap));
|
|
775
|
+
}
|
|
776
|
+
return gap;
|
|
777
|
+
}
|
|
778
|
+
_writeToMix(data, trackPos) {
|
|
779
|
+
if (this._fdMix === null) return;
|
|
780
|
+
const filePos = 44 + trackPos;
|
|
781
|
+
if (trackPos < this._mixWritten) {
|
|
782
|
+
const overlap = Math.min(data.length, this._mixWritten - trackPos);
|
|
783
|
+
const existing = Buffer.alloc(overlap);
|
|
784
|
+
fs__namespace.readSync(this._fdMix, existing, 0, overlap, filePos);
|
|
785
|
+
const mixed = mixSamples(existing, data.subarray(0, overlap));
|
|
786
|
+
fs__namespace.writeSync(this._fdMix, mixed, 0, mixed.length, filePos);
|
|
787
|
+
if (data.length > overlap) {
|
|
788
|
+
fs__namespace.writeSync(this._fdMix, data, overlap, data.length - overlap, filePos + overlap);
|
|
789
|
+
this._mixWritten = trackPos + data.length;
|
|
790
|
+
}
|
|
791
|
+
} else {
|
|
792
|
+
if (trackPos > this._mixWritten) {
|
|
793
|
+
let gap = trackPos - this._mixWritten;
|
|
794
|
+
gap = gap - gap % 2;
|
|
795
|
+
if (gap > 0) {
|
|
796
|
+
const silence = Buffer.alloc(gap);
|
|
797
|
+
fs__namespace.writeSync(this._fdMix, silence, 0, silence.length, 44 + this._mixWritten);
|
|
798
|
+
this._mixWritten += gap;
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
fs__namespace.writeSync(this._fdMix, data, 0, data.length, 44 + this._mixWritten);
|
|
802
|
+
this._mixWritten += data.length;
|
|
773
803
|
}
|
|
774
804
|
}
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
this.
|
|
805
|
+
writeInbound(pcm16_8k) {
|
|
806
|
+
if (!this._started || this._fdIn === null) return;
|
|
807
|
+
try {
|
|
808
|
+
const gap = this._padSilence(this._fdIn, this._inWritten);
|
|
809
|
+
this._inWritten += gap;
|
|
810
|
+
const posBefore = this._inWritten;
|
|
811
|
+
fs__namespace.writeSync(this._fdIn, pcm16_8k);
|
|
812
|
+
this._inWritten += pcm16_8k.length;
|
|
813
|
+
this._writeToMix(pcm16_8k, posBefore);
|
|
814
|
+
} catch (err) {
|
|
815
|
+
console.error("Error writing inbound audio:", err);
|
|
779
816
|
}
|
|
780
817
|
}
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
this.
|
|
818
|
+
writeOutbound(pcm16_8k) {
|
|
819
|
+
if (!this._started || this._fdOut === null) return;
|
|
820
|
+
try {
|
|
821
|
+
const gap = this._padSilence(this._fdOut, this._outWritten);
|
|
822
|
+
this._outWritten += gap;
|
|
823
|
+
const posBefore = this._outWritten;
|
|
824
|
+
fs__namespace.writeSync(this._fdOut, pcm16_8k);
|
|
825
|
+
this._outWritten += pcm16_8k.length;
|
|
826
|
+
this._writeToMix(pcm16_8k, posBefore);
|
|
827
|
+
} catch (err) {
|
|
828
|
+
console.error("Error writing outbound audio:", err);
|
|
785
829
|
}
|
|
786
830
|
}
|
|
787
|
-
/** Stop recording and write WAV files to disk. Returns file paths. */
|
|
788
831
|
stop() {
|
|
789
|
-
if (!this._started
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
fs__namespace.closeSync(fd);
|
|
832
|
+
if (!this._started) return;
|
|
833
|
+
try {
|
|
834
|
+
let maxWritten = Math.max(this._inWritten, this._outWritten, this._mixWritten);
|
|
835
|
+
maxWritten = maxWritten & ~1;
|
|
836
|
+
for (const [fd, written] of [
|
|
837
|
+
[this._fdIn, this._inWritten],
|
|
838
|
+
[this._fdOut, this._outWritten],
|
|
839
|
+
[this._fdMix, this._mixWritten]
|
|
840
|
+
]) {
|
|
841
|
+
if (fd === null) continue;
|
|
842
|
+
const pad = maxWritten - written;
|
|
843
|
+
if (pad > 0) {
|
|
844
|
+
fs__namespace.writeSync(fd, Buffer.alloc(pad), 0, pad, 44 + written);
|
|
845
|
+
}
|
|
846
|
+
fs__namespace.writeSync(fd, makeWavHeader(maxWritten), 0, 44, 0);
|
|
847
|
+
fs__namespace.closeSync(fd);
|
|
848
|
+
}
|
|
849
|
+
} catch (err) {
|
|
850
|
+
console.error("Error stopping recorder:", err);
|
|
851
|
+
} finally {
|
|
852
|
+
this._fdIn = null;
|
|
853
|
+
this._fdOut = null;
|
|
854
|
+
this._fdMix = null;
|
|
855
|
+
this._started = false;
|
|
856
|
+
}
|
|
815
857
|
}
|
|
816
858
|
};
|
|
817
859
|
|
|
@@ -891,23 +933,23 @@ var CallSession = class {
|
|
|
891
933
|
/** Mark the session as ended (called internally). */
|
|
892
934
|
_markEnded() {
|
|
893
935
|
this._status = "ended";
|
|
894
|
-
this._emit({ type: "ended" });
|
|
895
936
|
this._resolveEnded();
|
|
896
937
|
}
|
|
897
|
-
/** Emit an event to registered handlers. */
|
|
898
|
-
|
|
899
|
-
|
|
938
|
+
/** Emit an event to registered handlers. Matches Python SDK: _emit(event, ...args) */
|
|
939
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
940
|
+
_emit(event, ...args) {
|
|
941
|
+
const handlers = this._handlers.get(event);
|
|
900
942
|
if (handlers) {
|
|
901
943
|
for (const handler of handlers) {
|
|
902
944
|
try {
|
|
903
|
-
const result = handler(
|
|
945
|
+
const result = handler(this, ...args);
|
|
904
946
|
if (result && typeof result.catch === "function") {
|
|
905
947
|
result.catch((err) => {
|
|
906
|
-
console.error(`[CallSession] Error in ${event
|
|
948
|
+
console.error(`[CallSession] Error in ${event} handler:`, err);
|
|
907
949
|
});
|
|
908
950
|
}
|
|
909
951
|
} catch (err) {
|
|
910
|
-
console.error(`[CallSession] Error in ${event
|
|
952
|
+
console.error(`[CallSession] Error in ${event} handler:`, err);
|
|
911
953
|
}
|
|
912
954
|
}
|
|
913
955
|
}
|
|
@@ -1111,44 +1153,61 @@ var ATTR_AGENT_ID = "clawops.agent.id";
|
|
|
1111
1153
|
// src/agent/agent.ts
|
|
1112
1154
|
var ClawOpsAgent = class {
|
|
1113
1155
|
_apiKey;
|
|
1114
|
-
|
|
1156
|
+
_accountId;
|
|
1115
1157
|
_baseUrl;
|
|
1116
|
-
|
|
1158
|
+
_fromNumber;
|
|
1159
|
+
_session;
|
|
1117
1160
|
_tools = new ToolRegistry();
|
|
1118
1161
|
_handlers = /* @__PURE__ */ new Map();
|
|
1119
1162
|
_controlWs = null;
|
|
1120
|
-
|
|
1121
|
-
|
|
1163
|
+
_mcpServers;
|
|
1164
|
+
_recording;
|
|
1165
|
+
_recordingPath;
|
|
1122
1166
|
_activeSessions = /* @__PURE__ */ new Map();
|
|
1123
|
-
constructor(options
|
|
1167
|
+
constructor(options) {
|
|
1124
1168
|
this._apiKey = options.apiKey ?? process.env["CLAWOPS_API_KEY"] ?? "";
|
|
1125
|
-
this.
|
|
1169
|
+
this._accountId = options.accountId ?? process.env["CLAWOPS_ACCOUNT_ID"] ?? "";
|
|
1126
1170
|
this._baseUrl = options.baseUrl ?? chunkJNSHMKDI_cjs.DEFAULT_BASE_URL;
|
|
1127
|
-
this.
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
const sessionInstance = options.session;
|
|
1133
|
-
this._sessionFactory = () => sessionInstance;
|
|
1134
|
-
}
|
|
1135
|
-
}
|
|
1136
|
-
if (options.mcpServers) {
|
|
1137
|
-
this._mcpClient = new MCPClient();
|
|
1138
|
-
for (const [name, config] of Object.entries(options.mcpServers)) {
|
|
1139
|
-
this._mcpClient.addServer(name, config);
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1171
|
+
this._fromNumber = options.from;
|
|
1172
|
+
this._session = options.session;
|
|
1173
|
+
this._recording = options.recording ?? false;
|
|
1174
|
+
this._recordingPath = options.recordingPath ?? "./recordings";
|
|
1175
|
+
this._mcpServers = options.mcpServers ?? [];
|
|
1142
1176
|
if (options.tracing) {
|
|
1143
1177
|
setTracingConfig(options.tracing);
|
|
1144
1178
|
}
|
|
1145
1179
|
}
|
|
1146
|
-
/**
|
|
1147
|
-
|
|
1148
|
-
|
|
1180
|
+
/**
|
|
1181
|
+
* Register a function tool.
|
|
1182
|
+
*
|
|
1183
|
+
* Supports two signatures (matching Python SDK):
|
|
1184
|
+
* agent.tool(name, description, parameters, handler)
|
|
1185
|
+
* agent.tool(functionToolObject)
|
|
1186
|
+
*/
|
|
1187
|
+
tool(nameOrTool, description, parameters, handler) {
|
|
1188
|
+
if (typeof nameOrTool === "string") {
|
|
1189
|
+
if (!description || !parameters || !handler) {
|
|
1190
|
+
throw new chunkJNSHMKDI_cjs.AgentError("tool(name, description, parameters, handler) requires all arguments.");
|
|
1191
|
+
}
|
|
1192
|
+
this._tools.register({
|
|
1193
|
+
name: nameOrTool,
|
|
1194
|
+
description,
|
|
1195
|
+
parameters,
|
|
1196
|
+
required: Object.keys(parameters),
|
|
1197
|
+
handler
|
|
1198
|
+
});
|
|
1199
|
+
} else {
|
|
1200
|
+
this._tools.register(nameOrTool);
|
|
1201
|
+
}
|
|
1149
1202
|
return this;
|
|
1150
1203
|
}
|
|
1151
|
-
/**
|
|
1204
|
+
/**
|
|
1205
|
+
* Register an event handler.
|
|
1206
|
+
*
|
|
1207
|
+
* Matches Python SDK decorator style:
|
|
1208
|
+
* agent.on("call_start", (call) => { ... })
|
|
1209
|
+
* agent.on("transcript", (call, role, text) => { ... })
|
|
1210
|
+
*/
|
|
1152
1211
|
on(event, handler) {
|
|
1153
1212
|
let list = this._handlers.get(event);
|
|
1154
1213
|
if (!list) {
|
|
@@ -1163,21 +1222,14 @@ var ClawOpsAgent = class {
|
|
|
1163
1222
|
if (!this._apiKey) {
|
|
1164
1223
|
throw new chunkJNSHMKDI_cjs.AgentError("API key is required. Set CLAWOPS_API_KEY or pass apiKey option.");
|
|
1165
1224
|
}
|
|
1166
|
-
if (!this.
|
|
1167
|
-
throw new chunkJNSHMKDI_cjs.AgentError("
|
|
1168
|
-
}
|
|
1169
|
-
if (this._mcpClient) {
|
|
1170
|
-
try {
|
|
1171
|
-
const mcpTools = await this._mcpClient.connect();
|
|
1172
|
-
this._tools.registerMcpTools(mcpTools);
|
|
1173
|
-
} catch (err) {
|
|
1174
|
-
console.error("[ClawOpsAgent] MCP connection error:", err);
|
|
1175
|
-
}
|
|
1225
|
+
if (!this._accountId) {
|
|
1226
|
+
throw new chunkJNSHMKDI_cjs.AgentError("Account ID is required. Set CLAWOPS_ACCOUNT_ID or pass accountId option.");
|
|
1176
1227
|
}
|
|
1177
1228
|
this._controlWs = new ControlWebSocket({
|
|
1178
1229
|
baseUrl: this._baseUrl,
|
|
1179
1230
|
apiKey: this._apiKey,
|
|
1180
|
-
|
|
1231
|
+
accountId: this._accountId,
|
|
1232
|
+
number: this._fromNumber
|
|
1181
1233
|
});
|
|
1182
1234
|
this._controlWs.on("call.incoming", (event) => this._handleIncoming(event));
|
|
1183
1235
|
this._controlWs.on("call.ended", (event) => this._handleEnded(event));
|
|
@@ -1192,6 +1244,7 @@ var ClawOpsAgent = class {
|
|
|
1192
1244
|
`Failed to connect to ClawOps: ${err instanceof Error ? err.message : String(err)}`
|
|
1193
1245
|
);
|
|
1194
1246
|
}
|
|
1247
|
+
console.log(`[ClawOpsAgent] Connected on ${this._fromNumber}`);
|
|
1195
1248
|
}
|
|
1196
1249
|
/**
|
|
1197
1250
|
* Connect and block until disconnected.
|
|
@@ -1217,83 +1270,118 @@ var ClawOpsAgent = class {
|
|
|
1217
1270
|
session._markEnded();
|
|
1218
1271
|
}
|
|
1219
1272
|
this._activeSessions.clear();
|
|
1220
|
-
|
|
1221
|
-
await this._mcpClient.disconnect();
|
|
1222
|
-
this._tools.clearMcpTools();
|
|
1223
|
-
}
|
|
1273
|
+
console.log("[ClawOpsAgent] Disconnected");
|
|
1224
1274
|
}
|
|
1225
1275
|
/**
|
|
1226
1276
|
* Initiate an outbound call.
|
|
1277
|
+
* Matches Python SDK: agent.call(to, { timeout })
|
|
1227
1278
|
*/
|
|
1228
|
-
call(options) {
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
}
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1279
|
+
async call(to, options) {
|
|
1280
|
+
await this.connect();
|
|
1281
|
+
const url = `${this._baseUrl}/v1/accounts/${this._accountId}/calls`;
|
|
1282
|
+
const body = { To: to, From: this._fromNumber, Timeout: options?.timeout ?? 60 };
|
|
1283
|
+
const resp = await fetch(url, {
|
|
1284
|
+
method: "POST",
|
|
1285
|
+
headers: {
|
|
1286
|
+
Authorization: `Bearer ${this._apiKey}`,
|
|
1287
|
+
"Content-Type": "application/json"
|
|
1288
|
+
},
|
|
1289
|
+
body: JSON.stringify(body)
|
|
1290
|
+
});
|
|
1291
|
+
if (resp.status !== 201) {
|
|
1292
|
+
const error = await resp.json();
|
|
1293
|
+
throw new chunkJNSHMKDI_cjs.AgentError(`\uBC1C\uC2E0 \uC2E4\uD328 (${resp.status}): ${error["error"] ?? ""}`);
|
|
1294
|
+
}
|
|
1295
|
+
const data = await resp.json();
|
|
1296
|
+
const callSession = new CallSession({
|
|
1297
|
+
callId: data["callId"],
|
|
1298
|
+
fromNumber: this._fromNumber,
|
|
1299
|
+
toNumber: to,
|
|
1300
|
+
accountId: this._accountId,
|
|
1301
|
+
direction: "outbound"
|
|
1239
1302
|
});
|
|
1303
|
+
for (const [evt, handlers] of this._handlers) {
|
|
1304
|
+
for (const handler of handlers) {
|
|
1305
|
+
callSession.on(evt, handler);
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
this._activeSessions.set(callSession.callId, callSession);
|
|
1309
|
+
console.log(`[ClawOpsAgent] Outbound call initiated: ${this._fromNumber} -> ${to} (${callSession.callId})`);
|
|
1310
|
+
return callSession;
|
|
1240
1311
|
}
|
|
1241
1312
|
_handleIncoming(event) {
|
|
1242
|
-
const
|
|
1313
|
+
const callId = event["callId"];
|
|
1314
|
+
const fromNumber = event["from"] ?? "";
|
|
1315
|
+
const mediaUrl = event["mediaUrl"] ?? "";
|
|
1243
1316
|
const session = new CallSession({
|
|
1244
|
-
callId
|
|
1245
|
-
fromNumber
|
|
1246
|
-
toNumber:
|
|
1247
|
-
accountId:
|
|
1248
|
-
direction:
|
|
1249
|
-
metadata: data.metadata
|
|
1317
|
+
callId,
|
|
1318
|
+
fromNumber,
|
|
1319
|
+
toNumber: this._fromNumber,
|
|
1320
|
+
accountId: this._accountId,
|
|
1321
|
+
direction: "inbound"
|
|
1250
1322
|
});
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1323
|
+
for (const [evt, handlers] of this._handlers) {
|
|
1324
|
+
for (const handler of handlers) {
|
|
1325
|
+
session.on(evt, handler);
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
this._activeSessions.set(callId, session);
|
|
1329
|
+
if (this._controlWs) {
|
|
1330
|
+
this._controlWs.send({ event: "call.accept", callId });
|
|
1331
|
+
}
|
|
1332
|
+
if (mediaUrl) {
|
|
1333
|
+
this._startCallSession(session, mediaUrl).catch((err) => {
|
|
1334
|
+
console.error(`[ClawOpsAgent] Error in call session ${callId}:`, err);
|
|
1256
1335
|
});
|
|
1257
1336
|
}
|
|
1258
1337
|
}
|
|
1259
1338
|
_handleEnded(event) {
|
|
1260
|
-
const
|
|
1339
|
+
const callId = event["callId"];
|
|
1340
|
+
const session = this._activeSessions.get(callId);
|
|
1261
1341
|
if (session) {
|
|
1262
1342
|
session._markEnded();
|
|
1263
|
-
this._activeSessions.delete(
|
|
1264
|
-
this._emitEvent("call.ended", session);
|
|
1343
|
+
this._activeSessions.delete(callId);
|
|
1265
1344
|
}
|
|
1266
1345
|
}
|
|
1267
1346
|
_handleOutboundReady(event) {
|
|
1268
|
-
const
|
|
1269
|
-
const
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1347
|
+
const callId = event["callId"];
|
|
1348
|
+
const mediaUrl = event["mediaUrl"] ?? "";
|
|
1349
|
+
let session = this._activeSessions.get(callId);
|
|
1350
|
+
if (!session) {
|
|
1351
|
+
session = new CallSession({
|
|
1352
|
+
callId,
|
|
1353
|
+
fromNumber: this._fromNumber,
|
|
1354
|
+
toNumber: event["to"] ?? "",
|
|
1355
|
+
accountId: this._accountId,
|
|
1356
|
+
direction: "outbound"
|
|
1357
|
+
});
|
|
1358
|
+
for (const [evt, handlers] of this._handlers) {
|
|
1359
|
+
for (const handler of handlers) {
|
|
1360
|
+
session.on(evt, handler);
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
this._activeSessions.set(callId, session);
|
|
1364
|
+
}
|
|
1365
|
+
if (mediaUrl) {
|
|
1366
|
+
this._startCallSession(session, mediaUrl).catch((err) => {
|
|
1367
|
+
console.error(`[ClawOpsAgent] Error in call session ${callId}:`, err);
|
|
1282
1368
|
});
|
|
1283
1369
|
}
|
|
1284
1370
|
}
|
|
1285
1371
|
_handleRinging(event) {
|
|
1286
|
-
const
|
|
1372
|
+
const callId = event["callId"];
|
|
1373
|
+
const session = this._activeSessions.get(callId);
|
|
1287
1374
|
if (session) {
|
|
1288
|
-
|
|
1375
|
+
console.log(`[ClawOpsAgent] Outbound call ringing: ${callId}`);
|
|
1289
1376
|
}
|
|
1290
1377
|
}
|
|
1291
1378
|
_handleFailed(event) {
|
|
1292
|
-
const
|
|
1379
|
+
const callId = event["callId"];
|
|
1380
|
+
const session = this._activeSessions.get(callId);
|
|
1293
1381
|
if (session) {
|
|
1382
|
+
session._emit("call_failed", event["reason"] ?? "failed");
|
|
1294
1383
|
session._markEnded();
|
|
1295
|
-
this._activeSessions.delete(
|
|
1296
|
-
this._emitEvent("call.failed", session);
|
|
1384
|
+
this._activeSessions.delete(callId);
|
|
1297
1385
|
}
|
|
1298
1386
|
}
|
|
1299
1387
|
async _startCallSession(session, mediaWsUrl) {
|
|
@@ -1302,23 +1390,33 @@ var ClawOpsAgent = class {
|
|
|
1302
1390
|
{
|
|
1303
1391
|
[ATTR_CALL_ID]: session.callId,
|
|
1304
1392
|
[ATTR_CALL_DIRECTION]: session.direction,
|
|
1305
|
-
[ATTR_AGENT_ID]: this.
|
|
1393
|
+
[ATTR_AGENT_ID]: this._accountId
|
|
1306
1394
|
},
|
|
1307
1395
|
async () => {
|
|
1308
1396
|
const sessionTools = this._tools.fork();
|
|
1397
|
+
const mcpClients = [];
|
|
1398
|
+
if (this._mcpServers.length > 0) {
|
|
1399
|
+
for (const serverConfig of this._mcpServers) {
|
|
1400
|
+
const client = new MCPClient();
|
|
1401
|
+
client.addServer("mcp", serverConfig);
|
|
1402
|
+
try {
|
|
1403
|
+
const tools = await client.connect();
|
|
1404
|
+
sessionTools.registerMcpTools(tools);
|
|
1405
|
+
mcpClients.push(client);
|
|
1406
|
+
} catch (err) {
|
|
1407
|
+
console.error("[ClawOpsAgent] MCP connection error:", err);
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1309
1411
|
let recorder = null;
|
|
1310
|
-
if (this.
|
|
1311
|
-
recorder = new AudioRecorder(
|
|
1312
|
-
recorder.start(
|
|
1412
|
+
if (this._recording) {
|
|
1413
|
+
recorder = new AudioRecorder(this._recordingPath, session.callId);
|
|
1414
|
+
recorder.start();
|
|
1313
1415
|
}
|
|
1314
1416
|
const mediaWs = new MediaWebSocket();
|
|
1315
1417
|
session._bindTransport(
|
|
1316
1418
|
(audio) => {
|
|
1317
|
-
|
|
1318
|
-
mediaWs.sendAudio(ulaw.toString("base64"));
|
|
1319
|
-
if (recorder) {
|
|
1320
|
-
recorder.writeRawOutbound(audio);
|
|
1321
|
-
}
|
|
1419
|
+
mediaWs.sendAudio(audio.toString("base64"));
|
|
1322
1420
|
},
|
|
1323
1421
|
() => {
|
|
1324
1422
|
mediaWs.sendClear();
|
|
@@ -1327,15 +1425,19 @@ var ClawOpsAgent = class {
|
|
|
1327
1425
|
mediaWs.close();
|
|
1328
1426
|
}
|
|
1329
1427
|
);
|
|
1330
|
-
const sessionHandler = this.
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1428
|
+
const sessionHandler = this._session;
|
|
1429
|
+
if ("setToolRegistry" in sessionHandler && typeof sessionHandler.setToolRegistry === "function") {
|
|
1430
|
+
sessionHandler.setToolRegistry(sessionTools);
|
|
1431
|
+
}
|
|
1432
|
+
if (recorder && "setRecorder" in sessionHandler && typeof sessionHandler.setRecorder === "function") {
|
|
1433
|
+
sessionHandler.setRecorder(recorder);
|
|
1434
|
+
}
|
|
1435
|
+
mediaWs.onAudio((ulawAudio, _timestamp) => {
|
|
1336
1436
|
if (sessionHandler) {
|
|
1337
|
-
|
|
1338
|
-
|
|
1437
|
+
sessionHandler.feedAudio(ulawAudio);
|
|
1438
|
+
}
|
|
1439
|
+
if (recorder) {
|
|
1440
|
+
recorder.writeInbound(ulawToPcm16(ulawAudio));
|
|
1339
1441
|
}
|
|
1340
1442
|
});
|
|
1341
1443
|
mediaWs.onClose(() => {
|
|
@@ -1344,74 +1446,93 @@ var ClawOpsAgent = class {
|
|
|
1344
1446
|
}
|
|
1345
1447
|
session._markEnded();
|
|
1346
1448
|
});
|
|
1449
|
+
session._emit("call_start");
|
|
1347
1450
|
try {
|
|
1348
|
-
await mediaWs.connect(mediaWsUrl);
|
|
1349
|
-
|
|
1350
|
-
await sessionHandler.start(session, sessionTools);
|
|
1351
|
-
}
|
|
1451
|
+
await mediaWs.connect(mediaWsUrl, this._apiKey);
|
|
1452
|
+
await sessionHandler.start(session, sessionTools);
|
|
1352
1453
|
await session.wait();
|
|
1353
|
-
|
|
1354
|
-
await sessionHandler.stop();
|
|
1355
|
-
}
|
|
1454
|
+
await sessionHandler.stop();
|
|
1356
1455
|
} catch (err) {
|
|
1357
1456
|
console.error(`[ClawOpsAgent] Call session error:`, err);
|
|
1358
1457
|
} finally {
|
|
1458
|
+
if (mcpClients.length > 0) {
|
|
1459
|
+
sessionTools.clearMcpTools();
|
|
1460
|
+
for (const c of mcpClients) {
|
|
1461
|
+
await c.disconnect();
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1359
1464
|
mediaWs.close();
|
|
1360
1465
|
if (recorder) {
|
|
1361
1466
|
recorder.stop();
|
|
1362
1467
|
}
|
|
1468
|
+
session._emit("call_end");
|
|
1469
|
+
session._markEnded();
|
|
1470
|
+
this._activeSessions.delete(session.callId);
|
|
1363
1471
|
}
|
|
1364
1472
|
}
|
|
1365
1473
|
);
|
|
1366
1474
|
}
|
|
1367
|
-
_emitEvent(event, session) {
|
|
1368
|
-
const handlers = this._handlers.get(event);
|
|
1369
|
-
if (handlers) {
|
|
1370
|
-
for (const handler of handlers) {
|
|
1371
|
-
try {
|
|
1372
|
-
const result = handler(session);
|
|
1373
|
-
if (result && typeof result.catch === "function") {
|
|
1374
|
-
result.catch((err) => {
|
|
1375
|
-
console.error(`[ClawOpsAgent] Error in ${event} handler:`, err);
|
|
1376
|
-
});
|
|
1377
|
-
}
|
|
1378
|
-
} catch (err) {
|
|
1379
|
-
console.error(`[ClawOpsAgent] Error in ${event} handler:`, err);
|
|
1380
|
-
}
|
|
1381
|
-
}
|
|
1382
|
-
}
|
|
1383
|
-
}
|
|
1384
1475
|
};
|
|
1385
1476
|
|
|
1386
1477
|
// src/agent/pipeline/openai-realtime.ts
|
|
1478
|
+
var OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime?model=";
|
|
1479
|
+
var HANG_UP_TOOL = {
|
|
1480
|
+
type: "function",
|
|
1481
|
+
name: "hang_up",
|
|
1482
|
+
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1483
|
+
parameters: { type: "object", properties: {}, required: [] }
|
|
1484
|
+
};
|
|
1387
1485
|
var OpenAIRealtime = class {
|
|
1388
|
-
|
|
1486
|
+
_apiKey;
|
|
1487
|
+
_systemPrompt;
|
|
1488
|
+
_model;
|
|
1489
|
+
_voice;
|
|
1490
|
+
_language;
|
|
1491
|
+
_eagerness;
|
|
1492
|
+
_greeting;
|
|
1389
1493
|
_ws = null;
|
|
1390
|
-
|
|
1494
|
+
_call = null;
|
|
1391
1495
|
_tools = null;
|
|
1496
|
+
_recorder = null;
|
|
1392
1497
|
_closed = false;
|
|
1498
|
+
// Truncation / barge-in tracking (matching Python SDK)
|
|
1499
|
+
_lastAssistantItem = null;
|
|
1500
|
+
_responseStartTs = null;
|
|
1501
|
+
_sentAudioChunks = 0;
|
|
1502
|
+
_audioRemainder = Buffer.alloc(0);
|
|
1393
1503
|
constructor(options = {}) {
|
|
1394
|
-
this.
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1504
|
+
this._apiKey = options.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
|
|
1505
|
+
this._systemPrompt = options.systemPrompt ?? "";
|
|
1506
|
+
this._model = options.model ?? "gpt-realtime-1.5";
|
|
1507
|
+
this._voice = options.voice ?? "marin";
|
|
1508
|
+
this._language = options.language ?? "ko";
|
|
1509
|
+
this._eagerness = options.eagerness ?? "high";
|
|
1510
|
+
this._greeting = options.greeting ?? true;
|
|
1511
|
+
}
|
|
1512
|
+
/** Inject per-call ToolRegistry. */
|
|
1513
|
+
setToolRegistry(registry) {
|
|
1514
|
+
this._tools = registry;
|
|
1515
|
+
}
|
|
1516
|
+
/** Inject per-call AudioRecorder. */
|
|
1517
|
+
setRecorder(recorder) {
|
|
1518
|
+
this._recorder = recorder;
|
|
1401
1519
|
}
|
|
1402
1520
|
async start(callSession, tools) {
|
|
1403
|
-
this.
|
|
1404
|
-
this._tools = tools
|
|
1521
|
+
this._call = callSession;
|
|
1522
|
+
if (tools) this._tools = tools;
|
|
1405
1523
|
this._closed = false;
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1524
|
+
this._lastAssistantItem = null;
|
|
1525
|
+
this._responseStartTs = null;
|
|
1526
|
+
this._sentAudioChunks = 0;
|
|
1527
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1528
|
+
if (!this._apiKey) {
|
|
1529
|
+
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey option.");
|
|
1409
1530
|
}
|
|
1410
1531
|
const { WebSocket } = await import('ws');
|
|
1411
|
-
const url =
|
|
1532
|
+
const url = `${OPENAI_REALTIME_URL}${this._model}`;
|
|
1412
1533
|
this._ws = new WebSocket(url, {
|
|
1413
1534
|
headers: {
|
|
1414
|
-
Authorization: `Bearer ${
|
|
1535
|
+
Authorization: `Bearer ${this._apiKey}`,
|
|
1415
1536
|
"OpenAI-Beta": "realtime=v1"
|
|
1416
1537
|
}
|
|
1417
1538
|
});
|
|
@@ -1419,6 +1540,9 @@ var OpenAIRealtime = class {
|
|
|
1419
1540
|
const ws = this._ws;
|
|
1420
1541
|
ws.on("open", () => {
|
|
1421
1542
|
this._sendSessionUpdate();
|
|
1543
|
+
if (this._greeting) {
|
|
1544
|
+
this._send({ type: "response.create" });
|
|
1545
|
+
}
|
|
1422
1546
|
resolve();
|
|
1423
1547
|
});
|
|
1424
1548
|
ws.on("message", (data) => {
|
|
@@ -1441,12 +1565,10 @@ var OpenAIRealtime = class {
|
|
|
1441
1565
|
}
|
|
1442
1566
|
feedAudio(audio) {
|
|
1443
1567
|
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1444
|
-
this.
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
})
|
|
1449
|
-
);
|
|
1568
|
+
this._send({
|
|
1569
|
+
type: "input_audio_buffer.append",
|
|
1570
|
+
audio: audio.toString("base64")
|
|
1571
|
+
});
|
|
1450
1572
|
}
|
|
1451
1573
|
}
|
|
1452
1574
|
async stop() {
|
|
@@ -1458,52 +1580,71 @@ var OpenAIRealtime = class {
|
|
|
1458
1580
|
}
|
|
1459
1581
|
_sendSessionUpdate() {
|
|
1460
1582
|
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1461
|
-
const
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
);
|
|
1583
|
+
const toolSchemas = this._tools ? this._tools.toOpenAITools().map((t) => ({ type: "function", ...t.function })) : [];
|
|
1584
|
+
toolSchemas.push(HANG_UP_TOOL);
|
|
1585
|
+
this._send({
|
|
1586
|
+
type: "session.update",
|
|
1587
|
+
session: {
|
|
1588
|
+
modalities: ["text", "audio"],
|
|
1589
|
+
voice: this._voice,
|
|
1590
|
+
instructions: this._systemPrompt,
|
|
1591
|
+
input_audio_format: "g711_ulaw",
|
|
1592
|
+
output_audio_format: "g711_ulaw",
|
|
1593
|
+
input_audio_transcription: {
|
|
1594
|
+
model: "gpt-4o-mini-transcribe",
|
|
1595
|
+
language: this._language
|
|
1596
|
+
},
|
|
1597
|
+
input_audio_noise_reduction: { type: "far_field" },
|
|
1598
|
+
turn_detection: {
|
|
1599
|
+
type: "semantic_vad",
|
|
1600
|
+
interrupt_response: true,
|
|
1601
|
+
eagerness: this._eagerness
|
|
1602
|
+
},
|
|
1603
|
+
tools: toolSchemas
|
|
1604
|
+
}
|
|
1605
|
+
});
|
|
1485
1606
|
}
|
|
1486
1607
|
_handleMessage(msg) {
|
|
1487
1608
|
const type = msg["type"];
|
|
1488
1609
|
switch (type) {
|
|
1489
1610
|
case "response.audio.delta": {
|
|
1490
|
-
|
|
1491
|
-
if (delta && this._callSession) {
|
|
1492
|
-
const audio = Buffer.from(delta, "base64");
|
|
1493
|
-
this._callSession.sendAudio(audio);
|
|
1494
|
-
}
|
|
1611
|
+
this._handleAudioDelta(msg);
|
|
1495
1612
|
break;
|
|
1496
1613
|
}
|
|
1497
1614
|
case "response.audio.done": {
|
|
1615
|
+
if (this._audioRemainder.length > 0) {
|
|
1616
|
+
const padded = Buffer.concat([
|
|
1617
|
+
this._audioRemainder,
|
|
1618
|
+
Buffer.alloc(160 - this._audioRemainder.length, 255)
|
|
1619
|
+
]);
|
|
1620
|
+
if (this._call) {
|
|
1621
|
+
this._call.sendAudio(padded);
|
|
1622
|
+
}
|
|
1623
|
+
this._sentAudioChunks++;
|
|
1624
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1625
|
+
}
|
|
1498
1626
|
break;
|
|
1499
1627
|
}
|
|
1500
|
-
case "
|
|
1501
|
-
this.
|
|
1628
|
+
case "input_audio_buffer.speech_started": {
|
|
1629
|
+
this._handleTruncation();
|
|
1502
1630
|
break;
|
|
1503
1631
|
}
|
|
1504
|
-
case "
|
|
1505
|
-
|
|
1506
|
-
|
|
1632
|
+
case "response.output_item.done": {
|
|
1633
|
+
const item = msg["item"];
|
|
1634
|
+
if (item && item["type"] === "function_call") {
|
|
1635
|
+
this._handleToolCall(item);
|
|
1636
|
+
}
|
|
1637
|
+
break;
|
|
1638
|
+
}
|
|
1639
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
1640
|
+
if (this._call) {
|
|
1641
|
+
this._call._emit("transcript", "user", msg["transcript"] ?? "");
|
|
1642
|
+
}
|
|
1643
|
+
break;
|
|
1644
|
+
}
|
|
1645
|
+
case "response.audio_transcript.done": {
|
|
1646
|
+
if (this._call) {
|
|
1647
|
+
this._call._emit("transcript", "assistant", msg["transcript"] ?? "");
|
|
1507
1648
|
}
|
|
1508
1649
|
break;
|
|
1509
1650
|
}
|
|
@@ -1513,73 +1654,147 @@ var OpenAIRealtime = class {
|
|
|
1513
1654
|
}
|
|
1514
1655
|
}
|
|
1515
1656
|
}
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1657
|
+
_handleAudioDelta(msg) {
|
|
1658
|
+
if (this._responseStartTs === null) {
|
|
1659
|
+
this._responseStartTs = Date.now();
|
|
1660
|
+
this._sentAudioChunks = 0;
|
|
1661
|
+
}
|
|
1662
|
+
if (msg["item_id"]) {
|
|
1663
|
+
this._lastAssistantItem = msg["item_id"];
|
|
1664
|
+
}
|
|
1665
|
+
const ulaw = Buffer.from(msg["delta"], "base64");
|
|
1666
|
+
if (this._recorder) {
|
|
1667
|
+
this._recorder.writeOutbound(ulawToPcm16(ulaw));
|
|
1668
|
+
}
|
|
1669
|
+
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
1670
|
+
const chunkSize = 160;
|
|
1671
|
+
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
1672
|
+
for (let off = 0; off < fullEnd; off += chunkSize) {
|
|
1673
|
+
if (this._call) {
|
|
1674
|
+
this._call.sendAudio(combined.subarray(off, off + chunkSize));
|
|
1675
|
+
}
|
|
1676
|
+
this._sentAudioChunks++;
|
|
1677
|
+
}
|
|
1678
|
+
this._audioRemainder = combined.subarray(fullEnd);
|
|
1679
|
+
}
|
|
1680
|
+
_handleTruncation() {
|
|
1681
|
+
if (!this._lastAssistantItem || this._responseStartTs === null) {
|
|
1522
1682
|
return;
|
|
1523
1683
|
}
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1684
|
+
const audioEndMs = Math.max(0, this._sentAudioChunks * 20);
|
|
1685
|
+
this._send({
|
|
1686
|
+
type: "conversation.item.truncate",
|
|
1687
|
+
item_id: this._lastAssistantItem,
|
|
1688
|
+
content_index: 0,
|
|
1689
|
+
audio_end_ms: audioEndMs
|
|
1690
|
+
});
|
|
1691
|
+
if (this._call) {
|
|
1692
|
+
this._call.clearAudio();
|
|
1693
|
+
}
|
|
1694
|
+
this._lastAssistantItem = null;
|
|
1695
|
+
this._responseStartTs = null;
|
|
1696
|
+
this._sentAudioChunks = 0;
|
|
1697
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1698
|
+
}
|
|
1699
|
+
async _handleToolCall(item) {
|
|
1700
|
+
const funcName = item["name"];
|
|
1701
|
+
const callId = item["call_id"];
|
|
1702
|
+
if (funcName === "hang_up") {
|
|
1703
|
+
if (this._call) {
|
|
1704
|
+
this._call.hangup();
|
|
1539
1705
|
}
|
|
1706
|
+
return;
|
|
1707
|
+
}
|
|
1708
|
+
if (!this._tools || !this._tools.has(funcName)) {
|
|
1709
|
+
console.error(`[OpenAIRealtime] Unknown tool: ${funcName}`);
|
|
1710
|
+
return;
|
|
1711
|
+
}
|
|
1712
|
+
let result;
|
|
1713
|
+
try {
|
|
1714
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
1715
|
+
result = await this._tools.call(funcName, args);
|
|
1540
1716
|
} catch (err) {
|
|
1541
|
-
console.error(`[OpenAIRealtime] Tool call
|
|
1717
|
+
console.error(`[OpenAIRealtime] Tool call failed: ${funcName}:`, err);
|
|
1718
|
+
result = `Error: ${err}`;
|
|
1719
|
+
}
|
|
1720
|
+
this._send({
|
|
1721
|
+
type: "conversation.item.create",
|
|
1722
|
+
item: {
|
|
1723
|
+
type: "function_call_output",
|
|
1724
|
+
call_id: callId,
|
|
1725
|
+
output: typeof result === "string" ? result : JSON.stringify(result)
|
|
1726
|
+
}
|
|
1727
|
+
});
|
|
1728
|
+
this._send({ type: "response.create" });
|
|
1729
|
+
}
|
|
1730
|
+
_send(data) {
|
|
1731
|
+
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1732
|
+
this._ws.send(JSON.stringify(data));
|
|
1542
1733
|
}
|
|
1543
1734
|
}
|
|
1544
1735
|
};
|
|
1545
1736
|
|
|
1546
1737
|
// src/agent/pipeline/gemini-realtime.ts
|
|
1738
|
+
var HANG_UP_TOOL2 = {
|
|
1739
|
+
name: "hang_up",
|
|
1740
|
+
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1741
|
+
parameters: { type: "object", properties: {} }
|
|
1742
|
+
};
|
|
1547
1743
|
var GeminiRealtime = class {
|
|
1548
|
-
|
|
1744
|
+
_apiKey;
|
|
1745
|
+
_systemPrompt;
|
|
1746
|
+
_model;
|
|
1747
|
+
_voice;
|
|
1748
|
+
_language;
|
|
1749
|
+
_greeting;
|
|
1750
|
+
_generationConfig;
|
|
1549
1751
|
_ws = null;
|
|
1550
|
-
|
|
1752
|
+
_call = null;
|
|
1551
1753
|
_tools = null;
|
|
1754
|
+
_recorder = null;
|
|
1552
1755
|
_closed = false;
|
|
1756
|
+
_sentAudioChunks = 0;
|
|
1757
|
+
_audioRemainder = Buffer.alloc(0);
|
|
1553
1758
|
constructor(options = {}) {
|
|
1554
|
-
this.
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1759
|
+
this._apiKey = options.apiKey ?? process.env["GOOGLE_API_KEY"] ?? "";
|
|
1760
|
+
this._systemPrompt = options.systemPrompt ?? "";
|
|
1761
|
+
this._model = options.model ?? "gemini-2.5-flash-native-audio-preview-12-2025";
|
|
1762
|
+
this._voice = options.voice ?? "Kore";
|
|
1763
|
+
this._language = options.language ?? "ko";
|
|
1764
|
+
this._greeting = options.greeting ?? true;
|
|
1765
|
+
this._generationConfig = options.generationConfig;
|
|
1766
|
+
}
|
|
1767
|
+
/** Inject per-call ToolRegistry. */
|
|
1768
|
+
setToolRegistry(registry) {
|
|
1769
|
+
this._tools = registry;
|
|
1770
|
+
}
|
|
1771
|
+
/** Inject per-call AudioRecorder. */
|
|
1772
|
+
setRecorder(recorder) {
|
|
1773
|
+
this._recorder = recorder;
|
|
1558
1774
|
}
|
|
1559
1775
|
async start(callSession, tools) {
|
|
1560
|
-
this.
|
|
1561
|
-
this._tools = tools
|
|
1776
|
+
this._call = callSession;
|
|
1777
|
+
if (tools) this._tools = tools;
|
|
1562
1778
|
this._closed = false;
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1779
|
+
this._sentAudioChunks = 0;
|
|
1780
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1781
|
+
if (!this._apiKey) {
|
|
1782
|
+
throw new Error("Google API key is required. Set GOOGLE_API_KEY or pass apiKey option.");
|
|
1566
1783
|
}
|
|
1567
1784
|
const { WebSocket } = await import('ws');
|
|
1568
|
-
this.
|
|
1569
|
-
const url = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=${apiKey}`;
|
|
1785
|
+
const url = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=${this._apiKey}`;
|
|
1570
1786
|
this._ws = new WebSocket(url);
|
|
1571
1787
|
return new Promise((resolve, reject) => {
|
|
1572
1788
|
const ws = this._ws;
|
|
1573
1789
|
ws.on("open", () => {
|
|
1574
1790
|
this._sendSetup();
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
}
|
|
1582
|
-
}
|
|
1791
|
+
this._waitSetupComplete().then(() => {
|
|
1792
|
+
if (this._greeting) {
|
|
1793
|
+
this._sendGreeting();
|
|
1794
|
+
}
|
|
1795
|
+
this._receiveLoop();
|
|
1796
|
+
resolve();
|
|
1797
|
+
}).catch(reject);
|
|
1583
1798
|
});
|
|
1584
1799
|
ws.on("close", () => {
|
|
1585
1800
|
this._closed = true;
|
|
@@ -1594,13 +1809,15 @@ var GeminiRealtime = class {
|
|
|
1594
1809
|
}
|
|
1595
1810
|
feedAudio(audio) {
|
|
1596
1811
|
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1812
|
+
const pcm8k = ulawToPcm16(audio);
|
|
1813
|
+
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
1597
1814
|
this._ws.send(
|
|
1598
1815
|
JSON.stringify({
|
|
1599
1816
|
realtimeInput: {
|
|
1600
1817
|
mediaChunks: [
|
|
1601
1818
|
{
|
|
1602
1819
|
mimeType: "audio/pcm;rate=16000",
|
|
1603
|
-
data:
|
|
1820
|
+
data: pcm16k.toString("base64")
|
|
1604
1821
|
}
|
|
1605
1822
|
]
|
|
1606
1823
|
}
|
|
@@ -1618,35 +1835,85 @@ var GeminiRealtime = class {
|
|
|
1618
1835
|
_sendSetup() {
|
|
1619
1836
|
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1620
1837
|
const setupConfig = {
|
|
1621
|
-
model: `models/${this.
|
|
1838
|
+
model: `models/${this._model}`,
|
|
1622
1839
|
generationConfig: {
|
|
1623
1840
|
responseModalities: ["AUDIO"],
|
|
1624
1841
|
speechConfig: {
|
|
1625
1842
|
voiceConfig: {
|
|
1626
1843
|
prebuiltVoiceConfig: {
|
|
1627
|
-
voiceName: this.
|
|
1844
|
+
voiceName: this._voice
|
|
1628
1845
|
}
|
|
1629
1846
|
}
|
|
1630
1847
|
},
|
|
1631
|
-
...this.
|
|
1848
|
+
...this._generationConfig
|
|
1849
|
+
},
|
|
1850
|
+
realtimeInputConfig: {
|
|
1851
|
+
automaticActivityDetection: {
|
|
1852
|
+
disabled: false
|
|
1853
|
+
}
|
|
1632
1854
|
}
|
|
1633
1855
|
};
|
|
1634
|
-
if (this.
|
|
1856
|
+
if (this._systemPrompt) {
|
|
1635
1857
|
setupConfig["systemInstruction"] = {
|
|
1636
|
-
parts: [{ text: this.
|
|
1858
|
+
parts: [{ text: this._systemPrompt }]
|
|
1637
1859
|
};
|
|
1638
1860
|
}
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
}
|
|
1861
|
+
const toolDefs = this._tools ? this._tools.toOpenAITools().map((t) => ({
|
|
1862
|
+
name: t.function.name,
|
|
1863
|
+
description: t.function.description,
|
|
1864
|
+
parameters: t.function.parameters
|
|
1865
|
+
})) : [];
|
|
1866
|
+
toolDefs.push(HANG_UP_TOOL2);
|
|
1867
|
+
setupConfig["tools"] = [{ functionDeclarations: toolDefs }];
|
|
1647
1868
|
this._ws.send(JSON.stringify({ setup: setupConfig }));
|
|
1648
1869
|
}
|
|
1870
|
+
_waitSetupComplete() {
|
|
1871
|
+
return new Promise((resolve, reject) => {
|
|
1872
|
+
if (!this._ws) {
|
|
1873
|
+
reject(new Error("WebSocket not connected"));
|
|
1874
|
+
return;
|
|
1875
|
+
}
|
|
1876
|
+
const onMessage = (data) => {
|
|
1877
|
+
try {
|
|
1878
|
+
const msg = JSON.parse(data.toString());
|
|
1879
|
+
if ("setupComplete" in msg) {
|
|
1880
|
+
this._ws?.removeListener("message", onMessage);
|
|
1881
|
+
resolve();
|
|
1882
|
+
}
|
|
1883
|
+
} catch {
|
|
1884
|
+
}
|
|
1885
|
+
};
|
|
1886
|
+
this._ws.on("message", onMessage);
|
|
1887
|
+
});
|
|
1888
|
+
}
|
|
1889
|
+
_sendGreeting() {
|
|
1890
|
+
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1891
|
+
this._ws.send(
|
|
1892
|
+
JSON.stringify({
|
|
1893
|
+
clientContent: {
|
|
1894
|
+
turns: [
|
|
1895
|
+
{
|
|
1896
|
+
role: "user",
|
|
1897
|
+
parts: [{ text: "\uC778\uC0AC\uD574 \uC8FC\uC138\uC694." }]
|
|
1898
|
+
}
|
|
1899
|
+
],
|
|
1900
|
+
turnComplete: true
|
|
1901
|
+
}
|
|
1902
|
+
})
|
|
1903
|
+
);
|
|
1904
|
+
}
|
|
1905
|
+
_receiveLoop() {
|
|
1906
|
+
if (!this._ws) return;
|
|
1907
|
+
this._ws.on("message", (data) => {
|
|
1908
|
+
try {
|
|
1909
|
+
const msg = JSON.parse(data.toString());
|
|
1910
|
+
this._handleMessage(msg);
|
|
1911
|
+
} catch {
|
|
1912
|
+
}
|
|
1913
|
+
});
|
|
1914
|
+
}
|
|
1649
1915
|
_handleMessage(msg) {
|
|
1916
|
+
if (!this._call) return;
|
|
1650
1917
|
const serverContent = msg["serverContent"];
|
|
1651
1918
|
if (serverContent) {
|
|
1652
1919
|
const modelTurn = serverContent["modelTurn"];
|
|
@@ -1655,35 +1922,108 @@ var GeminiRealtime = class {
|
|
|
1655
1922
|
if (parts) {
|
|
1656
1923
|
for (const part of parts) {
|
|
1657
1924
|
const inlineData = part["inlineData"];
|
|
1658
|
-
if (inlineData && inlineData["data"]
|
|
1659
|
-
const
|
|
1660
|
-
|
|
1925
|
+
if (inlineData && inlineData["data"]) {
|
|
1926
|
+
const mimeType = inlineData["mimeType"] ?? "";
|
|
1927
|
+
if (mimeType.includes("audio")) {
|
|
1928
|
+
this._handleAudioData(inlineData["data"]);
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
const text = part["text"];
|
|
1932
|
+
if (text && this._call) {
|
|
1933
|
+
this._call._emit("transcript", "assistant", text);
|
|
1661
1934
|
}
|
|
1662
1935
|
}
|
|
1663
1936
|
}
|
|
1664
1937
|
}
|
|
1938
|
+
if (serverContent["turnComplete"]) {
|
|
1939
|
+
this._flushAudioRemainder();
|
|
1940
|
+
}
|
|
1941
|
+
if (serverContent["interrupted"]) {
|
|
1942
|
+
if (this._call) {
|
|
1943
|
+
this._call.clearAudio();
|
|
1944
|
+
}
|
|
1945
|
+
this._sentAudioChunks = 0;
|
|
1946
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1947
|
+
}
|
|
1948
|
+
}
|
|
1949
|
+
const inputTranscription = msg["inputTranscription"];
|
|
1950
|
+
if (inputTranscription) {
|
|
1951
|
+
const text = inputTranscription["text"];
|
|
1952
|
+
if (text && this._call) {
|
|
1953
|
+
this._call._emit("transcript", "user", text);
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
const outputTranscription = msg["outputTranscription"];
|
|
1957
|
+
if (outputTranscription) {
|
|
1958
|
+
const text = outputTranscription["text"];
|
|
1959
|
+
if (text && this._call) {
|
|
1960
|
+
this._call._emit("transcript", "assistant", text);
|
|
1961
|
+
}
|
|
1665
1962
|
}
|
|
1666
1963
|
const toolCall = msg["toolCall"];
|
|
1667
1964
|
if (toolCall) {
|
|
1668
1965
|
this._handleToolCall(toolCall);
|
|
1669
1966
|
}
|
|
1967
|
+
if (msg["toolCallCancellation"]) ;
|
|
1968
|
+
}
|
|
1969
|
+
_handleAudioData(b64Data) {
|
|
1970
|
+
if (!this._call) return;
|
|
1971
|
+
const pcm24k = Buffer.from(b64Data, "base64");
|
|
1972
|
+
if (this._recorder) {
|
|
1973
|
+
this._recorder.writeOutbound(resamplePcm16(pcm24k, 24e3, 8e3));
|
|
1974
|
+
}
|
|
1975
|
+
const pcm8k = resamplePcm16(pcm24k, 24e3, 8e3);
|
|
1976
|
+
const ulaw = pcm16ToUlaw(pcm8k);
|
|
1977
|
+
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
1978
|
+
const chunkSize = 160;
|
|
1979
|
+
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
1980
|
+
for (let off = 0; off < fullEnd; off += chunkSize) {
|
|
1981
|
+
this._call.sendAudio(combined.subarray(off, off + chunkSize));
|
|
1982
|
+
this._sentAudioChunks++;
|
|
1983
|
+
}
|
|
1984
|
+
this._audioRemainder = combined.subarray(fullEnd);
|
|
1985
|
+
}
|
|
1986
|
+
_flushAudioRemainder() {
|
|
1987
|
+
if (this._audioRemainder.length > 0 && this._call) {
|
|
1988
|
+
const padded = Buffer.concat([
|
|
1989
|
+
this._audioRemainder,
|
|
1990
|
+
Buffer.alloc(160 - this._audioRemainder.length, 255)
|
|
1991
|
+
]);
|
|
1992
|
+
this._call.sendAudio(padded);
|
|
1993
|
+
this._sentAudioChunks++;
|
|
1994
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
1995
|
+
}
|
|
1670
1996
|
}
|
|
1671
1997
|
async _handleToolCall(toolCall) {
|
|
1672
1998
|
const functionCalls = toolCall["functionCalls"];
|
|
1673
|
-
if (!functionCalls
|
|
1999
|
+
if (!functionCalls) return;
|
|
1674
2000
|
const responses = [];
|
|
1675
2001
|
for (const fc of functionCalls) {
|
|
1676
2002
|
const name = fc["name"];
|
|
2003
|
+
const fcId = fc["id"] ?? "";
|
|
1677
2004
|
const args = fc["args"] ?? {};
|
|
2005
|
+
if (name === "hang_up") {
|
|
2006
|
+
if (this._call) {
|
|
2007
|
+
this._call.hangup();
|
|
2008
|
+
}
|
|
2009
|
+
return;
|
|
2010
|
+
}
|
|
2011
|
+
if (!this._tools || !this._tools.has(name)) {
|
|
2012
|
+
console.error(`[GeminiRealtime] Unknown tool: ${name}`);
|
|
2013
|
+
responses.push({ id: fcId, name, response: { error: `Unknown tool: ${name}` } });
|
|
2014
|
+
continue;
|
|
2015
|
+
}
|
|
1678
2016
|
try {
|
|
1679
2017
|
const result = await this._tools.call(name, args);
|
|
1680
2018
|
responses.push({
|
|
2019
|
+
id: fcId,
|
|
1681
2020
|
name,
|
|
1682
2021
|
response: { result: typeof result === "string" ? result : JSON.stringify(result) }
|
|
1683
2022
|
});
|
|
1684
2023
|
} catch (err) {
|
|
1685
2024
|
console.error(`[GeminiRealtime] Tool call error for ${name}:`, err);
|
|
1686
2025
|
responses.push({
|
|
2026
|
+
id: fcId,
|
|
1687
2027
|
name,
|
|
1688
2028
|
response: { error: String(err) }
|
|
1689
2029
|
});
|
|
@@ -1707,12 +2047,15 @@ var PipelineSession = class {
|
|
|
1707
2047
|
_llm;
|
|
1708
2048
|
_tts;
|
|
1709
2049
|
_systemPrompt;
|
|
2050
|
+
_greeting;
|
|
2051
|
+
_language;
|
|
1710
2052
|
_temperature;
|
|
1711
2053
|
_maxTokens;
|
|
1712
2054
|
_sampleRate;
|
|
1713
2055
|
_interruptOnSpeech;
|
|
1714
2056
|
_callSession = null;
|
|
1715
2057
|
_tools = null;
|
|
2058
|
+
_recorder = null;
|
|
1716
2059
|
_conversation = [];
|
|
1717
2060
|
_audioBuffer = [];
|
|
1718
2061
|
_running = false;
|
|
@@ -1722,10 +2065,20 @@ var PipelineSession = class {
|
|
|
1722
2065
|
this._llm = options.llm;
|
|
1723
2066
|
this._tts = options.tts;
|
|
1724
2067
|
this._systemPrompt = options.systemPrompt;
|
|
2068
|
+
this._greeting = options.greeting ?? true;
|
|
2069
|
+
this._language = options.language ?? "ko";
|
|
1725
2070
|
this._temperature = options.temperature;
|
|
1726
2071
|
this._maxTokens = options.maxTokens;
|
|
1727
2072
|
this._sampleRate = options.sampleRate ?? 8e3;
|
|
1728
2073
|
this._interruptOnSpeech = options.interruptOnSpeech ?? true;
|
|
2074
|
+
if (options.toolRegistry) this._tools = options.toolRegistry;
|
|
2075
|
+
if (options.recorder) this._recorder = options.recorder;
|
|
2076
|
+
}
|
|
2077
|
+
setToolRegistry(registry) {
|
|
2078
|
+
this._tools = registry;
|
|
2079
|
+
}
|
|
2080
|
+
setRecorder(recorder) {
|
|
2081
|
+
this._recorder = recorder;
|
|
1729
2082
|
}
|
|
1730
2083
|
async start(callSession, tools) {
|
|
1731
2084
|
this._callSession = callSession;
|
|
@@ -1738,6 +2091,11 @@ var PipelineSession = class {
|
|
|
1738
2091
|
content: this._systemPrompt
|
|
1739
2092
|
});
|
|
1740
2093
|
}
|
|
2094
|
+
if (this._greeting) {
|
|
2095
|
+
this._generateGreeting().catch((err) => {
|
|
2096
|
+
console.error("[PipelineSession] Greeting error:", err);
|
|
2097
|
+
});
|
|
2098
|
+
}
|
|
1741
2099
|
this._runSttLoop().catch((err) => {
|
|
1742
2100
|
console.error("[PipelineSession] STT loop error:", err);
|
|
1743
2101
|
});
|
|
@@ -1771,14 +2129,24 @@ var PipelineSession = class {
|
|
|
1771
2129
|
async *_createAudioStream() {
|
|
1772
2130
|
while (this._running) {
|
|
1773
2131
|
if (this._audioBuffer.length > 0) {
|
|
1774
|
-
|
|
2132
|
+
const ulaw = this._audioBuffer.shift();
|
|
2133
|
+
const pcm8k = ulawToPcm16(ulaw);
|
|
2134
|
+
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
2135
|
+
yield pcm16k;
|
|
1775
2136
|
} else {
|
|
1776
2137
|
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1777
2138
|
}
|
|
1778
2139
|
}
|
|
1779
2140
|
}
|
|
2141
|
+
async _generateGreeting() {
|
|
2142
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
2143
|
+
await this._respond();
|
|
2144
|
+
}
|
|
1780
2145
|
async _handleUserSpeech(transcript) {
|
|
1781
2146
|
this._conversation.push({ role: "user", content: transcript });
|
|
2147
|
+
await this._respond();
|
|
2148
|
+
}
|
|
2149
|
+
async _respond() {
|
|
1782
2150
|
let fullResponse = "";
|
|
1783
2151
|
const textChunks = [];
|
|
1784
2152
|
const llmStream = this._llm.generate(this._conversation, {
|
|
@@ -1845,7 +2213,19 @@ var PipelineSession = class {
|
|
|
1845
2213
|
sampleRate: this._sampleRate
|
|
1846
2214
|
})) {
|
|
1847
2215
|
if (!this._running || !this._speaking) break;
|
|
1848
|
-
this.
|
|
2216
|
+
if (this._recorder) {
|
|
2217
|
+
const pcm8k2 = this._sampleRate !== 8e3 ? resamplePcm16(audioChunk, this._sampleRate, 8e3) : audioChunk;
|
|
2218
|
+
this._recorder.writeOutbound(pcm8k2);
|
|
2219
|
+
}
|
|
2220
|
+
const pcm8k = resamplePcm16(audioChunk, this._sampleRate, 8e3);
|
|
2221
|
+
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2222
|
+
for (let off = 0; off < ulaw.length; off += 160) {
|
|
2223
|
+
let chunk = ulaw.subarray(off, off + 160);
|
|
2224
|
+
if (chunk.length < 160) {
|
|
2225
|
+
chunk = Buffer.concat([chunk, Buffer.alloc(160 - chunk.length, 255)]);
|
|
2226
|
+
}
|
|
2227
|
+
this._callSession.sendAudio(chunk);
|
|
2228
|
+
}
|
|
1849
2229
|
}
|
|
1850
2230
|
} catch (err) {
|
|
1851
2231
|
console.error("[PipelineSession] TTS error:", err);
|
|
@@ -1860,14 +2240,14 @@ var DeepgramSTT = class {
|
|
|
1860
2240
|
_options;
|
|
1861
2241
|
constructor(options = {}) {
|
|
1862
2242
|
this._options = {
|
|
1863
|
-
model: "nova-
|
|
2243
|
+
model: "nova-3",
|
|
1864
2244
|
language: "ko",
|
|
1865
|
-
|
|
1866
|
-
punctuate: true,
|
|
1867
|
-
smartFormat: true,
|
|
2245
|
+
sampleRate: 16e3,
|
|
1868
2246
|
encoding: "linear16",
|
|
1869
|
-
|
|
1870
|
-
|
|
2247
|
+
punctuate: true,
|
|
2248
|
+
interimResults: true,
|
|
2249
|
+
endpointing: 300,
|
|
2250
|
+
utteranceEndMs: 1e3,
|
|
1871
2251
|
...options
|
|
1872
2252
|
};
|
|
1873
2253
|
}
|
|
@@ -1884,10 +2264,10 @@ var DeepgramSTT = class {
|
|
|
1884
2264
|
language,
|
|
1885
2265
|
punctuate: String(this._options.punctuate),
|
|
1886
2266
|
interim_results: String(this._options.interimResults),
|
|
1887
|
-
smart_format: String(this._options.smartFormat),
|
|
1888
2267
|
encoding: this._options.encoding,
|
|
1889
2268
|
sample_rate: String(sampleRate),
|
|
1890
|
-
|
|
2269
|
+
endpointing: String(this._options.endpointing),
|
|
2270
|
+
utterance_end_ms: String(this._options.utteranceEndMs)
|
|
1891
2271
|
});
|
|
1892
2272
|
const url = `wss://api.deepgram.com/v1/listen?${params.toString()}`;
|
|
1893
2273
|
const ws = new WebSocket(url, {
|
|
@@ -1976,13 +2356,12 @@ var ElevenLabsTTS = class {
|
|
|
1976
2356
|
_options;
|
|
1977
2357
|
constructor(options = {}) {
|
|
1978
2358
|
this._options = {
|
|
1979
|
-
voiceId: "
|
|
1980
|
-
|
|
1981
|
-
outputFormat: "
|
|
2359
|
+
voiceId: "EXAVITQu4vr4xnSDxMaL",
|
|
2360
|
+
model: "eleven_flash_v2_5",
|
|
2361
|
+
outputFormat: "pcm_24000",
|
|
1982
2362
|
stability: 0.5,
|
|
1983
2363
|
similarityBoost: 0.75,
|
|
1984
|
-
|
|
1985
|
-
useSpeakerBoost: true,
|
|
2364
|
+
languageCode: "ko",
|
|
1986
2365
|
...options
|
|
1987
2366
|
};
|
|
1988
2367
|
}
|
|
@@ -2008,12 +2387,10 @@ var ElevenLabsTTS = class {
|
|
|
2008
2387
|
},
|
|
2009
2388
|
body: JSON.stringify({
|
|
2010
2389
|
text,
|
|
2011
|
-
model_id: this._options.
|
|
2390
|
+
model_id: this._options.model,
|
|
2012
2391
|
voice_settings: {
|
|
2013
2392
|
stability: this._options.stability,
|
|
2014
|
-
similarity_boost: this._options.similarityBoost
|
|
2015
|
-
style: this._options.style,
|
|
2016
|
-
use_speaker_boost: this._options.useSpeakerBoost
|
|
2393
|
+
similarity_boost: this._options.similarityBoost
|
|
2017
2394
|
}
|
|
2018
2395
|
})
|
|
2019
2396
|
});
|
|
@@ -2037,7 +2414,7 @@ var ElevenLabsTTS = class {
|
|
|
2037
2414
|
}
|
|
2038
2415
|
async *_synthesizeStreaming(apiKey, voiceId, textStream) {
|
|
2039
2416
|
const { WebSocket } = await import('ws');
|
|
2040
|
-
const url = `wss://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream-input?model_id=${this._options.
|
|
2417
|
+
const url = `wss://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream-input?model_id=${this._options.model}&output_format=${this._options.outputFormat}`;
|
|
2041
2418
|
const ws = new WebSocket(url);
|
|
2042
2419
|
const audioQueue = [];
|
|
2043
2420
|
let resolveWait = null;
|
|
@@ -2048,9 +2425,7 @@ var ElevenLabsTTS = class {
|
|
|
2048
2425
|
text: " ",
|
|
2049
2426
|
voice_settings: {
|
|
2050
2427
|
stability: this._options.stability,
|
|
2051
|
-
similarity_boost: this._options.similarityBoost
|
|
2052
|
-
style: this._options.style,
|
|
2053
|
-
use_speaker_boost: this._options.useSpeakerBoost
|
|
2428
|
+
similarity_boost: this._options.similarityBoost
|
|
2054
2429
|
},
|
|
2055
2430
|
xi_api_key: apiKey
|
|
2056
2431
|
})
|
|
@@ -2127,7 +2502,9 @@ var OpenAILLM = class {
|
|
|
2127
2502
|
_options;
|
|
2128
2503
|
constructor(options = {}) {
|
|
2129
2504
|
this._options = {
|
|
2130
|
-
model: "gpt-4o",
|
|
2505
|
+
model: "gpt-4o-mini",
|
|
2506
|
+
temperature: 0.8,
|
|
2507
|
+
maxTokens: 4096,
|
|
2131
2508
|
...options
|
|
2132
2509
|
};
|
|
2133
2510
|
}
|
|
@@ -2211,8 +2588,9 @@ var AnthropicLLM = class {
|
|
|
2211
2588
|
_options;
|
|
2212
2589
|
constructor(options = {}) {
|
|
2213
2590
|
this._options = {
|
|
2214
|
-
model: "claude-sonnet-4-
|
|
2215
|
-
|
|
2591
|
+
model: "claude-sonnet-4-6",
|
|
2592
|
+
temperature: 0.8,
|
|
2593
|
+
maxTokens: 4096,
|
|
2216
2594
|
...options
|
|
2217
2595
|
};
|
|
2218
2596
|
}
|
|
@@ -2315,7 +2693,9 @@ var GeminiLLM = class {
|
|
|
2315
2693
|
_options;
|
|
2316
2694
|
constructor(options = {}) {
|
|
2317
2695
|
this._options = {
|
|
2318
|
-
model: "gemini-2.
|
|
2696
|
+
model: "gemini-2.5-flash",
|
|
2697
|
+
temperature: 0.8,
|
|
2698
|
+
maxTokens: 4096,
|
|
2319
2699
|
...options
|
|
2320
2700
|
};
|
|
2321
2701
|
}
|
|
@@ -2479,13 +2859,13 @@ var OpenAICompatLLM = class {
|
|
|
2479
2859
|
var OllamaLLM = class {
|
|
2480
2860
|
_inner;
|
|
2481
2861
|
constructor(options = {}) {
|
|
2482
|
-
const baseUrl =
|
|
2862
|
+
const baseUrl = options.baseUrl ?? process.env["OLLAMA_BASE_URL"] ?? "http://localhost:11434/v1";
|
|
2483
2863
|
this._inner = new OpenAICompatLLM({
|
|
2484
|
-
baseUrl:
|
|
2485
|
-
model: options.model ?? "llama3.
|
|
2864
|
+
baseUrl: baseUrl.replace(/\/$/, ""),
|
|
2865
|
+
model: options.model ?? "llama3.2",
|
|
2486
2866
|
apiKey: "ollama",
|
|
2487
|
-
temperature: options.temperature,
|
|
2488
|
-
maxTokens: options.maxTokens
|
|
2867
|
+
temperature: options.temperature ?? 0.8,
|
|
2868
|
+
maxTokens: options.maxTokens ?? 4096
|
|
2489
2869
|
});
|
|
2490
2870
|
}
|
|
2491
2871
|
async *generate(messages, options) {
|