@teamlearners/clawops 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/agent/index.cjs +575 -25
- package/dist/agent/index.cjs.map +1 -1
- package/dist/agent/index.d.cts +68 -6
- package/dist/agent/index.d.ts +68 -6
- package/dist/agent/index.js +575 -26
- package/dist/agent/index.js.map +1 -1
- package/dist/index.cjs +2 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +8 -5
- package/dist/index.d.ts +8 -5
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/package.json +36 -11
package/dist/agent/index.js
CHANGED
|
@@ -555,6 +555,20 @@ function buildMediaResponse(audioBase64) {
|
|
|
555
555
|
}
|
|
556
556
|
});
|
|
557
557
|
}
|
|
558
|
+
var VALID_DTMF_DIGITS = new Set("0123456789*#");
|
|
559
|
+
function parseDtmfEvent(data) {
|
|
560
|
+
const dtmf = data["dtmf"];
|
|
561
|
+
return {
|
|
562
|
+
digit: dtmf["digit"] ?? "",
|
|
563
|
+
track: dtmf["track"] ?? ""
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
function buildDtmfMessage(digit) {
|
|
567
|
+
if (!VALID_DTMF_DIGITS.has(digit)) {
|
|
568
|
+
throw new Error(`\uC720\uD6A8\uD558\uC9C0 \uC54A\uC740 DTMF digit: ${digit}`);
|
|
569
|
+
}
|
|
570
|
+
return JSON.stringify({ event: "dtmf", dtmf: { digit } });
|
|
571
|
+
}
|
|
558
572
|
var MediaWebSocket = class {
|
|
559
573
|
_ws = null;
|
|
560
574
|
_audioQueue = [];
|
|
@@ -563,6 +577,8 @@ var MediaWebSocket = class {
|
|
|
563
577
|
_onAudio = null;
|
|
564
578
|
_onStart = null;
|
|
565
579
|
_onClose = null;
|
|
580
|
+
_onDtmf = null;
|
|
581
|
+
_markWaiters = /* @__PURE__ */ new Map();
|
|
566
582
|
/** Set the handler for inbound audio data. */
|
|
567
583
|
onAudio(handler) {
|
|
568
584
|
this._onAudio = handler;
|
|
@@ -575,6 +591,20 @@ var MediaWebSocket = class {
|
|
|
575
591
|
onClose(handler) {
|
|
576
592
|
this._onClose = handler;
|
|
577
593
|
}
|
|
594
|
+
/** Set the handler for inbound DTMF events. */
|
|
595
|
+
onDtmf(handler) {
|
|
596
|
+
this._onDtmf = handler;
|
|
597
|
+
}
|
|
598
|
+
/** Send a single DTMF digit to the platform. */
|
|
599
|
+
sendDtmf(digit) {
|
|
600
|
+
if (this._ws && this._ws.readyState === 1) {
|
|
601
|
+
this._ws.send(buildDtmfMessage(digit));
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
/** Whether the WebSocket is connected. */
|
|
605
|
+
get isConnected() {
|
|
606
|
+
return this._ws !== null && this._ws.readyState === 1 && !this._closed;
|
|
607
|
+
}
|
|
578
608
|
/** Connect to a media WebSocket URL with Bearer authentication. */
|
|
579
609
|
async connect(url, apiKey) {
|
|
580
610
|
const { WebSocket } = await import('ws');
|
|
@@ -634,6 +664,34 @@ var MediaWebSocket = class {
|
|
|
634
664
|
);
|
|
635
665
|
}
|
|
636
666
|
}
|
|
667
|
+
/** Wait for all queued audio to be sent. */
|
|
668
|
+
flush() {
|
|
669
|
+
if (this._audioQueue.length === 0 || this._closed) return Promise.resolve();
|
|
670
|
+
return new Promise((resolve) => {
|
|
671
|
+
const check = () => {
|
|
672
|
+
if (this._audioQueue.length === 0 || this._closed) {
|
|
673
|
+
resolve();
|
|
674
|
+
} else {
|
|
675
|
+
setTimeout(check, 5);
|
|
676
|
+
}
|
|
677
|
+
};
|
|
678
|
+
setTimeout(check, 5);
|
|
679
|
+
});
|
|
680
|
+
}
|
|
681
|
+
/** Wait for a named mark to be echoed back by the server. */
|
|
682
|
+
waitForMark(name, timeoutMs = 5e3) {
|
|
683
|
+
if (this._closed) return Promise.resolve();
|
|
684
|
+
return new Promise((resolve) => {
|
|
685
|
+
const timer = setTimeout(() => {
|
|
686
|
+
this._markWaiters.delete(name);
|
|
687
|
+
resolve();
|
|
688
|
+
}, timeoutMs);
|
|
689
|
+
this._markWaiters.set(name, () => {
|
|
690
|
+
clearTimeout(timer);
|
|
691
|
+
resolve();
|
|
692
|
+
});
|
|
693
|
+
});
|
|
694
|
+
}
|
|
637
695
|
/** Close the media WebSocket. */
|
|
638
696
|
close() {
|
|
639
697
|
this._closed = true;
|
|
@@ -659,6 +717,24 @@ var MediaWebSocket = class {
|
|
|
659
717
|
}
|
|
660
718
|
break;
|
|
661
719
|
}
|
|
720
|
+
case "dtmf": {
|
|
721
|
+
const dtmfEvt = parseDtmfEvent(msg);
|
|
722
|
+
if (this._onDtmf) {
|
|
723
|
+
this._onDtmf(dtmfEvt.digit);
|
|
724
|
+
}
|
|
725
|
+
break;
|
|
726
|
+
}
|
|
727
|
+
case "mark": {
|
|
728
|
+
const markName = msg["mark"]?.["name"];
|
|
729
|
+
if (markName) {
|
|
730
|
+
const resolve = this._markWaiters.get(markName);
|
|
731
|
+
if (resolve) {
|
|
732
|
+
this._markWaiters.delete(markName);
|
|
733
|
+
resolve();
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
break;
|
|
737
|
+
}
|
|
662
738
|
case "stop": {
|
|
663
739
|
this.close();
|
|
664
740
|
break;
|
|
@@ -847,6 +923,13 @@ var CallSession = class {
|
|
|
847
923
|
_sendAudioFn = null;
|
|
848
924
|
_clearAudioFn = null;
|
|
849
925
|
_hangupFn = null;
|
|
926
|
+
/** @internal */
|
|
927
|
+
_sendDtmfFn = null;
|
|
928
|
+
/** @internal */
|
|
929
|
+
_isTransportConnected = null;
|
|
930
|
+
_dtmfCollectorActive = false;
|
|
931
|
+
_dtmfResolvers = [];
|
|
932
|
+
_dtmfBuffer = [];
|
|
850
933
|
_handlers = /* @__PURE__ */ new Map();
|
|
851
934
|
_endedPromise;
|
|
852
935
|
_resolveEnded;
|
|
@@ -870,10 +953,12 @@ var CallSession = class {
|
|
|
870
953
|
return (Date.now() - this.startTime.getTime()) / 1e3;
|
|
871
954
|
}
|
|
872
955
|
/** Bind transport functions (called internally by the agent). */
|
|
873
|
-
_bindTransport(send, clear, hangup) {
|
|
956
|
+
_bindTransport(send, clear, hangup, sendDtmf, isConnected) {
|
|
874
957
|
this._sendAudioFn = send;
|
|
875
958
|
this._clearAudioFn = clear;
|
|
876
959
|
this._hangupFn = hangup;
|
|
960
|
+
if (sendDtmf) this._sendDtmfFn = sendDtmf;
|
|
961
|
+
if (isConnected) this._isTransportConnected = isConnected;
|
|
877
962
|
this._status = "active";
|
|
878
963
|
}
|
|
879
964
|
/** Send PCM16 or ulaw audio to the caller. */
|
|
@@ -888,10 +973,76 @@ var CallSession = class {
|
|
|
888
973
|
this._clearAudioFn();
|
|
889
974
|
}
|
|
890
975
|
}
|
|
891
|
-
/** Hang up the call. */
|
|
892
|
-
hangup() {
|
|
976
|
+
/** Hang up the call, waiting for pending audio to finish. */
|
|
977
|
+
async hangup() {
|
|
893
978
|
if (this._hangupFn) {
|
|
894
|
-
this._hangupFn();
|
|
979
|
+
await this._hangupFn();
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
/** @internal Route a received DTMF digit to an active collector or buffer. */
|
|
983
|
+
_routeDtmf(digit) {
|
|
984
|
+
if (this._dtmfCollectorActive && this._dtmfResolvers.length > 0) {
|
|
985
|
+
const resolve = this._dtmfResolvers.shift();
|
|
986
|
+
resolve(digit);
|
|
987
|
+
} else {
|
|
988
|
+
this._dtmfBuffer.push(digit);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
/** Collect DTMF digits from the caller. */
|
|
992
|
+
async collectDtmf(options) {
|
|
993
|
+
if (this._dtmfCollectorActive) {
|
|
994
|
+
throw new Error("\uC774\uBBF8 DTMF \uC218\uC9D1 \uC911\uC785\uB2C8\uB2E4");
|
|
995
|
+
}
|
|
996
|
+
const { maxDigits, finishOnKey = "#", timeout = 5 } = options;
|
|
997
|
+
this._dtmfCollectorActive = true;
|
|
998
|
+
const collected = [];
|
|
999
|
+
try {
|
|
1000
|
+
while (collected.length < maxDigits) {
|
|
1001
|
+
if (this._dtmfBuffer.length > 0) {
|
|
1002
|
+
const digit2 = this._dtmfBuffer.shift();
|
|
1003
|
+
if (digit2 === finishOnKey) break;
|
|
1004
|
+
collected.push(digit2);
|
|
1005
|
+
continue;
|
|
1006
|
+
}
|
|
1007
|
+
const digit = await Promise.race([
|
|
1008
|
+
new Promise((resolve) => {
|
|
1009
|
+
this._dtmfResolvers.push(resolve);
|
|
1010
|
+
}),
|
|
1011
|
+
new Promise((resolve) => {
|
|
1012
|
+
setTimeout(() => resolve(null), timeout * 1e3);
|
|
1013
|
+
})
|
|
1014
|
+
]);
|
|
1015
|
+
if (digit === null) break;
|
|
1016
|
+
if (digit === finishOnKey) break;
|
|
1017
|
+
collected.push(digit);
|
|
1018
|
+
}
|
|
1019
|
+
} finally {
|
|
1020
|
+
this._dtmfCollectorActive = false;
|
|
1021
|
+
this._dtmfResolvers = [];
|
|
1022
|
+
this._dtmfBuffer = [];
|
|
1023
|
+
}
|
|
1024
|
+
return collected.join("");
|
|
1025
|
+
}
|
|
1026
|
+
/** Send a sequence of DTMF digits. */
|
|
1027
|
+
async sendDtmfSequence(digits) {
|
|
1028
|
+
if (!this._sendDtmfFn) {
|
|
1029
|
+
throw new Error("DTMF \uC804\uC1A1 \uD568\uC218\uAC00 \uBC14\uC778\uB529\uB418\uC9C0 \uC54A\uC558\uC2B5\uB2C8\uB2E4");
|
|
1030
|
+
}
|
|
1031
|
+
for (const ch of digits) {
|
|
1032
|
+
if (this._isTransportConnected && !this._isTransportConnected()) {
|
|
1033
|
+
throw new Error("DTMF \uC804\uC1A1 \uC911 \uC5F0\uACB0\uC774 \uB04A\uC5B4\uC84C\uC2B5\uB2C8\uB2E4");
|
|
1034
|
+
}
|
|
1035
|
+
if (ch === "w") {
|
|
1036
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
1037
|
+
} else if (ch === "W") {
|
|
1038
|
+
await new Promise((r) => setTimeout(r, 1e3));
|
|
1039
|
+
} else if ("0123456789*#".includes(ch)) {
|
|
1040
|
+
if (this._sendDtmfFn) {
|
|
1041
|
+
await this._sendDtmfFn(ch);
|
|
1042
|
+
}
|
|
1043
|
+
} else {
|
|
1044
|
+
throw new Error(`\uC720\uD6A8\uD558\uC9C0 \uC54A\uC740 DTMF \uBB38\uC790: ${ch}`);
|
|
1045
|
+
}
|
|
895
1046
|
}
|
|
896
1047
|
}
|
|
897
1048
|
/** Register an event handler. */
|
|
@@ -933,6 +1084,33 @@ var CallSession = class {
|
|
|
933
1084
|
}
|
|
934
1085
|
};
|
|
935
1086
|
|
|
1087
|
+
// src/agent/builtin-tool.ts
|
|
1088
|
+
var BuiltinTool = /* @__PURE__ */ ((BuiltinTool2) => {
|
|
1089
|
+
BuiltinTool2["HANG_UP"] = "hang_up";
|
|
1090
|
+
BuiltinTool2["COLLECT_DTMF"] = "collect_dtmf";
|
|
1091
|
+
BuiltinTool2["SEND_DTMF"] = "send_dtmf";
|
|
1092
|
+
BuiltinTool2["ALL"] = "all";
|
|
1093
|
+
BuiltinTool2["NONE"] = "none";
|
|
1094
|
+
return BuiltinTool2;
|
|
1095
|
+
})(BuiltinTool || {});
|
|
1096
|
+
var INDIVIDUAL_TOOLS = /* @__PURE__ */ new Set([
|
|
1097
|
+
"hang_up" /* HANG_UP */,
|
|
1098
|
+
"collect_dtmf" /* COLLECT_DTMF */,
|
|
1099
|
+
"send_dtmf" /* SEND_DTMF */
|
|
1100
|
+
]);
|
|
1101
|
+
function resolveBuiltinTools(value) {
|
|
1102
|
+
if (typeof value === "string") {
|
|
1103
|
+
if (value === "all" /* ALL */) {
|
|
1104
|
+
return new Set(INDIVIDUAL_TOOLS);
|
|
1105
|
+
}
|
|
1106
|
+
if (value === "none" /* NONE */) {
|
|
1107
|
+
return /* @__PURE__ */ new Set();
|
|
1108
|
+
}
|
|
1109
|
+
return /* @__PURE__ */ new Set([value]);
|
|
1110
|
+
}
|
|
1111
|
+
return new Set(value.filter((t) => INDIVIDUAL_TOOLS.has(t)));
|
|
1112
|
+
}
|
|
1113
|
+
|
|
936
1114
|
// src/agent/tool.ts
|
|
937
1115
|
function functionTool(fn) {
|
|
938
1116
|
return fn;
|
|
@@ -1141,6 +1319,12 @@ var ClawOpsAgent = class {
|
|
|
1141
1319
|
_recording;
|
|
1142
1320
|
_recordingPath;
|
|
1143
1321
|
_activeSessions = /* @__PURE__ */ new Map();
|
|
1322
|
+
_builtinTools;
|
|
1323
|
+
_passiveDtmfDebounceMs;
|
|
1324
|
+
_passiveDtmfBuffer = [];
|
|
1325
|
+
_passiveDtmfTimer = null;
|
|
1326
|
+
_passiveDtmfCallId = null;
|
|
1327
|
+
_callSessions = /* @__PURE__ */ new Map();
|
|
1144
1328
|
constructor(options) {
|
|
1145
1329
|
this._apiKey = options.apiKey ?? process.env["CLAWOPS_API_KEY"] ?? "";
|
|
1146
1330
|
this._accountId = options.accountId ?? process.env["CLAWOPS_ACCOUNT_ID"] ?? "";
|
|
@@ -1150,6 +1334,8 @@ var ClawOpsAgent = class {
|
|
|
1150
1334
|
this._recording = options.recording ?? false;
|
|
1151
1335
|
this._recordingPath = options.recordingPath ?? "./recordings";
|
|
1152
1336
|
this._mcpServers = options.mcpServers ?? [];
|
|
1337
|
+
this._builtinTools = resolveBuiltinTools(options.builtinTools ?? "all" /* ALL */);
|
|
1338
|
+
this._passiveDtmfDebounceMs = options.passiveDtmfDebounceMs ?? 500;
|
|
1153
1339
|
if (options.tracing) {
|
|
1154
1340
|
setTracingConfig(options.tracing);
|
|
1155
1341
|
}
|
|
@@ -1164,7 +1350,9 @@ var ClawOpsAgent = class {
|
|
|
1164
1350
|
tool(nameOrTool, description, parameters, handler) {
|
|
1165
1351
|
if (typeof nameOrTool === "string") {
|
|
1166
1352
|
if (!description || !parameters || !handler) {
|
|
1167
|
-
throw new AgentError(
|
|
1353
|
+
throw new AgentError(
|
|
1354
|
+
"tool(name, description, parameters, handler) requires all arguments."
|
|
1355
|
+
);
|
|
1168
1356
|
}
|
|
1169
1357
|
this._tools.register({
|
|
1170
1358
|
name: nameOrTool,
|
|
@@ -1200,7 +1388,9 @@ var ClawOpsAgent = class {
|
|
|
1200
1388
|
throw new AgentError("API key is required. Set CLAWOPS_API_KEY or pass apiKey option.");
|
|
1201
1389
|
}
|
|
1202
1390
|
if (!this._accountId) {
|
|
1203
|
-
throw new AgentError(
|
|
1391
|
+
throw new AgentError(
|
|
1392
|
+
"Account ID is required. Set CLAWOPS_ACCOUNT_ID or pass accountId option."
|
|
1393
|
+
);
|
|
1204
1394
|
}
|
|
1205
1395
|
this._controlWs = new ControlWebSocket({
|
|
1206
1396
|
baseUrl: this._baseUrl,
|
|
@@ -1247,6 +1437,7 @@ var ClawOpsAgent = class {
|
|
|
1247
1437
|
session._markEnded();
|
|
1248
1438
|
}
|
|
1249
1439
|
this._activeSessions.clear();
|
|
1440
|
+
this._callSessions.clear();
|
|
1250
1441
|
console.log("[ClawOpsAgent] Disconnected");
|
|
1251
1442
|
}
|
|
1252
1443
|
/**
|
|
@@ -1283,7 +1474,9 @@ var ClawOpsAgent = class {
|
|
|
1283
1474
|
}
|
|
1284
1475
|
}
|
|
1285
1476
|
this._activeSessions.set(callSession.callId, callSession);
|
|
1286
|
-
console.log(
|
|
1477
|
+
console.log(
|
|
1478
|
+
`[ClawOpsAgent] Outbound call initiated: ${this._fromNumber} -> ${to} (${callSession.callId})`
|
|
1479
|
+
);
|
|
1287
1480
|
return callSession;
|
|
1288
1481
|
}
|
|
1289
1482
|
_handleIncoming(event) {
|
|
@@ -1361,6 +1554,30 @@ var ClawOpsAgent = class {
|
|
|
1361
1554
|
this._activeSessions.delete(callId);
|
|
1362
1555
|
}
|
|
1363
1556
|
}
|
|
1557
|
+
_onDtmfEvent(callSession, digit) {
|
|
1558
|
+
callSession._emit("dtmf", digit);
|
|
1559
|
+
callSession._routeDtmf(digit);
|
|
1560
|
+
if (callSession._dtmfCollectorActive) {
|
|
1561
|
+
callSession.clearAudio();
|
|
1562
|
+
return;
|
|
1563
|
+
}
|
|
1564
|
+
this._passiveDtmfBuffer.push(digit);
|
|
1565
|
+
this._passiveDtmfCallId = callSession.callId;
|
|
1566
|
+
if (this._passiveDtmfTimer) {
|
|
1567
|
+
clearTimeout(this._passiveDtmfTimer);
|
|
1568
|
+
}
|
|
1569
|
+
this._passiveDtmfTimer = setTimeout(() => {
|
|
1570
|
+
const digits = this._passiveDtmfBuffer.join("");
|
|
1571
|
+
this._passiveDtmfBuffer = [];
|
|
1572
|
+
const sessionHandler = this._passiveDtmfCallId ? this._callSessions.get(this._passiveDtmfCallId) : null;
|
|
1573
|
+
this._passiveDtmfCallId = null;
|
|
1574
|
+
if (digits && sessionHandler && sessionHandler.feedDtmf) {
|
|
1575
|
+
sessionHandler.feedDtmf(digits).catch((err) => {
|
|
1576
|
+
console.error("[ClawOpsAgent] feedDtmf error:", err);
|
|
1577
|
+
});
|
|
1578
|
+
}
|
|
1579
|
+
}, this._passiveDtmfDebounceMs);
|
|
1580
|
+
}
|
|
1364
1581
|
async _startCallSession(session, mediaWsUrl) {
|
|
1365
1582
|
await withSpan(
|
|
1366
1583
|
"clawops.call_session",
|
|
@@ -1398,9 +1615,17 @@ var ClawOpsAgent = class {
|
|
|
1398
1615
|
() => {
|
|
1399
1616
|
mediaWs.sendClear();
|
|
1400
1617
|
},
|
|
1401
|
-
() => {
|
|
1618
|
+
async () => {
|
|
1619
|
+
await mediaWs.flush();
|
|
1620
|
+
const markName = `hangup-${Date.now()}`;
|
|
1621
|
+
mediaWs.sendMark(markName);
|
|
1622
|
+
await mediaWs.waitForMark(markName, 5e3);
|
|
1402
1623
|
mediaWs.close();
|
|
1403
|
-
}
|
|
1624
|
+
},
|
|
1625
|
+
async (digit) => {
|
|
1626
|
+
mediaWs.sendDtmf(digit);
|
|
1627
|
+
},
|
|
1628
|
+
() => mediaWs.isConnected
|
|
1404
1629
|
);
|
|
1405
1630
|
const sessionHandler = this._session;
|
|
1406
1631
|
if ("setToolRegistry" in sessionHandler && typeof sessionHandler.setToolRegistry === "function") {
|
|
@@ -1409,6 +1634,10 @@ var ClawOpsAgent = class {
|
|
|
1409
1634
|
if (recorder && "setRecorder" in sessionHandler && typeof sessionHandler.setRecorder === "function") {
|
|
1410
1635
|
sessionHandler.setRecorder(recorder);
|
|
1411
1636
|
}
|
|
1637
|
+
if ("setBuiltinTools" in sessionHandler && typeof sessionHandler.setBuiltinTools === "function") {
|
|
1638
|
+
sessionHandler.setBuiltinTools(this._builtinTools);
|
|
1639
|
+
}
|
|
1640
|
+
this._callSessions.set(session.callId, sessionHandler);
|
|
1412
1641
|
mediaWs.onAudio((ulawAudio, _timestamp) => {
|
|
1413
1642
|
if (sessionHandler) {
|
|
1414
1643
|
sessionHandler.feedAudio(ulawAudio);
|
|
@@ -1417,6 +1646,9 @@ var ClawOpsAgent = class {
|
|
|
1417
1646
|
recorder.writeInbound(ulawToPcm16(ulawAudio));
|
|
1418
1647
|
}
|
|
1419
1648
|
});
|
|
1649
|
+
mediaWs.onDtmf((digit) => {
|
|
1650
|
+
this._onDtmfEvent(session, digit);
|
|
1651
|
+
});
|
|
1420
1652
|
mediaWs.onClose(() => {
|
|
1421
1653
|
if (recorder) {
|
|
1422
1654
|
recorder.stop();
|
|
@@ -1445,6 +1677,7 @@ var ClawOpsAgent = class {
|
|
|
1445
1677
|
session._emit("call_end");
|
|
1446
1678
|
session._markEnded();
|
|
1447
1679
|
this._activeSessions.delete(session.callId);
|
|
1680
|
+
this._callSessions.delete(session.callId);
|
|
1448
1681
|
}
|
|
1449
1682
|
}
|
|
1450
1683
|
);
|
|
@@ -1459,6 +1692,32 @@ var HANG_UP_TOOL = {
|
|
|
1459
1692
|
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1460
1693
|
parameters: { type: "object", properties: {}, required: [] }
|
|
1461
1694
|
};
|
|
1695
|
+
var COLLECT_DTMF_TOOL = {
|
|
1696
|
+
type: "function",
|
|
1697
|
+
name: "collect_dtmf",
|
|
1698
|
+
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
|
|
1699
|
+
parameters: {
|
|
1700
|
+
type: "object",
|
|
1701
|
+
properties: {
|
|
1702
|
+
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
1703
|
+
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
1704
|
+
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
1705
|
+
},
|
|
1706
|
+
required: ["max_digits"]
|
|
1707
|
+
}
|
|
1708
|
+
};
|
|
1709
|
+
var SEND_DTMF_TOOL = {
|
|
1710
|
+
type: "function",
|
|
1711
|
+
name: "send_dtmf",
|
|
1712
|
+
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
1713
|
+
parameters: {
|
|
1714
|
+
type: "object",
|
|
1715
|
+
properties: {
|
|
1716
|
+
digits: { type: "string", description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30." }
|
|
1717
|
+
},
|
|
1718
|
+
required: ["digits"]
|
|
1719
|
+
}
|
|
1720
|
+
};
|
|
1462
1721
|
var OpenAIRealtime = class {
|
|
1463
1722
|
_apiKey;
|
|
1464
1723
|
_systemPrompt;
|
|
@@ -1467,6 +1726,10 @@ var OpenAIRealtime = class {
|
|
|
1467
1726
|
_language;
|
|
1468
1727
|
_eagerness;
|
|
1469
1728
|
_greeting;
|
|
1729
|
+
_builtinTools = null;
|
|
1730
|
+
setBuiltinTools(tools) {
|
|
1731
|
+
this._builtinTools = tools;
|
|
1732
|
+
}
|
|
1470
1733
|
_ws = null;
|
|
1471
1734
|
_call = null;
|
|
1472
1735
|
_tools = null;
|
|
@@ -1477,6 +1740,9 @@ var OpenAIRealtime = class {
|
|
|
1477
1740
|
_responseStartTs = null;
|
|
1478
1741
|
_sentAudioChunks = 0;
|
|
1479
1742
|
_audioRemainder = Buffer.alloc(0);
|
|
1743
|
+
// Response state tracking — prevent sending response.create while one is active
|
|
1744
|
+
_responseInProgress = false;
|
|
1745
|
+
_onResponseDone = null;
|
|
1480
1746
|
constructor(options = {}) {
|
|
1481
1747
|
this._apiKey = options.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
|
|
1482
1748
|
this._systemPrompt = options.systemPrompt ?? "";
|
|
@@ -1540,6 +1806,18 @@ var OpenAIRealtime = class {
|
|
|
1540
1806
|
});
|
|
1541
1807
|
});
|
|
1542
1808
|
}
|
|
1809
|
+
async feedDtmf(digits) {
|
|
1810
|
+
await this._waitForResponseDone();
|
|
1811
|
+
this._send({
|
|
1812
|
+
type: "conversation.item.create",
|
|
1813
|
+
item: {
|
|
1814
|
+
type: "message",
|
|
1815
|
+
role: "user",
|
|
1816
|
+
content: [{ type: "input_text", text: `[DTMF \uC785\uB825: ${digits}]` }]
|
|
1817
|
+
}
|
|
1818
|
+
});
|
|
1819
|
+
this._send({ type: "response.create" });
|
|
1820
|
+
}
|
|
1543
1821
|
feedAudio(audio) {
|
|
1544
1822
|
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1545
1823
|
this._send({
|
|
@@ -1558,7 +1836,9 @@ var OpenAIRealtime = class {
|
|
|
1558
1836
|
_sendSessionUpdate() {
|
|
1559
1837
|
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1560
1838
|
const toolSchemas = this._tools ? this._tools.toOpenAITools().map((t) => ({ type: "function", ...t.function })) : [];
|
|
1561
|
-
toolSchemas.push(HANG_UP_TOOL);
|
|
1839
|
+
if (!this._builtinTools || this._builtinTools.has("hang_up" /* HANG_UP */)) toolSchemas.push(HANG_UP_TOOL);
|
|
1840
|
+
if (!this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */)) toolSchemas.push(COLLECT_DTMF_TOOL);
|
|
1841
|
+
if (!this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */)) toolSchemas.push(SEND_DTMF_TOOL);
|
|
1562
1842
|
this._send({
|
|
1563
1843
|
type: "session.update",
|
|
1564
1844
|
session: {
|
|
@@ -1625,6 +1905,19 @@ var OpenAIRealtime = class {
|
|
|
1625
1905
|
}
|
|
1626
1906
|
break;
|
|
1627
1907
|
}
|
|
1908
|
+
case "response.created": {
|
|
1909
|
+
this._responseInProgress = true;
|
|
1910
|
+
break;
|
|
1911
|
+
}
|
|
1912
|
+
case "response.done": {
|
|
1913
|
+
this._responseInProgress = false;
|
|
1914
|
+
if (this._onResponseDone) {
|
|
1915
|
+
const cb = this._onResponseDone;
|
|
1916
|
+
this._onResponseDone = null;
|
|
1917
|
+
cb();
|
|
1918
|
+
}
|
|
1919
|
+
break;
|
|
1920
|
+
}
|
|
1628
1921
|
case "error": {
|
|
1629
1922
|
console.error("[OpenAIRealtime] API error:", msg["error"]);
|
|
1630
1923
|
break;
|
|
@@ -1678,7 +1971,56 @@ var OpenAIRealtime = class {
|
|
|
1678
1971
|
const callId = item["call_id"];
|
|
1679
1972
|
if (funcName === "hang_up") {
|
|
1680
1973
|
if (this._call) {
|
|
1681
|
-
this._call.hangup();
|
|
1974
|
+
await this._call.hangup();
|
|
1975
|
+
}
|
|
1976
|
+
return;
|
|
1977
|
+
}
|
|
1978
|
+
if (funcName === "collect_dtmf") {
|
|
1979
|
+
if (this._call) {
|
|
1980
|
+
let result2;
|
|
1981
|
+
try {
|
|
1982
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
1983
|
+
result2 = await this._call.collectDtmf({
|
|
1984
|
+
maxDigits: args["max_digits"] ?? 4,
|
|
1985
|
+
finishOnKey: args["finish_on_key"] ?? "#",
|
|
1986
|
+
timeout: args["timeout"] ?? 5
|
|
1987
|
+
});
|
|
1988
|
+
} catch (err) {
|
|
1989
|
+
result2 = `Error: ${err}`;
|
|
1990
|
+
}
|
|
1991
|
+
await this._waitForResponseDone();
|
|
1992
|
+
this._send({
|
|
1993
|
+
type: "conversation.item.create",
|
|
1994
|
+
item: {
|
|
1995
|
+
type: "function_call_output",
|
|
1996
|
+
call_id: callId,
|
|
1997
|
+
output: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)"
|
|
1998
|
+
}
|
|
1999
|
+
});
|
|
2000
|
+
this._send({ type: "response.create" });
|
|
2001
|
+
}
|
|
2002
|
+
return;
|
|
2003
|
+
}
|
|
2004
|
+
if (funcName === "send_dtmf") {
|
|
2005
|
+
if (this._call) {
|
|
2006
|
+
let result2;
|
|
2007
|
+
try {
|
|
2008
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2009
|
+
await this._call.sendDtmfSequence(args["digits"] ?? "");
|
|
2010
|
+
result2 = "sent";
|
|
2011
|
+
} catch (err) {
|
|
2012
|
+
result2 = `Error: ${err}`;
|
|
2013
|
+
}
|
|
2014
|
+
await this._waitForResponseDone();
|
|
2015
|
+
this._send({
|
|
2016
|
+
type: "conversation.item.create",
|
|
2017
|
+
item: {
|
|
2018
|
+
type: "function_call_output",
|
|
2019
|
+
call_id: callId,
|
|
2020
|
+
output: result2
|
|
2021
|
+
}
|
|
2022
|
+
});
|
|
2023
|
+
this._send({ type: "response.create" });
|
|
1682
2024
|
}
|
|
1683
2025
|
return;
|
|
1684
2026
|
}
|
|
@@ -1694,6 +2036,7 @@ var OpenAIRealtime = class {
|
|
|
1694
2036
|
console.error(`[OpenAIRealtime] Tool call failed: ${funcName}:`, err);
|
|
1695
2037
|
result = `Error: ${err}`;
|
|
1696
2038
|
}
|
|
2039
|
+
await this._waitForResponseDone();
|
|
1697
2040
|
this._send({
|
|
1698
2041
|
type: "conversation.item.create",
|
|
1699
2042
|
item: {
|
|
@@ -1704,6 +2047,12 @@ var OpenAIRealtime = class {
|
|
|
1704
2047
|
});
|
|
1705
2048
|
this._send({ type: "response.create" });
|
|
1706
2049
|
}
|
|
2050
|
+
_waitForResponseDone() {
|
|
2051
|
+
if (!this._responseInProgress) return Promise.resolve();
|
|
2052
|
+
return new Promise((resolve) => {
|
|
2053
|
+
this._onResponseDone = resolve;
|
|
2054
|
+
});
|
|
2055
|
+
}
|
|
1707
2056
|
_send(data) {
|
|
1708
2057
|
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1709
2058
|
this._ws.send(JSON.stringify(data));
|
|
@@ -1717,6 +2066,33 @@ var HANG_UP_TOOL2 = {
|
|
|
1717
2066
|
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1718
2067
|
parameters: { type: "object", properties: {} }
|
|
1719
2068
|
};
|
|
2069
|
+
var COLLECT_DTMF_TOOL2 = {
|
|
2070
|
+
name: "collect_dtmf",
|
|
2071
|
+
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
|
|
2072
|
+
parameters: {
|
|
2073
|
+
type: "object",
|
|
2074
|
+
properties: {
|
|
2075
|
+
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
2076
|
+
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
2077
|
+
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
2078
|
+
},
|
|
2079
|
+
required: ["max_digits"]
|
|
2080
|
+
}
|
|
2081
|
+
};
|
|
2082
|
+
var SEND_DTMF_TOOL2 = {
|
|
2083
|
+
name: "send_dtmf",
|
|
2084
|
+
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
2085
|
+
parameters: {
|
|
2086
|
+
type: "object",
|
|
2087
|
+
properties: {
|
|
2088
|
+
digits: {
|
|
2089
|
+
type: "string",
|
|
2090
|
+
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30."
|
|
2091
|
+
}
|
|
2092
|
+
},
|
|
2093
|
+
required: ["digits"]
|
|
2094
|
+
}
|
|
2095
|
+
};
|
|
1720
2096
|
function resolveRef(ref, defs) {
|
|
1721
2097
|
const parts = ref.replace(/^#\//, "").split("/");
|
|
1722
2098
|
let result = defs;
|
|
@@ -1779,7 +2155,11 @@ function sanitizeSchemaForGemini(schema, defs, depth = 0) {
|
|
|
1779
2155
|
result["properties"] = props;
|
|
1780
2156
|
}
|
|
1781
2157
|
if (schema["items"] && typeof schema["items"] === "object" && !Array.isArray(schema["items"])) {
|
|
1782
|
-
result["items"] = sanitizeSchemaForGemini(
|
|
2158
|
+
result["items"] = sanitizeSchemaForGemini(
|
|
2159
|
+
schema["items"],
|
|
2160
|
+
defs,
|
|
2161
|
+
depth + 1
|
|
2162
|
+
);
|
|
1783
2163
|
}
|
|
1784
2164
|
if (!result["type"] && result["properties"]) result["type"] = "object";
|
|
1785
2165
|
if (result["type"] === "object" && !result["properties"]) result["properties"] = {};
|
|
@@ -1800,6 +2180,8 @@ var GeminiRealtime = class {
|
|
|
1800
2180
|
_closed = false;
|
|
1801
2181
|
_sentAudioChunks = 0;
|
|
1802
2182
|
_audioRemainder = Buffer.alloc(0);
|
|
2183
|
+
_builtinTools = null;
|
|
2184
|
+
_toolCallInProgress = false;
|
|
1803
2185
|
constructor(options = {}) {
|
|
1804
2186
|
this._apiKey = options.apiKey ?? process.env["GOOGLE_API_KEY"] ?? "";
|
|
1805
2187
|
this._systemPrompt = options.systemPrompt ?? "";
|
|
@@ -1816,6 +2198,9 @@ var GeminiRealtime = class {
|
|
|
1816
2198
|
setRecorder(recorder) {
|
|
1817
2199
|
this._recorder = recorder;
|
|
1818
2200
|
}
|
|
2201
|
+
setBuiltinTools(tools) {
|
|
2202
|
+
this._builtinTools = tools;
|
|
2203
|
+
}
|
|
1819
2204
|
async start(callSession, tools) {
|
|
1820
2205
|
this._call = callSession;
|
|
1821
2206
|
if (tools) this._tools = tools;
|
|
@@ -1854,7 +2239,10 @@ var GeminiRealtime = class {
|
|
|
1854
2239
|
onerror: (err) => {
|
|
1855
2240
|
console.error("[GeminiRealtime] SDK error:", err);
|
|
1856
2241
|
},
|
|
1857
|
-
onclose: () => {
|
|
2242
|
+
onclose: (ev) => {
|
|
2243
|
+
console.log(
|
|
2244
|
+
`[GeminiRealtime] Connection closed: code=${ev?.code ?? "unknown"}`
|
|
2245
|
+
);
|
|
1858
2246
|
this._closed = true;
|
|
1859
2247
|
}
|
|
1860
2248
|
}
|
|
@@ -1872,7 +2260,7 @@ var GeminiRealtime = class {
|
|
|
1872
2260
|
}
|
|
1873
2261
|
}
|
|
1874
2262
|
feedAudio(audio) {
|
|
1875
|
-
if (this._session && !this._closed) {
|
|
2263
|
+
if (this._session && !this._closed && !this._toolCallInProgress) {
|
|
1876
2264
|
const pcm8k = ulawToPcm16(audio);
|
|
1877
2265
|
if (this._recorder) {
|
|
1878
2266
|
this._recorder.writeInbound(pcm8k);
|
|
@@ -1886,6 +2274,14 @@ var GeminiRealtime = class {
|
|
|
1886
2274
|
});
|
|
1887
2275
|
}
|
|
1888
2276
|
}
|
|
2277
|
+
async feedDtmf(digits) {
|
|
2278
|
+
if (this._session) {
|
|
2279
|
+
this._session.sendClientContent({
|
|
2280
|
+
turns: [{ role: "user", parts: [{ text: `[DTMF \uC785\uB825: ${digits}]` }] }],
|
|
2281
|
+
turnComplete: true
|
|
2282
|
+
});
|
|
2283
|
+
}
|
|
2284
|
+
}
|
|
1889
2285
|
async stop() {
|
|
1890
2286
|
this._closed = true;
|
|
1891
2287
|
if (this._session) {
|
|
@@ -1904,7 +2300,9 @@ var GeminiRealtime = class {
|
|
|
1904
2300
|
t.function.parameters ?? { type: "object", properties: {} }
|
|
1905
2301
|
)
|
|
1906
2302
|
})) : [];
|
|
1907
|
-
toolDefs.push(HANG_UP_TOOL2);
|
|
2303
|
+
if (!this._builtinTools || this._builtinTools.has("hang_up" /* HANG_UP */)) toolDefs.push(HANG_UP_TOOL2);
|
|
2304
|
+
if (!this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */)) toolDefs.push(COLLECT_DTMF_TOOL2);
|
|
2305
|
+
if (!this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */)) toolDefs.push(SEND_DTMF_TOOL2);
|
|
1908
2306
|
return toolDefs;
|
|
1909
2307
|
}
|
|
1910
2308
|
_handleMessage(msg) {
|
|
@@ -1924,9 +2322,11 @@ var GeminiRealtime = class {
|
|
|
1924
2322
|
}
|
|
1925
2323
|
}
|
|
1926
2324
|
if (serverContent.turnComplete) {
|
|
2325
|
+
console.log("[GeminiRealtime] Turn complete");
|
|
1927
2326
|
this._flushAudioRemainder();
|
|
1928
2327
|
}
|
|
1929
2328
|
if (serverContent.interrupted) {
|
|
2329
|
+
console.log("[GeminiRealtime] Barge-in detected");
|
|
1930
2330
|
if (this._call) {
|
|
1931
2331
|
this._call.clearAudio();
|
|
1932
2332
|
}
|
|
@@ -1935,16 +2335,24 @@ var GeminiRealtime = class {
|
|
|
1935
2335
|
}
|
|
1936
2336
|
const inputText = serverContent.inputTranscription?.text;
|
|
1937
2337
|
if (inputText && this._call) {
|
|
2338
|
+
console.log(`[GeminiRealtime] [TRANSCRIPT-USER] ${inputText}`);
|
|
1938
2339
|
this._call._emit("transcript", "user", inputText);
|
|
1939
2340
|
}
|
|
1940
2341
|
const outputText = serverContent.outputTranscription?.text;
|
|
1941
2342
|
if (outputText && this._call) {
|
|
2343
|
+
console.log(`[GeminiRealtime] [TRANSCRIPT-ASSISTANT] ${outputText}`);
|
|
1942
2344
|
this._call._emit("transcript", "assistant", outputText);
|
|
1943
2345
|
}
|
|
1944
2346
|
}
|
|
1945
2347
|
if (msg.toolCall) {
|
|
1946
2348
|
this._handleToolCall(msg.toolCall);
|
|
1947
2349
|
}
|
|
2350
|
+
const toolCancellation = msg["toolCallCancellation"];
|
|
2351
|
+
if (toolCancellation) {
|
|
2352
|
+
console.log(
|
|
2353
|
+
`[GeminiRealtime] Tool call cancelled: ${(toolCancellation.ids ?? []).join(", ")}`
|
|
2354
|
+
);
|
|
2355
|
+
}
|
|
1948
2356
|
}
|
|
1949
2357
|
_handleAudioData(b64Data) {
|
|
1950
2358
|
if (!this._call) return;
|
|
@@ -1957,9 +2365,9 @@ var GeminiRealtime = class {
|
|
|
1957
2365
|
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
1958
2366
|
const chunkSize = 160;
|
|
1959
2367
|
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
1960
|
-
|
|
1961
|
-
this._call.sendAudio(combined.subarray(
|
|
1962
|
-
this._sentAudioChunks
|
|
2368
|
+
if (fullEnd > 0) {
|
|
2369
|
+
this._call.sendAudio(combined.subarray(0, fullEnd));
|
|
2370
|
+
this._sentAudioChunks += fullEnd / chunkSize;
|
|
1963
2371
|
}
|
|
1964
2372
|
this._audioRemainder = combined.subarray(fullEnd);
|
|
1965
2373
|
}
|
|
@@ -1977,17 +2385,64 @@ var GeminiRealtime = class {
|
|
|
1977
2385
|
async _handleToolCall(toolCall) {
|
|
1978
2386
|
const functionCalls = toolCall.functionCalls;
|
|
1979
2387
|
if (!functionCalls) return;
|
|
2388
|
+
this._toolCallInProgress = true;
|
|
2389
|
+
console.log(
|
|
2390
|
+
`[GeminiRealtime] toolCall: ${functionCalls.map((fc) => fc.name).join(", ")}`
|
|
2391
|
+
);
|
|
1980
2392
|
const responses = [];
|
|
1981
2393
|
for (const fc of functionCalls) {
|
|
1982
2394
|
const name = fc.name ?? "";
|
|
1983
2395
|
const fcId = fc.id ?? "";
|
|
1984
2396
|
const args = fc.args ?? {};
|
|
2397
|
+
console.log(`[GeminiRealtime] Tool call: ${name}(${JSON.stringify(args)})`);
|
|
1985
2398
|
if (name === "hang_up") {
|
|
2399
|
+
console.log("[GeminiRealtime] hang_up: ending call");
|
|
1986
2400
|
if (this._call) {
|
|
1987
|
-
this._call.hangup();
|
|
2401
|
+
await this._call.hangup();
|
|
1988
2402
|
}
|
|
1989
2403
|
return;
|
|
1990
2404
|
}
|
|
2405
|
+
if (name === "collect_dtmf") {
|
|
2406
|
+
if (this._call) {
|
|
2407
|
+
let result;
|
|
2408
|
+
try {
|
|
2409
|
+
console.log(
|
|
2410
|
+
`[GeminiRealtime] collect_dtmf: waiting for digits (maxDigits=${args["max_digits"] ?? 4}, timeout=${args["timeout"] ?? 5})`
|
|
2411
|
+
);
|
|
2412
|
+
result = await this._call.collectDtmf({
|
|
2413
|
+
maxDigits: args["max_digits"] ?? 4,
|
|
2414
|
+
finishOnKey: args["finish_on_key"] ?? "#",
|
|
2415
|
+
timeout: args["timeout"] ?? 5
|
|
2416
|
+
});
|
|
2417
|
+
console.log(`[GeminiRealtime] DTMF collected: ${result || "(empty)"}`);
|
|
2418
|
+
} catch (err) {
|
|
2419
|
+
console.error(`[GeminiRealtime] collect_dtmf error:`, err);
|
|
2420
|
+
result = `Error: ${err}`;
|
|
2421
|
+
}
|
|
2422
|
+
responses.push({
|
|
2423
|
+
id: fcId,
|
|
2424
|
+
name,
|
|
2425
|
+
response: { result: result || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)" }
|
|
2426
|
+
});
|
|
2427
|
+
}
|
|
2428
|
+
continue;
|
|
2429
|
+
}
|
|
2430
|
+
if (name === "send_dtmf") {
|
|
2431
|
+
if (this._call) {
|
|
2432
|
+
let result;
|
|
2433
|
+
try {
|
|
2434
|
+
console.log(`[GeminiRealtime] send_dtmf: digits="${args["digits"] ?? ""}"`);
|
|
2435
|
+
await this._call.sendDtmfSequence(args["digits"] ?? "");
|
|
2436
|
+
result = "sent";
|
|
2437
|
+
console.log(`[GeminiRealtime] send_dtmf: sent`);
|
|
2438
|
+
} catch (err) {
|
|
2439
|
+
console.error(`[GeminiRealtime] send_dtmf error:`, err);
|
|
2440
|
+
result = `Error: ${err}`;
|
|
2441
|
+
}
|
|
2442
|
+
responses.push({ id: fcId, name, response: { result } });
|
|
2443
|
+
}
|
|
2444
|
+
continue;
|
|
2445
|
+
}
|
|
1991
2446
|
if (!this._tools || !this._tools.has(name)) {
|
|
1992
2447
|
console.error(`[GeminiRealtime] Unknown tool: ${name}`);
|
|
1993
2448
|
responses.push({ id: fcId, name, response: { error: `Unknown tool: ${name}` } });
|
|
@@ -1995,13 +2450,15 @@ var GeminiRealtime = class {
|
|
|
1995
2450
|
}
|
|
1996
2451
|
try {
|
|
1997
2452
|
const result = await this._tools.call(name, args);
|
|
2453
|
+
const resultStr = typeof result === "string" ? result : JSON.stringify(result);
|
|
2454
|
+
console.log(`[GeminiRealtime] Tool result: ${name} -> ${resultStr.substring(0, 200)}`);
|
|
1998
2455
|
responses.push({
|
|
1999
2456
|
id: fcId,
|
|
2000
2457
|
name,
|
|
2001
|
-
response: { result:
|
|
2458
|
+
response: { result: resultStr }
|
|
2002
2459
|
});
|
|
2003
2460
|
} catch (err) {
|
|
2004
|
-
console.error(`[GeminiRealtime] Tool call
|
|
2461
|
+
console.error(`[GeminiRealtime] Tool call failed: ${name}:`, err);
|
|
2005
2462
|
responses.push({
|
|
2006
2463
|
id: fcId,
|
|
2007
2464
|
name,
|
|
@@ -2009,15 +2466,41 @@ var GeminiRealtime = class {
|
|
|
2009
2466
|
});
|
|
2010
2467
|
}
|
|
2011
2468
|
}
|
|
2012
|
-
if (this._session) {
|
|
2469
|
+
if (responses.length > 0 && this._session) {
|
|
2470
|
+
console.log(`[GeminiRealtime] Sending ${responses.length} tool response(s)`);
|
|
2013
2471
|
this._session.sendToolResponse({
|
|
2014
2472
|
functionResponses: responses
|
|
2015
2473
|
});
|
|
2016
2474
|
}
|
|
2475
|
+
this._toolCallInProgress = false;
|
|
2017
2476
|
}
|
|
2018
2477
|
};
|
|
2019
2478
|
|
|
2020
2479
|
// src/agent/pipeline/pipeline-session.ts
|
|
2480
|
+
var COLLECT_DTMF_TOOL3 = {
|
|
2481
|
+
function: {
|
|
2482
|
+
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4.",
|
|
2483
|
+
parameters: {
|
|
2484
|
+
properties: {
|
|
2485
|
+
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
2486
|
+
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
2487
|
+
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
2488
|
+
},
|
|
2489
|
+
required: ["max_digits"]
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
2492
|
+
};
|
|
2493
|
+
var SEND_DTMF_TOOL3 = {
|
|
2494
|
+
function: {
|
|
2495
|
+
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
2496
|
+
parameters: {
|
|
2497
|
+
properties: {
|
|
2498
|
+
digits: { type: "string", description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30." }
|
|
2499
|
+
},
|
|
2500
|
+
required: ["digits"]
|
|
2501
|
+
}
|
|
2502
|
+
}
|
|
2503
|
+
};
|
|
2021
2504
|
var PipelineSession = class {
|
|
2022
2505
|
_stt;
|
|
2023
2506
|
_llm;
|
|
@@ -2036,6 +2519,7 @@ var PipelineSession = class {
|
|
|
2036
2519
|
_audioBuffer = [];
|
|
2037
2520
|
_running = false;
|
|
2038
2521
|
_speaking = false;
|
|
2522
|
+
_builtinTools = null;
|
|
2039
2523
|
constructor(options) {
|
|
2040
2524
|
this._stt = options.stt;
|
|
2041
2525
|
this._llm = options.llm;
|
|
@@ -2056,6 +2540,9 @@ var PipelineSession = class {
|
|
|
2056
2540
|
setRecorder(recorder) {
|
|
2057
2541
|
this._recorder = recorder;
|
|
2058
2542
|
}
|
|
2543
|
+
setBuiltinTools(tools) {
|
|
2544
|
+
this._builtinTools = tools;
|
|
2545
|
+
}
|
|
2059
2546
|
async start(callSession, tools) {
|
|
2060
2547
|
this._callSession = callSession;
|
|
2061
2548
|
this._tools = tools ?? null;
|
|
@@ -2081,6 +2568,13 @@ var PipelineSession = class {
|
|
|
2081
2568
|
this._audioBuffer.push(audio);
|
|
2082
2569
|
}
|
|
2083
2570
|
}
|
|
2571
|
+
async feedDtmf(digits) {
|
|
2572
|
+
this._conversation.push({
|
|
2573
|
+
role: "user",
|
|
2574
|
+
content: `[DTMF \uC785\uB825: ${digits}]`
|
|
2575
|
+
});
|
|
2576
|
+
await this._respond();
|
|
2577
|
+
}
|
|
2084
2578
|
async stop() {
|
|
2085
2579
|
this._running = false;
|
|
2086
2580
|
this._audioBuffer = [];
|
|
@@ -2122,11 +2616,37 @@ var PipelineSession = class {
|
|
|
2122
2616
|
this._conversation.push({ role: "user", content: transcript });
|
|
2123
2617
|
await this._respond();
|
|
2124
2618
|
}
|
|
2619
|
+
_buildEffectiveTools() {
|
|
2620
|
+
const includeCollectDtmf = !this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */);
|
|
2621
|
+
const includeSendDtmf = !this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */);
|
|
2622
|
+
if (!includeCollectDtmf && !includeSendDtmf) return this._tools ?? void 0;
|
|
2623
|
+
const base = this._tools ? this._tools.fork() : new ToolRegistry();
|
|
2624
|
+
if (includeCollectDtmf) {
|
|
2625
|
+
base.register({
|
|
2626
|
+
name: "collect_dtmf",
|
|
2627
|
+
description: COLLECT_DTMF_TOOL3.function.description,
|
|
2628
|
+
parameters: COLLECT_DTMF_TOOL3.function.parameters.properties,
|
|
2629
|
+
required: COLLECT_DTMF_TOOL3.function.parameters.required,
|
|
2630
|
+
handler: async () => ""
|
|
2631
|
+
});
|
|
2632
|
+
}
|
|
2633
|
+
if (includeSendDtmf) {
|
|
2634
|
+
base.register({
|
|
2635
|
+
name: "send_dtmf",
|
|
2636
|
+
description: SEND_DTMF_TOOL3.function.description,
|
|
2637
|
+
parameters: SEND_DTMF_TOOL3.function.parameters.properties,
|
|
2638
|
+
required: SEND_DTMF_TOOL3.function.parameters.required,
|
|
2639
|
+
handler: async () => ""
|
|
2640
|
+
});
|
|
2641
|
+
}
|
|
2642
|
+
return base;
|
|
2643
|
+
}
|
|
2125
2644
|
async _respond() {
|
|
2126
2645
|
let fullResponse = "";
|
|
2127
2646
|
const textChunks = [];
|
|
2647
|
+
const effectiveTools = this._buildEffectiveTools();
|
|
2128
2648
|
const llmStream = this._llm.generate(this._conversation, {
|
|
2129
|
-
tools:
|
|
2649
|
+
tools: effectiveTools,
|
|
2130
2650
|
temperature: this._temperature,
|
|
2131
2651
|
maxTokens: this._maxTokens
|
|
2132
2652
|
});
|
|
@@ -2145,10 +2665,38 @@ var PipelineSession = class {
|
|
|
2145
2665
|
}
|
|
2146
2666
|
}
|
|
2147
2667
|
async _handleToolCall(chunk) {
|
|
2148
|
-
if (!chunk.toolCall
|
|
2668
|
+
if (!chunk.toolCall) return;
|
|
2149
2669
|
const { id, name, arguments: argsStr } = chunk.toolCall;
|
|
2150
2670
|
try {
|
|
2151
2671
|
const args = JSON.parse(argsStr);
|
|
2672
|
+
if (name === "collect_dtmf" && this._callSession) {
|
|
2673
|
+
let result2;
|
|
2674
|
+
try {
|
|
2675
|
+
result2 = await this._callSession.collectDtmf({
|
|
2676
|
+
maxDigits: args["max_digits"] ?? 4,
|
|
2677
|
+
finishOnKey: args["finish_on_key"] ?? "#",
|
|
2678
|
+
timeout: args["timeout"] ?? 5
|
|
2679
|
+
});
|
|
2680
|
+
} catch (err) {
|
|
2681
|
+
result2 = `Error: ${err}`;
|
|
2682
|
+
}
|
|
2683
|
+
this._conversation.push({ role: "tool", content: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)", tool_call_id: id, name });
|
|
2684
|
+
await this._respond();
|
|
2685
|
+
return;
|
|
2686
|
+
}
|
|
2687
|
+
if (name === "send_dtmf" && this._callSession) {
|
|
2688
|
+
let result2;
|
|
2689
|
+
try {
|
|
2690
|
+
await this._callSession.sendDtmfSequence(args["digits"] ?? "");
|
|
2691
|
+
result2 = "sent";
|
|
2692
|
+
} catch (err) {
|
|
2693
|
+
result2 = `Error: ${err}`;
|
|
2694
|
+
}
|
|
2695
|
+
this._conversation.push({ role: "tool", content: result2, tool_call_id: id, name });
|
|
2696
|
+
await this._respond();
|
|
2697
|
+
return;
|
|
2698
|
+
}
|
|
2699
|
+
if (!this._tools) return;
|
|
2152
2700
|
const result = await this._tools.call(name, args);
|
|
2153
2701
|
this._conversation.push({
|
|
2154
2702
|
role: "assistant",
|
|
@@ -2161,9 +2709,10 @@ var PipelineSession = class {
|
|
|
2161
2709
|
tool_call_id: id,
|
|
2162
2710
|
name
|
|
2163
2711
|
});
|
|
2712
|
+
const effectiveTools = this._buildEffectiveTools();
|
|
2164
2713
|
let followUpText = "";
|
|
2165
2714
|
const followUpStream = this._llm.generate(this._conversation, {
|
|
2166
|
-
tools:
|
|
2715
|
+
tools: effectiveTools,
|
|
2167
2716
|
temperature: this._temperature,
|
|
2168
2717
|
maxTokens: this._maxTokens
|
|
2169
2718
|
});
|
|
@@ -2987,6 +3536,6 @@ function mcpServerHTTP(options) {
|
|
|
2987
3536
|
};
|
|
2988
3537
|
}
|
|
2989
3538
|
|
|
2990
|
-
export { AnthropicLLM, AudioRecorder, CallSession, ClawOpsAgent, DECODE_TABLE, DeepSeekLLM, DeepgramSTT, ElevenLabsTTS, FireworksLLM, GeminiLLM, GeminiRealtime, GroqLLM, MCPClient, MistralLLM, OllamaLLM, OpenAICompatLLM, OpenAILLM, OpenAIRealtime, PerplexityLLM, PipelineSession, TogetherLLM, ToolRegistry, XaiLLM, functionTool, getTracingConfig, mcpServerHTTP, mcpServerStdio, pcm16ToUlaw, resamplePcm16, resetTracingConfig, setTracingConfig, ulawToPcm16, zodToToolParams };
|
|
3539
|
+
export { AnthropicLLM, AudioRecorder, BuiltinTool, CallSession, ClawOpsAgent, DECODE_TABLE, DeepSeekLLM, DeepgramSTT, ElevenLabsTTS, FireworksLLM, GeminiLLM, GeminiRealtime, GroqLLM, MCPClient, MistralLLM, OllamaLLM, OpenAICompatLLM, OpenAILLM, OpenAIRealtime, PerplexityLLM, PipelineSession, TogetherLLM, ToolRegistry, XaiLLM, functionTool, getTracingConfig, mcpServerHTTP, mcpServerStdio, pcm16ToUlaw, resamplePcm16, resetTracingConfig, setTracingConfig, ulawToPcm16, zodToToolParams };
|
|
2991
3540
|
//# sourceMappingURL=index.js.map
|
|
2992
3541
|
//# sourceMappingURL=index.js.map
|