@teamlearners/clawops 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/agent/index.cjs +575 -25
- package/dist/agent/index.cjs.map +1 -1
- package/dist/agent/index.d.cts +68 -6
- package/dist/agent/index.d.ts +68 -6
- package/dist/agent/index.js +575 -26
- package/dist/agent/index.js.map +1 -1
- package/dist/index.cjs +2 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +8 -5
- package/dist/index.d.ts +8 -5
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/package.json +36 -11
package/dist/agent/index.cjs
CHANGED
|
@@ -578,6 +578,20 @@ function buildMediaResponse(audioBase64) {
|
|
|
578
578
|
}
|
|
579
579
|
});
|
|
580
580
|
}
|
|
581
|
+
var VALID_DTMF_DIGITS = new Set("0123456789*#");
|
|
582
|
+
function parseDtmfEvent(data) {
|
|
583
|
+
const dtmf = data["dtmf"];
|
|
584
|
+
return {
|
|
585
|
+
digit: dtmf["digit"] ?? "",
|
|
586
|
+
track: dtmf["track"] ?? ""
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
function buildDtmfMessage(digit) {
|
|
590
|
+
if (!VALID_DTMF_DIGITS.has(digit)) {
|
|
591
|
+
throw new Error(`\uC720\uD6A8\uD558\uC9C0 \uC54A\uC740 DTMF digit: ${digit}`);
|
|
592
|
+
}
|
|
593
|
+
return JSON.stringify({ event: "dtmf", dtmf: { digit } });
|
|
594
|
+
}
|
|
581
595
|
var MediaWebSocket = class {
|
|
582
596
|
_ws = null;
|
|
583
597
|
_audioQueue = [];
|
|
@@ -586,6 +600,8 @@ var MediaWebSocket = class {
|
|
|
586
600
|
_onAudio = null;
|
|
587
601
|
_onStart = null;
|
|
588
602
|
_onClose = null;
|
|
603
|
+
_onDtmf = null;
|
|
604
|
+
_markWaiters = /* @__PURE__ */ new Map();
|
|
589
605
|
/** Set the handler for inbound audio data. */
|
|
590
606
|
onAudio(handler) {
|
|
591
607
|
this._onAudio = handler;
|
|
@@ -598,6 +614,20 @@ var MediaWebSocket = class {
|
|
|
598
614
|
onClose(handler) {
|
|
599
615
|
this._onClose = handler;
|
|
600
616
|
}
|
|
617
|
+
/** Set the handler for inbound DTMF events. */
|
|
618
|
+
onDtmf(handler) {
|
|
619
|
+
this._onDtmf = handler;
|
|
620
|
+
}
|
|
621
|
+
/** Send a single DTMF digit to the platform. */
|
|
622
|
+
sendDtmf(digit) {
|
|
623
|
+
if (this._ws && this._ws.readyState === 1) {
|
|
624
|
+
this._ws.send(buildDtmfMessage(digit));
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
/** Whether the WebSocket is connected. */
|
|
628
|
+
get isConnected() {
|
|
629
|
+
return this._ws !== null && this._ws.readyState === 1 && !this._closed;
|
|
630
|
+
}
|
|
601
631
|
/** Connect to a media WebSocket URL with Bearer authentication. */
|
|
602
632
|
async connect(url, apiKey) {
|
|
603
633
|
const { WebSocket } = await import('ws');
|
|
@@ -657,6 +687,34 @@ var MediaWebSocket = class {
|
|
|
657
687
|
);
|
|
658
688
|
}
|
|
659
689
|
}
|
|
690
|
+
/** Wait for all queued audio to be sent. */
|
|
691
|
+
flush() {
|
|
692
|
+
if (this._audioQueue.length === 0 || this._closed) return Promise.resolve();
|
|
693
|
+
return new Promise((resolve) => {
|
|
694
|
+
const check = () => {
|
|
695
|
+
if (this._audioQueue.length === 0 || this._closed) {
|
|
696
|
+
resolve();
|
|
697
|
+
} else {
|
|
698
|
+
setTimeout(check, 5);
|
|
699
|
+
}
|
|
700
|
+
};
|
|
701
|
+
setTimeout(check, 5);
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
/** Wait for a named mark to be echoed back by the server. */
|
|
705
|
+
waitForMark(name, timeoutMs = 5e3) {
|
|
706
|
+
if (this._closed) return Promise.resolve();
|
|
707
|
+
return new Promise((resolve) => {
|
|
708
|
+
const timer = setTimeout(() => {
|
|
709
|
+
this._markWaiters.delete(name);
|
|
710
|
+
resolve();
|
|
711
|
+
}, timeoutMs);
|
|
712
|
+
this._markWaiters.set(name, () => {
|
|
713
|
+
clearTimeout(timer);
|
|
714
|
+
resolve();
|
|
715
|
+
});
|
|
716
|
+
});
|
|
717
|
+
}
|
|
660
718
|
/** Close the media WebSocket. */
|
|
661
719
|
close() {
|
|
662
720
|
this._closed = true;
|
|
@@ -682,6 +740,24 @@ var MediaWebSocket = class {
|
|
|
682
740
|
}
|
|
683
741
|
break;
|
|
684
742
|
}
|
|
743
|
+
case "dtmf": {
|
|
744
|
+
const dtmfEvt = parseDtmfEvent(msg);
|
|
745
|
+
if (this._onDtmf) {
|
|
746
|
+
this._onDtmf(dtmfEvt.digit);
|
|
747
|
+
}
|
|
748
|
+
break;
|
|
749
|
+
}
|
|
750
|
+
case "mark": {
|
|
751
|
+
const markName = msg["mark"]?.["name"];
|
|
752
|
+
if (markName) {
|
|
753
|
+
const resolve = this._markWaiters.get(markName);
|
|
754
|
+
if (resolve) {
|
|
755
|
+
this._markWaiters.delete(markName);
|
|
756
|
+
resolve();
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
break;
|
|
760
|
+
}
|
|
685
761
|
case "stop": {
|
|
686
762
|
this.close();
|
|
687
763
|
break;
|
|
@@ -870,6 +946,13 @@ var CallSession = class {
|
|
|
870
946
|
_sendAudioFn = null;
|
|
871
947
|
_clearAudioFn = null;
|
|
872
948
|
_hangupFn = null;
|
|
949
|
+
/** @internal */
|
|
950
|
+
_sendDtmfFn = null;
|
|
951
|
+
/** @internal */
|
|
952
|
+
_isTransportConnected = null;
|
|
953
|
+
_dtmfCollectorActive = false;
|
|
954
|
+
_dtmfResolvers = [];
|
|
955
|
+
_dtmfBuffer = [];
|
|
873
956
|
_handlers = /* @__PURE__ */ new Map();
|
|
874
957
|
_endedPromise;
|
|
875
958
|
_resolveEnded;
|
|
@@ -893,10 +976,12 @@ var CallSession = class {
|
|
|
893
976
|
return (Date.now() - this.startTime.getTime()) / 1e3;
|
|
894
977
|
}
|
|
895
978
|
/** Bind transport functions (called internally by the agent). */
|
|
896
|
-
_bindTransport(send, clear, hangup) {
|
|
979
|
+
_bindTransport(send, clear, hangup, sendDtmf, isConnected) {
|
|
897
980
|
this._sendAudioFn = send;
|
|
898
981
|
this._clearAudioFn = clear;
|
|
899
982
|
this._hangupFn = hangup;
|
|
983
|
+
if (sendDtmf) this._sendDtmfFn = sendDtmf;
|
|
984
|
+
if (isConnected) this._isTransportConnected = isConnected;
|
|
900
985
|
this._status = "active";
|
|
901
986
|
}
|
|
902
987
|
/** Send PCM16 or ulaw audio to the caller. */
|
|
@@ -911,10 +996,76 @@ var CallSession = class {
|
|
|
911
996
|
this._clearAudioFn();
|
|
912
997
|
}
|
|
913
998
|
}
|
|
914
|
-
/** Hang up the call. */
|
|
915
|
-
hangup() {
|
|
999
|
+
/** Hang up the call, waiting for pending audio to finish. */
|
|
1000
|
+
async hangup() {
|
|
916
1001
|
if (this._hangupFn) {
|
|
917
|
-
this._hangupFn();
|
|
1002
|
+
await this._hangupFn();
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
/** @internal Route a received DTMF digit to an active collector or buffer. */
|
|
1006
|
+
_routeDtmf(digit) {
|
|
1007
|
+
if (this._dtmfCollectorActive && this._dtmfResolvers.length > 0) {
|
|
1008
|
+
const resolve = this._dtmfResolvers.shift();
|
|
1009
|
+
resolve(digit);
|
|
1010
|
+
} else {
|
|
1011
|
+
this._dtmfBuffer.push(digit);
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
/** Collect DTMF digits from the caller. */
|
|
1015
|
+
async collectDtmf(options) {
|
|
1016
|
+
if (this._dtmfCollectorActive) {
|
|
1017
|
+
throw new Error("\uC774\uBBF8 DTMF \uC218\uC9D1 \uC911\uC785\uB2C8\uB2E4");
|
|
1018
|
+
}
|
|
1019
|
+
const { maxDigits, finishOnKey = "#", timeout = 5 } = options;
|
|
1020
|
+
this._dtmfCollectorActive = true;
|
|
1021
|
+
const collected = [];
|
|
1022
|
+
try {
|
|
1023
|
+
while (collected.length < maxDigits) {
|
|
1024
|
+
if (this._dtmfBuffer.length > 0) {
|
|
1025
|
+
const digit2 = this._dtmfBuffer.shift();
|
|
1026
|
+
if (digit2 === finishOnKey) break;
|
|
1027
|
+
collected.push(digit2);
|
|
1028
|
+
continue;
|
|
1029
|
+
}
|
|
1030
|
+
const digit = await Promise.race([
|
|
1031
|
+
new Promise((resolve) => {
|
|
1032
|
+
this._dtmfResolvers.push(resolve);
|
|
1033
|
+
}),
|
|
1034
|
+
new Promise((resolve) => {
|
|
1035
|
+
setTimeout(() => resolve(null), timeout * 1e3);
|
|
1036
|
+
})
|
|
1037
|
+
]);
|
|
1038
|
+
if (digit === null) break;
|
|
1039
|
+
if (digit === finishOnKey) break;
|
|
1040
|
+
collected.push(digit);
|
|
1041
|
+
}
|
|
1042
|
+
} finally {
|
|
1043
|
+
this._dtmfCollectorActive = false;
|
|
1044
|
+
this._dtmfResolvers = [];
|
|
1045
|
+
this._dtmfBuffer = [];
|
|
1046
|
+
}
|
|
1047
|
+
return collected.join("");
|
|
1048
|
+
}
|
|
1049
|
+
/** Send a sequence of DTMF digits. */
|
|
1050
|
+
async sendDtmfSequence(digits) {
|
|
1051
|
+
if (!this._sendDtmfFn) {
|
|
1052
|
+
throw new Error("DTMF \uC804\uC1A1 \uD568\uC218\uAC00 \uBC14\uC778\uB529\uB418\uC9C0 \uC54A\uC558\uC2B5\uB2C8\uB2E4");
|
|
1053
|
+
}
|
|
1054
|
+
for (const ch of digits) {
|
|
1055
|
+
if (this._isTransportConnected && !this._isTransportConnected()) {
|
|
1056
|
+
throw new Error("DTMF \uC804\uC1A1 \uC911 \uC5F0\uACB0\uC774 \uB04A\uC5B4\uC84C\uC2B5\uB2C8\uB2E4");
|
|
1057
|
+
}
|
|
1058
|
+
if (ch === "w") {
|
|
1059
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
1060
|
+
} else if (ch === "W") {
|
|
1061
|
+
await new Promise((r) => setTimeout(r, 1e3));
|
|
1062
|
+
} else if ("0123456789*#".includes(ch)) {
|
|
1063
|
+
if (this._sendDtmfFn) {
|
|
1064
|
+
await this._sendDtmfFn(ch);
|
|
1065
|
+
}
|
|
1066
|
+
} else {
|
|
1067
|
+
throw new Error(`\uC720\uD6A8\uD558\uC9C0 \uC54A\uC740 DTMF \uBB38\uC790: ${ch}`);
|
|
1068
|
+
}
|
|
918
1069
|
}
|
|
919
1070
|
}
|
|
920
1071
|
/** Register an event handler. */
|
|
@@ -956,6 +1107,33 @@ var CallSession = class {
|
|
|
956
1107
|
}
|
|
957
1108
|
};
|
|
958
1109
|
|
|
1110
|
+
// src/agent/builtin-tool.ts
|
|
1111
|
+
var BuiltinTool = /* @__PURE__ */ ((BuiltinTool2) => {
|
|
1112
|
+
BuiltinTool2["HANG_UP"] = "hang_up";
|
|
1113
|
+
BuiltinTool2["COLLECT_DTMF"] = "collect_dtmf";
|
|
1114
|
+
BuiltinTool2["SEND_DTMF"] = "send_dtmf";
|
|
1115
|
+
BuiltinTool2["ALL"] = "all";
|
|
1116
|
+
BuiltinTool2["NONE"] = "none";
|
|
1117
|
+
return BuiltinTool2;
|
|
1118
|
+
})(BuiltinTool || {});
|
|
1119
|
+
var INDIVIDUAL_TOOLS = /* @__PURE__ */ new Set([
|
|
1120
|
+
"hang_up" /* HANG_UP */,
|
|
1121
|
+
"collect_dtmf" /* COLLECT_DTMF */,
|
|
1122
|
+
"send_dtmf" /* SEND_DTMF */
|
|
1123
|
+
]);
|
|
1124
|
+
function resolveBuiltinTools(value) {
|
|
1125
|
+
if (typeof value === "string") {
|
|
1126
|
+
if (value === "all" /* ALL */) {
|
|
1127
|
+
return new Set(INDIVIDUAL_TOOLS);
|
|
1128
|
+
}
|
|
1129
|
+
if (value === "none" /* NONE */) {
|
|
1130
|
+
return /* @__PURE__ */ new Set();
|
|
1131
|
+
}
|
|
1132
|
+
return /* @__PURE__ */ new Set([value]);
|
|
1133
|
+
}
|
|
1134
|
+
return new Set(value.filter((t) => INDIVIDUAL_TOOLS.has(t)));
|
|
1135
|
+
}
|
|
1136
|
+
|
|
959
1137
|
// src/agent/tool.ts
|
|
960
1138
|
function functionTool(fn) {
|
|
961
1139
|
return fn;
|
|
@@ -1164,6 +1342,12 @@ var ClawOpsAgent = class {
|
|
|
1164
1342
|
_recording;
|
|
1165
1343
|
_recordingPath;
|
|
1166
1344
|
_activeSessions = /* @__PURE__ */ new Map();
|
|
1345
|
+
_builtinTools;
|
|
1346
|
+
_passiveDtmfDebounceMs;
|
|
1347
|
+
_passiveDtmfBuffer = [];
|
|
1348
|
+
_passiveDtmfTimer = null;
|
|
1349
|
+
_passiveDtmfCallId = null;
|
|
1350
|
+
_callSessions = /* @__PURE__ */ new Map();
|
|
1167
1351
|
constructor(options) {
|
|
1168
1352
|
this._apiKey = options.apiKey ?? process.env["CLAWOPS_API_KEY"] ?? "";
|
|
1169
1353
|
this._accountId = options.accountId ?? process.env["CLAWOPS_ACCOUNT_ID"] ?? "";
|
|
@@ -1173,6 +1357,8 @@ var ClawOpsAgent = class {
|
|
|
1173
1357
|
this._recording = options.recording ?? false;
|
|
1174
1358
|
this._recordingPath = options.recordingPath ?? "./recordings";
|
|
1175
1359
|
this._mcpServers = options.mcpServers ?? [];
|
|
1360
|
+
this._builtinTools = resolveBuiltinTools(options.builtinTools ?? "all" /* ALL */);
|
|
1361
|
+
this._passiveDtmfDebounceMs = options.passiveDtmfDebounceMs ?? 500;
|
|
1176
1362
|
if (options.tracing) {
|
|
1177
1363
|
setTracingConfig(options.tracing);
|
|
1178
1364
|
}
|
|
@@ -1187,7 +1373,9 @@ var ClawOpsAgent = class {
|
|
|
1187
1373
|
tool(nameOrTool, description, parameters, handler) {
|
|
1188
1374
|
if (typeof nameOrTool === "string") {
|
|
1189
1375
|
if (!description || !parameters || !handler) {
|
|
1190
|
-
throw new chunk6IQN5RQD_cjs.AgentError(
|
|
1376
|
+
throw new chunk6IQN5RQD_cjs.AgentError(
|
|
1377
|
+
"tool(name, description, parameters, handler) requires all arguments."
|
|
1378
|
+
);
|
|
1191
1379
|
}
|
|
1192
1380
|
this._tools.register({
|
|
1193
1381
|
name: nameOrTool,
|
|
@@ -1223,7 +1411,9 @@ var ClawOpsAgent = class {
|
|
|
1223
1411
|
throw new chunk6IQN5RQD_cjs.AgentError("API key is required. Set CLAWOPS_API_KEY or pass apiKey option.");
|
|
1224
1412
|
}
|
|
1225
1413
|
if (!this._accountId) {
|
|
1226
|
-
throw new chunk6IQN5RQD_cjs.AgentError(
|
|
1414
|
+
throw new chunk6IQN5RQD_cjs.AgentError(
|
|
1415
|
+
"Account ID is required. Set CLAWOPS_ACCOUNT_ID or pass accountId option."
|
|
1416
|
+
);
|
|
1227
1417
|
}
|
|
1228
1418
|
this._controlWs = new ControlWebSocket({
|
|
1229
1419
|
baseUrl: this._baseUrl,
|
|
@@ -1270,6 +1460,7 @@ var ClawOpsAgent = class {
|
|
|
1270
1460
|
session._markEnded();
|
|
1271
1461
|
}
|
|
1272
1462
|
this._activeSessions.clear();
|
|
1463
|
+
this._callSessions.clear();
|
|
1273
1464
|
console.log("[ClawOpsAgent] Disconnected");
|
|
1274
1465
|
}
|
|
1275
1466
|
/**
|
|
@@ -1306,7 +1497,9 @@ var ClawOpsAgent = class {
|
|
|
1306
1497
|
}
|
|
1307
1498
|
}
|
|
1308
1499
|
this._activeSessions.set(callSession.callId, callSession);
|
|
1309
|
-
console.log(
|
|
1500
|
+
console.log(
|
|
1501
|
+
`[ClawOpsAgent] Outbound call initiated: ${this._fromNumber} -> ${to} (${callSession.callId})`
|
|
1502
|
+
);
|
|
1310
1503
|
return callSession;
|
|
1311
1504
|
}
|
|
1312
1505
|
_handleIncoming(event) {
|
|
@@ -1384,6 +1577,30 @@ var ClawOpsAgent = class {
|
|
|
1384
1577
|
this._activeSessions.delete(callId);
|
|
1385
1578
|
}
|
|
1386
1579
|
}
|
|
1580
|
+
_onDtmfEvent(callSession, digit) {
|
|
1581
|
+
callSession._emit("dtmf", digit);
|
|
1582
|
+
callSession._routeDtmf(digit);
|
|
1583
|
+
if (callSession._dtmfCollectorActive) {
|
|
1584
|
+
callSession.clearAudio();
|
|
1585
|
+
return;
|
|
1586
|
+
}
|
|
1587
|
+
this._passiveDtmfBuffer.push(digit);
|
|
1588
|
+
this._passiveDtmfCallId = callSession.callId;
|
|
1589
|
+
if (this._passiveDtmfTimer) {
|
|
1590
|
+
clearTimeout(this._passiveDtmfTimer);
|
|
1591
|
+
}
|
|
1592
|
+
this._passiveDtmfTimer = setTimeout(() => {
|
|
1593
|
+
const digits = this._passiveDtmfBuffer.join("");
|
|
1594
|
+
this._passiveDtmfBuffer = [];
|
|
1595
|
+
const sessionHandler = this._passiveDtmfCallId ? this._callSessions.get(this._passiveDtmfCallId) : null;
|
|
1596
|
+
this._passiveDtmfCallId = null;
|
|
1597
|
+
if (digits && sessionHandler && sessionHandler.feedDtmf) {
|
|
1598
|
+
sessionHandler.feedDtmf(digits).catch((err) => {
|
|
1599
|
+
console.error("[ClawOpsAgent] feedDtmf error:", err);
|
|
1600
|
+
});
|
|
1601
|
+
}
|
|
1602
|
+
}, this._passiveDtmfDebounceMs);
|
|
1603
|
+
}
|
|
1387
1604
|
async _startCallSession(session, mediaWsUrl) {
|
|
1388
1605
|
await withSpan(
|
|
1389
1606
|
"clawops.call_session",
|
|
@@ -1421,9 +1638,17 @@ var ClawOpsAgent = class {
|
|
|
1421
1638
|
() => {
|
|
1422
1639
|
mediaWs.sendClear();
|
|
1423
1640
|
},
|
|
1424
|
-
() => {
|
|
1641
|
+
async () => {
|
|
1642
|
+
await mediaWs.flush();
|
|
1643
|
+
const markName = `hangup-${Date.now()}`;
|
|
1644
|
+
mediaWs.sendMark(markName);
|
|
1645
|
+
await mediaWs.waitForMark(markName, 5e3);
|
|
1425
1646
|
mediaWs.close();
|
|
1426
|
-
}
|
|
1647
|
+
},
|
|
1648
|
+
async (digit) => {
|
|
1649
|
+
mediaWs.sendDtmf(digit);
|
|
1650
|
+
},
|
|
1651
|
+
() => mediaWs.isConnected
|
|
1427
1652
|
);
|
|
1428
1653
|
const sessionHandler = this._session;
|
|
1429
1654
|
if ("setToolRegistry" in sessionHandler && typeof sessionHandler.setToolRegistry === "function") {
|
|
@@ -1432,6 +1657,10 @@ var ClawOpsAgent = class {
|
|
|
1432
1657
|
if (recorder && "setRecorder" in sessionHandler && typeof sessionHandler.setRecorder === "function") {
|
|
1433
1658
|
sessionHandler.setRecorder(recorder);
|
|
1434
1659
|
}
|
|
1660
|
+
if ("setBuiltinTools" in sessionHandler && typeof sessionHandler.setBuiltinTools === "function") {
|
|
1661
|
+
sessionHandler.setBuiltinTools(this._builtinTools);
|
|
1662
|
+
}
|
|
1663
|
+
this._callSessions.set(session.callId, sessionHandler);
|
|
1435
1664
|
mediaWs.onAudio((ulawAudio, _timestamp) => {
|
|
1436
1665
|
if (sessionHandler) {
|
|
1437
1666
|
sessionHandler.feedAudio(ulawAudio);
|
|
@@ -1440,6 +1669,9 @@ var ClawOpsAgent = class {
|
|
|
1440
1669
|
recorder.writeInbound(ulawToPcm16(ulawAudio));
|
|
1441
1670
|
}
|
|
1442
1671
|
});
|
|
1672
|
+
mediaWs.onDtmf((digit) => {
|
|
1673
|
+
this._onDtmfEvent(session, digit);
|
|
1674
|
+
});
|
|
1443
1675
|
mediaWs.onClose(() => {
|
|
1444
1676
|
if (recorder) {
|
|
1445
1677
|
recorder.stop();
|
|
@@ -1468,6 +1700,7 @@ var ClawOpsAgent = class {
|
|
|
1468
1700
|
session._emit("call_end");
|
|
1469
1701
|
session._markEnded();
|
|
1470
1702
|
this._activeSessions.delete(session.callId);
|
|
1703
|
+
this._callSessions.delete(session.callId);
|
|
1471
1704
|
}
|
|
1472
1705
|
}
|
|
1473
1706
|
);
|
|
@@ -1482,6 +1715,32 @@ var HANG_UP_TOOL = {
|
|
|
1482
1715
|
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1483
1716
|
parameters: { type: "object", properties: {}, required: [] }
|
|
1484
1717
|
};
|
|
1718
|
+
var COLLECT_DTMF_TOOL = {
|
|
1719
|
+
type: "function",
|
|
1720
|
+
name: "collect_dtmf",
|
|
1721
|
+
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
|
|
1722
|
+
parameters: {
|
|
1723
|
+
type: "object",
|
|
1724
|
+
properties: {
|
|
1725
|
+
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
1726
|
+
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
1727
|
+
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
1728
|
+
},
|
|
1729
|
+
required: ["max_digits"]
|
|
1730
|
+
}
|
|
1731
|
+
};
|
|
1732
|
+
var SEND_DTMF_TOOL = {
|
|
1733
|
+
type: "function",
|
|
1734
|
+
name: "send_dtmf",
|
|
1735
|
+
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
1736
|
+
parameters: {
|
|
1737
|
+
type: "object",
|
|
1738
|
+
properties: {
|
|
1739
|
+
digits: { type: "string", description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30." }
|
|
1740
|
+
},
|
|
1741
|
+
required: ["digits"]
|
|
1742
|
+
}
|
|
1743
|
+
};
|
|
1485
1744
|
var OpenAIRealtime = class {
|
|
1486
1745
|
_apiKey;
|
|
1487
1746
|
_systemPrompt;
|
|
@@ -1490,6 +1749,10 @@ var OpenAIRealtime = class {
|
|
|
1490
1749
|
_language;
|
|
1491
1750
|
_eagerness;
|
|
1492
1751
|
_greeting;
|
|
1752
|
+
_builtinTools = null;
|
|
1753
|
+
setBuiltinTools(tools) {
|
|
1754
|
+
this._builtinTools = tools;
|
|
1755
|
+
}
|
|
1493
1756
|
_ws = null;
|
|
1494
1757
|
_call = null;
|
|
1495
1758
|
_tools = null;
|
|
@@ -1500,6 +1763,9 @@ var OpenAIRealtime = class {
|
|
|
1500
1763
|
_responseStartTs = null;
|
|
1501
1764
|
_sentAudioChunks = 0;
|
|
1502
1765
|
_audioRemainder = Buffer.alloc(0);
|
|
1766
|
+
// Response state tracking — prevent sending response.create while one is active
|
|
1767
|
+
_responseInProgress = false;
|
|
1768
|
+
_onResponseDone = null;
|
|
1503
1769
|
constructor(options = {}) {
|
|
1504
1770
|
this._apiKey = options.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
|
|
1505
1771
|
this._systemPrompt = options.systemPrompt ?? "";
|
|
@@ -1563,6 +1829,18 @@ var OpenAIRealtime = class {
|
|
|
1563
1829
|
});
|
|
1564
1830
|
});
|
|
1565
1831
|
}
|
|
1832
|
+
async feedDtmf(digits) {
|
|
1833
|
+
await this._waitForResponseDone();
|
|
1834
|
+
this._send({
|
|
1835
|
+
type: "conversation.item.create",
|
|
1836
|
+
item: {
|
|
1837
|
+
type: "message",
|
|
1838
|
+
role: "user",
|
|
1839
|
+
content: [{ type: "input_text", text: `[DTMF \uC785\uB825: ${digits}]` }]
|
|
1840
|
+
}
|
|
1841
|
+
});
|
|
1842
|
+
this._send({ type: "response.create" });
|
|
1843
|
+
}
|
|
1566
1844
|
feedAudio(audio) {
|
|
1567
1845
|
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1568
1846
|
this._send({
|
|
@@ -1581,7 +1859,9 @@ var OpenAIRealtime = class {
|
|
|
1581
1859
|
_sendSessionUpdate() {
|
|
1582
1860
|
if (!this._ws || this._ws.readyState !== 1) return;
|
|
1583
1861
|
const toolSchemas = this._tools ? this._tools.toOpenAITools().map((t) => ({ type: "function", ...t.function })) : [];
|
|
1584
|
-
toolSchemas.push(HANG_UP_TOOL);
|
|
1862
|
+
if (!this._builtinTools || this._builtinTools.has("hang_up" /* HANG_UP */)) toolSchemas.push(HANG_UP_TOOL);
|
|
1863
|
+
if (!this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */)) toolSchemas.push(COLLECT_DTMF_TOOL);
|
|
1864
|
+
if (!this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */)) toolSchemas.push(SEND_DTMF_TOOL);
|
|
1585
1865
|
this._send({
|
|
1586
1866
|
type: "session.update",
|
|
1587
1867
|
session: {
|
|
@@ -1648,6 +1928,19 @@ var OpenAIRealtime = class {
|
|
|
1648
1928
|
}
|
|
1649
1929
|
break;
|
|
1650
1930
|
}
|
|
1931
|
+
case "response.created": {
|
|
1932
|
+
this._responseInProgress = true;
|
|
1933
|
+
break;
|
|
1934
|
+
}
|
|
1935
|
+
case "response.done": {
|
|
1936
|
+
this._responseInProgress = false;
|
|
1937
|
+
if (this._onResponseDone) {
|
|
1938
|
+
const cb = this._onResponseDone;
|
|
1939
|
+
this._onResponseDone = null;
|
|
1940
|
+
cb();
|
|
1941
|
+
}
|
|
1942
|
+
break;
|
|
1943
|
+
}
|
|
1651
1944
|
case "error": {
|
|
1652
1945
|
console.error("[OpenAIRealtime] API error:", msg["error"]);
|
|
1653
1946
|
break;
|
|
@@ -1701,7 +1994,56 @@ var OpenAIRealtime = class {
|
|
|
1701
1994
|
const callId = item["call_id"];
|
|
1702
1995
|
if (funcName === "hang_up") {
|
|
1703
1996
|
if (this._call) {
|
|
1704
|
-
this._call.hangup();
|
|
1997
|
+
await this._call.hangup();
|
|
1998
|
+
}
|
|
1999
|
+
return;
|
|
2000
|
+
}
|
|
2001
|
+
if (funcName === "collect_dtmf") {
|
|
2002
|
+
if (this._call) {
|
|
2003
|
+
let result2;
|
|
2004
|
+
try {
|
|
2005
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2006
|
+
result2 = await this._call.collectDtmf({
|
|
2007
|
+
maxDigits: args["max_digits"] ?? 4,
|
|
2008
|
+
finishOnKey: args["finish_on_key"] ?? "#",
|
|
2009
|
+
timeout: args["timeout"] ?? 5
|
|
2010
|
+
});
|
|
2011
|
+
} catch (err) {
|
|
2012
|
+
result2 = `Error: ${err}`;
|
|
2013
|
+
}
|
|
2014
|
+
await this._waitForResponseDone();
|
|
2015
|
+
this._send({
|
|
2016
|
+
type: "conversation.item.create",
|
|
2017
|
+
item: {
|
|
2018
|
+
type: "function_call_output",
|
|
2019
|
+
call_id: callId,
|
|
2020
|
+
output: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)"
|
|
2021
|
+
}
|
|
2022
|
+
});
|
|
2023
|
+
this._send({ type: "response.create" });
|
|
2024
|
+
}
|
|
2025
|
+
return;
|
|
2026
|
+
}
|
|
2027
|
+
if (funcName === "send_dtmf") {
|
|
2028
|
+
if (this._call) {
|
|
2029
|
+
let result2;
|
|
2030
|
+
try {
|
|
2031
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2032
|
+
await this._call.sendDtmfSequence(args["digits"] ?? "");
|
|
2033
|
+
result2 = "sent";
|
|
2034
|
+
} catch (err) {
|
|
2035
|
+
result2 = `Error: ${err}`;
|
|
2036
|
+
}
|
|
2037
|
+
await this._waitForResponseDone();
|
|
2038
|
+
this._send({
|
|
2039
|
+
type: "conversation.item.create",
|
|
2040
|
+
item: {
|
|
2041
|
+
type: "function_call_output",
|
|
2042
|
+
call_id: callId,
|
|
2043
|
+
output: result2
|
|
2044
|
+
}
|
|
2045
|
+
});
|
|
2046
|
+
this._send({ type: "response.create" });
|
|
1705
2047
|
}
|
|
1706
2048
|
return;
|
|
1707
2049
|
}
|
|
@@ -1717,6 +2059,7 @@ var OpenAIRealtime = class {
|
|
|
1717
2059
|
console.error(`[OpenAIRealtime] Tool call failed: ${funcName}:`, err);
|
|
1718
2060
|
result = `Error: ${err}`;
|
|
1719
2061
|
}
|
|
2062
|
+
await this._waitForResponseDone();
|
|
1720
2063
|
this._send({
|
|
1721
2064
|
type: "conversation.item.create",
|
|
1722
2065
|
item: {
|
|
@@ -1727,6 +2070,12 @@ var OpenAIRealtime = class {
|
|
|
1727
2070
|
});
|
|
1728
2071
|
this._send({ type: "response.create" });
|
|
1729
2072
|
}
|
|
2073
|
+
_waitForResponseDone() {
|
|
2074
|
+
if (!this._responseInProgress) return Promise.resolve();
|
|
2075
|
+
return new Promise((resolve) => {
|
|
2076
|
+
this._onResponseDone = resolve;
|
|
2077
|
+
});
|
|
2078
|
+
}
|
|
1730
2079
|
_send(data) {
|
|
1731
2080
|
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
1732
2081
|
this._ws.send(JSON.stringify(data));
|
|
@@ -1740,6 +2089,33 @@ var HANG_UP_TOOL2 = {
|
|
|
1740
2089
|
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1741
2090
|
parameters: { type: "object", properties: {} }
|
|
1742
2091
|
};
|
|
2092
|
+
var COLLECT_DTMF_TOOL2 = {
|
|
2093
|
+
name: "collect_dtmf",
|
|
2094
|
+
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
|
|
2095
|
+
parameters: {
|
|
2096
|
+
type: "object",
|
|
2097
|
+
properties: {
|
|
2098
|
+
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
2099
|
+
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
2100
|
+
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
2101
|
+
},
|
|
2102
|
+
required: ["max_digits"]
|
|
2103
|
+
}
|
|
2104
|
+
};
|
|
2105
|
+
var SEND_DTMF_TOOL2 = {
|
|
2106
|
+
name: "send_dtmf",
|
|
2107
|
+
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
2108
|
+
parameters: {
|
|
2109
|
+
type: "object",
|
|
2110
|
+
properties: {
|
|
2111
|
+
digits: {
|
|
2112
|
+
type: "string",
|
|
2113
|
+
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30."
|
|
2114
|
+
}
|
|
2115
|
+
},
|
|
2116
|
+
required: ["digits"]
|
|
2117
|
+
}
|
|
2118
|
+
};
|
|
1743
2119
|
function resolveRef(ref, defs) {
|
|
1744
2120
|
const parts = ref.replace(/^#\//, "").split("/");
|
|
1745
2121
|
let result = defs;
|
|
@@ -1802,7 +2178,11 @@ function sanitizeSchemaForGemini(schema, defs, depth = 0) {
|
|
|
1802
2178
|
result["properties"] = props;
|
|
1803
2179
|
}
|
|
1804
2180
|
if (schema["items"] && typeof schema["items"] === "object" && !Array.isArray(schema["items"])) {
|
|
1805
|
-
result["items"] = sanitizeSchemaForGemini(
|
|
2181
|
+
result["items"] = sanitizeSchemaForGemini(
|
|
2182
|
+
schema["items"],
|
|
2183
|
+
defs,
|
|
2184
|
+
depth + 1
|
|
2185
|
+
);
|
|
1806
2186
|
}
|
|
1807
2187
|
if (!result["type"] && result["properties"]) result["type"] = "object";
|
|
1808
2188
|
if (result["type"] === "object" && !result["properties"]) result["properties"] = {};
|
|
@@ -1823,6 +2203,8 @@ var GeminiRealtime = class {
|
|
|
1823
2203
|
_closed = false;
|
|
1824
2204
|
_sentAudioChunks = 0;
|
|
1825
2205
|
_audioRemainder = Buffer.alloc(0);
|
|
2206
|
+
_builtinTools = null;
|
|
2207
|
+
_toolCallInProgress = false;
|
|
1826
2208
|
constructor(options = {}) {
|
|
1827
2209
|
this._apiKey = options.apiKey ?? process.env["GOOGLE_API_KEY"] ?? "";
|
|
1828
2210
|
this._systemPrompt = options.systemPrompt ?? "";
|
|
@@ -1839,6 +2221,9 @@ var GeminiRealtime = class {
|
|
|
1839
2221
|
setRecorder(recorder) {
|
|
1840
2222
|
this._recorder = recorder;
|
|
1841
2223
|
}
|
|
2224
|
+
setBuiltinTools(tools) {
|
|
2225
|
+
this._builtinTools = tools;
|
|
2226
|
+
}
|
|
1842
2227
|
async start(callSession, tools) {
|
|
1843
2228
|
this._call = callSession;
|
|
1844
2229
|
if (tools) this._tools = tools;
|
|
@@ -1877,7 +2262,10 @@ var GeminiRealtime = class {
|
|
|
1877
2262
|
onerror: (err) => {
|
|
1878
2263
|
console.error("[GeminiRealtime] SDK error:", err);
|
|
1879
2264
|
},
|
|
1880
|
-
onclose: () => {
|
|
2265
|
+
onclose: (ev) => {
|
|
2266
|
+
console.log(
|
|
2267
|
+
`[GeminiRealtime] Connection closed: code=${ev?.code ?? "unknown"}`
|
|
2268
|
+
);
|
|
1881
2269
|
this._closed = true;
|
|
1882
2270
|
}
|
|
1883
2271
|
}
|
|
@@ -1895,7 +2283,7 @@ var GeminiRealtime = class {
|
|
|
1895
2283
|
}
|
|
1896
2284
|
}
|
|
1897
2285
|
feedAudio(audio) {
|
|
1898
|
-
if (this._session && !this._closed) {
|
|
2286
|
+
if (this._session && !this._closed && !this._toolCallInProgress) {
|
|
1899
2287
|
const pcm8k = ulawToPcm16(audio);
|
|
1900
2288
|
if (this._recorder) {
|
|
1901
2289
|
this._recorder.writeInbound(pcm8k);
|
|
@@ -1909,6 +2297,14 @@ var GeminiRealtime = class {
|
|
|
1909
2297
|
});
|
|
1910
2298
|
}
|
|
1911
2299
|
}
|
|
2300
|
+
async feedDtmf(digits) {
|
|
2301
|
+
if (this._session) {
|
|
2302
|
+
this._session.sendClientContent({
|
|
2303
|
+
turns: [{ role: "user", parts: [{ text: `[DTMF \uC785\uB825: ${digits}]` }] }],
|
|
2304
|
+
turnComplete: true
|
|
2305
|
+
});
|
|
2306
|
+
}
|
|
2307
|
+
}
|
|
1912
2308
|
async stop() {
|
|
1913
2309
|
this._closed = true;
|
|
1914
2310
|
if (this._session) {
|
|
@@ -1927,7 +2323,9 @@ var GeminiRealtime = class {
|
|
|
1927
2323
|
t.function.parameters ?? { type: "object", properties: {} }
|
|
1928
2324
|
)
|
|
1929
2325
|
})) : [];
|
|
1930
|
-
toolDefs.push(HANG_UP_TOOL2);
|
|
2326
|
+
if (!this._builtinTools || this._builtinTools.has("hang_up" /* HANG_UP */)) toolDefs.push(HANG_UP_TOOL2);
|
|
2327
|
+
if (!this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */)) toolDefs.push(COLLECT_DTMF_TOOL2);
|
|
2328
|
+
if (!this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */)) toolDefs.push(SEND_DTMF_TOOL2);
|
|
1931
2329
|
return toolDefs;
|
|
1932
2330
|
}
|
|
1933
2331
|
_handleMessage(msg) {
|
|
@@ -1947,9 +2345,11 @@ var GeminiRealtime = class {
|
|
|
1947
2345
|
}
|
|
1948
2346
|
}
|
|
1949
2347
|
if (serverContent.turnComplete) {
|
|
2348
|
+
console.log("[GeminiRealtime] Turn complete");
|
|
1950
2349
|
this._flushAudioRemainder();
|
|
1951
2350
|
}
|
|
1952
2351
|
if (serverContent.interrupted) {
|
|
2352
|
+
console.log("[GeminiRealtime] Barge-in detected");
|
|
1953
2353
|
if (this._call) {
|
|
1954
2354
|
this._call.clearAudio();
|
|
1955
2355
|
}
|
|
@@ -1958,16 +2358,24 @@ var GeminiRealtime = class {
|
|
|
1958
2358
|
}
|
|
1959
2359
|
const inputText = serverContent.inputTranscription?.text;
|
|
1960
2360
|
if (inputText && this._call) {
|
|
2361
|
+
console.log(`[GeminiRealtime] [TRANSCRIPT-USER] ${inputText}`);
|
|
1961
2362
|
this._call._emit("transcript", "user", inputText);
|
|
1962
2363
|
}
|
|
1963
2364
|
const outputText = serverContent.outputTranscription?.text;
|
|
1964
2365
|
if (outputText && this._call) {
|
|
2366
|
+
console.log(`[GeminiRealtime] [TRANSCRIPT-ASSISTANT] ${outputText}`);
|
|
1965
2367
|
this._call._emit("transcript", "assistant", outputText);
|
|
1966
2368
|
}
|
|
1967
2369
|
}
|
|
1968
2370
|
if (msg.toolCall) {
|
|
1969
2371
|
this._handleToolCall(msg.toolCall);
|
|
1970
2372
|
}
|
|
2373
|
+
const toolCancellation = msg["toolCallCancellation"];
|
|
2374
|
+
if (toolCancellation) {
|
|
2375
|
+
console.log(
|
|
2376
|
+
`[GeminiRealtime] Tool call cancelled: ${(toolCancellation.ids ?? []).join(", ")}`
|
|
2377
|
+
);
|
|
2378
|
+
}
|
|
1971
2379
|
}
|
|
1972
2380
|
_handleAudioData(b64Data) {
|
|
1973
2381
|
if (!this._call) return;
|
|
@@ -1980,9 +2388,9 @@ var GeminiRealtime = class {
|
|
|
1980
2388
|
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
1981
2389
|
const chunkSize = 160;
|
|
1982
2390
|
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
1983
|
-
|
|
1984
|
-
this._call.sendAudio(combined.subarray(
|
|
1985
|
-
this._sentAudioChunks
|
|
2391
|
+
if (fullEnd > 0) {
|
|
2392
|
+
this._call.sendAudio(combined.subarray(0, fullEnd));
|
|
2393
|
+
this._sentAudioChunks += fullEnd / chunkSize;
|
|
1986
2394
|
}
|
|
1987
2395
|
this._audioRemainder = combined.subarray(fullEnd);
|
|
1988
2396
|
}
|
|
@@ -2000,17 +2408,64 @@ var GeminiRealtime = class {
|
|
|
2000
2408
|
async _handleToolCall(toolCall) {
|
|
2001
2409
|
const functionCalls = toolCall.functionCalls;
|
|
2002
2410
|
if (!functionCalls) return;
|
|
2411
|
+
this._toolCallInProgress = true;
|
|
2412
|
+
console.log(
|
|
2413
|
+
`[GeminiRealtime] toolCall: ${functionCalls.map((fc) => fc.name).join(", ")}`
|
|
2414
|
+
);
|
|
2003
2415
|
const responses = [];
|
|
2004
2416
|
for (const fc of functionCalls) {
|
|
2005
2417
|
const name = fc.name ?? "";
|
|
2006
2418
|
const fcId = fc.id ?? "";
|
|
2007
2419
|
const args = fc.args ?? {};
|
|
2420
|
+
console.log(`[GeminiRealtime] Tool call: ${name}(${JSON.stringify(args)})`);
|
|
2008
2421
|
if (name === "hang_up") {
|
|
2422
|
+
console.log("[GeminiRealtime] hang_up: ending call");
|
|
2009
2423
|
if (this._call) {
|
|
2010
|
-
this._call.hangup();
|
|
2424
|
+
await this._call.hangup();
|
|
2011
2425
|
}
|
|
2012
2426
|
return;
|
|
2013
2427
|
}
|
|
2428
|
+
if (name === "collect_dtmf") {
|
|
2429
|
+
if (this._call) {
|
|
2430
|
+
let result;
|
|
2431
|
+
try {
|
|
2432
|
+
console.log(
|
|
2433
|
+
`[GeminiRealtime] collect_dtmf: waiting for digits (maxDigits=${args["max_digits"] ?? 4}, timeout=${args["timeout"] ?? 5})`
|
|
2434
|
+
);
|
|
2435
|
+
result = await this._call.collectDtmf({
|
|
2436
|
+
maxDigits: args["max_digits"] ?? 4,
|
|
2437
|
+
finishOnKey: args["finish_on_key"] ?? "#",
|
|
2438
|
+
timeout: args["timeout"] ?? 5
|
|
2439
|
+
});
|
|
2440
|
+
console.log(`[GeminiRealtime] DTMF collected: ${result || "(empty)"}`);
|
|
2441
|
+
} catch (err) {
|
|
2442
|
+
console.error(`[GeminiRealtime] collect_dtmf error:`, err);
|
|
2443
|
+
result = `Error: ${err}`;
|
|
2444
|
+
}
|
|
2445
|
+
responses.push({
|
|
2446
|
+
id: fcId,
|
|
2447
|
+
name,
|
|
2448
|
+
response: { result: result || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)" }
|
|
2449
|
+
});
|
|
2450
|
+
}
|
|
2451
|
+
continue;
|
|
2452
|
+
}
|
|
2453
|
+
if (name === "send_dtmf") {
|
|
2454
|
+
if (this._call) {
|
|
2455
|
+
let result;
|
|
2456
|
+
try {
|
|
2457
|
+
console.log(`[GeminiRealtime] send_dtmf: digits="${args["digits"] ?? ""}"`);
|
|
2458
|
+
await this._call.sendDtmfSequence(args["digits"] ?? "");
|
|
2459
|
+
result = "sent";
|
|
2460
|
+
console.log(`[GeminiRealtime] send_dtmf: sent`);
|
|
2461
|
+
} catch (err) {
|
|
2462
|
+
console.error(`[GeminiRealtime] send_dtmf error:`, err);
|
|
2463
|
+
result = `Error: ${err}`;
|
|
2464
|
+
}
|
|
2465
|
+
responses.push({ id: fcId, name, response: { result } });
|
|
2466
|
+
}
|
|
2467
|
+
continue;
|
|
2468
|
+
}
|
|
2014
2469
|
if (!this._tools || !this._tools.has(name)) {
|
|
2015
2470
|
console.error(`[GeminiRealtime] Unknown tool: ${name}`);
|
|
2016
2471
|
responses.push({ id: fcId, name, response: { error: `Unknown tool: ${name}` } });
|
|
@@ -2018,13 +2473,15 @@ var GeminiRealtime = class {
|
|
|
2018
2473
|
}
|
|
2019
2474
|
try {
|
|
2020
2475
|
const result = await this._tools.call(name, args);
|
|
2476
|
+
const resultStr = typeof result === "string" ? result : JSON.stringify(result);
|
|
2477
|
+
console.log(`[GeminiRealtime] Tool result: ${name} -> ${resultStr.substring(0, 200)}`);
|
|
2021
2478
|
responses.push({
|
|
2022
2479
|
id: fcId,
|
|
2023
2480
|
name,
|
|
2024
|
-
response: { result:
|
|
2481
|
+
response: { result: resultStr }
|
|
2025
2482
|
});
|
|
2026
2483
|
} catch (err) {
|
|
2027
|
-
console.error(`[GeminiRealtime] Tool call
|
|
2484
|
+
console.error(`[GeminiRealtime] Tool call failed: ${name}:`, err);
|
|
2028
2485
|
responses.push({
|
|
2029
2486
|
id: fcId,
|
|
2030
2487
|
name,
|
|
@@ -2032,15 +2489,41 @@ var GeminiRealtime = class {
|
|
|
2032
2489
|
});
|
|
2033
2490
|
}
|
|
2034
2491
|
}
|
|
2035
|
-
if (this._session) {
|
|
2492
|
+
if (responses.length > 0 && this._session) {
|
|
2493
|
+
console.log(`[GeminiRealtime] Sending ${responses.length} tool response(s)`);
|
|
2036
2494
|
this._session.sendToolResponse({
|
|
2037
2495
|
functionResponses: responses
|
|
2038
2496
|
});
|
|
2039
2497
|
}
|
|
2498
|
+
this._toolCallInProgress = false;
|
|
2040
2499
|
}
|
|
2041
2500
|
};
|
|
2042
2501
|
|
|
2043
2502
|
// src/agent/pipeline/pipeline-session.ts
|
|
2503
|
+
var COLLECT_DTMF_TOOL3 = {
|
|
2504
|
+
function: {
|
|
2505
|
+
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4.",
|
|
2506
|
+
parameters: {
|
|
2507
|
+
properties: {
|
|
2508
|
+
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
2509
|
+
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
2510
|
+
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
2511
|
+
},
|
|
2512
|
+
required: ["max_digits"]
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
};
|
|
2516
|
+
var SEND_DTMF_TOOL3 = {
|
|
2517
|
+
function: {
|
|
2518
|
+
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
2519
|
+
parameters: {
|
|
2520
|
+
properties: {
|
|
2521
|
+
digits: { type: "string", description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30." }
|
|
2522
|
+
},
|
|
2523
|
+
required: ["digits"]
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
};
|
|
2044
2527
|
var PipelineSession = class {
|
|
2045
2528
|
_stt;
|
|
2046
2529
|
_llm;
|
|
@@ -2059,6 +2542,7 @@ var PipelineSession = class {
|
|
|
2059
2542
|
_audioBuffer = [];
|
|
2060
2543
|
_running = false;
|
|
2061
2544
|
_speaking = false;
|
|
2545
|
+
_builtinTools = null;
|
|
2062
2546
|
constructor(options) {
|
|
2063
2547
|
this._stt = options.stt;
|
|
2064
2548
|
this._llm = options.llm;
|
|
@@ -2079,6 +2563,9 @@ var PipelineSession = class {
|
|
|
2079
2563
|
setRecorder(recorder) {
|
|
2080
2564
|
this._recorder = recorder;
|
|
2081
2565
|
}
|
|
2566
|
+
setBuiltinTools(tools) {
|
|
2567
|
+
this._builtinTools = tools;
|
|
2568
|
+
}
|
|
2082
2569
|
async start(callSession, tools) {
|
|
2083
2570
|
this._callSession = callSession;
|
|
2084
2571
|
this._tools = tools ?? null;
|
|
@@ -2104,6 +2591,13 @@ var PipelineSession = class {
|
|
|
2104
2591
|
this._audioBuffer.push(audio);
|
|
2105
2592
|
}
|
|
2106
2593
|
}
|
|
2594
|
+
async feedDtmf(digits) {
|
|
2595
|
+
this._conversation.push({
|
|
2596
|
+
role: "user",
|
|
2597
|
+
content: `[DTMF \uC785\uB825: ${digits}]`
|
|
2598
|
+
});
|
|
2599
|
+
await this._respond();
|
|
2600
|
+
}
|
|
2107
2601
|
async stop() {
|
|
2108
2602
|
this._running = false;
|
|
2109
2603
|
this._audioBuffer = [];
|
|
@@ -2145,11 +2639,37 @@ var PipelineSession = class {
|
|
|
2145
2639
|
this._conversation.push({ role: "user", content: transcript });
|
|
2146
2640
|
await this._respond();
|
|
2147
2641
|
}
|
|
2642
|
+
_buildEffectiveTools() {
|
|
2643
|
+
const includeCollectDtmf = !this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */);
|
|
2644
|
+
const includeSendDtmf = !this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */);
|
|
2645
|
+
if (!includeCollectDtmf && !includeSendDtmf) return this._tools ?? void 0;
|
|
2646
|
+
const base = this._tools ? this._tools.fork() : new ToolRegistry();
|
|
2647
|
+
if (includeCollectDtmf) {
|
|
2648
|
+
base.register({
|
|
2649
|
+
name: "collect_dtmf",
|
|
2650
|
+
description: COLLECT_DTMF_TOOL3.function.description,
|
|
2651
|
+
parameters: COLLECT_DTMF_TOOL3.function.parameters.properties,
|
|
2652
|
+
required: COLLECT_DTMF_TOOL3.function.parameters.required,
|
|
2653
|
+
handler: async () => ""
|
|
2654
|
+
});
|
|
2655
|
+
}
|
|
2656
|
+
if (includeSendDtmf) {
|
|
2657
|
+
base.register({
|
|
2658
|
+
name: "send_dtmf",
|
|
2659
|
+
description: SEND_DTMF_TOOL3.function.description,
|
|
2660
|
+
parameters: SEND_DTMF_TOOL3.function.parameters.properties,
|
|
2661
|
+
required: SEND_DTMF_TOOL3.function.parameters.required,
|
|
2662
|
+
handler: async () => ""
|
|
2663
|
+
});
|
|
2664
|
+
}
|
|
2665
|
+
return base;
|
|
2666
|
+
}
|
|
2148
2667
|
async _respond() {
|
|
2149
2668
|
let fullResponse = "";
|
|
2150
2669
|
const textChunks = [];
|
|
2670
|
+
const effectiveTools = this._buildEffectiveTools();
|
|
2151
2671
|
const llmStream = this._llm.generate(this._conversation, {
|
|
2152
|
-
tools:
|
|
2672
|
+
tools: effectiveTools,
|
|
2153
2673
|
temperature: this._temperature,
|
|
2154
2674
|
maxTokens: this._maxTokens
|
|
2155
2675
|
});
|
|
@@ -2168,10 +2688,38 @@ var PipelineSession = class {
|
|
|
2168
2688
|
}
|
|
2169
2689
|
}
|
|
2170
2690
|
async _handleToolCall(chunk) {
|
|
2171
|
-
if (!chunk.toolCall
|
|
2691
|
+
if (!chunk.toolCall) return;
|
|
2172
2692
|
const { id, name, arguments: argsStr } = chunk.toolCall;
|
|
2173
2693
|
try {
|
|
2174
2694
|
const args = JSON.parse(argsStr);
|
|
2695
|
+
if (name === "collect_dtmf" && this._callSession) {
|
|
2696
|
+
let result2;
|
|
2697
|
+
try {
|
|
2698
|
+
result2 = await this._callSession.collectDtmf({
|
|
2699
|
+
maxDigits: args["max_digits"] ?? 4,
|
|
2700
|
+
finishOnKey: args["finish_on_key"] ?? "#",
|
|
2701
|
+
timeout: args["timeout"] ?? 5
|
|
2702
|
+
});
|
|
2703
|
+
} catch (err) {
|
|
2704
|
+
result2 = `Error: ${err}`;
|
|
2705
|
+
}
|
|
2706
|
+
this._conversation.push({ role: "tool", content: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)", tool_call_id: id, name });
|
|
2707
|
+
await this._respond();
|
|
2708
|
+
return;
|
|
2709
|
+
}
|
|
2710
|
+
if (name === "send_dtmf" && this._callSession) {
|
|
2711
|
+
let result2;
|
|
2712
|
+
try {
|
|
2713
|
+
await this._callSession.sendDtmfSequence(args["digits"] ?? "");
|
|
2714
|
+
result2 = "sent";
|
|
2715
|
+
} catch (err) {
|
|
2716
|
+
result2 = `Error: ${err}`;
|
|
2717
|
+
}
|
|
2718
|
+
this._conversation.push({ role: "tool", content: result2, tool_call_id: id, name });
|
|
2719
|
+
await this._respond();
|
|
2720
|
+
return;
|
|
2721
|
+
}
|
|
2722
|
+
if (!this._tools) return;
|
|
2175
2723
|
const result = await this._tools.call(name, args);
|
|
2176
2724
|
this._conversation.push({
|
|
2177
2725
|
role: "assistant",
|
|
@@ -2184,9 +2732,10 @@ var PipelineSession = class {
|
|
|
2184
2732
|
tool_call_id: id,
|
|
2185
2733
|
name
|
|
2186
2734
|
});
|
|
2735
|
+
const effectiveTools = this._buildEffectiveTools();
|
|
2187
2736
|
let followUpText = "";
|
|
2188
2737
|
const followUpStream = this._llm.generate(this._conversation, {
|
|
2189
|
-
tools:
|
|
2738
|
+
tools: effectiveTools,
|
|
2190
2739
|
temperature: this._temperature,
|
|
2191
2740
|
maxTokens: this._maxTokens
|
|
2192
2741
|
});
|
|
@@ -3012,6 +3561,7 @@ function mcpServerHTTP(options) {
|
|
|
3012
3561
|
|
|
3013
3562
|
exports.AnthropicLLM = AnthropicLLM;
|
|
3014
3563
|
exports.AudioRecorder = AudioRecorder;
|
|
3564
|
+
exports.BuiltinTool = BuiltinTool;
|
|
3015
3565
|
exports.CallSession = CallSession;
|
|
3016
3566
|
exports.ClawOpsAgent = ClawOpsAgent;
|
|
3017
3567
|
exports.DECODE_TABLE = DECODE_TABLE;
|