@teamlearners/clawops 0.5.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/index.cjs +1103 -919
- package/dist/agent/index.cjs.map +1 -1
- package/dist/agent/index.d.cts +172 -74
- package/dist/agent/index.d.ts +172 -74
- package/dist/agent/index.js +1093 -914
- package/dist/agent/index.js.map +1 -1
- package/dist/{chunk-7S2OEBS6.js → chunk-2B74QZZJ.js} +6 -3
- package/dist/chunk-2B74QZZJ.js.map +1 -0
- package/dist/{chunk-6IQN5RQD.cjs → chunk-R742GPVL.cjs} +6 -2
- package/dist/chunk-R742GPVL.cjs.map +1 -0
- package/dist/index.cjs +38 -38
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +3 -6
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/dist/chunk-6IQN5RQD.cjs.map +0 -1
- package/dist/chunk-7S2OEBS6.js.map +0 -1
package/dist/agent/index.cjs
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var
|
|
3
|
+
var chunkR742GPVL_cjs = require('../chunk-R742GPVL.cjs');
|
|
4
4
|
var pino = require('pino');
|
|
5
5
|
var fs = require('fs');
|
|
6
6
|
var path = require('path');
|
|
7
|
+
var os = require('os');
|
|
7
8
|
|
|
8
9
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
9
10
|
|
|
@@ -28,6 +29,7 @@ function _interopNamespace(e) {
|
|
|
28
29
|
var pino__default = /*#__PURE__*/_interopDefault(pino);
|
|
29
30
|
var fs__namespace = /*#__PURE__*/_interopNamespace(fs);
|
|
30
31
|
var path__namespace = /*#__PURE__*/_interopNamespace(path);
|
|
32
|
+
var os__default = /*#__PURE__*/_interopDefault(os);
|
|
31
33
|
|
|
32
34
|
// src/agent/audio.ts
|
|
33
35
|
var _BIAS = 132;
|
|
@@ -998,6 +1000,36 @@ var AudioRecorder = class {
|
|
|
998
1000
|
}
|
|
999
1001
|
}
|
|
1000
1002
|
};
|
|
1003
|
+
var MAX_ERRORS = 20;
|
|
1004
|
+
var MAX_ERROR_MESSAGE_LENGTH = 200;
|
|
1005
|
+
function getSdkInfo() {
|
|
1006
|
+
return {
|
|
1007
|
+
name: "clawops-node",
|
|
1008
|
+
version: chunkR742GPVL_cjs.VERSION,
|
|
1009
|
+
runtime: `node/${process.versions.node}`,
|
|
1010
|
+
os: `${process.platform}/${os__default.default.arch()}`
|
|
1011
|
+
};
|
|
1012
|
+
}
|
|
1013
|
+
function createCallMetrics() {
|
|
1014
|
+
return {
|
|
1015
|
+
firstResponseMs: null,
|
|
1016
|
+
turnCount: 0,
|
|
1017
|
+
toolCallCount: 0,
|
|
1018
|
+
toolErrorCount: 0,
|
|
1019
|
+
bargeInCount: 0,
|
|
1020
|
+
endReason: null,
|
|
1021
|
+
errors: []
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
function addMetricError(metrics, err) {
|
|
1025
|
+
metrics.toolErrorCount++;
|
|
1026
|
+
if (metrics.errors.length < MAX_ERRORS) {
|
|
1027
|
+
metrics.errors.push({
|
|
1028
|
+
type: err.name || "Error",
|
|
1029
|
+
message: (err.message || "").slice(0, MAX_ERROR_MESSAGE_LENGTH)
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1001
1033
|
|
|
1002
1034
|
// src/agent/session.ts
|
|
1003
1035
|
var CallSession = class {
|
|
@@ -1023,6 +1055,8 @@ var CallSession = class {
|
|
|
1023
1055
|
_handlers = /* @__PURE__ */ new Map();
|
|
1024
1056
|
_endedPromise;
|
|
1025
1057
|
_resolveEnded;
|
|
1058
|
+
_metrics = createCallMetrics();
|
|
1059
|
+
_firstResponseSent = false;
|
|
1026
1060
|
constructor(options) {
|
|
1027
1061
|
this.callId = options.callId;
|
|
1028
1062
|
this.fromNumber = options.fromNumber;
|
|
@@ -1045,6 +1079,30 @@ var CallSession = class {
|
|
|
1045
1079
|
get duration() {
|
|
1046
1080
|
return (Date.now() - this.startTime.getTime()) / 1e3;
|
|
1047
1081
|
}
|
|
1082
|
+
get metrics() {
|
|
1083
|
+
return this._metrics;
|
|
1084
|
+
}
|
|
1085
|
+
recordFirstResponse() {
|
|
1086
|
+
if (!this._firstResponseSent) {
|
|
1087
|
+
this._firstResponseSent = true;
|
|
1088
|
+
this._metrics.firstResponseMs = Date.now() - this.startTime.getTime();
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
recordTurn() {
|
|
1092
|
+
this._metrics.turnCount++;
|
|
1093
|
+
}
|
|
1094
|
+
recordToolCall() {
|
|
1095
|
+
this._metrics.toolCallCount++;
|
|
1096
|
+
}
|
|
1097
|
+
recordToolError(err) {
|
|
1098
|
+
addMetricError(this._metrics, err);
|
|
1099
|
+
}
|
|
1100
|
+
recordBargeIn() {
|
|
1101
|
+
this._metrics.bargeInCount++;
|
|
1102
|
+
}
|
|
1103
|
+
recordEndReason(reason) {
|
|
1104
|
+
this._metrics.endReason = reason;
|
|
1105
|
+
}
|
|
1048
1106
|
/** Bind transport functions (called internally by the agent). */
|
|
1049
1107
|
_bindTransport(send, clear, hangup, sendDtmf, isConnected) {
|
|
1050
1108
|
this._sendAudioFn = send;
|
|
@@ -1426,7 +1484,7 @@ var ClawOpsAgent = class {
|
|
|
1426
1484
|
constructor(options) {
|
|
1427
1485
|
this._apiKey = options.apiKey ?? process.env["CLAWOPS_API_KEY"] ?? "";
|
|
1428
1486
|
this._accountId = options.accountId ?? process.env["CLAWOPS_ACCOUNT_ID"] ?? "";
|
|
1429
|
-
this._baseUrl = options.baseUrl ??
|
|
1487
|
+
this._baseUrl = options.baseUrl ?? chunkR742GPVL_cjs.DEFAULT_BASE_URL;
|
|
1430
1488
|
this._fromNumber = options.from;
|
|
1431
1489
|
this._session = options.session;
|
|
1432
1490
|
this._recording = options.recording ?? false;
|
|
@@ -1451,7 +1509,7 @@ var ClawOpsAgent = class {
|
|
|
1451
1509
|
tool(nameOrTool, description, parameters, handler) {
|
|
1452
1510
|
if (typeof nameOrTool === "string") {
|
|
1453
1511
|
if (!description || !parameters || !handler) {
|
|
1454
|
-
throw new
|
|
1512
|
+
throw new chunkR742GPVL_cjs.AgentError(
|
|
1455
1513
|
"tool(name, description, parameters, handler) requires all arguments."
|
|
1456
1514
|
);
|
|
1457
1515
|
}
|
|
@@ -1485,11 +1543,12 @@ var ClawOpsAgent = class {
|
|
|
1485
1543
|
}
|
|
1486
1544
|
/** Connect to the ClawOps platform and start listening for calls. */
|
|
1487
1545
|
async connect() {
|
|
1546
|
+
if (this._controlWs) return;
|
|
1488
1547
|
if (!this._apiKey) {
|
|
1489
|
-
throw new
|
|
1548
|
+
throw new chunkR742GPVL_cjs.AgentError("API key is required. Set CLAWOPS_API_KEY or pass apiKey option.");
|
|
1490
1549
|
}
|
|
1491
1550
|
if (!this._accountId) {
|
|
1492
|
-
throw new
|
|
1551
|
+
throw new chunkR742GPVL_cjs.AgentError(
|
|
1493
1552
|
"Account ID is required. Set CLAWOPS_ACCOUNT_ID or pass accountId option."
|
|
1494
1553
|
);
|
|
1495
1554
|
}
|
|
@@ -1508,8 +1567,12 @@ var ClawOpsAgent = class {
|
|
|
1508
1567
|
try {
|
|
1509
1568
|
await this._controlWs.connect();
|
|
1510
1569
|
await this._controlWs.waitConnected();
|
|
1570
|
+
try {
|
|
1571
|
+
this._controlWs.send({ event: "agent.hello", sdk: getSdkInfo() });
|
|
1572
|
+
} catch {
|
|
1573
|
+
}
|
|
1511
1574
|
} catch (err) {
|
|
1512
|
-
throw new
|
|
1575
|
+
throw new chunkR742GPVL_cjs.AgentConnectionError(
|
|
1513
1576
|
`Failed to connect to ClawOps: ${err instanceof Error ? err.message : String(err)}`
|
|
1514
1577
|
);
|
|
1515
1578
|
}
|
|
@@ -1560,7 +1623,7 @@ var ClawOpsAgent = class {
|
|
|
1560
1623
|
});
|
|
1561
1624
|
if (resp.status !== 201) {
|
|
1562
1625
|
const error = await resp.json();
|
|
1563
|
-
throw new
|
|
1626
|
+
throw new chunkR742GPVL_cjs.AgentError(`\uBC1C\uC2E0 \uC2E4\uD328 (${resp.status}): ${error["error"] ?? ""}`);
|
|
1564
1627
|
}
|
|
1565
1628
|
const data = await resp.json();
|
|
1566
1629
|
const callSession = new CallSession({
|
|
@@ -1721,9 +1784,11 @@ var ClawOpsAgent = class {
|
|
|
1721
1784
|
session._bindTransport(
|
|
1722
1785
|
(audio) => {
|
|
1723
1786
|
mediaWs.sendAudio(audio.toString("base64"));
|
|
1787
|
+
session.recordFirstResponse();
|
|
1724
1788
|
},
|
|
1725
1789
|
() => {
|
|
1726
1790
|
mediaWs.sendClear();
|
|
1791
|
+
session.recordBargeIn();
|
|
1727
1792
|
},
|
|
1728
1793
|
async () => {
|
|
1729
1794
|
await mediaWs.flush();
|
|
@@ -1774,10 +1839,22 @@ var ClawOpsAgent = class {
|
|
|
1774
1839
|
await mediaWs.connect(mediaWsUrl, this._apiKey);
|
|
1775
1840
|
this._log.info("Media stream started: %s", session.callId);
|
|
1776
1841
|
await sessionHandler.start(session, sessionTools);
|
|
1842
|
+
const telemetry = sessionHandler.getTelemetry?.() ?? null;
|
|
1843
|
+
if (telemetry) {
|
|
1844
|
+
telemetry.toolCount = sessionTools?.size ?? 0;
|
|
1845
|
+
telemetry.mcpServerCount = this._mcpServers?.length ?? 0;
|
|
1846
|
+
telemetry.builtinTools = this._builtinTools ? [...this._builtinTools].map((t) => t.toString()) : [];
|
|
1847
|
+
telemetry.recordingEnabled = this._recording;
|
|
1848
|
+
try {
|
|
1849
|
+
this._controlWs.send({ event: "call.telemetry", callId: session.callId, telemetry });
|
|
1850
|
+
} catch {
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1777
1853
|
await session.wait();
|
|
1778
1854
|
await sessionHandler.stop();
|
|
1779
1855
|
} catch (err) {
|
|
1780
1856
|
this._log.error({ err }, "Call session error: %s", session.callId);
|
|
1857
|
+
session.recordEndReason("error");
|
|
1781
1858
|
} finally {
|
|
1782
1859
|
if (mcpClients.length > 0) {
|
|
1783
1860
|
sessionTools.clearMcpTools();
|
|
@@ -1789,6 +1866,13 @@ var ClawOpsAgent = class {
|
|
|
1789
1866
|
if (recorder) {
|
|
1790
1867
|
recorder.stop();
|
|
1791
1868
|
}
|
|
1869
|
+
if (!session.metrics.endReason) {
|
|
1870
|
+
session.recordEndReason(session.status === "ended" ? "user_hangup" : "agent_hangup");
|
|
1871
|
+
}
|
|
1872
|
+
try {
|
|
1873
|
+
this._controlWs?.send({ event: "call.metrics", callId: session.callId, metrics: session.metrics });
|
|
1874
|
+
} catch {
|
|
1875
|
+
}
|
|
1792
1876
|
session._emit("call_end");
|
|
1793
1877
|
session._markEnded();
|
|
1794
1878
|
this._activeSessions.delete(session.callId);
|
|
@@ -1799,20 +1883,13 @@ var ClawOpsAgent = class {
|
|
|
1799
1883
|
}
|
|
1800
1884
|
};
|
|
1801
1885
|
|
|
1802
|
-
// src/agent/pipeline/
|
|
1803
|
-
var
|
|
1804
|
-
var HANG_UP_TOOL = {
|
|
1805
|
-
type: "function",
|
|
1886
|
+
// src/agent/pipeline/builtin-tool-schemas.ts
|
|
1887
|
+
var HANG_UP = {
|
|
1806
1888
|
name: "hang_up",
|
|
1807
1889
|
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1808
|
-
parameters: {
|
|
1809
|
-
type: "object",
|
|
1810
|
-
properties: {},
|
|
1811
|
-
required: []
|
|
1812
|
-
}
|
|
1890
|
+
parameters: { type: "object", properties: {} }
|
|
1813
1891
|
};
|
|
1814
|
-
var
|
|
1815
|
-
type: "function",
|
|
1892
|
+
var COLLECT_DTMF = {
|
|
1816
1893
|
name: "collect_dtmf",
|
|
1817
1894
|
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
|
|
1818
1895
|
parameters: {
|
|
@@ -1825,8 +1902,7 @@ var COLLECT_DTMF_TOOL = {
|
|
|
1825
1902
|
required: ["max_digits"]
|
|
1826
1903
|
}
|
|
1827
1904
|
};
|
|
1828
|
-
var
|
|
1829
|
-
type: "function",
|
|
1905
|
+
var SEND_DTMF = {
|
|
1830
1906
|
name: "send_dtmf",
|
|
1831
1907
|
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
1832
1908
|
parameters: {
|
|
@@ -1834,492 +1910,419 @@ var SEND_DTMF_TOOL = {
|
|
|
1834
1910
|
properties: {
|
|
1835
1911
|
digits: {
|
|
1836
1912
|
type: "string",
|
|
1837
|
-
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30."
|
|
1913
|
+
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30. \uC608: '1', '1234#', '1w2'"
|
|
1838
1914
|
}
|
|
1839
1915
|
},
|
|
1840
1916
|
required: ["digits"]
|
|
1841
1917
|
}
|
|
1842
1918
|
};
|
|
1843
|
-
var
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1919
|
+
var TOOL_MAP = /* @__PURE__ */ new Map([
|
|
1920
|
+
["hang_up" /* HANG_UP */, HANG_UP],
|
|
1921
|
+
["collect_dtmf" /* COLLECT_DTMF */, COLLECT_DTMF],
|
|
1922
|
+
["send_dtmf" /* SEND_DTMF */, SEND_DTMF]
|
|
1923
|
+
]);
|
|
1924
|
+
var BUILTIN_TOOL_NAMES = new Set(
|
|
1925
|
+
Array.from(TOOL_MAP.values()).map((s) => s.name)
|
|
1926
|
+
);
|
|
1927
|
+
function toChatCompletions(schema) {
|
|
1928
|
+
return {
|
|
1929
|
+
type: "function",
|
|
1930
|
+
function: {
|
|
1931
|
+
name: schema.name,
|
|
1932
|
+
description: schema.description,
|
|
1933
|
+
parameters: schema.parameters
|
|
1934
|
+
}
|
|
1935
|
+
};
|
|
1936
|
+
}
|
|
1937
|
+
function toRealtime(schema) {
|
|
1938
|
+
return {
|
|
1939
|
+
type: "function",
|
|
1940
|
+
name: schema.name,
|
|
1941
|
+
description: schema.description,
|
|
1942
|
+
parameters: schema.parameters
|
|
1943
|
+
};
|
|
1944
|
+
}
|
|
1945
|
+
function toGemini(schema) {
|
|
1946
|
+
return {
|
|
1947
|
+
name: schema.name,
|
|
1948
|
+
description: schema.description,
|
|
1949
|
+
parameters: schema.parameters
|
|
1950
|
+
};
|
|
1951
|
+
}
|
|
1952
|
+
var CONVERTERS = {
|
|
1953
|
+
chat: toChatCompletions,
|
|
1954
|
+
realtime: toRealtime,
|
|
1955
|
+
gemini: toGemini
|
|
1956
|
+
};
|
|
1957
|
+
function getBuiltinToolSchemas(builtinTools, fmt) {
|
|
1958
|
+
const converter = CONVERTERS[fmt];
|
|
1959
|
+
const result = [];
|
|
1960
|
+
for (const [toolEnum, schema] of TOOL_MAP) {
|
|
1961
|
+
if (builtinTools === null || builtinTools.has(toolEnum)) {
|
|
1962
|
+
result.push(converter(schema));
|
|
1963
|
+
}
|
|
1855
1964
|
}
|
|
1856
|
-
|
|
1857
|
-
|
|
1965
|
+
return result;
|
|
1966
|
+
}
|
|
1967
|
+
function isBuiltinTool(name) {
|
|
1968
|
+
return BUILTIN_TOOL_NAMES.has(name);
|
|
1969
|
+
}
|
|
1970
|
+
async function executeBuiltinTool(funcName, args, call) {
|
|
1971
|
+
if (funcName === "hang_up") {
|
|
1972
|
+
await call.hangup();
|
|
1973
|
+
return "";
|
|
1858
1974
|
}
|
|
1859
|
-
|
|
1860
|
-
|
|
1975
|
+
if (funcName === "collect_dtmf") {
|
|
1976
|
+
try {
|
|
1977
|
+
const result = await call.collectDtmf({
|
|
1978
|
+
maxDigits: args["max_digits"] ?? 4,
|
|
1979
|
+
finishOnKey: args["finish_on_key"] ?? "#",
|
|
1980
|
+
timeout: args["timeout"] ?? 5
|
|
1981
|
+
});
|
|
1982
|
+
return result || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)";
|
|
1983
|
+
} catch (e) {
|
|
1984
|
+
return `Error: ${e}`;
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
if (funcName === "send_dtmf") {
|
|
1988
|
+
try {
|
|
1989
|
+
await call.sendDtmfSequence(args["digits"] ?? "");
|
|
1990
|
+
return "sent";
|
|
1991
|
+
} catch (e) {
|
|
1992
|
+
return `Error: ${e}`;
|
|
1993
|
+
}
|
|
1994
|
+
}
|
|
1995
|
+
return null;
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
// src/agent/pipeline/pipeline-session.ts
|
|
1999
|
+
var PipelineSession = class {
|
|
2000
|
+
_stt;
|
|
2001
|
+
_llm;
|
|
2002
|
+
_tts;
|
|
2003
|
+
_systemPrompt;
|
|
2004
|
+
_greeting;
|
|
2005
|
+
_language;
|
|
2006
|
+
_temperature;
|
|
2007
|
+
_maxTokens;
|
|
2008
|
+
_sampleRate;
|
|
2009
|
+
_interruptOnSpeech;
|
|
2010
|
+
_callSession = null;
|
|
1861
2011
|
_tools = null;
|
|
1862
2012
|
_recorder = null;
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
this.
|
|
1874
|
-
this._systemPrompt = options.systemPrompt ?? "";
|
|
1875
|
-
this._model = options.model ?? "gpt-realtime-1.5";
|
|
1876
|
-
this._voice = options.voice ?? "marin";
|
|
1877
|
-
this._language = options.language ?? "ko";
|
|
1878
|
-
this._eagerness = options.eagerness ?? "high";
|
|
2013
|
+
_conversation = [];
|
|
2014
|
+
_audioBuffer = [];
|
|
2015
|
+
_running = false;
|
|
2016
|
+
_speaking = false;
|
|
2017
|
+
_builtinTools = null;
|
|
2018
|
+
_log = NOOP_LOGGER;
|
|
2019
|
+
constructor(options) {
|
|
2020
|
+
this._stt = options.stt;
|
|
2021
|
+
this._llm = options.llm;
|
|
2022
|
+
this._tts = options.tts;
|
|
2023
|
+
this._systemPrompt = options.systemPrompt;
|
|
1879
2024
|
this._greeting = options.greeting ?? true;
|
|
2025
|
+
this._language = options.language ?? "ko";
|
|
2026
|
+
this._temperature = options.temperature;
|
|
2027
|
+
this._maxTokens = options.maxTokens;
|
|
2028
|
+
this._sampleRate = options.sampleRate ?? 8e3;
|
|
2029
|
+
this._interruptOnSpeech = options.interruptOnSpeech ?? true;
|
|
2030
|
+
if (options.toolRegistry) this._tools = options.toolRegistry;
|
|
2031
|
+
if (options.recorder) this._recorder = options.recorder;
|
|
1880
2032
|
}
|
|
1881
|
-
/** Inject per-call ToolRegistry. */
|
|
1882
2033
|
setToolRegistry(registry) {
|
|
1883
2034
|
this._tools = registry;
|
|
1884
2035
|
}
|
|
1885
|
-
/** Inject per-call AudioRecorder. */
|
|
1886
2036
|
setRecorder(recorder) {
|
|
1887
2037
|
this._recorder = recorder;
|
|
1888
2038
|
}
|
|
1889
|
-
|
|
1890
|
-
this.
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
2039
|
+
setBuiltinTools(tools) {
|
|
2040
|
+
this._builtinTools = tools;
|
|
2041
|
+
}
|
|
2042
|
+
getTelemetry() {
|
|
2043
|
+
const llm = this._llm;
|
|
2044
|
+
const stt = this._stt;
|
|
2045
|
+
const tts = this._tts;
|
|
2046
|
+
return {
|
|
2047
|
+
sessionType: "pipeline",
|
|
2048
|
+
llm: llm.provider && llm.model ? { provider: llm.provider, model: llm.model } : null,
|
|
2049
|
+
stt: stt.provider && stt.model ? { provider: stt.provider, model: stt.model } : null,
|
|
2050
|
+
tts: tts.provider && tts.model ? { provider: tts.provider, model: tts.model } : null,
|
|
2051
|
+
voice: tts.voiceId ?? null,
|
|
2052
|
+
language: this._language,
|
|
2053
|
+
greetingEnabled: this._greeting,
|
|
2054
|
+
recordingEnabled: !!this._recorder,
|
|
2055
|
+
toolCount: this._tools?.size ?? 0,
|
|
2056
|
+
mcpServerCount: 0,
|
|
2057
|
+
builtinTools: []
|
|
2058
|
+
};
|
|
2059
|
+
}
|
|
2060
|
+
setLogger(logger) {
|
|
2061
|
+
this._log = logger;
|
|
2062
|
+
if ("setLogger" in this._stt && typeof this._stt.setLogger === "function") {
|
|
2063
|
+
this._stt.setLogger(logger);
|
|
1899
2064
|
}
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
this._send({ type: "response.create" });
|
|
1915
|
-
}
|
|
1916
|
-
resolve();
|
|
1917
|
-
});
|
|
1918
|
-
ws.on("message", (data) => {
|
|
1919
|
-
try {
|
|
1920
|
-
const msg = JSON.parse(data.toString());
|
|
1921
|
-
this._handleMessage(msg);
|
|
1922
|
-
} catch {
|
|
1923
|
-
}
|
|
1924
|
-
});
|
|
1925
|
-
ws.on("close", () => {
|
|
1926
|
-
this._closed = true;
|
|
2065
|
+
if ("setLogger" in this._tts && typeof this._tts.setLogger === "function") {
|
|
2066
|
+
this._tts.setLogger(logger);
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
async start(callSession, tools) {
|
|
2070
|
+
this._callSession = callSession;
|
|
2071
|
+
this._tools = tools ?? null;
|
|
2072
|
+
this._running = true;
|
|
2073
|
+
this._log.info("PipelineSession started");
|
|
2074
|
+
this._conversation = [];
|
|
2075
|
+
if (this._systemPrompt) {
|
|
2076
|
+
this._conversation.push({
|
|
2077
|
+
role: "system",
|
|
2078
|
+
content: this._systemPrompt
|
|
1927
2079
|
});
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
}
|
|
1932
|
-
this._log.error({ err }, "OpenAI Realtime WS error");
|
|
2080
|
+
}
|
|
2081
|
+
if (this._greeting) {
|
|
2082
|
+
this._generateGreeting().catch((err) => {
|
|
2083
|
+
this._log.error({ err }, "Greeting error");
|
|
1933
2084
|
});
|
|
2085
|
+
}
|
|
2086
|
+
this._runSttLoop().catch((err) => {
|
|
2087
|
+
this._log.error({ err }, "STT loop error");
|
|
1934
2088
|
});
|
|
1935
2089
|
}
|
|
1936
|
-
async feedDtmf(digits) {
|
|
1937
|
-
await this._waitForResponseDone();
|
|
1938
|
-
this._send({
|
|
1939
|
-
type: "conversation.item.create",
|
|
1940
|
-
item: {
|
|
1941
|
-
type: "message",
|
|
1942
|
-
role: "user",
|
|
1943
|
-
content: [{ type: "input_text", text: `[DTMF \uC785\uB825: ${digits}]` }]
|
|
1944
|
-
}
|
|
1945
|
-
});
|
|
1946
|
-
this._send({ type: "response.create" });
|
|
1947
|
-
}
|
|
1948
2090
|
feedAudio(audio) {
|
|
1949
|
-
if (this.
|
|
1950
|
-
this.
|
|
1951
|
-
type: "input_audio_buffer.append",
|
|
1952
|
-
audio: audio.toString("base64")
|
|
1953
|
-
});
|
|
2091
|
+
if (this._running) {
|
|
2092
|
+
this._audioBuffer.push(audio);
|
|
1954
2093
|
}
|
|
1955
2094
|
}
|
|
2095
|
+
async feedDtmf(digits) {
|
|
2096
|
+
this._conversation.push({
|
|
2097
|
+
role: "user",
|
|
2098
|
+
content: `[DTMF \uC785\uB825: ${digits}]`
|
|
2099
|
+
});
|
|
2100
|
+
await this._respond();
|
|
2101
|
+
}
|
|
1956
2102
|
async stop() {
|
|
1957
|
-
this.
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
this._ws = null;
|
|
1961
|
-
}
|
|
2103
|
+
this._running = false;
|
|
2104
|
+
this._log.info("PipelineSession stopped");
|
|
2105
|
+
this._audioBuffer = [];
|
|
1962
2106
|
}
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
modalities: ["text", "audio"],
|
|
1976
|
-
voice: this._voice,
|
|
1977
|
-
instructions: this._systemPrompt,
|
|
1978
|
-
input_audio_format: "g711_ulaw",
|
|
1979
|
-
output_audio_format: "g711_ulaw",
|
|
1980
|
-
input_audio_transcription: {
|
|
1981
|
-
model: "whisper-1",
|
|
1982
|
-
language: this._language
|
|
1983
|
-
},
|
|
1984
|
-
input_audio_noise_reduction: { type: "far_field" },
|
|
1985
|
-
turn_detection: {
|
|
1986
|
-
type: "semantic_vad",
|
|
1987
|
-
interrupt_response: true,
|
|
1988
|
-
eagerness: this._eagerness
|
|
1989
|
-
},
|
|
1990
|
-
tools: toolSchemas
|
|
2107
|
+
async _runSttLoop() {
|
|
2108
|
+
const audioStream = this._createAudioStream();
|
|
2109
|
+
for await (const event of this._stt.transcribe(audioStream, {
|
|
2110
|
+
sampleRate: this._sampleRate
|
|
2111
|
+
})) {
|
|
2112
|
+
if (!this._running) break;
|
|
2113
|
+
if (event.type === "interim" && this._speaking && this._interruptOnSpeech) {
|
|
2114
|
+
this._speaking = false;
|
|
2115
|
+
if (this._callSession) {
|
|
2116
|
+
this._callSession.clearAudio();
|
|
2117
|
+
}
|
|
2118
|
+
this._log.info('Barge-in: "%s"', event.transcript.substring(0, 30));
|
|
1991
2119
|
}
|
|
1992
|
-
|
|
2120
|
+
if (event.type === "final" && event.transcript.trim()) {
|
|
2121
|
+
this._log.info("STT: %s", event.transcript);
|
|
2122
|
+
await this._handleUserSpeech(event.transcript);
|
|
2123
|
+
}
|
|
2124
|
+
}
|
|
1993
2125
|
}
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
const padded = Buffer.concat([
|
|
2004
|
-
this._audioRemainder,
|
|
2005
|
-
Buffer.alloc(160 - this._audioRemainder.length, 255)
|
|
2006
|
-
]);
|
|
2007
|
-
if (this._call) {
|
|
2008
|
-
this._call.sendAudio(padded);
|
|
2009
|
-
}
|
|
2010
|
-
this._sentAudioChunks++;
|
|
2011
|
-
this._audioRemainder = Buffer.alloc(0);
|
|
2012
|
-
}
|
|
2013
|
-
break;
|
|
2014
|
-
}
|
|
2015
|
-
case "input_audio_buffer.speech_started": {
|
|
2016
|
-
this._handleTruncation();
|
|
2017
|
-
break;
|
|
2018
|
-
}
|
|
2019
|
-
case "response.output_item.done": {
|
|
2020
|
-
const item = msg["item"];
|
|
2021
|
-
if (item && item["type"] === "function_call") {
|
|
2022
|
-
this._handleToolCall(item);
|
|
2023
|
-
}
|
|
2024
|
-
break;
|
|
2025
|
-
}
|
|
2026
|
-
case "conversation.item.input_audio_transcription.completed": {
|
|
2027
|
-
if (this._call) {
|
|
2028
|
-
this._call._emit("transcript", "user", msg["transcript"] ?? "");
|
|
2029
|
-
}
|
|
2030
|
-
break;
|
|
2031
|
-
}
|
|
2032
|
-
case "response.audio_transcript.done": {
|
|
2033
|
-
if (this._call) {
|
|
2034
|
-
this._call._emit("transcript", "assistant", msg["transcript"] ?? "");
|
|
2035
|
-
}
|
|
2036
|
-
break;
|
|
2037
|
-
}
|
|
2038
|
-
case "response.created": {
|
|
2039
|
-
this._responseInProgress = true;
|
|
2040
|
-
break;
|
|
2041
|
-
}
|
|
2042
|
-
case "response.done": {
|
|
2043
|
-
this._responseInProgress = false;
|
|
2044
|
-
if (this._onResponseDone) {
|
|
2045
|
-
const cb = this._onResponseDone;
|
|
2046
|
-
this._onResponseDone = null;
|
|
2047
|
-
cb();
|
|
2048
|
-
}
|
|
2049
|
-
break;
|
|
2050
|
-
}
|
|
2051
|
-
case "error": {
|
|
2052
|
-
this._log.error({ apiError: msg["error"] }, "OpenAI error");
|
|
2053
|
-
break;
|
|
2126
|
+
async *_createAudioStream() {
|
|
2127
|
+
while (this._running) {
|
|
2128
|
+
if (this._audioBuffer.length > 0) {
|
|
2129
|
+
const ulaw = this._audioBuffer.shift();
|
|
2130
|
+
const pcm8k = ulawToPcm16(ulaw);
|
|
2131
|
+
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
2132
|
+
yield pcm16k;
|
|
2133
|
+
} else {
|
|
2134
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
2054
2135
|
}
|
|
2055
2136
|
}
|
|
2056
2137
|
}
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
this._sentAudioChunks = 0;
|
|
2061
|
-
}
|
|
2062
|
-
if (msg["item_id"]) {
|
|
2063
|
-
this._lastAssistantItem = msg["item_id"];
|
|
2064
|
-
}
|
|
2065
|
-
const ulaw = Buffer.from(msg["delta"], "base64");
|
|
2066
|
-
if (this._recorder) {
|
|
2067
|
-
this._recorder.writeOutbound(ulawToPcm16(ulaw));
|
|
2068
|
-
}
|
|
2069
|
-
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
2070
|
-
const chunkSize = 160;
|
|
2071
|
-
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
2072
|
-
for (let off = 0; off < fullEnd; off += chunkSize) {
|
|
2073
|
-
if (this._call) {
|
|
2074
|
-
this._call.sendAudio(combined.subarray(off, off + chunkSize));
|
|
2075
|
-
}
|
|
2076
|
-
this._sentAudioChunks++;
|
|
2077
|
-
}
|
|
2078
|
-
this._audioRemainder = combined.subarray(fullEnd);
|
|
2138
|
+
async _generateGreeting() {
|
|
2139
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
2140
|
+
await this._respond();
|
|
2079
2141
|
}
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
this.
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
audio_end_ms: audioEndMs
|
|
2142
|
+
async _handleUserSpeech(transcript) {
|
|
2143
|
+
this._conversation.push({ role: "user", content: transcript });
|
|
2144
|
+
await this._respond();
|
|
2145
|
+
}
|
|
2146
|
+
_buildEffectiveTools() {
|
|
2147
|
+
const builtinSchemas = getBuiltinToolSchemas(this._builtinTools, "chat");
|
|
2148
|
+
const dtmfSchemas = builtinSchemas.filter((s) => {
|
|
2149
|
+
const name = s["function"]?.["name"];
|
|
2150
|
+
return name === "collect_dtmf" || name === "send_dtmf";
|
|
2090
2151
|
});
|
|
2091
|
-
if (
|
|
2092
|
-
|
|
2152
|
+
if (dtmfSchemas.length === 0) return this._tools ?? void 0;
|
|
2153
|
+
const base = this._tools ? this._tools.fork() : new ToolRegistry();
|
|
2154
|
+
for (const schema of dtmfSchemas) {
|
|
2155
|
+
const fn = schema["function"];
|
|
2156
|
+
const params = fn["parameters"];
|
|
2157
|
+
base.register({
|
|
2158
|
+
name: fn["name"],
|
|
2159
|
+
description: fn["description"],
|
|
2160
|
+
parameters: params["properties"] ?? {},
|
|
2161
|
+
required: params["required"] ?? [],
|
|
2162
|
+
handler: async () => ""
|
|
2163
|
+
});
|
|
2093
2164
|
}
|
|
2094
|
-
|
|
2095
|
-
this._responseStartTs = null;
|
|
2096
|
-
this._sentAudioChunks = 0;
|
|
2097
|
-
this._audioRemainder = Buffer.alloc(0);
|
|
2165
|
+
return base;
|
|
2098
2166
|
}
|
|
2099
|
-
async
|
|
2100
|
-
|
|
2101
|
-
const
|
|
2102
|
-
this.
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2167
|
+
async _respond() {
|
|
2168
|
+
let fullResponse = "";
|
|
2169
|
+
const textChunks = [];
|
|
2170
|
+
const effectiveTools = this._buildEffectiveTools();
|
|
2171
|
+
const llmStream = this._llm.generate(this._conversation, {
|
|
2172
|
+
tools: effectiveTools,
|
|
2173
|
+
temperature: this._temperature,
|
|
2174
|
+
maxTokens: this._maxTokens
|
|
2175
|
+
});
|
|
2176
|
+
for await (const chunk of llmStream) {
|
|
2177
|
+
if (!this._running) break;
|
|
2178
|
+
if (chunk.type === "text" && chunk.text) {
|
|
2179
|
+
textChunks.push(chunk.text);
|
|
2180
|
+
fullResponse += chunk.text;
|
|
2181
|
+
} else if (chunk.type === "tool_call" && chunk.toolCall) {
|
|
2182
|
+
await this._handleToolCall(chunk);
|
|
2106
2183
|
}
|
|
2107
|
-
return;
|
|
2108
2184
|
}
|
|
2109
|
-
if (
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2185
|
+
if (fullResponse.trim()) {
|
|
2186
|
+
this._log.info("Assistant: %s", fullResponse.substring(0, 100));
|
|
2187
|
+
this._conversation.push({ role: "assistant", content: fullResponse });
|
|
2188
|
+
await this._synthesizeAndSend(fullResponse);
|
|
2189
|
+
}
|
|
2190
|
+
}
|
|
2191
|
+
async _handleToolCall(chunk) {
|
|
2192
|
+
if (!chunk.toolCall) return;
|
|
2193
|
+
const { id, name, arguments: argsStr } = chunk.toolCall;
|
|
2194
|
+
try {
|
|
2195
|
+
const args = JSON.parse(argsStr);
|
|
2196
|
+
if (BUILTIN_TOOL_NAMES.has(name) && this._callSession) {
|
|
2197
|
+
const result2 = await executeBuiltinTool(name, args, this._callSession);
|
|
2198
|
+
if (result2 !== null) {
|
|
2199
|
+
if (name === "hang_up") return;
|
|
2200
|
+
this._conversation.push({ role: "tool", content: result2, tool_call_id: id, name });
|
|
2201
|
+
await this._respond();
|
|
2202
|
+
return;
|
|
2121
2203
|
}
|
|
2122
|
-
await this._waitForResponseDone();
|
|
2123
|
-
this._send({
|
|
2124
|
-
type: "conversation.item.create",
|
|
2125
|
-
item: {
|
|
2126
|
-
type: "function_call_output",
|
|
2127
|
-
call_id: callId,
|
|
2128
|
-
output: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)"
|
|
2129
|
-
}
|
|
2130
|
-
});
|
|
2131
|
-
this._send({ type: "response.create" });
|
|
2132
2204
|
}
|
|
2133
|
-
return;
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2205
|
+
if (!this._tools) return;
|
|
2206
|
+
this._callSession?.recordToolCall();
|
|
2207
|
+
const result = await this._tools.call(name, args);
|
|
2208
|
+
this._conversation.push({
|
|
2209
|
+
role: "assistant",
|
|
2210
|
+
content: ""
|
|
2211
|
+
// Tool call info stored in the message flow
|
|
2212
|
+
});
|
|
2213
|
+
this._conversation.push({
|
|
2214
|
+
role: "tool",
|
|
2215
|
+
content: typeof result === "string" ? result : JSON.stringify(result),
|
|
2216
|
+
tool_call_id: id,
|
|
2217
|
+
name
|
|
2218
|
+
});
|
|
2219
|
+
const effectiveTools = this._buildEffectiveTools();
|
|
2220
|
+
let followUpText = "";
|
|
2221
|
+
const followUpStream = this._llm.generate(this._conversation, {
|
|
2222
|
+
tools: effectiveTools,
|
|
2223
|
+
temperature: this._temperature,
|
|
2224
|
+
maxTokens: this._maxTokens
|
|
2225
|
+
});
|
|
2226
|
+
for await (const followChunk of followUpStream) {
|
|
2227
|
+
if (!this._running) break;
|
|
2228
|
+
if (followChunk.type === "text" && followChunk.text) {
|
|
2229
|
+
followUpText += followChunk.text;
|
|
2144
2230
|
}
|
|
2145
|
-
await this._waitForResponseDone();
|
|
2146
|
-
this._send({
|
|
2147
|
-
type: "conversation.item.create",
|
|
2148
|
-
item: {
|
|
2149
|
-
type: "function_call_output",
|
|
2150
|
-
call_id: callId,
|
|
2151
|
-
output: result2
|
|
2152
|
-
}
|
|
2153
|
-
});
|
|
2154
|
-
this._send({ type: "response.create" });
|
|
2155
2231
|
}
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
this._log.error("Unknown tool: %s", funcName);
|
|
2160
|
-
return;
|
|
2161
|
-
}
|
|
2162
|
-
let result;
|
|
2163
|
-
try {
|
|
2164
|
-
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2165
|
-
result = await this._tools.call(funcName, args);
|
|
2166
|
-
} catch (err) {
|
|
2167
|
-
this._log.error({ err }, "Tool call failed: %s", funcName);
|
|
2168
|
-
result = `Error: ${err}`;
|
|
2169
|
-
}
|
|
2170
|
-
await this._waitForResponseDone();
|
|
2171
|
-
this._send({
|
|
2172
|
-
type: "conversation.item.create",
|
|
2173
|
-
item: {
|
|
2174
|
-
type: "function_call_output",
|
|
2175
|
-
call_id: callId,
|
|
2176
|
-
output: typeof result === "string" ? result : JSON.stringify(result)
|
|
2232
|
+
if (followUpText.trim()) {
|
|
2233
|
+
this._conversation.push({ role: "assistant", content: followUpText });
|
|
2234
|
+
await this._synthesizeAndSend(followUpText);
|
|
2177
2235
|
}
|
|
2178
|
-
})
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
if (!this._responseInProgress) return Promise.resolve();
|
|
2183
|
-
return new Promise((resolve) => {
|
|
2184
|
-
this._onResponseDone = resolve;
|
|
2185
|
-
});
|
|
2186
|
-
}
|
|
2187
|
-
_send(data) {
|
|
2188
|
-
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
2189
|
-
this._ws.send(JSON.stringify(data));
|
|
2190
|
-
}
|
|
2191
|
-
}
|
|
2192
|
-
};
|
|
2193
|
-
|
|
2194
|
-
// src/agent/pipeline/gemini-realtime.ts
|
|
2195
|
-
var HANG_UP_TOOL2 = {
|
|
2196
|
-
name: "hang_up",
|
|
2197
|
-
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
2198
|
-
parameters: { type: "object", properties: {} }
|
|
2199
|
-
};
|
|
2200
|
-
var COLLECT_DTMF_TOOL2 = {
|
|
2201
|
-
name: "collect_dtmf",
|
|
2202
|
-
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
|
|
2203
|
-
parameters: {
|
|
2204
|
-
type: "object",
|
|
2205
|
-
properties: {
|
|
2206
|
-
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
2207
|
-
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
2208
|
-
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
2209
|
-
},
|
|
2210
|
-
required: ["max_digits"]
|
|
2211
|
-
}
|
|
2212
|
-
};
|
|
2213
|
-
var SEND_DTMF_TOOL2 = {
|
|
2214
|
-
name: "send_dtmf",
|
|
2215
|
-
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
2216
|
-
parameters: {
|
|
2217
|
-
type: "object",
|
|
2218
|
-
properties: {
|
|
2219
|
-
digits: {
|
|
2220
|
-
type: "string",
|
|
2221
|
-
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30."
|
|
2236
|
+
} catch (err) {
|
|
2237
|
+
this._log.error({ err }, "Tool call failed: %s", name);
|
|
2238
|
+
if (err instanceof Error) {
|
|
2239
|
+
this._callSession?.recordToolError(err);
|
|
2222
2240
|
}
|
|
2223
|
-
},
|
|
2224
|
-
required: ["digits"]
|
|
2225
|
-
}
|
|
2226
|
-
};
|
|
2227
|
-
function resolveRef(ref, defs) {
|
|
2228
|
-
const parts = ref.replace(/^#\//, "").split("/");
|
|
2229
|
-
let result = defs;
|
|
2230
|
-
for (const part of parts) {
|
|
2231
|
-
if (result && typeof result === "object" && !Array.isArray(result)) {
|
|
2232
|
-
result = result[part];
|
|
2233
|
-
} else {
|
|
2234
|
-
return {};
|
|
2235
2241
|
}
|
|
2236
2242
|
}
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
for (const v of variants) {
|
|
2256
|
-
if (v && typeof v === "object") {
|
|
2257
|
-
const resolved = sanitizeSchemaForGemini(v, defs, depth + 1);
|
|
2258
|
-
if (resolved["type"] === "object" && resolved["properties"]) {
|
|
2259
|
-
return resolved;
|
|
2243
|
+
async _synthesizeAndSend(text) {
|
|
2244
|
+
if (!this._callSession || !this._running) return;
|
|
2245
|
+
this._speaking = true;
|
|
2246
|
+
try {
|
|
2247
|
+
for await (const audioChunk of this._tts.synthesize(text, {
|
|
2248
|
+
sampleRate: this._sampleRate
|
|
2249
|
+
})) {
|
|
2250
|
+
if (!this._running || !this._speaking) break;
|
|
2251
|
+
if (this._recorder) {
|
|
2252
|
+
const pcm8k2 = this._sampleRate !== 8e3 ? resamplePcm16(audioChunk, this._sampleRate, 8e3) : audioChunk;
|
|
2253
|
+
this._recorder.writeOutbound(pcm8k2);
|
|
2254
|
+
}
|
|
2255
|
+
const pcm8k = resamplePcm16(audioChunk, this._sampleRate, 8e3);
|
|
2256
|
+
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2257
|
+
for (let off = 0; off < ulaw.length; off += 160) {
|
|
2258
|
+
let chunk = ulaw.subarray(off, off + 160);
|
|
2259
|
+
if (chunk.length < 160) {
|
|
2260
|
+
chunk = Buffer.concat([chunk, Buffer.alloc(160 - chunk.length, 255)]);
|
|
2260
2261
|
}
|
|
2262
|
+
this._callSession.sendAudio(chunk);
|
|
2261
2263
|
}
|
|
2262
2264
|
}
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
}
|
|
2268
|
-
}
|
|
2269
|
-
const result = {};
|
|
2270
|
-
let schemaType = schema["type"];
|
|
2271
|
-
if (Array.isArray(schemaType)) {
|
|
2272
|
-
const nonNull = schemaType.filter((t) => t !== "null");
|
|
2273
|
-
schemaType = nonNull[0] ?? "string";
|
|
2274
|
-
}
|
|
2275
|
-
if (schemaType) result["type"] = schemaType;
|
|
2276
|
-
if (schema["description"]) result["description"] = schema["description"];
|
|
2277
|
-
if (schema["enum"]) result["enum"] = schema["enum"];
|
|
2278
|
-
if (schema["required"]) result["required"] = schema["required"];
|
|
2279
|
-
if (schema["properties"] && typeof schema["properties"] === "object") {
|
|
2280
|
-
const props = {};
|
|
2281
|
-
for (const [key, val] of Object.entries(schema["properties"])) {
|
|
2282
|
-
if (val && typeof val === "object") {
|
|
2283
|
-
props[key] = sanitizeSchemaForGemini(val, defs, depth + 1);
|
|
2284
|
-
}
|
|
2265
|
+
} catch (err) {
|
|
2266
|
+
this._log.error({ err }, "TTS error");
|
|
2267
|
+
} finally {
|
|
2268
|
+
this._speaking = false;
|
|
2285
2269
|
}
|
|
2286
|
-
result["properties"] = props;
|
|
2287
|
-
}
|
|
2288
|
-
if (schema["items"] && typeof schema["items"] === "object" && !Array.isArray(schema["items"])) {
|
|
2289
|
-
result["items"] = sanitizeSchemaForGemini(
|
|
2290
|
-
schema["items"],
|
|
2291
|
-
defs,
|
|
2292
|
-
depth + 1
|
|
2293
|
-
);
|
|
2294
2270
|
}
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
var
|
|
2271
|
+
};
|
|
2272
|
+
|
|
2273
|
+
// src/agent/pipeline/realtime/openai-realtime.ts
|
|
2274
|
+
var OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime?model=";
|
|
2275
|
+
var OpenAIRealtime = class {
|
|
2300
2276
|
_apiKey;
|
|
2301
2277
|
_systemPrompt;
|
|
2302
2278
|
_model;
|
|
2303
2279
|
_voice;
|
|
2304
2280
|
_language;
|
|
2281
|
+
_turnDetection;
|
|
2305
2282
|
_greeting;
|
|
2306
|
-
|
|
2307
|
-
|
|
2283
|
+
_log = NOOP_LOGGER;
|
|
2284
|
+
_builtinTools = null;
|
|
2285
|
+
setLogger(logger) {
|
|
2286
|
+
this._log = logger;
|
|
2287
|
+
}
|
|
2288
|
+
setBuiltinTools(tools) {
|
|
2289
|
+
this._builtinTools = tools;
|
|
2290
|
+
}
|
|
2291
|
+
getTelemetry() {
|
|
2292
|
+
return {
|
|
2293
|
+
sessionType: "openai_realtime",
|
|
2294
|
+
llm: { provider: "openai", model: this._model },
|
|
2295
|
+
stt: null,
|
|
2296
|
+
tts: null,
|
|
2297
|
+
voice: this._voice,
|
|
2298
|
+
language: this._language,
|
|
2299
|
+
greetingEnabled: this._greeting,
|
|
2300
|
+
recordingEnabled: !!this._recorder,
|
|
2301
|
+
toolCount: this._tools?.size ?? 0,
|
|
2302
|
+
mcpServerCount: 0,
|
|
2303
|
+
builtinTools: []
|
|
2304
|
+
};
|
|
2305
|
+
}
|
|
2306
|
+
_ws = null;
|
|
2308
2307
|
_call = null;
|
|
2309
2308
|
_tools = null;
|
|
2310
2309
|
_recorder = null;
|
|
2311
2310
|
_closed = false;
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2311
|
+
// PlaybackState — 현재 재생 중인 응답 상태
|
|
2312
|
+
_playback = null;
|
|
2313
|
+
_latestMediaTs = 0;
|
|
2314
|
+
// Pending tool call tracking — 인터럽트 시 취소용
|
|
2315
|
+
_pendingToolCalls = /* @__PURE__ */ new Map();
|
|
2316
|
+
// Response state tracking — prevent sending response.create while one is active
|
|
2317
|
+
_responseInProgress = false;
|
|
2318
|
+
_onResponseDone = null;
|
|
2317
2319
|
constructor(options = {}) {
|
|
2318
|
-
this._apiKey = options.apiKey ?? process.env["
|
|
2320
|
+
this._apiKey = options.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
|
|
2319
2321
|
this._systemPrompt = options.systemPrompt ?? "";
|
|
2320
|
-
this._model = options.model ?? "
|
|
2321
|
-
this._voice = options.voice ?? "
|
|
2322
|
+
this._model = options.model ?? "gpt-realtime-1.5";
|
|
2323
|
+
this._voice = options.voice ?? "marin";
|
|
2322
2324
|
this._language = options.language ?? "ko";
|
|
2325
|
+
this._turnDetection = options.turnDetection !== void 0 ? options.turnDetection : { type: "semantic_vad", eagerness: "medium", interrupt_response: true };
|
|
2323
2326
|
this._greeting = options.greeting ?? true;
|
|
2324
2327
|
}
|
|
2325
2328
|
/** Inject per-call ToolRegistry. */
|
|
@@ -2330,340 +2333,409 @@ var GeminiRealtime = class {
|
|
|
2330
2333
|
setRecorder(recorder) {
|
|
2331
2334
|
this._recorder = recorder;
|
|
2332
2335
|
}
|
|
2333
|
-
setBuiltinTools(tools) {
|
|
2334
|
-
this._builtinTools = tools;
|
|
2335
|
-
}
|
|
2336
|
-
setLogger(logger) {
|
|
2337
|
-
this._log = logger;
|
|
2338
|
-
}
|
|
2339
2336
|
async start(callSession, tools) {
|
|
2340
2337
|
this._call = callSession;
|
|
2341
2338
|
if (tools) this._tools = tools;
|
|
2342
2339
|
this._closed = false;
|
|
2343
|
-
this.
|
|
2344
|
-
this.
|
|
2340
|
+
this._playback = null;
|
|
2341
|
+
this._latestMediaTs = 0;
|
|
2345
2342
|
if (!this._apiKey) {
|
|
2346
|
-
throw new Error("
|
|
2347
|
-
}
|
|
2348
|
-
const { GoogleGenAI } = await import('@google/genai/node');
|
|
2349
|
-
const client = new GoogleGenAI({ apiKey: this._apiKey });
|
|
2350
|
-
const config = {
|
|
2351
|
-
responseModalities: ["AUDIO"],
|
|
2352
|
-
speechConfig: {
|
|
2353
|
-
voiceConfig: {
|
|
2354
|
-
prebuiltVoiceConfig: {
|
|
2355
|
-
voiceName: this._voice
|
|
2356
|
-
}
|
|
2357
|
-
}
|
|
2358
|
-
},
|
|
2359
|
-
inputAudioTranscription: {},
|
|
2360
|
-
outputAudioTranscription: {}
|
|
2361
|
-
};
|
|
2362
|
-
if (this._systemPrompt) {
|
|
2363
|
-
config["systemInstruction"] = this._systemPrompt;
|
|
2364
|
-
}
|
|
2365
|
-
const toolSchemas = this._buildToolSchemas();
|
|
2366
|
-
if (toolSchemas.length > 0) {
|
|
2367
|
-
config["tools"] = [{ functionDeclarations: toolSchemas }];
|
|
2343
|
+
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey option.");
|
|
2368
2344
|
}
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
this._log.error({ err }, "Gemini SDK error");
|
|
2376
|
-
},
|
|
2377
|
-
onclose: (ev) => {
|
|
2378
|
-
this._log.info({ code: ev?.code ?? "unknown" }, "Gemini connection closed");
|
|
2379
|
-
this._closed = true;
|
|
2380
|
-
}
|
|
2345
|
+
const { WebSocket } = await import('ws');
|
|
2346
|
+
const url = `${OPENAI_REALTIME_URL}${this._model}`;
|
|
2347
|
+
this._ws = new WebSocket(url, {
|
|
2348
|
+
headers: {
|
|
2349
|
+
Authorization: `Bearer ${this._apiKey}`,
|
|
2350
|
+
"OpenAI-Beta": "realtime=v1"
|
|
2381
2351
|
}
|
|
2382
2352
|
});
|
|
2383
|
-
|
|
2384
|
-
this.
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
}
|
|
2390
|
-
|
|
2391
|
-
|
|
2353
|
+
return new Promise((resolve, reject) => {
|
|
2354
|
+
const ws = this._ws;
|
|
2355
|
+
ws.on("open", () => {
|
|
2356
|
+
this._sendSessionUpdate();
|
|
2357
|
+
this._log.info("OpenAI Realtime connected");
|
|
2358
|
+
if (this._greeting) {
|
|
2359
|
+
this._send({ type: "response.create" });
|
|
2360
|
+
}
|
|
2361
|
+
resolve();
|
|
2392
2362
|
});
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2363
|
+
ws.on("message", (data) => {
|
|
2364
|
+
try {
|
|
2365
|
+
const msg = JSON.parse(data.toString());
|
|
2366
|
+
this._handleMessage(msg);
|
|
2367
|
+
} catch {
|
|
2368
|
+
}
|
|
2369
|
+
});
|
|
2370
|
+
ws.on("close", () => {
|
|
2371
|
+
this._closed = true;
|
|
2372
|
+
});
|
|
2373
|
+
ws.on("error", (err) => {
|
|
2374
|
+
if (!this._ws) {
|
|
2375
|
+
reject(err);
|
|
2406
2376
|
}
|
|
2377
|
+
this._log.error({ err }, "OpenAI Realtime WS error");
|
|
2407
2378
|
});
|
|
2408
|
-
}
|
|
2379
|
+
});
|
|
2409
2380
|
}
|
|
2410
2381
|
async feedDtmf(digits) {
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2382
|
+
await this._waitForResponseDone();
|
|
2383
|
+
this._send({
|
|
2384
|
+
type: "conversation.item.create",
|
|
2385
|
+
item: {
|
|
2386
|
+
type: "message",
|
|
2387
|
+
role: "user",
|
|
2388
|
+
content: [{ type: "input_text", text: `[DTMF \uC785\uB825: ${digits}]` }]
|
|
2389
|
+
}
|
|
2390
|
+
});
|
|
2391
|
+
this._send({ type: "response.create" });
|
|
2392
|
+
}
|
|
2393
|
+
feedAudio(audio) {
|
|
2394
|
+
this._latestMediaTs = Date.now();
|
|
2395
|
+
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
2396
|
+
this._send({
|
|
2397
|
+
type: "input_audio_buffer.append",
|
|
2398
|
+
audio: audio.toString("base64")
|
|
2415
2399
|
});
|
|
2416
2400
|
}
|
|
2417
2401
|
}
|
|
2418
2402
|
async stop() {
|
|
2419
2403
|
this._closed = true;
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
this.
|
|
2404
|
+
for (const [, controller] of this._pendingToolCalls) {
|
|
2405
|
+
controller.abort();
|
|
2406
|
+
}
|
|
2407
|
+
this._pendingToolCalls.clear();
|
|
2408
|
+
if (this._ws) {
|
|
2409
|
+
this._ws.close();
|
|
2410
|
+
this._ws = null;
|
|
2426
2411
|
}
|
|
2427
2412
|
}
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2413
|
+
_sendSessionUpdate() {
|
|
2414
|
+
if (!this._ws || this._ws.readyState !== 1) return;
|
|
2415
|
+
const toolSchemas = this._tools ? this._tools.toOpenAITools().map((t) => ({ type: "function", ...t.function })) : [];
|
|
2416
|
+
toolSchemas.push(...getBuiltinToolSchemas(this._builtinTools, "realtime"));
|
|
2417
|
+
this._send({
|
|
2418
|
+
type: "session.update",
|
|
2419
|
+
session: {
|
|
2420
|
+
modalities: ["text", "audio"],
|
|
2421
|
+
voice: this._voice,
|
|
2422
|
+
instructions: this._systemPrompt,
|
|
2423
|
+
input_audio_format: "g711_ulaw",
|
|
2424
|
+
output_audio_format: "g711_ulaw",
|
|
2425
|
+
input_audio_transcription: {
|
|
2426
|
+
model: "whisper-1",
|
|
2427
|
+
language: this._language
|
|
2428
|
+
},
|
|
2429
|
+
input_audio_noise_reduction: { type: "far_field" },
|
|
2430
|
+
turn_detection: this._turnDetection,
|
|
2431
|
+
tools: toolSchemas
|
|
2432
|
+
}
|
|
2433
|
+
});
|
|
2440
2434
|
}
|
|
2441
2435
|
_handleMessage(msg) {
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2436
|
+
const type = msg["type"];
|
|
2437
|
+
switch (type) {
|
|
2438
|
+
case "response.audio.delta": {
|
|
2439
|
+
this._handleAudioDelta(msg);
|
|
2440
|
+
break;
|
|
2441
|
+
}
|
|
2442
|
+
case "response.audio.done": {
|
|
2443
|
+
if (this._playback) {
|
|
2444
|
+
this._playback.generating = false;
|
|
2445
|
+
if (this._playback.audioRemainder.length > 0) {
|
|
2446
|
+
const padded = Buffer.concat([
|
|
2447
|
+
this._playback.audioRemainder,
|
|
2448
|
+
Buffer.alloc(160 - this._playback.audioRemainder.length, 255)
|
|
2449
|
+
]);
|
|
2450
|
+
if (this._call) {
|
|
2451
|
+
this._call.sendAudio(padded);
|
|
2453
2452
|
}
|
|
2453
|
+
this._playback.sentChunks++;
|
|
2454
|
+
this._playback.audioRemainder = Buffer.alloc(0);
|
|
2454
2455
|
}
|
|
2455
2456
|
}
|
|
2457
|
+
break;
|
|
2456
2458
|
}
|
|
2457
|
-
|
|
2458
|
-
this.
|
|
2459
|
-
|
|
2459
|
+
case "input_audio_buffer.speech_started": {
|
|
2460
|
+
this._handleTruncation();
|
|
2461
|
+
break;
|
|
2460
2462
|
}
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
+
case "response.output_item.done": {
|
|
2464
|
+
const item = msg["item"];
|
|
2465
|
+
if (item && item["type"] === "function_call") {
|
|
2466
|
+
this._handleToolCall(item);
|
|
2467
|
+
}
|
|
2468
|
+
break;
|
|
2469
|
+
}
|
|
2470
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
2463
2471
|
if (this._call) {
|
|
2464
|
-
this._call.
|
|
2472
|
+
this._call._emit("transcript", "user", msg["transcript"] ?? "");
|
|
2465
2473
|
}
|
|
2466
|
-
|
|
2467
|
-
this._audioRemainder = Buffer.alloc(0);
|
|
2474
|
+
break;
|
|
2468
2475
|
}
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2476
|
+
case "response.audio_transcript.done": {
|
|
2477
|
+
if (this._call) {
|
|
2478
|
+
this._call._emit("transcript", "assistant", msg["transcript"] ?? "");
|
|
2479
|
+
}
|
|
2480
|
+
break;
|
|
2473
2481
|
}
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2482
|
+
case "response.created": {
|
|
2483
|
+
this._responseInProgress = true;
|
|
2484
|
+
break;
|
|
2485
|
+
}
|
|
2486
|
+
case "response.done": {
|
|
2487
|
+
this._responseInProgress = false;
|
|
2488
|
+
if (this._onResponseDone) {
|
|
2489
|
+
const cb = this._onResponseDone;
|
|
2490
|
+
this._onResponseDone = null;
|
|
2491
|
+
cb();
|
|
2492
|
+
}
|
|
2493
|
+
break;
|
|
2494
|
+
}
|
|
2495
|
+
case "error": {
|
|
2496
|
+
this._log.error({ apiError: msg["error"] }, "OpenAI error");
|
|
2497
|
+
break;
|
|
2478
2498
|
}
|
|
2479
|
-
}
|
|
2480
|
-
if (msg.toolCall) {
|
|
2481
|
-
this._handleToolCall(msg.toolCall);
|
|
2482
|
-
}
|
|
2483
|
-
const toolCancellation = msg["toolCallCancellation"];
|
|
2484
|
-
if (toolCancellation) {
|
|
2485
|
-
this._log.info({ ids: toolCancellation.ids }, "Tool call cancelled");
|
|
2486
2499
|
}
|
|
2487
2500
|
}
|
|
2488
|
-
|
|
2489
|
-
if (
|
|
2490
|
-
|
|
2501
|
+
_handleAudioDelta(msg) {
|
|
2502
|
+
if (this._playback === null) {
|
|
2503
|
+
this._playback = {
|
|
2504
|
+
itemId: msg["item_id"] || "",
|
|
2505
|
+
startTs: this._latestMediaTs || Date.now(),
|
|
2506
|
+
sentChunks: 0,
|
|
2507
|
+
generating: true,
|
|
2508
|
+
audioRemainder: Buffer.alloc(0)
|
|
2509
|
+
};
|
|
2510
|
+
} else if (msg["item_id"]) {
|
|
2511
|
+
this._playback.itemId = msg["item_id"];
|
|
2512
|
+
}
|
|
2513
|
+
const pb = this._playback;
|
|
2514
|
+
const ulaw = Buffer.from(msg["delta"], "base64");
|
|
2491
2515
|
if (this._recorder) {
|
|
2492
|
-
this._recorder.writeOutbound(
|
|
2516
|
+
this._recorder.writeOutbound(ulawToPcm16(ulaw));
|
|
2493
2517
|
}
|
|
2494
|
-
const
|
|
2495
|
-
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2496
|
-
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
2518
|
+
const combined = Buffer.concat([pb.audioRemainder, ulaw]);
|
|
2497
2519
|
const chunkSize = 160;
|
|
2498
2520
|
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
2499
|
-
|
|
2500
|
-
this._call
|
|
2501
|
-
|
|
2521
|
+
for (let off = 0; off < fullEnd; off += chunkSize) {
|
|
2522
|
+
if (this._call) {
|
|
2523
|
+
this._call.sendAudio(combined.subarray(off, off + chunkSize));
|
|
2524
|
+
}
|
|
2525
|
+
pb.sentChunks++;
|
|
2502
2526
|
}
|
|
2503
|
-
|
|
2527
|
+
pb.audioRemainder = combined.subarray(fullEnd);
|
|
2504
2528
|
}
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
this._call.
|
|
2512
|
-
|
|
2513
|
-
|
|
2529
|
+
_handleTruncation() {
|
|
2530
|
+
for (const [, controller] of this._pendingToolCalls) {
|
|
2531
|
+
controller.abort();
|
|
2532
|
+
}
|
|
2533
|
+
this._pendingToolCalls.clear();
|
|
2534
|
+
if (this._call) {
|
|
2535
|
+
this._call.clearAudio();
|
|
2536
|
+
}
|
|
2537
|
+
const pb = this._playback;
|
|
2538
|
+
if (pb === null) {
|
|
2539
|
+
return;
|
|
2514
2540
|
}
|
|
2541
|
+
const playedMs = Math.max(0, (this._latestMediaTs || Date.now()) - pb.startTs);
|
|
2542
|
+
this._log.info(
|
|
2543
|
+
"[Interrupt] item=%s played=%dms total=%dms",
|
|
2544
|
+
pb.itemId,
|
|
2545
|
+
playedMs,
|
|
2546
|
+
pb.sentChunks * 20
|
|
2547
|
+
);
|
|
2548
|
+
this._playback = null;
|
|
2515
2549
|
}
|
|
2516
|
-
async _handleToolCall(
|
|
2517
|
-
const
|
|
2518
|
-
|
|
2519
|
-
this.
|
|
2520
|
-
const
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
await this.
|
|
2550
|
+
async _handleToolCall(item) {
|
|
2551
|
+
const funcName = item["name"];
|
|
2552
|
+
const callId = item["call_id"];
|
|
2553
|
+
this._log.info("Tool call: %s", funcName);
|
|
2554
|
+
const controller = new AbortController();
|
|
2555
|
+
this._pendingToolCalls.set(callId, controller);
|
|
2556
|
+
try {
|
|
2557
|
+
if (BUILTIN_TOOL_NAMES.has(funcName) && this._call) {
|
|
2558
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2559
|
+
const result2 = await executeBuiltinTool(funcName, args, this._call);
|
|
2560
|
+
if (result2 !== null) {
|
|
2561
|
+
if (funcName === "hang_up") return;
|
|
2562
|
+
if (controller.signal.aborted) return;
|
|
2563
|
+
await this._waitForResponseDone();
|
|
2564
|
+
this._send({
|
|
2565
|
+
type: "conversation.item.create",
|
|
2566
|
+
item: {
|
|
2567
|
+
type: "function_call_output",
|
|
2568
|
+
call_id: callId,
|
|
2569
|
+
output: result2
|
|
2570
|
+
}
|
|
2571
|
+
});
|
|
2572
|
+
this._send({ type: "response.create" });
|
|
2573
|
+
return;
|
|
2530
2574
|
}
|
|
2531
|
-
return;
|
|
2532
2575
|
}
|
|
2533
|
-
if (
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
});
|
|
2543
|
-
this._log.info("DTMF collected: %s", result || "(empty)");
|
|
2544
|
-
} catch (err) {
|
|
2545
|
-
this._log.error({ err }, "collect_dtmf error");
|
|
2546
|
-
result = `Error: ${err}`;
|
|
2576
|
+
if (!this._tools || !this._tools.has(funcName)) {
|
|
2577
|
+
this._log.error("Unknown tool: %s", funcName);
|
|
2578
|
+
await this._waitForResponseDone();
|
|
2579
|
+
this._send({
|
|
2580
|
+
type: "conversation.item.create",
|
|
2581
|
+
item: {
|
|
2582
|
+
type: "function_call_output",
|
|
2583
|
+
call_id: callId,
|
|
2584
|
+
output: JSON.stringify({ error: `Unknown tool: ${funcName}` })
|
|
2547
2585
|
}
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2586
|
+
});
|
|
2587
|
+
this._send({ type: "response.create" });
|
|
2588
|
+
return;
|
|
2589
|
+
}
|
|
2590
|
+
let result;
|
|
2591
|
+
try {
|
|
2592
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2593
|
+
this._call?.recordToolCall();
|
|
2594
|
+
result = await this._tools.call(funcName, args);
|
|
2595
|
+
} catch (err) {
|
|
2596
|
+
this._log.error({ err }, "Tool call failed: %s", funcName);
|
|
2597
|
+
if (err instanceof Error) {
|
|
2598
|
+
this._call?.recordToolError(err);
|
|
2553
2599
|
}
|
|
2554
|
-
|
|
2600
|
+
result = `Error: ${err}`;
|
|
2555
2601
|
}
|
|
2556
|
-
if (
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2602
|
+
if (controller.signal.aborted) {
|
|
2603
|
+
this._log.info("Tool call cancelled (user interrupted): %s", funcName);
|
|
2604
|
+
return;
|
|
2605
|
+
}
|
|
2606
|
+
const resultStr = typeof result === "string" ? result : JSON.stringify(result);
|
|
2607
|
+
await this._waitForResponseDone();
|
|
2608
|
+
this._send({
|
|
2609
|
+
type: "conversation.item.create",
|
|
2610
|
+
item: {
|
|
2611
|
+
type: "function_call_output",
|
|
2612
|
+
call_id: callId,
|
|
2613
|
+
output: resultStr
|
|
2614
|
+
}
|
|
2615
|
+
});
|
|
2616
|
+
this._log.info("[ToolResult] %s call_id=%s len=%d", funcName, callId, resultStr.length);
|
|
2617
|
+
this._send({ type: "response.create" });
|
|
2618
|
+
} finally {
|
|
2619
|
+
this._pendingToolCalls.delete(callId);
|
|
2620
|
+
}
|
|
2621
|
+
}
|
|
2622
|
+
_waitForResponseDone() {
|
|
2623
|
+
if (!this._responseInProgress) return Promise.resolve();
|
|
2624
|
+
return new Promise((resolve) => {
|
|
2625
|
+
this._onResponseDone = resolve;
|
|
2626
|
+
});
|
|
2627
|
+
}
|
|
2628
|
+
_send(data) {
|
|
2629
|
+
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
2630
|
+
this._ws.send(JSON.stringify(data));
|
|
2631
|
+
}
|
|
2632
|
+
}
|
|
2633
|
+
};
|
|
2634
|
+
|
|
2635
|
+
// src/agent/pipeline/realtime/gemini-realtime.ts
|
|
2636
|
+
function resolveRef(ref, defs) {
|
|
2637
|
+
const parts = ref.replace(/^#\//, "").split("/");
|
|
2638
|
+
let result = defs;
|
|
2639
|
+
for (const part of parts) {
|
|
2640
|
+
if (result && typeof result === "object" && !Array.isArray(result)) {
|
|
2641
|
+
result = result[part];
|
|
2642
|
+
} else {
|
|
2643
|
+
return {};
|
|
2644
|
+
}
|
|
2645
|
+
}
|
|
2646
|
+
return typeof result === "object" && result !== null && !Array.isArray(result) ? result : {};
|
|
2647
|
+
}
|
|
2648
|
+
function sanitizeSchemaForGemini(schema, defs, depth = 0) {
|
|
2649
|
+
if (depth > 15) return { type: "object", properties: {} };
|
|
2650
|
+
if (!schema || typeof schema !== "object") return { type: "object", properties: {} };
|
|
2651
|
+
if (defs === void 0) {
|
|
2652
|
+
defs = schema["$defs"] ?? schema["definitions"] ?? {};
|
|
2653
|
+
}
|
|
2654
|
+
if (typeof schema["$ref"] === "string") {
|
|
2655
|
+
const resolved = resolveRef(schema["$ref"], { $defs: defs, definitions: defs });
|
|
2656
|
+
if (resolved && Object.keys(resolved).length > 0) {
|
|
2657
|
+
return sanitizeSchemaForGemini(resolved, defs, depth + 1);
|
|
2658
|
+
}
|
|
2659
|
+
return { type: "object", properties: {} };
|
|
2660
|
+
}
|
|
2661
|
+
for (const comboKey of ["oneOf", "anyOf", "allOf"]) {
|
|
2662
|
+
const variants = schema[comboKey];
|
|
2663
|
+
if (Array.isArray(variants) && variants.length > 0) {
|
|
2664
|
+
for (const v of variants) {
|
|
2665
|
+
if (v && typeof v === "object") {
|
|
2666
|
+
const resolved = sanitizeSchemaForGemini(v, defs, depth + 1);
|
|
2667
|
+
if (resolved["type"] === "object" && resolved["properties"]) {
|
|
2668
|
+
return resolved;
|
|
2567
2669
|
}
|
|
2568
|
-
responses.push({ id: fcId, name, response: { result } });
|
|
2569
2670
|
}
|
|
2570
|
-
continue;
|
|
2571
|
-
}
|
|
2572
|
-
if (!this._tools || !this._tools.has(name)) {
|
|
2573
|
-
this._log.error("Unknown tool: %s", name);
|
|
2574
|
-
responses.push({ id: fcId, name, response: { error: `Unknown tool: ${name}` } });
|
|
2575
|
-
continue;
|
|
2576
2671
|
}
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
this._log.info("Tool result: %s -> %s", name, resultStr.substring(0, 200));
|
|
2581
|
-
responses.push({
|
|
2582
|
-
id: fcId,
|
|
2583
|
-
name,
|
|
2584
|
-
response: { result: resultStr }
|
|
2585
|
-
});
|
|
2586
|
-
} catch (err) {
|
|
2587
|
-
this._log.error({ err }, "Tool call failed: %s", name);
|
|
2588
|
-
responses.push({
|
|
2589
|
-
id: fcId,
|
|
2590
|
-
name,
|
|
2591
|
-
response: { error: String(err) }
|
|
2592
|
-
});
|
|
2672
|
+
const first = variants[0];
|
|
2673
|
+
if (first && typeof first === "object") {
|
|
2674
|
+
return sanitizeSchemaForGemini(first, defs, depth + 1);
|
|
2593
2675
|
}
|
|
2594
2676
|
}
|
|
2595
|
-
if (responses.length > 0 && this._session) {
|
|
2596
|
-
this._log.debug("Sending %d tool response(s)", responses.length);
|
|
2597
|
-
this._session.sendToolResponse({
|
|
2598
|
-
functionResponses: responses
|
|
2599
|
-
});
|
|
2600
|
-
}
|
|
2601
|
-
this._toolCallInProgress = false;
|
|
2602
2677
|
}
|
|
2603
|
-
};
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4.",
|
|
2609
|
-
parameters: {
|
|
2610
|
-
properties: {
|
|
2611
|
-
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
2612
|
-
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
2613
|
-
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
2614
|
-
},
|
|
2615
|
-
required: ["max_digits"]
|
|
2616
|
-
}
|
|
2678
|
+
const result = {};
|
|
2679
|
+
let schemaType = schema["type"];
|
|
2680
|
+
if (Array.isArray(schemaType)) {
|
|
2681
|
+
const nonNull = schemaType.filter((t) => t !== "null");
|
|
2682
|
+
schemaType = nonNull[0] ?? "string";
|
|
2617
2683
|
}
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2684
|
+
if (schemaType) result["type"] = schemaType;
|
|
2685
|
+
if (schema["description"]) result["description"] = schema["description"];
|
|
2686
|
+
if (schema["enum"]) result["enum"] = schema["enum"];
|
|
2687
|
+
if (schema["required"]) result["required"] = schema["required"];
|
|
2688
|
+
if (schema["properties"] && typeof schema["properties"] === "object") {
|
|
2689
|
+
const props = {};
|
|
2690
|
+
for (const [key, val] of Object.entries(schema["properties"])) {
|
|
2691
|
+
if (val && typeof val === "object") {
|
|
2692
|
+
props[key] = sanitizeSchemaForGemini(val, defs, depth + 1);
|
|
2693
|
+
}
|
|
2627
2694
|
}
|
|
2695
|
+
result["properties"] = props;
|
|
2628
2696
|
}
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2697
|
+
if (schema["items"] && typeof schema["items"] === "object" && !Array.isArray(schema["items"])) {
|
|
2698
|
+
result["items"] = sanitizeSchemaForGemini(
|
|
2699
|
+
schema["items"],
|
|
2700
|
+
defs,
|
|
2701
|
+
depth + 1
|
|
2702
|
+
);
|
|
2703
|
+
}
|
|
2704
|
+
if (!result["type"] && result["properties"]) result["type"] = "object";
|
|
2705
|
+
if (result["type"] === "object" && !result["properties"]) result["properties"] = {};
|
|
2706
|
+
return result;
|
|
2707
|
+
}
|
|
2708
|
+
var GeminiRealtime = class {
|
|
2709
|
+
_apiKey;
|
|
2634
2710
|
_systemPrompt;
|
|
2635
|
-
|
|
2711
|
+
_model;
|
|
2712
|
+
_voice;
|
|
2636
2713
|
_language;
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
_callSession = null;
|
|
2714
|
+
_greeting;
|
|
2715
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
2716
|
+
_session = null;
|
|
2717
|
+
_call = null;
|
|
2642
2718
|
_tools = null;
|
|
2643
2719
|
_recorder = null;
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
_speaking = false;
|
|
2720
|
+
_closed = false;
|
|
2721
|
+
_sentAudioChunks = 0;
|
|
2722
|
+
_audioRemainder = Buffer.alloc(0);
|
|
2648
2723
|
_builtinTools = null;
|
|
2724
|
+
_toolCallInProgress = false;
|
|
2649
2725
|
_log = NOOP_LOGGER;
|
|
2650
|
-
constructor(options) {
|
|
2651
|
-
this.
|
|
2652
|
-
this.
|
|
2653
|
-
this.
|
|
2654
|
-
this.
|
|
2655
|
-
this._greeting = options.greeting ?? true;
|
|
2726
|
+
constructor(options = {}) {
|
|
2727
|
+
this._apiKey = options.apiKey ?? process.env["GOOGLE_API_KEY"] ?? "";
|
|
2728
|
+
this._systemPrompt = options.systemPrompt ?? "";
|
|
2729
|
+
this._model = options.model ?? "gemini-2.5-flash-native-audio-preview-12-2025";
|
|
2730
|
+
this._voice = options.voice ?? "Kore";
|
|
2656
2731
|
this._language = options.language ?? "ko";
|
|
2657
|
-
this.
|
|
2658
|
-
this._maxTokens = options.maxTokens;
|
|
2659
|
-
this._sampleRate = options.sampleRate ?? 8e3;
|
|
2660
|
-
this._interruptOnSpeech = options.interruptOnSpeech ?? true;
|
|
2661
|
-
if (options.toolRegistry) this._tools = options.toolRegistry;
|
|
2662
|
-
if (options.recorder) this._recorder = options.recorder;
|
|
2732
|
+
this._greeting = options.greeting ?? true;
|
|
2663
2733
|
}
|
|
2734
|
+
/** Inject per-call ToolRegistry. */
|
|
2664
2735
|
setToolRegistry(registry) {
|
|
2665
2736
|
this._tools = registry;
|
|
2666
2737
|
}
|
|
2738
|
+
/** Inject per-call AudioRecorder. */
|
|
2667
2739
|
setRecorder(recorder) {
|
|
2668
2740
|
this._recorder = recorder;
|
|
2669
2741
|
}
|
|
@@ -2672,239 +2744,266 @@ var PipelineSession = class {
|
|
|
2672
2744
|
}
|
|
2673
2745
|
setLogger(logger) {
|
|
2674
2746
|
this._log = logger;
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
this.
|
|
2680
|
-
|
|
2747
|
+
}
|
|
2748
|
+
getTelemetry() {
|
|
2749
|
+
return {
|
|
2750
|
+
sessionType: "gemini_realtime",
|
|
2751
|
+
llm: { provider: "gemini", model: this._model },
|
|
2752
|
+
stt: null,
|
|
2753
|
+
tts: null,
|
|
2754
|
+
voice: this._voice,
|
|
2755
|
+
language: this._language,
|
|
2756
|
+
greetingEnabled: this._greeting,
|
|
2757
|
+
recordingEnabled: !!this._recorder,
|
|
2758
|
+
toolCount: this._tools?.size ?? 0,
|
|
2759
|
+
mcpServerCount: 0,
|
|
2760
|
+
builtinTools: []
|
|
2761
|
+
};
|
|
2681
2762
|
}
|
|
2682
2763
|
async start(callSession, tools) {
|
|
2683
|
-
this.
|
|
2684
|
-
this._tools = tools
|
|
2685
|
-
this.
|
|
2686
|
-
this.
|
|
2687
|
-
this.
|
|
2688
|
-
if (this.
|
|
2689
|
-
|
|
2690
|
-
role: "system",
|
|
2691
|
-
content: this._systemPrompt
|
|
2692
|
-
});
|
|
2764
|
+
this._call = callSession;
|
|
2765
|
+
if (tools) this._tools = tools;
|
|
2766
|
+
this._closed = false;
|
|
2767
|
+
this._sentAudioChunks = 0;
|
|
2768
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
2769
|
+
if (!this._apiKey) {
|
|
2770
|
+
throw new Error("Google API key is required. Set GOOGLE_API_KEY or pass apiKey option.");
|
|
2693
2771
|
}
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2772
|
+
const { GoogleGenAI } = await import('@google/genai/node');
|
|
2773
|
+
const client = new GoogleGenAI({ apiKey: this._apiKey });
|
|
2774
|
+
const config = {
|
|
2775
|
+
responseModalities: ["AUDIO"],
|
|
2776
|
+
speechConfig: {
|
|
2777
|
+
voiceConfig: {
|
|
2778
|
+
prebuiltVoiceConfig: {
|
|
2779
|
+
voiceName: this._voice
|
|
2780
|
+
}
|
|
2781
|
+
}
|
|
2782
|
+
},
|
|
2783
|
+
inputAudioTranscription: {},
|
|
2784
|
+
outputAudioTranscription: {}
|
|
2785
|
+
};
|
|
2786
|
+
if (this._systemPrompt) {
|
|
2787
|
+
config["systemInstruction"] = this._systemPrompt;
|
|
2698
2788
|
}
|
|
2699
|
-
this.
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
}
|
|
2703
|
-
feedAudio(audio) {
|
|
2704
|
-
if (this._running) {
|
|
2705
|
-
this._audioBuffer.push(audio);
|
|
2789
|
+
const toolSchemas = this._buildToolSchemas();
|
|
2790
|
+
if (toolSchemas.length > 0) {
|
|
2791
|
+
config["tools"] = [{ functionDeclarations: toolSchemas }];
|
|
2706
2792
|
}
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
this._audioBuffer = [];
|
|
2719
|
-
}
|
|
2720
|
-
async _runSttLoop() {
|
|
2721
|
-
const audioStream = this._createAudioStream();
|
|
2722
|
-
for await (const event of this._stt.transcribe(audioStream, {
|
|
2723
|
-
sampleRate: this._sampleRate
|
|
2724
|
-
})) {
|
|
2725
|
-
if (!this._running) break;
|
|
2726
|
-
if (event.type === "interim" && this._speaking && this._interruptOnSpeech) {
|
|
2727
|
-
this._speaking = false;
|
|
2728
|
-
if (this._callSession) {
|
|
2729
|
-
this._callSession.clearAudio();
|
|
2793
|
+
this._session = await client.live.connect({
|
|
2794
|
+
model: this._model,
|
|
2795
|
+
config,
|
|
2796
|
+
callbacks: {
|
|
2797
|
+
onmessage: (msg) => this._handleMessage(msg),
|
|
2798
|
+
onerror: (err) => {
|
|
2799
|
+
this._log.error({ err }, "Gemini SDK error");
|
|
2800
|
+
},
|
|
2801
|
+
onclose: (ev) => {
|
|
2802
|
+
this._log.info({ code: ev?.code ?? "unknown" }, "Gemini connection closed");
|
|
2803
|
+
this._closed = true;
|
|
2730
2804
|
}
|
|
2731
|
-
this._log.info('Barge-in: "%s"', event.transcript.substring(0, 30));
|
|
2732
|
-
}
|
|
2733
|
-
if (event.type === "final" && event.transcript.trim()) {
|
|
2734
|
-
this._log.info("STT: %s", event.transcript);
|
|
2735
|
-
await this._handleUserSpeech(event.transcript);
|
|
2736
2805
|
}
|
|
2806
|
+
});
|
|
2807
|
+
if (this._greeting) {
|
|
2808
|
+
this._session.sendClientContent({
|
|
2809
|
+
turns: [
|
|
2810
|
+
{
|
|
2811
|
+
role: "user",
|
|
2812
|
+
parts: [{ text: "\uC778\uC0AC\uD574 \uC8FC\uC138\uC694." }]
|
|
2813
|
+
}
|
|
2814
|
+
],
|
|
2815
|
+
turnComplete: true
|
|
2816
|
+
});
|
|
2737
2817
|
}
|
|
2738
2818
|
}
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
2745
|
-
yield pcm16k;
|
|
2746
|
-
} else {
|
|
2747
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
2819
|
+
feedAudio(audio) {
|
|
2820
|
+
if (this._session && !this._closed && !this._toolCallInProgress) {
|
|
2821
|
+
const pcm8k = ulawToPcm16(audio);
|
|
2822
|
+
if (this._recorder) {
|
|
2823
|
+
this._recorder.writeInbound(pcm8k);
|
|
2748
2824
|
}
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
async _handleUserSpeech(transcript) {
|
|
2756
|
-
this._conversation.push({ role: "user", content: transcript });
|
|
2757
|
-
await this._respond();
|
|
2758
|
-
}
|
|
2759
|
-
_buildEffectiveTools() {
|
|
2760
|
-
const includeCollectDtmf = !this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */);
|
|
2761
|
-
const includeSendDtmf = !this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */);
|
|
2762
|
-
if (!includeCollectDtmf && !includeSendDtmf) return this._tools ?? void 0;
|
|
2763
|
-
const base = this._tools ? this._tools.fork() : new ToolRegistry();
|
|
2764
|
-
if (includeCollectDtmf) {
|
|
2765
|
-
base.register({
|
|
2766
|
-
name: "collect_dtmf",
|
|
2767
|
-
description: COLLECT_DTMF_TOOL3.function.description,
|
|
2768
|
-
parameters: COLLECT_DTMF_TOOL3.function.parameters.properties,
|
|
2769
|
-
required: COLLECT_DTMF_TOOL3.function.parameters.required,
|
|
2770
|
-
handler: async () => ""
|
|
2825
|
+
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
2826
|
+
this._session.sendRealtimeInput({
|
|
2827
|
+
audio: {
|
|
2828
|
+
data: Buffer.from(pcm16k).toString("base64"),
|
|
2829
|
+
mimeType: "audio/pcm;rate=16000"
|
|
2830
|
+
}
|
|
2771
2831
|
});
|
|
2772
2832
|
}
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
handler: async () => ""
|
|
2833
|
+
}
|
|
2834
|
+
async feedDtmf(digits) {
|
|
2835
|
+
if (this._session) {
|
|
2836
|
+
this._session.sendClientContent({
|
|
2837
|
+
turns: [{ role: "user", parts: [{ text: `[DTMF \uC785\uB825: ${digits}]` }] }],
|
|
2838
|
+
turnComplete: true
|
|
2780
2839
|
});
|
|
2781
2840
|
}
|
|
2782
|
-
return base;
|
|
2783
2841
|
}
|
|
2784
|
-
async
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
temperature: this._temperature,
|
|
2791
|
-
maxTokens: this._maxTokens
|
|
2792
|
-
});
|
|
2793
|
-
for await (const chunk of llmStream) {
|
|
2794
|
-
if (!this._running) break;
|
|
2795
|
-
if (chunk.type === "text" && chunk.text) {
|
|
2796
|
-
textChunks.push(chunk.text);
|
|
2797
|
-
fullResponse += chunk.text;
|
|
2798
|
-
} else if (chunk.type === "tool_call" && chunk.toolCall) {
|
|
2799
|
-
await this._handleToolCall(chunk);
|
|
2842
|
+
async stop() {
|
|
2843
|
+
this._closed = true;
|
|
2844
|
+
if (this._session) {
|
|
2845
|
+
try {
|
|
2846
|
+
this._session.close();
|
|
2847
|
+
} catch {
|
|
2800
2848
|
}
|
|
2801
|
-
|
|
2802
|
-
if (fullResponse.trim()) {
|
|
2803
|
-
this._log.info("Assistant: %s", fullResponse.substring(0, 100));
|
|
2804
|
-
this._conversation.push({ role: "assistant", content: fullResponse });
|
|
2805
|
-
await this._synthesizeAndSend(fullResponse);
|
|
2849
|
+
this._session = null;
|
|
2806
2850
|
}
|
|
2807
2851
|
}
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2852
|
+
_buildToolSchemas() {
|
|
2853
|
+
const toolDefs = this._tools ? this._tools.toOpenAITools().map((t) => ({
|
|
2854
|
+
name: t.function.name,
|
|
2855
|
+
description: t.function.description,
|
|
2856
|
+
parameters: sanitizeSchemaForGemini(
|
|
2857
|
+
t.function.parameters ?? { type: "object", properties: {} }
|
|
2858
|
+
)
|
|
2859
|
+
})) : [];
|
|
2860
|
+
toolDefs.push(...getBuiltinToolSchemas(this._builtinTools, "gemini"));
|
|
2861
|
+
return toolDefs;
|
|
2862
|
+
}
|
|
2863
|
+
_handleMessage(msg) {
|
|
2864
|
+
if (!this._call) return;
|
|
2865
|
+
const serverContent = msg.serverContent;
|
|
2866
|
+
if (serverContent) {
|
|
2867
|
+
const modelTurn = serverContent.modelTurn;
|
|
2868
|
+
if (modelTurn) {
|
|
2869
|
+
for (const part of modelTurn.parts ?? []) {
|
|
2870
|
+
const inlineData = part.inlineData;
|
|
2871
|
+
if (inlineData?.data) {
|
|
2872
|
+
const mimeType = inlineData.mimeType ?? "";
|
|
2873
|
+
if (mimeType.includes("audio")) {
|
|
2874
|
+
this._handleAudioData(inlineData.data);
|
|
2875
|
+
}
|
|
2876
|
+
}
|
|
2823
2877
|
}
|
|
2824
|
-
this._conversation.push({ role: "tool", content: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)", tool_call_id: id, name });
|
|
2825
|
-
await this._respond();
|
|
2826
|
-
return;
|
|
2827
2878
|
}
|
|
2828
|
-
if (
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
await this._callSession.sendDtmfSequence(args["digits"] ?? "");
|
|
2832
|
-
result2 = "sent";
|
|
2833
|
-
} catch (err) {
|
|
2834
|
-
result2 = `Error: ${err}`;
|
|
2835
|
-
}
|
|
2836
|
-
this._conversation.push({ role: "tool", content: result2, tool_call_id: id, name });
|
|
2837
|
-
await this._respond();
|
|
2838
|
-
return;
|
|
2879
|
+
if (serverContent.turnComplete) {
|
|
2880
|
+
this._log.debug("Turn complete");
|
|
2881
|
+
this._flushAudioRemainder();
|
|
2839
2882
|
}
|
|
2840
|
-
if (
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
content: ""
|
|
2845
|
-
// Tool call info stored in the message flow
|
|
2846
|
-
});
|
|
2847
|
-
this._conversation.push({
|
|
2848
|
-
role: "tool",
|
|
2849
|
-
content: typeof result === "string" ? result : JSON.stringify(result),
|
|
2850
|
-
tool_call_id: id,
|
|
2851
|
-
name
|
|
2852
|
-
});
|
|
2853
|
-
const effectiveTools = this._buildEffectiveTools();
|
|
2854
|
-
let followUpText = "";
|
|
2855
|
-
const followUpStream = this._llm.generate(this._conversation, {
|
|
2856
|
-
tools: effectiveTools,
|
|
2857
|
-
temperature: this._temperature,
|
|
2858
|
-
maxTokens: this._maxTokens
|
|
2859
|
-
});
|
|
2860
|
-
for await (const followChunk of followUpStream) {
|
|
2861
|
-
if (!this._running) break;
|
|
2862
|
-
if (followChunk.type === "text" && followChunk.text) {
|
|
2863
|
-
followUpText += followChunk.text;
|
|
2883
|
+
if (serverContent.interrupted) {
|
|
2884
|
+
this._log.info("Barge-in detected");
|
|
2885
|
+
if (this._call) {
|
|
2886
|
+
this._call.clearAudio();
|
|
2864
2887
|
}
|
|
2888
|
+
this._sentAudioChunks = 0;
|
|
2889
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
2865
2890
|
}
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2891
|
+
const inputText = serverContent.inputTranscription?.text;
|
|
2892
|
+
if (inputText && this._call) {
|
|
2893
|
+
this._log.info("User: %s", inputText);
|
|
2894
|
+
this._call._emit("transcript", "user", inputText);
|
|
2869
2895
|
}
|
|
2870
|
-
|
|
2871
|
-
|
|
2896
|
+
const outputText = serverContent.outputTranscription?.text;
|
|
2897
|
+
if (outputText && this._call) {
|
|
2898
|
+
this._log.info("Assistant: %s", outputText);
|
|
2899
|
+
this._call._emit("transcript", "assistant", outputText);
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2902
|
+
if (msg.toolCall) {
|
|
2903
|
+
this._handleToolCall(msg.toolCall);
|
|
2904
|
+
}
|
|
2905
|
+
const toolCancellation = msg["toolCallCancellation"];
|
|
2906
|
+
if (toolCancellation) {
|
|
2907
|
+
this._log.info({ ids: toolCancellation.ids }, "Tool call cancelled");
|
|
2872
2908
|
}
|
|
2873
2909
|
}
|
|
2874
|
-
|
|
2875
|
-
if (!this.
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2910
|
+
_handleAudioData(b64Data) {
|
|
2911
|
+
if (!this._call) return;
|
|
2912
|
+
const pcm24k = Buffer.from(b64Data, "base64");
|
|
2913
|
+
if (this._recorder) {
|
|
2914
|
+
this._recorder.writeOutbound(resamplePcm16(pcm24k, 24e3, 8e3));
|
|
2915
|
+
}
|
|
2916
|
+
const pcm8k = resamplePcm16(pcm24k, 24e3, 8e3);
|
|
2917
|
+
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2918
|
+
const combined = Buffer.concat([this._audioRemainder, ulaw]);
|
|
2919
|
+
const chunkSize = 160;
|
|
2920
|
+
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
2921
|
+
if (fullEnd > 0) {
|
|
2922
|
+
this._call.sendAudio(combined.subarray(0, fullEnd));
|
|
2923
|
+
this._sentAudioChunks += fullEnd / chunkSize;
|
|
2924
|
+
}
|
|
2925
|
+
this._audioRemainder = combined.subarray(fullEnd);
|
|
2926
|
+
}
|
|
2927
|
+
_flushAudioRemainder() {
|
|
2928
|
+
if (this._audioRemainder.length > 0 && this._call) {
|
|
2929
|
+
const padded = Buffer.concat([
|
|
2930
|
+
this._audioRemainder,
|
|
2931
|
+
Buffer.alloc(160 - this._audioRemainder.length, 255)
|
|
2932
|
+
]);
|
|
2933
|
+
this._call.sendAudio(padded);
|
|
2934
|
+
this._sentAudioChunks++;
|
|
2935
|
+
this._audioRemainder = Buffer.alloc(0);
|
|
2936
|
+
}
|
|
2937
|
+
}
|
|
2938
|
+
async _handleToolCall(toolCall) {
|
|
2939
|
+
const functionCalls = toolCall.functionCalls;
|
|
2940
|
+
if (!functionCalls) return;
|
|
2941
|
+
this._toolCallInProgress = true;
|
|
2942
|
+
const responses = [];
|
|
2943
|
+
for (const fc of functionCalls) {
|
|
2944
|
+
const name = fc.name ?? "";
|
|
2945
|
+
const fcId = fc.id ?? "";
|
|
2946
|
+
const args = fc.args ?? {};
|
|
2947
|
+
this._log.info({ tool: name, args }, "Tool call: %s", name);
|
|
2948
|
+
if (BUILTIN_TOOL_NAMES.has(name) && this._call) {
|
|
2949
|
+
const result = await executeBuiltinTool(name, args, this._call);
|
|
2950
|
+
if (result !== null) {
|
|
2951
|
+
if (name === "hang_up") {
|
|
2952
|
+
this._log.info("hang_up: ending call");
|
|
2953
|
+
return;
|
|
2892
2954
|
}
|
|
2893
|
-
this.
|
|
2955
|
+
this._log.info("Builtin tool result: %s -> %s", name, result);
|
|
2956
|
+
responses.push({ id: fcId, name, response: { result } });
|
|
2957
|
+
continue;
|
|
2894
2958
|
}
|
|
2895
2959
|
}
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
2960
|
+
if (!this._tools || !this._tools.has(name)) {
|
|
2961
|
+
this._log.error("Unknown tool: %s", name);
|
|
2962
|
+
responses.push({ id: fcId, name, response: { error: `Unknown tool: ${name}` } });
|
|
2963
|
+
continue;
|
|
2964
|
+
}
|
|
2965
|
+
try {
|
|
2966
|
+
this._call?.recordToolCall();
|
|
2967
|
+
const result = await this._tools.call(name, args);
|
|
2968
|
+
const resultStr = typeof result === "string" ? result : JSON.stringify(result);
|
|
2969
|
+
this._log.info("Tool result: %s -> %s", name, resultStr.substring(0, 200));
|
|
2970
|
+
responses.push({
|
|
2971
|
+
id: fcId,
|
|
2972
|
+
name,
|
|
2973
|
+
response: { result: resultStr }
|
|
2974
|
+
});
|
|
2975
|
+
} catch (err) {
|
|
2976
|
+
this._log.error({ err }, "Tool call failed: %s", name);
|
|
2977
|
+
if (err instanceof Error) {
|
|
2978
|
+
this._call?.recordToolError(err);
|
|
2979
|
+
}
|
|
2980
|
+
responses.push({
|
|
2981
|
+
id: fcId,
|
|
2982
|
+
name,
|
|
2983
|
+
response: { error: String(err) }
|
|
2984
|
+
});
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
2987
|
+
if (responses.length > 0 && this._session) {
|
|
2988
|
+
this._log.debug("Sending %d tool response(s)", responses.length);
|
|
2989
|
+
this._session.sendToolResponse({
|
|
2990
|
+
functionResponses: responses
|
|
2991
|
+
});
|
|
2900
2992
|
}
|
|
2993
|
+
this._toolCallInProgress = false;
|
|
2901
2994
|
}
|
|
2902
2995
|
};
|
|
2903
2996
|
|
|
2904
|
-
// src/agent/pipeline/deepgram-stt.ts
|
|
2997
|
+
// src/agent/pipeline/stt/deepgram-stt.ts
|
|
2905
2998
|
var DeepgramSTT = class {
|
|
2906
2999
|
_options;
|
|
2907
3000
|
_log = NOOP_LOGGER;
|
|
3001
|
+
get provider() {
|
|
3002
|
+
return "deepgram";
|
|
3003
|
+
}
|
|
3004
|
+
get model() {
|
|
3005
|
+
return this._options.model ?? "nova-3";
|
|
3006
|
+
}
|
|
2908
3007
|
setLogger(logger) {
|
|
2909
3008
|
this._log = logger;
|
|
2910
3009
|
}
|
|
@@ -3022,10 +3121,19 @@ var DeepgramSTT = class {
|
|
|
3022
3121
|
}
|
|
3023
3122
|
};
|
|
3024
3123
|
|
|
3025
|
-
// src/agent/pipeline/elevenlabs-tts.ts
|
|
3124
|
+
// src/agent/pipeline/tts/elevenlabs-tts.ts
|
|
3026
3125
|
var ElevenLabsTTS = class {
|
|
3027
3126
|
_options;
|
|
3028
3127
|
_log = NOOP_LOGGER;
|
|
3128
|
+
get provider() {
|
|
3129
|
+
return "elevenlabs";
|
|
3130
|
+
}
|
|
3131
|
+
get model() {
|
|
3132
|
+
return this._options.model ?? "eleven_flash_v2_5";
|
|
3133
|
+
}
|
|
3134
|
+
get voiceId() {
|
|
3135
|
+
return this._options.voiceId ?? "EXAVITQu4vr4xnSDxMaL";
|
|
3136
|
+
}
|
|
3029
3137
|
setLogger(logger) {
|
|
3030
3138
|
this._log = logger;
|
|
3031
3139
|
}
|
|
@@ -3175,9 +3283,15 @@ var ElevenLabsTTS = class {
|
|
|
3175
3283
|
}
|
|
3176
3284
|
};
|
|
3177
3285
|
|
|
3178
|
-
// src/agent/pipeline/openai-llm.ts
|
|
3286
|
+
// src/agent/pipeline/llm/openai-llm.ts
|
|
3179
3287
|
var OpenAILLM = class {
|
|
3180
3288
|
_options;
|
|
3289
|
+
get provider() {
|
|
3290
|
+
return "openai";
|
|
3291
|
+
}
|
|
3292
|
+
get model() {
|
|
3293
|
+
return this._options.model;
|
|
3294
|
+
}
|
|
3181
3295
|
constructor(options = {}) {
|
|
3182
3296
|
this._options = {
|
|
3183
3297
|
model: "gpt-4o-mini",
|
|
@@ -3261,9 +3375,15 @@ var OpenAILLM = class {
|
|
|
3261
3375
|
}
|
|
3262
3376
|
};
|
|
3263
3377
|
|
|
3264
|
-
// src/agent/pipeline/anthropic-llm.ts
|
|
3378
|
+
// src/agent/pipeline/llm/anthropic-llm.ts
|
|
3265
3379
|
var AnthropicLLM = class {
|
|
3266
3380
|
_options;
|
|
3381
|
+
get provider() {
|
|
3382
|
+
return "anthropic";
|
|
3383
|
+
}
|
|
3384
|
+
get model() {
|
|
3385
|
+
return this._options.model;
|
|
3386
|
+
}
|
|
3267
3387
|
constructor(options = {}) {
|
|
3268
3388
|
this._options = {
|
|
3269
3389
|
model: "claude-sonnet-4-6",
|
|
@@ -3366,9 +3486,15 @@ var AnthropicLLM = class {
|
|
|
3366
3486
|
}
|
|
3367
3487
|
};
|
|
3368
3488
|
|
|
3369
|
-
// src/agent/pipeline/gemini-llm.ts
|
|
3489
|
+
// src/agent/pipeline/llm/gemini-llm.ts
|
|
3370
3490
|
var GeminiLLM = class {
|
|
3371
3491
|
_options;
|
|
3492
|
+
get provider() {
|
|
3493
|
+
return "gemini";
|
|
3494
|
+
}
|
|
3495
|
+
get model() {
|
|
3496
|
+
return this._options.model;
|
|
3497
|
+
}
|
|
3372
3498
|
constructor(options = {}) {
|
|
3373
3499
|
this._options = {
|
|
3374
3500
|
model: "gemini-2.5-flash",
|
|
@@ -3450,9 +3576,15 @@ var GeminiLLM = class {
|
|
|
3450
3576
|
}
|
|
3451
3577
|
};
|
|
3452
3578
|
|
|
3453
|
-
// src/agent/pipeline/openai-compat-llm.ts
|
|
3579
|
+
// src/agent/pipeline/llm/openai-compat-llm.ts
|
|
3454
3580
|
var OpenAICompatLLM = class {
|
|
3455
3581
|
_options;
|
|
3582
|
+
get provider() {
|
|
3583
|
+
return "openai_compatible";
|
|
3584
|
+
}
|
|
3585
|
+
get model() {
|
|
3586
|
+
return this._options.model;
|
|
3587
|
+
}
|
|
3456
3588
|
constructor(options) {
|
|
3457
3589
|
this._options = options;
|
|
3458
3590
|
}
|
|
@@ -3533,9 +3665,15 @@ var OpenAICompatLLM = class {
|
|
|
3533
3665
|
}
|
|
3534
3666
|
};
|
|
3535
3667
|
|
|
3536
|
-
// src/agent/pipeline/ollama-llm.ts
|
|
3668
|
+
// src/agent/pipeline/llm/ollama-llm.ts
|
|
3537
3669
|
var OllamaLLM = class {
|
|
3538
3670
|
_inner;
|
|
3671
|
+
get provider() {
|
|
3672
|
+
return "ollama";
|
|
3673
|
+
}
|
|
3674
|
+
get model() {
|
|
3675
|
+
return this._inner.model;
|
|
3676
|
+
}
|
|
3539
3677
|
constructor(options = {}) {
|
|
3540
3678
|
const baseUrl = options.baseUrl ?? process.env["OLLAMA_BASE_URL"] ?? "http://localhost:11434/v1";
|
|
3541
3679
|
this._inner = new OpenAICompatLLM({
|
|
@@ -3551,9 +3689,15 @@ var OllamaLLM = class {
|
|
|
3551
3689
|
}
|
|
3552
3690
|
};
|
|
3553
3691
|
|
|
3554
|
-
// src/agent/pipeline/mistral-llm.ts
|
|
3692
|
+
// src/agent/pipeline/llm/mistral-llm.ts
|
|
3555
3693
|
var MistralLLM = class {
|
|
3556
3694
|
_inner;
|
|
3695
|
+
get provider() {
|
|
3696
|
+
return "mistral";
|
|
3697
|
+
}
|
|
3698
|
+
get model() {
|
|
3699
|
+
return this._inner.model;
|
|
3700
|
+
}
|
|
3557
3701
|
constructor(options = {}) {
|
|
3558
3702
|
this._inner = new OpenAICompatLLM({
|
|
3559
3703
|
apiKey: options.apiKey ?? process.env["MISTRAL_API_KEY"],
|
|
@@ -3568,9 +3712,15 @@ var MistralLLM = class {
|
|
|
3568
3712
|
}
|
|
3569
3713
|
};
|
|
3570
3714
|
|
|
3571
|
-
// src/agent/pipeline/groq-llm.ts
|
|
3715
|
+
// src/agent/pipeline/llm/groq-llm.ts
|
|
3572
3716
|
var GroqLLM = class {
|
|
3573
3717
|
_inner;
|
|
3718
|
+
get provider() {
|
|
3719
|
+
return "groq";
|
|
3720
|
+
}
|
|
3721
|
+
get model() {
|
|
3722
|
+
return this._inner.model;
|
|
3723
|
+
}
|
|
3574
3724
|
constructor(options = {}) {
|
|
3575
3725
|
this._inner = new OpenAICompatLLM({
|
|
3576
3726
|
apiKey: options.apiKey ?? process.env["GROQ_API_KEY"],
|
|
@@ -3585,9 +3735,15 @@ var GroqLLM = class {
|
|
|
3585
3735
|
}
|
|
3586
3736
|
};
|
|
3587
3737
|
|
|
3588
|
-
// src/agent/pipeline/perplexity-llm.ts
|
|
3738
|
+
// src/agent/pipeline/llm/perplexity-llm.ts
|
|
3589
3739
|
var PerplexityLLM = class {
|
|
3590
3740
|
_inner;
|
|
3741
|
+
get provider() {
|
|
3742
|
+
return "perplexity";
|
|
3743
|
+
}
|
|
3744
|
+
get model() {
|
|
3745
|
+
return this._inner.model;
|
|
3746
|
+
}
|
|
3591
3747
|
constructor(options = {}) {
|
|
3592
3748
|
this._inner = new OpenAICompatLLM({
|
|
3593
3749
|
apiKey: options.apiKey ?? process.env["PERPLEXITY_API_KEY"],
|
|
@@ -3602,9 +3758,15 @@ var PerplexityLLM = class {
|
|
|
3602
3758
|
}
|
|
3603
3759
|
};
|
|
3604
3760
|
|
|
3605
|
-
// src/agent/pipeline/together-llm.ts
|
|
3761
|
+
// src/agent/pipeline/llm/together-llm.ts
|
|
3606
3762
|
var TogetherLLM = class {
|
|
3607
3763
|
_inner;
|
|
3764
|
+
get provider() {
|
|
3765
|
+
return "together";
|
|
3766
|
+
}
|
|
3767
|
+
get model() {
|
|
3768
|
+
return this._inner.model;
|
|
3769
|
+
}
|
|
3608
3770
|
constructor(options = {}) {
|
|
3609
3771
|
this._inner = new OpenAICompatLLM({
|
|
3610
3772
|
apiKey: options.apiKey ?? process.env["TOGETHER_API_KEY"],
|
|
@@ -3619,9 +3781,15 @@ var TogetherLLM = class {
|
|
|
3619
3781
|
}
|
|
3620
3782
|
};
|
|
3621
3783
|
|
|
3622
|
-
// src/agent/pipeline/fireworks-llm.ts
|
|
3784
|
+
// src/agent/pipeline/llm/fireworks-llm.ts
|
|
3623
3785
|
var FireworksLLM = class {
|
|
3624
3786
|
_inner;
|
|
3787
|
+
get provider() {
|
|
3788
|
+
return "fireworks";
|
|
3789
|
+
}
|
|
3790
|
+
get model() {
|
|
3791
|
+
return this._inner.model;
|
|
3792
|
+
}
|
|
3625
3793
|
constructor(options = {}) {
|
|
3626
3794
|
this._inner = new OpenAICompatLLM({
|
|
3627
3795
|
apiKey: options.apiKey ?? process.env["FIREWORKS_API_KEY"],
|
|
@@ -3636,9 +3804,15 @@ var FireworksLLM = class {
|
|
|
3636
3804
|
}
|
|
3637
3805
|
};
|
|
3638
3806
|
|
|
3639
|
-
// src/agent/pipeline/deepseek-llm.ts
|
|
3807
|
+
// src/agent/pipeline/llm/deepseek-llm.ts
|
|
3640
3808
|
var DeepSeekLLM = class {
|
|
3641
3809
|
_inner;
|
|
3810
|
+
get provider() {
|
|
3811
|
+
return "deepseek";
|
|
3812
|
+
}
|
|
3813
|
+
get model() {
|
|
3814
|
+
return this._inner.model;
|
|
3815
|
+
}
|
|
3642
3816
|
constructor(options = {}) {
|
|
3643
3817
|
this._inner = new OpenAICompatLLM({
|
|
3644
3818
|
apiKey: options.apiKey ?? process.env["DEEPSEEK_API_KEY"],
|
|
@@ -3653,9 +3827,15 @@ var DeepSeekLLM = class {
|
|
|
3653
3827
|
}
|
|
3654
3828
|
};
|
|
3655
3829
|
|
|
3656
|
-
// src/agent/pipeline/xai-llm.ts
|
|
3830
|
+
// src/agent/pipeline/llm/xai-llm.ts
|
|
3657
3831
|
var XaiLLM = class {
|
|
3658
3832
|
_inner;
|
|
3833
|
+
get provider() {
|
|
3834
|
+
return "xai";
|
|
3835
|
+
}
|
|
3836
|
+
get model() {
|
|
3837
|
+
return this._inner.model;
|
|
3838
|
+
}
|
|
3659
3839
|
constructor(options = {}) {
|
|
3660
3840
|
this._inner = new OpenAICompatLLM({
|
|
3661
3841
|
apiKey: options.apiKey ?? process.env["XAI_API_KEY"],
|
|
@@ -3691,6 +3871,7 @@ function mcpServerHTTP(options) {
|
|
|
3691
3871
|
|
|
3692
3872
|
exports.AnthropicLLM = AnthropicLLM;
|
|
3693
3873
|
exports.AudioRecorder = AudioRecorder;
|
|
3874
|
+
exports.BUILTIN_TOOL_NAMES = BUILTIN_TOOL_NAMES;
|
|
3694
3875
|
exports.BuiltinTool = BuiltinTool;
|
|
3695
3876
|
exports.CallSession = CallSession;
|
|
3696
3877
|
exports.ClawOpsAgent = ClawOpsAgent;
|
|
@@ -3715,8 +3896,11 @@ exports.ToolRegistry = ToolRegistry;
|
|
|
3715
3896
|
exports.XaiLLM = XaiLLM;
|
|
3716
3897
|
exports.createAgentLogger = createAgentLogger;
|
|
3717
3898
|
exports.createPipelineLogger = createPipelineLogger;
|
|
3899
|
+
exports.executeBuiltinTool = executeBuiltinTool;
|
|
3718
3900
|
exports.functionTool = functionTool;
|
|
3901
|
+
exports.getBuiltinToolSchemas = getBuiltinToolSchemas;
|
|
3719
3902
|
exports.getTracingConfig = getTracingConfig;
|
|
3903
|
+
exports.isBuiltinTool = isBuiltinTool;
|
|
3720
3904
|
exports.mcpServerHTTP = mcpServerHTTP;
|
|
3721
3905
|
exports.mcpServerStdio = mcpServerStdio;
|
|
3722
3906
|
exports.pcm16ToUlaw = pcm16ToUlaw;
|