@teamlearners/clawops 0.5.4 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/index.cjs +629 -652
- package/dist/agent/index.cjs.map +1 -1
- package/dist/agent/index.d.cts +99 -74
- package/dist/agent/index.d.ts +99 -74
- package/dist/agent/index.js +626 -653
- package/dist/agent/index.js.map +1 -1
- package/package.json +2 -2
package/dist/agent/index.cjs
CHANGED
|
@@ -361,6 +361,7 @@ function createPipelineLogger(parent) {
|
|
|
361
361
|
// src/agent/control-ws.ts
|
|
362
362
|
var INITIAL_RECONNECT_DELAY = 1e3;
|
|
363
363
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
364
|
+
var PING_TIMEOUT = 6e4;
|
|
364
365
|
function buildControlWsUrl(options) {
|
|
365
366
|
const scheme = options.baseUrl.startsWith("https") ? "wss" : "ws";
|
|
366
367
|
const host = options.baseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "");
|
|
@@ -386,6 +387,7 @@ var ControlWebSocket = class {
|
|
|
386
387
|
_connectedResolve = null;
|
|
387
388
|
_connectedPromise;
|
|
388
389
|
_log = NOOP_LOGGER;
|
|
390
|
+
_pingTimer = null;
|
|
389
391
|
setLogger(logger) {
|
|
390
392
|
this._log = logger;
|
|
391
393
|
}
|
|
@@ -416,6 +418,7 @@ var ControlWebSocket = class {
|
|
|
416
418
|
/** Close the WebSocket and stop reconnecting. */
|
|
417
419
|
close() {
|
|
418
420
|
this._closed = true;
|
|
421
|
+
this._clearPingTimer();
|
|
419
422
|
if (this._ws) {
|
|
420
423
|
this._ws.close();
|
|
421
424
|
this._ws = null;
|
|
@@ -432,12 +435,16 @@ var ControlWebSocket = class {
|
|
|
432
435
|
this._ws = ws;
|
|
433
436
|
ws.on("open", () => {
|
|
434
437
|
this._reconnectDelay = INITIAL_RECONNECT_DELAY;
|
|
438
|
+
this._resetPingTimer();
|
|
435
439
|
if (this._connectedResolve) {
|
|
436
440
|
this._connectedResolve();
|
|
437
441
|
this._connectedResolve = null;
|
|
438
442
|
}
|
|
439
443
|
this._log.info("Control WS connected: %s", this._url);
|
|
440
444
|
});
|
|
445
|
+
ws.on("ping", () => {
|
|
446
|
+
this._resetPingTimer();
|
|
447
|
+
});
|
|
441
448
|
ws.on("message", (data) => {
|
|
442
449
|
try {
|
|
443
450
|
const msg = JSON.parse(data.toString());
|
|
@@ -447,6 +454,7 @@ var ControlWebSocket = class {
|
|
|
447
454
|
}
|
|
448
455
|
});
|
|
449
456
|
ws.on("close", () => {
|
|
457
|
+
this._clearPingTimer();
|
|
450
458
|
if (!this._closed) {
|
|
451
459
|
this._scheduleReconnect();
|
|
452
460
|
}
|
|
@@ -472,6 +480,21 @@ var ControlWebSocket = class {
|
|
|
472
480
|
}
|
|
473
481
|
}
|
|
474
482
|
}
|
|
483
|
+
_resetPingTimer() {
|
|
484
|
+
this._clearPingTimer();
|
|
485
|
+
this._pingTimer = setTimeout(() => {
|
|
486
|
+
this._log.warn("Control WS ping timeout, closing connection");
|
|
487
|
+
if (this._ws) {
|
|
488
|
+
this._ws.terminate();
|
|
489
|
+
}
|
|
490
|
+
}, PING_TIMEOUT);
|
|
491
|
+
}
|
|
492
|
+
_clearPingTimer() {
|
|
493
|
+
if (this._pingTimer) {
|
|
494
|
+
clearTimeout(this._pingTimer);
|
|
495
|
+
this._pingTimer = null;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
475
498
|
_scheduleReconnect() {
|
|
476
499
|
const delay = this._reconnectDelay;
|
|
477
500
|
this._reconnectDelay = Math.min(this._reconnectDelay * 2, MAX_RECONNECT_DELAY);
|
|
@@ -1462,6 +1485,7 @@ var ClawOpsAgent = class {
|
|
|
1462
1485
|
}
|
|
1463
1486
|
/** Connect to the ClawOps platform and start listening for calls. */
|
|
1464
1487
|
async connect() {
|
|
1488
|
+
if (this._controlWs) return;
|
|
1465
1489
|
if (!this._apiKey) {
|
|
1466
1490
|
throw new chunk6IQN5RQD_cjs.AgentError("API key is required. Set CLAWOPS_API_KEY or pass apiKey option.");
|
|
1467
1491
|
}
|
|
@@ -1776,20 +1800,13 @@ var ClawOpsAgent = class {
|
|
|
1776
1800
|
}
|
|
1777
1801
|
};
|
|
1778
1802
|
|
|
1779
|
-
// src/agent/pipeline/
|
|
1780
|
-
var
|
|
1781
|
-
var HANG_UP_TOOL = {
|
|
1782
|
-
type: "function",
|
|
1803
|
+
// src/agent/pipeline/builtin-tool-schemas.ts
|
|
1804
|
+
var HANG_UP = {
|
|
1783
1805
|
name: "hang_up",
|
|
1784
1806
|
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1785
|
-
parameters: {
|
|
1786
|
-
type: "object",
|
|
1787
|
-
properties: {},
|
|
1788
|
-
required: []
|
|
1789
|
-
}
|
|
1807
|
+
parameters: { type: "object", properties: {} }
|
|
1790
1808
|
};
|
|
1791
|
-
var
|
|
1792
|
-
type: "function",
|
|
1809
|
+
var COLLECT_DTMF = {
|
|
1793
1810
|
name: "collect_dtmf",
|
|
1794
1811
|
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
|
|
1795
1812
|
parameters: {
|
|
@@ -1802,8 +1819,7 @@ var COLLECT_DTMF_TOOL = {
|
|
|
1802
1819
|
required: ["max_digits"]
|
|
1803
1820
|
}
|
|
1804
1821
|
};
|
|
1805
|
-
var
|
|
1806
|
-
type: "function",
|
|
1822
|
+
var SEND_DTMF = {
|
|
1807
1823
|
name: "send_dtmf",
|
|
1808
1824
|
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
1809
1825
|
parameters: {
|
|
@@ -1811,199 +1827,530 @@ var SEND_DTMF_TOOL = {
|
|
|
1811
1827
|
properties: {
|
|
1812
1828
|
digits: {
|
|
1813
1829
|
type: "string",
|
|
1814
|
-
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30."
|
|
1830
|
+
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30. \uC608: '1', '1234#', '1w2'"
|
|
1815
1831
|
}
|
|
1816
1832
|
},
|
|
1817
1833
|
required: ["digits"]
|
|
1818
1834
|
}
|
|
1819
1835
|
};
|
|
1820
|
-
var
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1836
|
+
var TOOL_MAP = /* @__PURE__ */ new Map([
|
|
1837
|
+
["hang_up" /* HANG_UP */, HANG_UP],
|
|
1838
|
+
["collect_dtmf" /* COLLECT_DTMF */, COLLECT_DTMF],
|
|
1839
|
+
["send_dtmf" /* SEND_DTMF */, SEND_DTMF]
|
|
1840
|
+
]);
|
|
1841
|
+
var BUILTIN_TOOL_NAMES = new Set(
|
|
1842
|
+
Array.from(TOOL_MAP.values()).map((s) => s.name)
|
|
1843
|
+
);
|
|
1844
|
+
function toChatCompletions(schema) {
|
|
1845
|
+
return {
|
|
1846
|
+
type: "function",
|
|
1847
|
+
function: {
|
|
1848
|
+
name: schema.name,
|
|
1849
|
+
description: schema.description,
|
|
1850
|
+
parameters: schema.parameters
|
|
1851
|
+
}
|
|
1852
|
+
};
|
|
1853
|
+
}
|
|
1854
|
+
function toRealtime(schema) {
|
|
1855
|
+
return {
|
|
1856
|
+
type: "function",
|
|
1857
|
+
name: schema.name,
|
|
1858
|
+
description: schema.description,
|
|
1859
|
+
parameters: schema.parameters
|
|
1860
|
+
};
|
|
1861
|
+
}
|
|
1862
|
+
function toGemini(schema) {
|
|
1863
|
+
return {
|
|
1864
|
+
name: schema.name,
|
|
1865
|
+
description: schema.description,
|
|
1866
|
+
parameters: schema.parameters
|
|
1867
|
+
};
|
|
1868
|
+
}
|
|
1869
|
+
var CONVERTERS = {
|
|
1870
|
+
chat: toChatCompletions,
|
|
1871
|
+
realtime: toRealtime,
|
|
1872
|
+
gemini: toGemini
|
|
1873
|
+
};
|
|
1874
|
+
function getBuiltinToolSchemas(builtinTools, fmt) {
|
|
1875
|
+
const converter = CONVERTERS[fmt];
|
|
1876
|
+
const result = [];
|
|
1877
|
+
for (const [toolEnum, schema] of TOOL_MAP) {
|
|
1878
|
+
if (builtinTools === null || builtinTools.has(toolEnum)) {
|
|
1879
|
+
result.push(converter(schema));
|
|
1880
|
+
}
|
|
1832
1881
|
}
|
|
1833
|
-
|
|
1834
|
-
|
|
1882
|
+
return result;
|
|
1883
|
+
}
|
|
1884
|
+
function isBuiltinTool(name) {
|
|
1885
|
+
return BUILTIN_TOOL_NAMES.has(name);
|
|
1886
|
+
}
|
|
1887
|
+
async function executeBuiltinTool(funcName, args, call) {
|
|
1888
|
+
if (funcName === "hang_up") {
|
|
1889
|
+
await call.hangup();
|
|
1890
|
+
return "";
|
|
1835
1891
|
}
|
|
1836
|
-
|
|
1837
|
-
|
|
1892
|
+
if (funcName === "collect_dtmf") {
|
|
1893
|
+
try {
|
|
1894
|
+
const result = await call.collectDtmf({
|
|
1895
|
+
maxDigits: args["max_digits"] ?? 4,
|
|
1896
|
+
finishOnKey: args["finish_on_key"] ?? "#",
|
|
1897
|
+
timeout: args["timeout"] ?? 5
|
|
1898
|
+
});
|
|
1899
|
+
return result || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)";
|
|
1900
|
+
} catch (e) {
|
|
1901
|
+
return `Error: ${e}`;
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
if (funcName === "send_dtmf") {
|
|
1905
|
+
try {
|
|
1906
|
+
await call.sendDtmfSequence(args["digits"] ?? "");
|
|
1907
|
+
return "sent";
|
|
1908
|
+
} catch (e) {
|
|
1909
|
+
return `Error: ${e}`;
|
|
1910
|
+
}
|
|
1911
|
+
}
|
|
1912
|
+
return null;
|
|
1913
|
+
}
|
|
1914
|
+
|
|
1915
|
+
// src/agent/pipeline/pipeline-session.ts
|
|
1916
|
+
var PipelineSession = class {
|
|
1917
|
+
_stt;
|
|
1918
|
+
_llm;
|
|
1919
|
+
_tts;
|
|
1920
|
+
_systemPrompt;
|
|
1921
|
+
_greeting;
|
|
1922
|
+
_language;
|
|
1923
|
+
_temperature;
|
|
1924
|
+
_maxTokens;
|
|
1925
|
+
_sampleRate;
|
|
1926
|
+
_interruptOnSpeech;
|
|
1927
|
+
_callSession = null;
|
|
1838
1928
|
_tools = null;
|
|
1839
1929
|
_recorder = null;
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
this.
|
|
1851
|
-
this._systemPrompt = options.systemPrompt ?? "";
|
|
1852
|
-
this._model = options.model ?? "gpt-realtime-1.5";
|
|
1853
|
-
this._voice = options.voice ?? "marin";
|
|
1854
|
-
this._language = options.language ?? "ko";
|
|
1855
|
-
this._eagerness = options.eagerness ?? "high";
|
|
1930
|
+
_conversation = [];
|
|
1931
|
+
_audioBuffer = [];
|
|
1932
|
+
_running = false;
|
|
1933
|
+
_speaking = false;
|
|
1934
|
+
_builtinTools = null;
|
|
1935
|
+
_log = NOOP_LOGGER;
|
|
1936
|
+
constructor(options) {
|
|
1937
|
+
this._stt = options.stt;
|
|
1938
|
+
this._llm = options.llm;
|
|
1939
|
+
this._tts = options.tts;
|
|
1940
|
+
this._systemPrompt = options.systemPrompt;
|
|
1856
1941
|
this._greeting = options.greeting ?? true;
|
|
1942
|
+
this._language = options.language ?? "ko";
|
|
1943
|
+
this._temperature = options.temperature;
|
|
1944
|
+
this._maxTokens = options.maxTokens;
|
|
1945
|
+
this._sampleRate = options.sampleRate ?? 8e3;
|
|
1946
|
+
this._interruptOnSpeech = options.interruptOnSpeech ?? true;
|
|
1947
|
+
if (options.toolRegistry) this._tools = options.toolRegistry;
|
|
1948
|
+
if (options.recorder) this._recorder = options.recorder;
|
|
1857
1949
|
}
|
|
1858
|
-
/** Inject per-call ToolRegistry. */
|
|
1859
1950
|
setToolRegistry(registry) {
|
|
1860
1951
|
this._tools = registry;
|
|
1861
1952
|
}
|
|
1862
|
-
/** Inject per-call AudioRecorder. */
|
|
1863
1953
|
setRecorder(recorder) {
|
|
1864
1954
|
this._recorder = recorder;
|
|
1865
1955
|
}
|
|
1866
|
-
|
|
1867
|
-
this.
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
this.
|
|
1871
|
-
this.
|
|
1872
|
-
|
|
1873
|
-
this._audioRemainder = Buffer.alloc(0);
|
|
1874
|
-
if (!this._apiKey) {
|
|
1875
|
-
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey option.");
|
|
1956
|
+
setBuiltinTools(tools) {
|
|
1957
|
+
this._builtinTools = tools;
|
|
1958
|
+
}
|
|
1959
|
+
setLogger(logger) {
|
|
1960
|
+
this._log = logger;
|
|
1961
|
+
if ("setLogger" in this._stt && typeof this._stt.setLogger === "function") {
|
|
1962
|
+
this._stt.setLogger(logger);
|
|
1876
1963
|
}
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
this._send({ type: "response.create" });
|
|
1892
|
-
}
|
|
1893
|
-
resolve();
|
|
1894
|
-
});
|
|
1895
|
-
ws.on("message", (data) => {
|
|
1896
|
-
try {
|
|
1897
|
-
const msg = JSON.parse(data.toString());
|
|
1898
|
-
this._handleMessage(msg);
|
|
1899
|
-
} catch {
|
|
1900
|
-
}
|
|
1901
|
-
});
|
|
1902
|
-
ws.on("close", () => {
|
|
1903
|
-
this._closed = true;
|
|
1964
|
+
if ("setLogger" in this._tts && typeof this._tts.setLogger === "function") {
|
|
1965
|
+
this._tts.setLogger(logger);
|
|
1966
|
+
}
|
|
1967
|
+
}
|
|
1968
|
+
async start(callSession, tools) {
|
|
1969
|
+
this._callSession = callSession;
|
|
1970
|
+
this._tools = tools ?? null;
|
|
1971
|
+
this._running = true;
|
|
1972
|
+
this._log.info("PipelineSession started");
|
|
1973
|
+
this._conversation = [];
|
|
1974
|
+
if (this._systemPrompt) {
|
|
1975
|
+
this._conversation.push({
|
|
1976
|
+
role: "system",
|
|
1977
|
+
content: this._systemPrompt
|
|
1904
1978
|
});
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
}
|
|
1909
|
-
this._log.error({ err }, "OpenAI Realtime WS error");
|
|
1979
|
+
}
|
|
1980
|
+
if (this._greeting) {
|
|
1981
|
+
this._generateGreeting().catch((err) => {
|
|
1982
|
+
this._log.error({ err }, "Greeting error");
|
|
1910
1983
|
});
|
|
1984
|
+
}
|
|
1985
|
+
this._runSttLoop().catch((err) => {
|
|
1986
|
+
this._log.error({ err }, "STT loop error");
|
|
1911
1987
|
});
|
|
1912
1988
|
}
|
|
1913
|
-
async feedDtmf(digits) {
|
|
1914
|
-
await this._waitForResponseDone();
|
|
1915
|
-
this._send({
|
|
1916
|
-
type: "conversation.item.create",
|
|
1917
|
-
item: {
|
|
1918
|
-
type: "message",
|
|
1919
|
-
role: "user",
|
|
1920
|
-
content: [{ type: "input_text", text: `[DTMF \uC785\uB825: ${digits}]` }]
|
|
1921
|
-
}
|
|
1922
|
-
});
|
|
1923
|
-
this._send({ type: "response.create" });
|
|
1924
|
-
}
|
|
1925
1989
|
feedAudio(audio) {
|
|
1926
|
-
if (this.
|
|
1927
|
-
this.
|
|
1928
|
-
type: "input_audio_buffer.append",
|
|
1929
|
-
audio: audio.toString("base64")
|
|
1930
|
-
});
|
|
1931
|
-
}
|
|
1932
|
-
}
|
|
1933
|
-
async stop() {
|
|
1934
|
-
this._closed = true;
|
|
1935
|
-
if (this._ws) {
|
|
1936
|
-
this._ws.close();
|
|
1937
|
-
this._ws = null;
|
|
1990
|
+
if (this._running) {
|
|
1991
|
+
this._audioBuffer.push(audio);
|
|
1938
1992
|
}
|
|
1939
1993
|
}
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
toolSchemas.push(HANG_UP_TOOL);
|
|
1945
|
-
if (!this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */))
|
|
1946
|
-
toolSchemas.push(COLLECT_DTMF_TOOL);
|
|
1947
|
-
if (!this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */))
|
|
1948
|
-
toolSchemas.push(SEND_DTMF_TOOL);
|
|
1949
|
-
this._send({
|
|
1950
|
-
type: "session.update",
|
|
1951
|
-
session: {
|
|
1952
|
-
modalities: ["text", "audio"],
|
|
1953
|
-
voice: this._voice,
|
|
1954
|
-
instructions: this._systemPrompt,
|
|
1955
|
-
input_audio_format: "g711_ulaw",
|
|
1956
|
-
output_audio_format: "g711_ulaw",
|
|
1957
|
-
input_audio_transcription: {
|
|
1958
|
-
model: "whisper-1",
|
|
1959
|
-
language: this._language
|
|
1960
|
-
},
|
|
1961
|
-
input_audio_noise_reduction: { type: "far_field" },
|
|
1962
|
-
turn_detection: {
|
|
1963
|
-
type: "semantic_vad",
|
|
1964
|
-
interrupt_response: true,
|
|
1965
|
-
eagerness: this._eagerness
|
|
1966
|
-
},
|
|
1967
|
-
tools: toolSchemas
|
|
1968
|
-
}
|
|
1994
|
+
async feedDtmf(digits) {
|
|
1995
|
+
this._conversation.push({
|
|
1996
|
+
role: "user",
|
|
1997
|
+
content: `[DTMF \uC785\uB825: ${digits}]`
|
|
1969
1998
|
});
|
|
1999
|
+
await this._respond();
|
|
1970
2000
|
}
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
}
|
|
1987
|
-
this._sentAudioChunks++;
|
|
1988
|
-
this._audioRemainder = Buffer.alloc(0);
|
|
2001
|
+
async stop() {
|
|
2002
|
+
this._running = false;
|
|
2003
|
+
this._log.info("PipelineSession stopped");
|
|
2004
|
+
this._audioBuffer = [];
|
|
2005
|
+
}
|
|
2006
|
+
async _runSttLoop() {
|
|
2007
|
+
const audioStream = this._createAudioStream();
|
|
2008
|
+
for await (const event of this._stt.transcribe(audioStream, {
|
|
2009
|
+
sampleRate: this._sampleRate
|
|
2010
|
+
})) {
|
|
2011
|
+
if (!this._running) break;
|
|
2012
|
+
if (event.type === "interim" && this._speaking && this._interruptOnSpeech) {
|
|
2013
|
+
this._speaking = false;
|
|
2014
|
+
if (this._callSession) {
|
|
2015
|
+
this._callSession.clearAudio();
|
|
1989
2016
|
}
|
|
1990
|
-
|
|
1991
|
-
}
|
|
1992
|
-
case "input_audio_buffer.speech_started": {
|
|
1993
|
-
this._handleTruncation();
|
|
1994
|
-
break;
|
|
2017
|
+
this._log.info('Barge-in: "%s"', event.transcript.substring(0, 30));
|
|
1995
2018
|
}
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
this._handleToolCall(item);
|
|
2000
|
-
}
|
|
2001
|
-
break;
|
|
2019
|
+
if (event.type === "final" && event.transcript.trim()) {
|
|
2020
|
+
this._log.info("STT: %s", event.transcript);
|
|
2021
|
+
await this._handleUserSpeech(event.transcript);
|
|
2002
2022
|
}
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
async *_createAudioStream() {
|
|
2026
|
+
while (this._running) {
|
|
2027
|
+
if (this._audioBuffer.length > 0) {
|
|
2028
|
+
const ulaw = this._audioBuffer.shift();
|
|
2029
|
+
const pcm8k = ulawToPcm16(ulaw);
|
|
2030
|
+
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
2031
|
+
yield pcm16k;
|
|
2032
|
+
} else {
|
|
2033
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
}
|
|
2037
|
+
async _generateGreeting() {
|
|
2038
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
2039
|
+
await this._respond();
|
|
2040
|
+
}
|
|
2041
|
+
async _handleUserSpeech(transcript) {
|
|
2042
|
+
this._conversation.push({ role: "user", content: transcript });
|
|
2043
|
+
await this._respond();
|
|
2044
|
+
}
|
|
2045
|
+
_buildEffectiveTools() {
|
|
2046
|
+
const builtinSchemas = getBuiltinToolSchemas(this._builtinTools, "chat");
|
|
2047
|
+
const dtmfSchemas = builtinSchemas.filter((s) => {
|
|
2048
|
+
const name = s["function"]?.["name"];
|
|
2049
|
+
return name === "collect_dtmf" || name === "send_dtmf";
|
|
2050
|
+
});
|
|
2051
|
+
if (dtmfSchemas.length === 0) return this._tools ?? void 0;
|
|
2052
|
+
const base = this._tools ? this._tools.fork() : new ToolRegistry();
|
|
2053
|
+
for (const schema of dtmfSchemas) {
|
|
2054
|
+
const fn = schema["function"];
|
|
2055
|
+
const params = fn["parameters"];
|
|
2056
|
+
base.register({
|
|
2057
|
+
name: fn["name"],
|
|
2058
|
+
description: fn["description"],
|
|
2059
|
+
parameters: params["properties"] ?? {},
|
|
2060
|
+
required: params["required"] ?? [],
|
|
2061
|
+
handler: async () => ""
|
|
2062
|
+
});
|
|
2063
|
+
}
|
|
2064
|
+
return base;
|
|
2065
|
+
}
|
|
2066
|
+
async _respond() {
|
|
2067
|
+
let fullResponse = "";
|
|
2068
|
+
const textChunks = [];
|
|
2069
|
+
const effectiveTools = this._buildEffectiveTools();
|
|
2070
|
+
const llmStream = this._llm.generate(this._conversation, {
|
|
2071
|
+
tools: effectiveTools,
|
|
2072
|
+
temperature: this._temperature,
|
|
2073
|
+
maxTokens: this._maxTokens
|
|
2074
|
+
});
|
|
2075
|
+
for await (const chunk of llmStream) {
|
|
2076
|
+
if (!this._running) break;
|
|
2077
|
+
if (chunk.type === "text" && chunk.text) {
|
|
2078
|
+
textChunks.push(chunk.text);
|
|
2079
|
+
fullResponse += chunk.text;
|
|
2080
|
+
} else if (chunk.type === "tool_call" && chunk.toolCall) {
|
|
2081
|
+
await this._handleToolCall(chunk);
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
if (fullResponse.trim()) {
|
|
2085
|
+
this._log.info("Assistant: %s", fullResponse.substring(0, 100));
|
|
2086
|
+
this._conversation.push({ role: "assistant", content: fullResponse });
|
|
2087
|
+
await this._synthesizeAndSend(fullResponse);
|
|
2088
|
+
}
|
|
2089
|
+
}
|
|
2090
|
+
async _handleToolCall(chunk) {
|
|
2091
|
+
if (!chunk.toolCall) return;
|
|
2092
|
+
const { id, name, arguments: argsStr } = chunk.toolCall;
|
|
2093
|
+
try {
|
|
2094
|
+
const args = JSON.parse(argsStr);
|
|
2095
|
+
if (BUILTIN_TOOL_NAMES.has(name) && this._callSession) {
|
|
2096
|
+
const result2 = await executeBuiltinTool(name, args, this._callSession);
|
|
2097
|
+
if (result2 !== null) {
|
|
2098
|
+
if (name === "hang_up") return;
|
|
2099
|
+
this._conversation.push({ role: "tool", content: result2, tool_call_id: id, name });
|
|
2100
|
+
await this._respond();
|
|
2101
|
+
return;
|
|
2102
|
+
}
|
|
2103
|
+
}
|
|
2104
|
+
if (!this._tools) return;
|
|
2105
|
+
const result = await this._tools.call(name, args);
|
|
2106
|
+
this._conversation.push({
|
|
2107
|
+
role: "assistant",
|
|
2108
|
+
content: ""
|
|
2109
|
+
// Tool call info stored in the message flow
|
|
2110
|
+
});
|
|
2111
|
+
this._conversation.push({
|
|
2112
|
+
role: "tool",
|
|
2113
|
+
content: typeof result === "string" ? result : JSON.stringify(result),
|
|
2114
|
+
tool_call_id: id,
|
|
2115
|
+
name
|
|
2116
|
+
});
|
|
2117
|
+
const effectiveTools = this._buildEffectiveTools();
|
|
2118
|
+
let followUpText = "";
|
|
2119
|
+
const followUpStream = this._llm.generate(this._conversation, {
|
|
2120
|
+
tools: effectiveTools,
|
|
2121
|
+
temperature: this._temperature,
|
|
2122
|
+
maxTokens: this._maxTokens
|
|
2123
|
+
});
|
|
2124
|
+
for await (const followChunk of followUpStream) {
|
|
2125
|
+
if (!this._running) break;
|
|
2126
|
+
if (followChunk.type === "text" && followChunk.text) {
|
|
2127
|
+
followUpText += followChunk.text;
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
if (followUpText.trim()) {
|
|
2131
|
+
this._conversation.push({ role: "assistant", content: followUpText });
|
|
2132
|
+
await this._synthesizeAndSend(followUpText);
|
|
2133
|
+
}
|
|
2134
|
+
} catch (err) {
|
|
2135
|
+
this._log.error({ err }, "Tool call failed: %s", name);
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
async _synthesizeAndSend(text) {
|
|
2139
|
+
if (!this._callSession || !this._running) return;
|
|
2140
|
+
this._speaking = true;
|
|
2141
|
+
try {
|
|
2142
|
+
for await (const audioChunk of this._tts.synthesize(text, {
|
|
2143
|
+
sampleRate: this._sampleRate
|
|
2144
|
+
})) {
|
|
2145
|
+
if (!this._running || !this._speaking) break;
|
|
2146
|
+
if (this._recorder) {
|
|
2147
|
+
const pcm8k2 = this._sampleRate !== 8e3 ? resamplePcm16(audioChunk, this._sampleRate, 8e3) : audioChunk;
|
|
2148
|
+
this._recorder.writeOutbound(pcm8k2);
|
|
2149
|
+
}
|
|
2150
|
+
const pcm8k = resamplePcm16(audioChunk, this._sampleRate, 8e3);
|
|
2151
|
+
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2152
|
+
for (let off = 0; off < ulaw.length; off += 160) {
|
|
2153
|
+
let chunk = ulaw.subarray(off, off + 160);
|
|
2154
|
+
if (chunk.length < 160) {
|
|
2155
|
+
chunk = Buffer.concat([chunk, Buffer.alloc(160 - chunk.length, 255)]);
|
|
2156
|
+
}
|
|
2157
|
+
this._callSession.sendAudio(chunk);
|
|
2158
|
+
}
|
|
2159
|
+
}
|
|
2160
|
+
} catch (err) {
|
|
2161
|
+
this._log.error({ err }, "TTS error");
|
|
2162
|
+
} finally {
|
|
2163
|
+
this._speaking = false;
|
|
2164
|
+
}
|
|
2165
|
+
}
|
|
2166
|
+
};
|
|
2167
|
+
|
|
2168
|
+
// src/agent/pipeline/realtime/openai-realtime.ts
|
|
2169
|
+
var OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime?model=";
|
|
2170
|
+
var OpenAIRealtime = class {
|
|
2171
|
+
_apiKey;
|
|
2172
|
+
_systemPrompt;
|
|
2173
|
+
_model;
|
|
2174
|
+
_voice;
|
|
2175
|
+
_language;
|
|
2176
|
+
_turnDetection;
|
|
2177
|
+
_greeting;
|
|
2178
|
+
_log = NOOP_LOGGER;
|
|
2179
|
+
_builtinTools = null;
|
|
2180
|
+
setLogger(logger) {
|
|
2181
|
+
this._log = logger;
|
|
2182
|
+
}
|
|
2183
|
+
setBuiltinTools(tools) {
|
|
2184
|
+
this._builtinTools = tools;
|
|
2185
|
+
}
|
|
2186
|
+
_ws = null;
|
|
2187
|
+
_call = null;
|
|
2188
|
+
_tools = null;
|
|
2189
|
+
_recorder = null;
|
|
2190
|
+
_closed = false;
|
|
2191
|
+
// PlaybackState — 현재 재생 중인 응답 상태
|
|
2192
|
+
_playback = null;
|
|
2193
|
+
_latestMediaTs = 0;
|
|
2194
|
+
// Pending tool call tracking — 인터럽트 시 취소용
|
|
2195
|
+
_pendingToolCalls = /* @__PURE__ */ new Map();
|
|
2196
|
+
// Response state tracking — prevent sending response.create while one is active
|
|
2197
|
+
_responseInProgress = false;
|
|
2198
|
+
_onResponseDone = null;
|
|
2199
|
+
constructor(options = {}) {
|
|
2200
|
+
this._apiKey = options.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
|
|
2201
|
+
this._systemPrompt = options.systemPrompt ?? "";
|
|
2202
|
+
this._model = options.model ?? "gpt-realtime-1.5";
|
|
2203
|
+
this._voice = options.voice ?? "marin";
|
|
2204
|
+
this._language = options.language ?? "ko";
|
|
2205
|
+
this._turnDetection = options.turnDetection !== void 0 ? options.turnDetection : { type: "semantic_vad", eagerness: "medium", interrupt_response: true };
|
|
2206
|
+
this._greeting = options.greeting ?? true;
|
|
2207
|
+
}
|
|
2208
|
+
/** Inject per-call ToolRegistry. */
|
|
2209
|
+
setToolRegistry(registry) {
|
|
2210
|
+
this._tools = registry;
|
|
2211
|
+
}
|
|
2212
|
+
/** Inject per-call AudioRecorder. */
|
|
2213
|
+
setRecorder(recorder) {
|
|
2214
|
+
this._recorder = recorder;
|
|
2215
|
+
}
|
|
2216
|
+
async start(callSession, tools) {
|
|
2217
|
+
this._call = callSession;
|
|
2218
|
+
if (tools) this._tools = tools;
|
|
2219
|
+
this._closed = false;
|
|
2220
|
+
this._playback = null;
|
|
2221
|
+
this._latestMediaTs = 0;
|
|
2222
|
+
if (!this._apiKey) {
|
|
2223
|
+
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey option.");
|
|
2224
|
+
}
|
|
2225
|
+
const { WebSocket } = await import('ws');
|
|
2226
|
+
const url = `${OPENAI_REALTIME_URL}${this._model}`;
|
|
2227
|
+
this._ws = new WebSocket(url, {
|
|
2228
|
+
headers: {
|
|
2229
|
+
Authorization: `Bearer ${this._apiKey}`,
|
|
2230
|
+
"OpenAI-Beta": "realtime=v1"
|
|
2231
|
+
}
|
|
2232
|
+
});
|
|
2233
|
+
return new Promise((resolve, reject) => {
|
|
2234
|
+
const ws = this._ws;
|
|
2235
|
+
ws.on("open", () => {
|
|
2236
|
+
this._sendSessionUpdate();
|
|
2237
|
+
this._log.info("OpenAI Realtime connected");
|
|
2238
|
+
if (this._greeting) {
|
|
2239
|
+
this._send({ type: "response.create" });
|
|
2240
|
+
}
|
|
2241
|
+
resolve();
|
|
2242
|
+
});
|
|
2243
|
+
ws.on("message", (data) => {
|
|
2244
|
+
try {
|
|
2245
|
+
const msg = JSON.parse(data.toString());
|
|
2246
|
+
this._handleMessage(msg);
|
|
2247
|
+
} catch {
|
|
2248
|
+
}
|
|
2249
|
+
});
|
|
2250
|
+
ws.on("close", () => {
|
|
2251
|
+
this._closed = true;
|
|
2252
|
+
});
|
|
2253
|
+
ws.on("error", (err) => {
|
|
2254
|
+
if (!this._ws) {
|
|
2255
|
+
reject(err);
|
|
2256
|
+
}
|
|
2257
|
+
this._log.error({ err }, "OpenAI Realtime WS error");
|
|
2258
|
+
});
|
|
2259
|
+
});
|
|
2260
|
+
}
|
|
2261
|
+
async feedDtmf(digits) {
|
|
2262
|
+
await this._waitForResponseDone();
|
|
2263
|
+
this._send({
|
|
2264
|
+
type: "conversation.item.create",
|
|
2265
|
+
item: {
|
|
2266
|
+
type: "message",
|
|
2267
|
+
role: "user",
|
|
2268
|
+
content: [{ type: "input_text", text: `[DTMF \uC785\uB825: ${digits}]` }]
|
|
2269
|
+
}
|
|
2270
|
+
});
|
|
2271
|
+
this._send({ type: "response.create" });
|
|
2272
|
+
}
|
|
2273
|
+
feedAudio(audio) {
|
|
2274
|
+
this._latestMediaTs = Date.now();
|
|
2275
|
+
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
2276
|
+
this._send({
|
|
2277
|
+
type: "input_audio_buffer.append",
|
|
2278
|
+
audio: audio.toString("base64")
|
|
2279
|
+
});
|
|
2280
|
+
}
|
|
2281
|
+
}
|
|
2282
|
+
async stop() {
|
|
2283
|
+
this._closed = true;
|
|
2284
|
+
for (const [, controller] of this._pendingToolCalls) {
|
|
2285
|
+
controller.abort();
|
|
2286
|
+
}
|
|
2287
|
+
this._pendingToolCalls.clear();
|
|
2288
|
+
if (this._ws) {
|
|
2289
|
+
this._ws.close();
|
|
2290
|
+
this._ws = null;
|
|
2291
|
+
}
|
|
2292
|
+
}
|
|
2293
|
+
_sendSessionUpdate() {
|
|
2294
|
+
if (!this._ws || this._ws.readyState !== 1) return;
|
|
2295
|
+
const toolSchemas = this._tools ? this._tools.toOpenAITools().map((t) => ({ type: "function", ...t.function })) : [];
|
|
2296
|
+
toolSchemas.push(...getBuiltinToolSchemas(this._builtinTools, "realtime"));
|
|
2297
|
+
this._send({
|
|
2298
|
+
type: "session.update",
|
|
2299
|
+
session: {
|
|
2300
|
+
modalities: ["text", "audio"],
|
|
2301
|
+
voice: this._voice,
|
|
2302
|
+
instructions: this._systemPrompt,
|
|
2303
|
+
input_audio_format: "g711_ulaw",
|
|
2304
|
+
output_audio_format: "g711_ulaw",
|
|
2305
|
+
input_audio_transcription: {
|
|
2306
|
+
model: "whisper-1",
|
|
2307
|
+
language: this._language
|
|
2308
|
+
},
|
|
2309
|
+
input_audio_noise_reduction: { type: "far_field" },
|
|
2310
|
+
turn_detection: this._turnDetection,
|
|
2311
|
+
tools: toolSchemas
|
|
2312
|
+
}
|
|
2313
|
+
});
|
|
2314
|
+
}
|
|
2315
|
+
_handleMessage(msg) {
|
|
2316
|
+
const type = msg["type"];
|
|
2317
|
+
switch (type) {
|
|
2318
|
+
case "response.audio.delta": {
|
|
2319
|
+
this._handleAudioDelta(msg);
|
|
2320
|
+
break;
|
|
2321
|
+
}
|
|
2322
|
+
case "response.audio.done": {
|
|
2323
|
+
if (this._playback) {
|
|
2324
|
+
this._playback.generating = false;
|
|
2325
|
+
if (this._playback.audioRemainder.length > 0) {
|
|
2326
|
+
const padded = Buffer.concat([
|
|
2327
|
+
this._playback.audioRemainder,
|
|
2328
|
+
Buffer.alloc(160 - this._playback.audioRemainder.length, 255)
|
|
2329
|
+
]);
|
|
2330
|
+
if (this._call) {
|
|
2331
|
+
this._call.sendAudio(padded);
|
|
2332
|
+
}
|
|
2333
|
+
this._playback.sentChunks++;
|
|
2334
|
+
this._playback.audioRemainder = Buffer.alloc(0);
|
|
2335
|
+
}
|
|
2336
|
+
}
|
|
2337
|
+
break;
|
|
2338
|
+
}
|
|
2339
|
+
case "input_audio_buffer.speech_started": {
|
|
2340
|
+
this._handleTruncation();
|
|
2341
|
+
break;
|
|
2342
|
+
}
|
|
2343
|
+
case "response.output_item.done": {
|
|
2344
|
+
const item = msg["item"];
|
|
2345
|
+
if (item && item["type"] === "function_call") {
|
|
2346
|
+
this._handleToolCall(item);
|
|
2347
|
+
}
|
|
2348
|
+
break;
|
|
2349
|
+
}
|
|
2350
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
2351
|
+
if (this._call) {
|
|
2352
|
+
this._call._emit("transcript", "user", msg["transcript"] ?? "");
|
|
2353
|
+
}
|
|
2007
2354
|
break;
|
|
2008
2355
|
}
|
|
2009
2356
|
case "response.audio_transcript.done": {
|
|
@@ -2032,128 +2379,121 @@ var OpenAIRealtime = class {
|
|
|
2032
2379
|
}
|
|
2033
2380
|
}
|
|
2034
2381
|
_handleAudioDelta(msg) {
|
|
2035
|
-
if (this.
|
|
2036
|
-
this.
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2382
|
+
if (this._playback === null) {
|
|
2383
|
+
this._playback = {
|
|
2384
|
+
itemId: msg["item_id"] || "",
|
|
2385
|
+
startTs: this._latestMediaTs || Date.now(),
|
|
2386
|
+
sentChunks: 0,
|
|
2387
|
+
generating: true,
|
|
2388
|
+
audioRemainder: Buffer.alloc(0)
|
|
2389
|
+
};
|
|
2390
|
+
} else if (msg["item_id"]) {
|
|
2391
|
+
this._playback.itemId = msg["item_id"];
|
|
2041
2392
|
}
|
|
2393
|
+
const pb = this._playback;
|
|
2042
2394
|
const ulaw = Buffer.from(msg["delta"], "base64");
|
|
2043
2395
|
if (this._recorder) {
|
|
2044
2396
|
this._recorder.writeOutbound(ulawToPcm16(ulaw));
|
|
2045
2397
|
}
|
|
2046
|
-
const combined = Buffer.concat([
|
|
2398
|
+
const combined = Buffer.concat([pb.audioRemainder, ulaw]);
|
|
2047
2399
|
const chunkSize = 160;
|
|
2048
2400
|
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
2049
2401
|
for (let off = 0; off < fullEnd; off += chunkSize) {
|
|
2050
2402
|
if (this._call) {
|
|
2051
2403
|
this._call.sendAudio(combined.subarray(off, off + chunkSize));
|
|
2052
2404
|
}
|
|
2053
|
-
|
|
2405
|
+
pb.sentChunks++;
|
|
2054
2406
|
}
|
|
2055
|
-
|
|
2407
|
+
pb.audioRemainder = combined.subarray(fullEnd);
|
|
2056
2408
|
}
|
|
2057
2409
|
_handleTruncation() {
|
|
2058
|
-
|
|
2059
|
-
|
|
2410
|
+
for (const [, controller] of this._pendingToolCalls) {
|
|
2411
|
+
controller.abort();
|
|
2060
2412
|
}
|
|
2061
|
-
|
|
2062
|
-
this._send({
|
|
2063
|
-
type: "conversation.item.truncate",
|
|
2064
|
-
item_id: this._lastAssistantItem,
|
|
2065
|
-
content_index: 0,
|
|
2066
|
-
audio_end_ms: audioEndMs
|
|
2067
|
-
});
|
|
2413
|
+
this._pendingToolCalls.clear();
|
|
2068
2414
|
if (this._call) {
|
|
2069
2415
|
this._call.clearAudio();
|
|
2070
2416
|
}
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2417
|
+
const pb = this._playback;
|
|
2418
|
+
if (pb === null) {
|
|
2419
|
+
return;
|
|
2420
|
+
}
|
|
2421
|
+
const playedMs = Math.max(0, (this._latestMediaTs || Date.now()) - pb.startTs);
|
|
2422
|
+
this._log.info(
|
|
2423
|
+
"[Interrupt] item=%s played=%dms total=%dms",
|
|
2424
|
+
pb.itemId,
|
|
2425
|
+
playedMs,
|
|
2426
|
+
pb.sentChunks * 20
|
|
2427
|
+
);
|
|
2428
|
+
this._playback = null;
|
|
2075
2429
|
}
|
|
2076
2430
|
async _handleToolCall(item) {
|
|
2077
2431
|
const funcName = item["name"];
|
|
2078
2432
|
const callId = item["call_id"];
|
|
2079
2433
|
this._log.info("Tool call: %s", funcName);
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2434
|
+
const controller = new AbortController();
|
|
2435
|
+
this._pendingToolCalls.set(callId, controller);
|
|
2436
|
+
try {
|
|
2437
|
+
if (BUILTIN_TOOL_NAMES.has(funcName) && this._call) {
|
|
2438
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2439
|
+
const result2 = await executeBuiltinTool(funcName, args, this._call);
|
|
2440
|
+
if (result2 !== null) {
|
|
2441
|
+
if (funcName === "hang_up") return;
|
|
2442
|
+
if (controller.signal.aborted) return;
|
|
2443
|
+
await this._waitForResponseDone();
|
|
2444
|
+
this._send({
|
|
2445
|
+
type: "conversation.item.create",
|
|
2446
|
+
item: {
|
|
2447
|
+
type: "function_call_output",
|
|
2448
|
+
call_id: callId,
|
|
2449
|
+
output: result2
|
|
2450
|
+
}
|
|
2095
2451
|
});
|
|
2096
|
-
|
|
2097
|
-
|
|
2452
|
+
this._send({ type: "response.create" });
|
|
2453
|
+
return;
|
|
2098
2454
|
}
|
|
2099
|
-
await this._waitForResponseDone();
|
|
2100
|
-
this._send({
|
|
2101
|
-
type: "conversation.item.create",
|
|
2102
|
-
item: {
|
|
2103
|
-
type: "function_call_output",
|
|
2104
|
-
call_id: callId,
|
|
2105
|
-
output: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)"
|
|
2106
|
-
}
|
|
2107
|
-
});
|
|
2108
|
-
this._send({ type: "response.create" });
|
|
2109
2455
|
}
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
if (funcName === "send_dtmf") {
|
|
2113
|
-
if (this._call) {
|
|
2114
|
-
let result2;
|
|
2115
|
-
try {
|
|
2116
|
-
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2117
|
-
await this._call.sendDtmfSequence(args["digits"] ?? "");
|
|
2118
|
-
result2 = "sent";
|
|
2119
|
-
} catch (err) {
|
|
2120
|
-
result2 = `Error: ${err}`;
|
|
2121
|
-
}
|
|
2456
|
+
if (!this._tools || !this._tools.has(funcName)) {
|
|
2457
|
+
this._log.error("Unknown tool: %s", funcName);
|
|
2122
2458
|
await this._waitForResponseDone();
|
|
2123
2459
|
this._send({
|
|
2124
2460
|
type: "conversation.item.create",
|
|
2125
2461
|
item: {
|
|
2126
2462
|
type: "function_call_output",
|
|
2127
2463
|
call_id: callId,
|
|
2128
|
-
output:
|
|
2464
|
+
output: JSON.stringify({ error: `Unknown tool: ${funcName}` })
|
|
2129
2465
|
}
|
|
2130
2466
|
});
|
|
2131
2467
|
this._send({ type: "response.create" });
|
|
2468
|
+
return;
|
|
2132
2469
|
}
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
try {
|
|
2141
|
-
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2142
|
-
result = await this._tools.call(funcName, args);
|
|
2143
|
-
} catch (err) {
|
|
2144
|
-
this._log.error({ err }, "Tool call failed: %s", funcName);
|
|
2145
|
-
result = `Error: ${err}`;
|
|
2146
|
-
}
|
|
2147
|
-
await this._waitForResponseDone();
|
|
2148
|
-
this._send({
|
|
2149
|
-
type: "conversation.item.create",
|
|
2150
|
-
item: {
|
|
2151
|
-
type: "function_call_output",
|
|
2152
|
-
call_id: callId,
|
|
2153
|
-
output: typeof result === "string" ? result : JSON.stringify(result)
|
|
2470
|
+
let result;
|
|
2471
|
+
try {
|
|
2472
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2473
|
+
result = await this._tools.call(funcName, args);
|
|
2474
|
+
} catch (err) {
|
|
2475
|
+
this._log.error({ err }, "Tool call failed: %s", funcName);
|
|
2476
|
+
result = `Error: ${err}`;
|
|
2154
2477
|
}
|
|
2155
|
-
|
|
2156
|
-
|
|
2478
|
+
if (controller.signal.aborted) {
|
|
2479
|
+
this._log.info("Tool call cancelled (user interrupted): %s", funcName);
|
|
2480
|
+
return;
|
|
2481
|
+
}
|
|
2482
|
+
const resultStr = typeof result === "string" ? result : JSON.stringify(result);
|
|
2483
|
+
await this._waitForResponseDone();
|
|
2484
|
+
this._send({
|
|
2485
|
+
type: "conversation.item.create",
|
|
2486
|
+
item: {
|
|
2487
|
+
type: "function_call_output",
|
|
2488
|
+
call_id: callId,
|
|
2489
|
+
output: resultStr
|
|
2490
|
+
}
|
|
2491
|
+
});
|
|
2492
|
+
this._log.info("[ToolResult] %s call_id=%s len=%d", funcName, callId, resultStr.length);
|
|
2493
|
+
this._send({ type: "response.create" });
|
|
2494
|
+
} finally {
|
|
2495
|
+
this._pendingToolCalls.delete(callId);
|
|
2496
|
+
}
|
|
2157
2497
|
}
|
|
2158
2498
|
_waitForResponseDone() {
|
|
2159
2499
|
if (!this._responseInProgress) return Promise.resolve();
|
|
@@ -2168,39 +2508,7 @@ var OpenAIRealtime = class {
|
|
|
2168
2508
|
}
|
|
2169
2509
|
};
|
|
2170
2510
|
|
|
2171
|
-
// src/agent/pipeline/gemini-realtime.ts
|
|
2172
|
-
var HANG_UP_TOOL2 = {
|
|
2173
|
-
name: "hang_up",
|
|
2174
|
-
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
2175
|
-
parameters: { type: "object", properties: {} }
|
|
2176
|
-
};
|
|
2177
|
-
var COLLECT_DTMF_TOOL2 = {
|
|
2178
|
-
name: "collect_dtmf",
|
|
2179
|
-
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
|
|
2180
|
-
parameters: {
|
|
2181
|
-
type: "object",
|
|
2182
|
-
properties: {
|
|
2183
|
-
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
2184
|
-
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
2185
|
-
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
2186
|
-
},
|
|
2187
|
-
required: ["max_digits"]
|
|
2188
|
-
}
|
|
2189
|
-
};
|
|
2190
|
-
var SEND_DTMF_TOOL2 = {
|
|
2191
|
-
name: "send_dtmf",
|
|
2192
|
-
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
2193
|
-
parameters: {
|
|
2194
|
-
type: "object",
|
|
2195
|
-
properties: {
|
|
2196
|
-
digits: {
|
|
2197
|
-
type: "string",
|
|
2198
|
-
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30."
|
|
2199
|
-
}
|
|
2200
|
-
},
|
|
2201
|
-
required: ["digits"]
|
|
2202
|
-
}
|
|
2203
|
-
};
|
|
2511
|
+
// src/agent/pipeline/realtime/gemini-realtime.ts
|
|
2204
2512
|
function resolveRef(ref, defs) {
|
|
2205
2513
|
const parts = ref.replace(/^#\//, "").split("/");
|
|
2206
2514
|
let result = defs;
|
|
@@ -2410,9 +2718,7 @@ var GeminiRealtime = class {
|
|
|
2410
2718
|
t.function.parameters ?? { type: "object", properties: {} }
|
|
2411
2719
|
)
|
|
2412
2720
|
})) : [];
|
|
2413
|
-
|
|
2414
|
-
if (!this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */)) toolDefs.push(COLLECT_DTMF_TOOL2);
|
|
2415
|
-
if (!this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */)) toolDefs.push(SEND_DTMF_TOOL2);
|
|
2721
|
+
toolDefs.push(...getBuiltinToolSchemas(this._builtinTools, "gemini"));
|
|
2416
2722
|
return toolDefs;
|
|
2417
2723
|
}
|
|
2418
2724
|
_handleMessage(msg) {
|
|
@@ -2500,51 +2806,17 @@ var GeminiRealtime = class {
|
|
|
2500
2806
|
const fcId = fc.id ?? "";
|
|
2501
2807
|
const args = fc.args ?? {};
|
|
2502
2808
|
this._log.info({ tool: name, args }, "Tool call: %s", name);
|
|
2503
|
-
if (name
|
|
2504
|
-
|
|
2505
|
-
if (
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
}
|
|
2510
|
-
if (name === "collect_dtmf") {
|
|
2511
|
-
if (this._call) {
|
|
2512
|
-
let result;
|
|
2513
|
-
try {
|
|
2514
|
-
this._log.info({ maxDigits: args["max_digits"] ?? 4, timeout: args["timeout"] ?? 5 }, "collect_dtmf: waiting for digits");
|
|
2515
|
-
result = await this._call.collectDtmf({
|
|
2516
|
-
maxDigits: args["max_digits"] ?? 4,
|
|
2517
|
-
finishOnKey: args["finish_on_key"] ?? "#",
|
|
2518
|
-
timeout: args["timeout"] ?? 5
|
|
2519
|
-
});
|
|
2520
|
-
this._log.info("DTMF collected: %s", result || "(empty)");
|
|
2521
|
-
} catch (err) {
|
|
2522
|
-
this._log.error({ err }, "collect_dtmf error");
|
|
2523
|
-
result = `Error: ${err}`;
|
|
2524
|
-
}
|
|
2525
|
-
responses.push({
|
|
2526
|
-
id: fcId,
|
|
2527
|
-
name,
|
|
2528
|
-
response: { result: result || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)" }
|
|
2529
|
-
});
|
|
2530
|
-
}
|
|
2531
|
-
continue;
|
|
2532
|
-
}
|
|
2533
|
-
if (name === "send_dtmf") {
|
|
2534
|
-
if (this._call) {
|
|
2535
|
-
let result;
|
|
2536
|
-
try {
|
|
2537
|
-
this._log.info('send_dtmf: digits="%s"', args["digits"] ?? "");
|
|
2538
|
-
await this._call.sendDtmfSequence(args["digits"] ?? "");
|
|
2539
|
-
result = "sent";
|
|
2540
|
-
this._log.info("send_dtmf: sent");
|
|
2541
|
-
} catch (err) {
|
|
2542
|
-
this._log.error({ err }, "send_dtmf error");
|
|
2543
|
-
result = `Error: ${err}`;
|
|
2809
|
+
if (BUILTIN_TOOL_NAMES.has(name) && this._call) {
|
|
2810
|
+
const result = await executeBuiltinTool(name, args, this._call);
|
|
2811
|
+
if (result !== null) {
|
|
2812
|
+
if (name === "hang_up") {
|
|
2813
|
+
this._log.info("hang_up: ending call");
|
|
2814
|
+
return;
|
|
2544
2815
|
}
|
|
2816
|
+
this._log.info("Builtin tool result: %s -> %s", name, result);
|
|
2545
2817
|
responses.push({ id: fcId, name, response: { result } });
|
|
2818
|
+
continue;
|
|
2546
2819
|
}
|
|
2547
|
-
continue;
|
|
2548
2820
|
}
|
|
2549
2821
|
if (!this._tools || !this._tools.has(name)) {
|
|
2550
2822
|
this._log.error("Unknown tool: %s", name);
|
|
@@ -2579,306 +2851,7 @@ var GeminiRealtime = class {
|
|
|
2579
2851
|
}
|
|
2580
2852
|
};
|
|
2581
2853
|
|
|
2582
|
-
// src/agent/pipeline/
|
|
2583
|
-
var COLLECT_DTMF_TOOL3 = {
|
|
2584
|
-
function: {
|
|
2585
|
-
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4.",
|
|
2586
|
-
parameters: {
|
|
2587
|
-
properties: {
|
|
2588
|
-
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
2589
|
-
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
2590
|
-
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
2591
|
-
},
|
|
2592
|
-
required: ["max_digits"]
|
|
2593
|
-
}
|
|
2594
|
-
}
|
|
2595
|
-
};
|
|
2596
|
-
var SEND_DTMF_TOOL3 = {
|
|
2597
|
-
function: {
|
|
2598
|
-
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
2599
|
-
parameters: {
|
|
2600
|
-
properties: {
|
|
2601
|
-
digits: { type: "string", description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30." }
|
|
2602
|
-
},
|
|
2603
|
-
required: ["digits"]
|
|
2604
|
-
}
|
|
2605
|
-
}
|
|
2606
|
-
};
|
|
2607
|
-
var PipelineSession = class {
|
|
2608
|
-
_stt;
|
|
2609
|
-
_llm;
|
|
2610
|
-
_tts;
|
|
2611
|
-
_systemPrompt;
|
|
2612
|
-
_greeting;
|
|
2613
|
-
_language;
|
|
2614
|
-
_temperature;
|
|
2615
|
-
_maxTokens;
|
|
2616
|
-
_sampleRate;
|
|
2617
|
-
_interruptOnSpeech;
|
|
2618
|
-
_callSession = null;
|
|
2619
|
-
_tools = null;
|
|
2620
|
-
_recorder = null;
|
|
2621
|
-
_conversation = [];
|
|
2622
|
-
_audioBuffer = [];
|
|
2623
|
-
_running = false;
|
|
2624
|
-
_speaking = false;
|
|
2625
|
-
_builtinTools = null;
|
|
2626
|
-
_log = NOOP_LOGGER;
|
|
2627
|
-
constructor(options) {
|
|
2628
|
-
this._stt = options.stt;
|
|
2629
|
-
this._llm = options.llm;
|
|
2630
|
-
this._tts = options.tts;
|
|
2631
|
-
this._systemPrompt = options.systemPrompt;
|
|
2632
|
-
this._greeting = options.greeting ?? true;
|
|
2633
|
-
this._language = options.language ?? "ko";
|
|
2634
|
-
this._temperature = options.temperature;
|
|
2635
|
-
this._maxTokens = options.maxTokens;
|
|
2636
|
-
this._sampleRate = options.sampleRate ?? 8e3;
|
|
2637
|
-
this._interruptOnSpeech = options.interruptOnSpeech ?? true;
|
|
2638
|
-
if (options.toolRegistry) this._tools = options.toolRegistry;
|
|
2639
|
-
if (options.recorder) this._recorder = options.recorder;
|
|
2640
|
-
}
|
|
2641
|
-
setToolRegistry(registry) {
|
|
2642
|
-
this._tools = registry;
|
|
2643
|
-
}
|
|
2644
|
-
setRecorder(recorder) {
|
|
2645
|
-
this._recorder = recorder;
|
|
2646
|
-
}
|
|
2647
|
-
setBuiltinTools(tools) {
|
|
2648
|
-
this._builtinTools = tools;
|
|
2649
|
-
}
|
|
2650
|
-
setLogger(logger) {
|
|
2651
|
-
this._log = logger;
|
|
2652
|
-
if ("setLogger" in this._stt && typeof this._stt.setLogger === "function") {
|
|
2653
|
-
this._stt.setLogger(logger);
|
|
2654
|
-
}
|
|
2655
|
-
if ("setLogger" in this._tts && typeof this._tts.setLogger === "function") {
|
|
2656
|
-
this._tts.setLogger(logger);
|
|
2657
|
-
}
|
|
2658
|
-
}
|
|
2659
|
-
async start(callSession, tools) {
|
|
2660
|
-
this._callSession = callSession;
|
|
2661
|
-
this._tools = tools ?? null;
|
|
2662
|
-
this._running = true;
|
|
2663
|
-
this._log.info("PipelineSession started");
|
|
2664
|
-
this._conversation = [];
|
|
2665
|
-
if (this._systemPrompt) {
|
|
2666
|
-
this._conversation.push({
|
|
2667
|
-
role: "system",
|
|
2668
|
-
content: this._systemPrompt
|
|
2669
|
-
});
|
|
2670
|
-
}
|
|
2671
|
-
if (this._greeting) {
|
|
2672
|
-
this._generateGreeting().catch((err) => {
|
|
2673
|
-
this._log.error({ err }, "Greeting error");
|
|
2674
|
-
});
|
|
2675
|
-
}
|
|
2676
|
-
this._runSttLoop().catch((err) => {
|
|
2677
|
-
this._log.error({ err }, "STT loop error");
|
|
2678
|
-
});
|
|
2679
|
-
}
|
|
2680
|
-
feedAudio(audio) {
|
|
2681
|
-
if (this._running) {
|
|
2682
|
-
this._audioBuffer.push(audio);
|
|
2683
|
-
}
|
|
2684
|
-
}
|
|
2685
|
-
async feedDtmf(digits) {
|
|
2686
|
-
this._conversation.push({
|
|
2687
|
-
role: "user",
|
|
2688
|
-
content: `[DTMF \uC785\uB825: ${digits}]`
|
|
2689
|
-
});
|
|
2690
|
-
await this._respond();
|
|
2691
|
-
}
|
|
2692
|
-
async stop() {
|
|
2693
|
-
this._running = false;
|
|
2694
|
-
this._log.info("PipelineSession stopped");
|
|
2695
|
-
this._audioBuffer = [];
|
|
2696
|
-
}
|
|
2697
|
-
async _runSttLoop() {
|
|
2698
|
-
const audioStream = this._createAudioStream();
|
|
2699
|
-
for await (const event of this._stt.transcribe(audioStream, {
|
|
2700
|
-
sampleRate: this._sampleRate
|
|
2701
|
-
})) {
|
|
2702
|
-
if (!this._running) break;
|
|
2703
|
-
if (event.type === "interim" && this._speaking && this._interruptOnSpeech) {
|
|
2704
|
-
this._speaking = false;
|
|
2705
|
-
if (this._callSession) {
|
|
2706
|
-
this._callSession.clearAudio();
|
|
2707
|
-
}
|
|
2708
|
-
this._log.info('Barge-in: "%s"', event.transcript.substring(0, 30));
|
|
2709
|
-
}
|
|
2710
|
-
if (event.type === "final" && event.transcript.trim()) {
|
|
2711
|
-
this._log.info("STT: %s", event.transcript);
|
|
2712
|
-
await this._handleUserSpeech(event.transcript);
|
|
2713
|
-
}
|
|
2714
|
-
}
|
|
2715
|
-
}
|
|
2716
|
-
async *_createAudioStream() {
|
|
2717
|
-
while (this._running) {
|
|
2718
|
-
if (this._audioBuffer.length > 0) {
|
|
2719
|
-
const ulaw = this._audioBuffer.shift();
|
|
2720
|
-
const pcm8k = ulawToPcm16(ulaw);
|
|
2721
|
-
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
2722
|
-
yield pcm16k;
|
|
2723
|
-
} else {
|
|
2724
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
2725
|
-
}
|
|
2726
|
-
}
|
|
2727
|
-
}
|
|
2728
|
-
async _generateGreeting() {
|
|
2729
|
-
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
2730
|
-
await this._respond();
|
|
2731
|
-
}
|
|
2732
|
-
async _handleUserSpeech(transcript) {
|
|
2733
|
-
this._conversation.push({ role: "user", content: transcript });
|
|
2734
|
-
await this._respond();
|
|
2735
|
-
}
|
|
2736
|
-
_buildEffectiveTools() {
|
|
2737
|
-
const includeCollectDtmf = !this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */);
|
|
2738
|
-
const includeSendDtmf = !this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */);
|
|
2739
|
-
if (!includeCollectDtmf && !includeSendDtmf) return this._tools ?? void 0;
|
|
2740
|
-
const base = this._tools ? this._tools.fork() : new ToolRegistry();
|
|
2741
|
-
if (includeCollectDtmf) {
|
|
2742
|
-
base.register({
|
|
2743
|
-
name: "collect_dtmf",
|
|
2744
|
-
description: COLLECT_DTMF_TOOL3.function.description,
|
|
2745
|
-
parameters: COLLECT_DTMF_TOOL3.function.parameters.properties,
|
|
2746
|
-
required: COLLECT_DTMF_TOOL3.function.parameters.required,
|
|
2747
|
-
handler: async () => ""
|
|
2748
|
-
});
|
|
2749
|
-
}
|
|
2750
|
-
if (includeSendDtmf) {
|
|
2751
|
-
base.register({
|
|
2752
|
-
name: "send_dtmf",
|
|
2753
|
-
description: SEND_DTMF_TOOL3.function.description,
|
|
2754
|
-
parameters: SEND_DTMF_TOOL3.function.parameters.properties,
|
|
2755
|
-
required: SEND_DTMF_TOOL3.function.parameters.required,
|
|
2756
|
-
handler: async () => ""
|
|
2757
|
-
});
|
|
2758
|
-
}
|
|
2759
|
-
return base;
|
|
2760
|
-
}
|
|
2761
|
-
async _respond() {
|
|
2762
|
-
let fullResponse = "";
|
|
2763
|
-
const textChunks = [];
|
|
2764
|
-
const effectiveTools = this._buildEffectiveTools();
|
|
2765
|
-
const llmStream = this._llm.generate(this._conversation, {
|
|
2766
|
-
tools: effectiveTools,
|
|
2767
|
-
temperature: this._temperature,
|
|
2768
|
-
maxTokens: this._maxTokens
|
|
2769
|
-
});
|
|
2770
|
-
for await (const chunk of llmStream) {
|
|
2771
|
-
if (!this._running) break;
|
|
2772
|
-
if (chunk.type === "text" && chunk.text) {
|
|
2773
|
-
textChunks.push(chunk.text);
|
|
2774
|
-
fullResponse += chunk.text;
|
|
2775
|
-
} else if (chunk.type === "tool_call" && chunk.toolCall) {
|
|
2776
|
-
await this._handleToolCall(chunk);
|
|
2777
|
-
}
|
|
2778
|
-
}
|
|
2779
|
-
if (fullResponse.trim()) {
|
|
2780
|
-
this._log.info("Assistant: %s", fullResponse.substring(0, 100));
|
|
2781
|
-
this._conversation.push({ role: "assistant", content: fullResponse });
|
|
2782
|
-
await this._synthesizeAndSend(fullResponse);
|
|
2783
|
-
}
|
|
2784
|
-
}
|
|
2785
|
-
async _handleToolCall(chunk) {
|
|
2786
|
-
if (!chunk.toolCall) return;
|
|
2787
|
-
const { id, name, arguments: argsStr } = chunk.toolCall;
|
|
2788
|
-
try {
|
|
2789
|
-
const args = JSON.parse(argsStr);
|
|
2790
|
-
if (name === "collect_dtmf" && this._callSession) {
|
|
2791
|
-
let result2;
|
|
2792
|
-
try {
|
|
2793
|
-
result2 = await this._callSession.collectDtmf({
|
|
2794
|
-
maxDigits: args["max_digits"] ?? 4,
|
|
2795
|
-
finishOnKey: args["finish_on_key"] ?? "#",
|
|
2796
|
-
timeout: args["timeout"] ?? 5
|
|
2797
|
-
});
|
|
2798
|
-
} catch (err) {
|
|
2799
|
-
result2 = `Error: ${err}`;
|
|
2800
|
-
}
|
|
2801
|
-
this._conversation.push({ role: "tool", content: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)", tool_call_id: id, name });
|
|
2802
|
-
await this._respond();
|
|
2803
|
-
return;
|
|
2804
|
-
}
|
|
2805
|
-
if (name === "send_dtmf" && this._callSession) {
|
|
2806
|
-
let result2;
|
|
2807
|
-
try {
|
|
2808
|
-
await this._callSession.sendDtmfSequence(args["digits"] ?? "");
|
|
2809
|
-
result2 = "sent";
|
|
2810
|
-
} catch (err) {
|
|
2811
|
-
result2 = `Error: ${err}`;
|
|
2812
|
-
}
|
|
2813
|
-
this._conversation.push({ role: "tool", content: result2, tool_call_id: id, name });
|
|
2814
|
-
await this._respond();
|
|
2815
|
-
return;
|
|
2816
|
-
}
|
|
2817
|
-
if (!this._tools) return;
|
|
2818
|
-
const result = await this._tools.call(name, args);
|
|
2819
|
-
this._conversation.push({
|
|
2820
|
-
role: "assistant",
|
|
2821
|
-
content: ""
|
|
2822
|
-
// Tool call info stored in the message flow
|
|
2823
|
-
});
|
|
2824
|
-
this._conversation.push({
|
|
2825
|
-
role: "tool",
|
|
2826
|
-
content: typeof result === "string" ? result : JSON.stringify(result),
|
|
2827
|
-
tool_call_id: id,
|
|
2828
|
-
name
|
|
2829
|
-
});
|
|
2830
|
-
const effectiveTools = this._buildEffectiveTools();
|
|
2831
|
-
let followUpText = "";
|
|
2832
|
-
const followUpStream = this._llm.generate(this._conversation, {
|
|
2833
|
-
tools: effectiveTools,
|
|
2834
|
-
temperature: this._temperature,
|
|
2835
|
-
maxTokens: this._maxTokens
|
|
2836
|
-
});
|
|
2837
|
-
for await (const followChunk of followUpStream) {
|
|
2838
|
-
if (!this._running) break;
|
|
2839
|
-
if (followChunk.type === "text" && followChunk.text) {
|
|
2840
|
-
followUpText += followChunk.text;
|
|
2841
|
-
}
|
|
2842
|
-
}
|
|
2843
|
-
if (followUpText.trim()) {
|
|
2844
|
-
this._conversation.push({ role: "assistant", content: followUpText });
|
|
2845
|
-
await this._synthesizeAndSend(followUpText);
|
|
2846
|
-
}
|
|
2847
|
-
} catch (err) {
|
|
2848
|
-
this._log.error({ err }, "Tool call failed: %s", name);
|
|
2849
|
-
}
|
|
2850
|
-
}
|
|
2851
|
-
async _synthesizeAndSend(text) {
|
|
2852
|
-
if (!this._callSession || !this._running) return;
|
|
2853
|
-
this._speaking = true;
|
|
2854
|
-
try {
|
|
2855
|
-
for await (const audioChunk of this._tts.synthesize(text, {
|
|
2856
|
-
sampleRate: this._sampleRate
|
|
2857
|
-
})) {
|
|
2858
|
-
if (!this._running || !this._speaking) break;
|
|
2859
|
-
if (this._recorder) {
|
|
2860
|
-
const pcm8k2 = this._sampleRate !== 8e3 ? resamplePcm16(audioChunk, this._sampleRate, 8e3) : audioChunk;
|
|
2861
|
-
this._recorder.writeOutbound(pcm8k2);
|
|
2862
|
-
}
|
|
2863
|
-
const pcm8k = resamplePcm16(audioChunk, this._sampleRate, 8e3);
|
|
2864
|
-
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2865
|
-
for (let off = 0; off < ulaw.length; off += 160) {
|
|
2866
|
-
let chunk = ulaw.subarray(off, off + 160);
|
|
2867
|
-
if (chunk.length < 160) {
|
|
2868
|
-
chunk = Buffer.concat([chunk, Buffer.alloc(160 - chunk.length, 255)]);
|
|
2869
|
-
}
|
|
2870
|
-
this._callSession.sendAudio(chunk);
|
|
2871
|
-
}
|
|
2872
|
-
}
|
|
2873
|
-
} catch (err) {
|
|
2874
|
-
this._log.error({ err }, "TTS error");
|
|
2875
|
-
} finally {
|
|
2876
|
-
this._speaking = false;
|
|
2877
|
-
}
|
|
2878
|
-
}
|
|
2879
|
-
};
|
|
2880
|
-
|
|
2881
|
-
// src/agent/pipeline/deepgram-stt.ts
|
|
2854
|
+
// src/agent/pipeline/stt/deepgram-stt.ts
|
|
2882
2855
|
var DeepgramSTT = class {
|
|
2883
2856
|
_options;
|
|
2884
2857
|
_log = NOOP_LOGGER;
|
|
@@ -2999,7 +2972,7 @@ var DeepgramSTT = class {
|
|
|
2999
2972
|
}
|
|
3000
2973
|
};
|
|
3001
2974
|
|
|
3002
|
-
// src/agent/pipeline/elevenlabs-tts.ts
|
|
2975
|
+
// src/agent/pipeline/tts/elevenlabs-tts.ts
|
|
3003
2976
|
var ElevenLabsTTS = class {
|
|
3004
2977
|
_options;
|
|
3005
2978
|
_log = NOOP_LOGGER;
|
|
@@ -3152,7 +3125,7 @@ var ElevenLabsTTS = class {
|
|
|
3152
3125
|
}
|
|
3153
3126
|
};
|
|
3154
3127
|
|
|
3155
|
-
// src/agent/pipeline/openai-llm.ts
|
|
3128
|
+
// src/agent/pipeline/llm/openai-llm.ts
|
|
3156
3129
|
var OpenAILLM = class {
|
|
3157
3130
|
_options;
|
|
3158
3131
|
constructor(options = {}) {
|
|
@@ -3238,7 +3211,7 @@ var OpenAILLM = class {
|
|
|
3238
3211
|
}
|
|
3239
3212
|
};
|
|
3240
3213
|
|
|
3241
|
-
// src/agent/pipeline/anthropic-llm.ts
|
|
3214
|
+
// src/agent/pipeline/llm/anthropic-llm.ts
|
|
3242
3215
|
var AnthropicLLM = class {
|
|
3243
3216
|
_options;
|
|
3244
3217
|
constructor(options = {}) {
|
|
@@ -3343,7 +3316,7 @@ var AnthropicLLM = class {
|
|
|
3343
3316
|
}
|
|
3344
3317
|
};
|
|
3345
3318
|
|
|
3346
|
-
// src/agent/pipeline/gemini-llm.ts
|
|
3319
|
+
// src/agent/pipeline/llm/gemini-llm.ts
|
|
3347
3320
|
var GeminiLLM = class {
|
|
3348
3321
|
_options;
|
|
3349
3322
|
constructor(options = {}) {
|
|
@@ -3427,7 +3400,7 @@ var GeminiLLM = class {
|
|
|
3427
3400
|
}
|
|
3428
3401
|
};
|
|
3429
3402
|
|
|
3430
|
-
// src/agent/pipeline/openai-compat-llm.ts
|
|
3403
|
+
// src/agent/pipeline/llm/openai-compat-llm.ts
|
|
3431
3404
|
var OpenAICompatLLM = class {
|
|
3432
3405
|
_options;
|
|
3433
3406
|
constructor(options) {
|
|
@@ -3510,7 +3483,7 @@ var OpenAICompatLLM = class {
|
|
|
3510
3483
|
}
|
|
3511
3484
|
};
|
|
3512
3485
|
|
|
3513
|
-
// src/agent/pipeline/ollama-llm.ts
|
|
3486
|
+
// src/agent/pipeline/llm/ollama-llm.ts
|
|
3514
3487
|
var OllamaLLM = class {
|
|
3515
3488
|
_inner;
|
|
3516
3489
|
constructor(options = {}) {
|
|
@@ -3528,7 +3501,7 @@ var OllamaLLM = class {
|
|
|
3528
3501
|
}
|
|
3529
3502
|
};
|
|
3530
3503
|
|
|
3531
|
-
// src/agent/pipeline/mistral-llm.ts
|
|
3504
|
+
// src/agent/pipeline/llm/mistral-llm.ts
|
|
3532
3505
|
var MistralLLM = class {
|
|
3533
3506
|
_inner;
|
|
3534
3507
|
constructor(options = {}) {
|
|
@@ -3545,7 +3518,7 @@ var MistralLLM = class {
|
|
|
3545
3518
|
}
|
|
3546
3519
|
};
|
|
3547
3520
|
|
|
3548
|
-
// src/agent/pipeline/groq-llm.ts
|
|
3521
|
+
// src/agent/pipeline/llm/groq-llm.ts
|
|
3549
3522
|
var GroqLLM = class {
|
|
3550
3523
|
_inner;
|
|
3551
3524
|
constructor(options = {}) {
|
|
@@ -3562,7 +3535,7 @@ var GroqLLM = class {
|
|
|
3562
3535
|
}
|
|
3563
3536
|
};
|
|
3564
3537
|
|
|
3565
|
-
// src/agent/pipeline/perplexity-llm.ts
|
|
3538
|
+
// src/agent/pipeline/llm/perplexity-llm.ts
|
|
3566
3539
|
var PerplexityLLM = class {
|
|
3567
3540
|
_inner;
|
|
3568
3541
|
constructor(options = {}) {
|
|
@@ -3579,7 +3552,7 @@ var PerplexityLLM = class {
|
|
|
3579
3552
|
}
|
|
3580
3553
|
};
|
|
3581
3554
|
|
|
3582
|
-
// src/agent/pipeline/together-llm.ts
|
|
3555
|
+
// src/agent/pipeline/llm/together-llm.ts
|
|
3583
3556
|
var TogetherLLM = class {
|
|
3584
3557
|
_inner;
|
|
3585
3558
|
constructor(options = {}) {
|
|
@@ -3596,7 +3569,7 @@ var TogetherLLM = class {
|
|
|
3596
3569
|
}
|
|
3597
3570
|
};
|
|
3598
3571
|
|
|
3599
|
-
// src/agent/pipeline/fireworks-llm.ts
|
|
3572
|
+
// src/agent/pipeline/llm/fireworks-llm.ts
|
|
3600
3573
|
var FireworksLLM = class {
|
|
3601
3574
|
_inner;
|
|
3602
3575
|
constructor(options = {}) {
|
|
@@ -3613,7 +3586,7 @@ var FireworksLLM = class {
|
|
|
3613
3586
|
}
|
|
3614
3587
|
};
|
|
3615
3588
|
|
|
3616
|
-
// src/agent/pipeline/deepseek-llm.ts
|
|
3589
|
+
// src/agent/pipeline/llm/deepseek-llm.ts
|
|
3617
3590
|
var DeepSeekLLM = class {
|
|
3618
3591
|
_inner;
|
|
3619
3592
|
constructor(options = {}) {
|
|
@@ -3630,7 +3603,7 @@ var DeepSeekLLM = class {
|
|
|
3630
3603
|
}
|
|
3631
3604
|
};
|
|
3632
3605
|
|
|
3633
|
-
// src/agent/pipeline/xai-llm.ts
|
|
3606
|
+
// src/agent/pipeline/llm/xai-llm.ts
|
|
3634
3607
|
var XaiLLM = class {
|
|
3635
3608
|
_inner;
|
|
3636
3609
|
constructor(options = {}) {
|
|
@@ -3668,6 +3641,7 @@ function mcpServerHTTP(options) {
|
|
|
3668
3641
|
|
|
3669
3642
|
exports.AnthropicLLM = AnthropicLLM;
|
|
3670
3643
|
exports.AudioRecorder = AudioRecorder;
|
|
3644
|
+
exports.BUILTIN_TOOL_NAMES = BUILTIN_TOOL_NAMES;
|
|
3671
3645
|
exports.BuiltinTool = BuiltinTool;
|
|
3672
3646
|
exports.CallSession = CallSession;
|
|
3673
3647
|
exports.ClawOpsAgent = ClawOpsAgent;
|
|
@@ -3692,8 +3666,11 @@ exports.ToolRegistry = ToolRegistry;
|
|
|
3692
3666
|
exports.XaiLLM = XaiLLM;
|
|
3693
3667
|
exports.createAgentLogger = createAgentLogger;
|
|
3694
3668
|
exports.createPipelineLogger = createPipelineLogger;
|
|
3669
|
+
exports.executeBuiltinTool = executeBuiltinTool;
|
|
3695
3670
|
exports.functionTool = functionTool;
|
|
3671
|
+
exports.getBuiltinToolSchemas = getBuiltinToolSchemas;
|
|
3696
3672
|
exports.getTracingConfig = getTracingConfig;
|
|
3673
|
+
exports.isBuiltinTool = isBuiltinTool;
|
|
3697
3674
|
exports.mcpServerHTTP = mcpServerHTTP;
|
|
3698
3675
|
exports.mcpServerStdio = mcpServerStdio;
|
|
3699
3676
|
exports.pcm16ToUlaw = pcm16ToUlaw;
|