@teamlearners/clawops 0.5.4 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/index.cjs +629 -652
- package/dist/agent/index.cjs.map +1 -1
- package/dist/agent/index.d.cts +99 -74
- package/dist/agent/index.d.ts +99 -74
- package/dist/agent/index.js +626 -653
- package/dist/agent/index.js.map +1 -1
- package/package.json +2 -2
package/dist/agent/index.js
CHANGED
|
@@ -335,6 +335,7 @@ function createPipelineLogger(parent) {
|
|
|
335
335
|
// src/agent/control-ws.ts
|
|
336
336
|
var INITIAL_RECONNECT_DELAY = 1e3;
|
|
337
337
|
var MAX_RECONNECT_DELAY = 3e4;
|
|
338
|
+
var PING_TIMEOUT = 6e4;
|
|
338
339
|
function buildControlWsUrl(options) {
|
|
339
340
|
const scheme = options.baseUrl.startsWith("https") ? "wss" : "ws";
|
|
340
341
|
const host = options.baseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "");
|
|
@@ -360,6 +361,7 @@ var ControlWebSocket = class {
|
|
|
360
361
|
_connectedResolve = null;
|
|
361
362
|
_connectedPromise;
|
|
362
363
|
_log = NOOP_LOGGER;
|
|
364
|
+
_pingTimer = null;
|
|
363
365
|
setLogger(logger) {
|
|
364
366
|
this._log = logger;
|
|
365
367
|
}
|
|
@@ -390,6 +392,7 @@ var ControlWebSocket = class {
|
|
|
390
392
|
/** Close the WebSocket and stop reconnecting. */
|
|
391
393
|
close() {
|
|
392
394
|
this._closed = true;
|
|
395
|
+
this._clearPingTimer();
|
|
393
396
|
if (this._ws) {
|
|
394
397
|
this._ws.close();
|
|
395
398
|
this._ws = null;
|
|
@@ -406,12 +409,16 @@ var ControlWebSocket = class {
|
|
|
406
409
|
this._ws = ws;
|
|
407
410
|
ws.on("open", () => {
|
|
408
411
|
this._reconnectDelay = INITIAL_RECONNECT_DELAY;
|
|
412
|
+
this._resetPingTimer();
|
|
409
413
|
if (this._connectedResolve) {
|
|
410
414
|
this._connectedResolve();
|
|
411
415
|
this._connectedResolve = null;
|
|
412
416
|
}
|
|
413
417
|
this._log.info("Control WS connected: %s", this._url);
|
|
414
418
|
});
|
|
419
|
+
ws.on("ping", () => {
|
|
420
|
+
this._resetPingTimer();
|
|
421
|
+
});
|
|
415
422
|
ws.on("message", (data) => {
|
|
416
423
|
try {
|
|
417
424
|
const msg = JSON.parse(data.toString());
|
|
@@ -421,6 +428,7 @@ var ControlWebSocket = class {
|
|
|
421
428
|
}
|
|
422
429
|
});
|
|
423
430
|
ws.on("close", () => {
|
|
431
|
+
this._clearPingTimer();
|
|
424
432
|
if (!this._closed) {
|
|
425
433
|
this._scheduleReconnect();
|
|
426
434
|
}
|
|
@@ -446,6 +454,21 @@ var ControlWebSocket = class {
|
|
|
446
454
|
}
|
|
447
455
|
}
|
|
448
456
|
}
|
|
457
|
+
_resetPingTimer() {
|
|
458
|
+
this._clearPingTimer();
|
|
459
|
+
this._pingTimer = setTimeout(() => {
|
|
460
|
+
this._log.warn("Control WS ping timeout, closing connection");
|
|
461
|
+
if (this._ws) {
|
|
462
|
+
this._ws.terminate();
|
|
463
|
+
}
|
|
464
|
+
}, PING_TIMEOUT);
|
|
465
|
+
}
|
|
466
|
+
_clearPingTimer() {
|
|
467
|
+
if (this._pingTimer) {
|
|
468
|
+
clearTimeout(this._pingTimer);
|
|
469
|
+
this._pingTimer = null;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
449
472
|
_scheduleReconnect() {
|
|
450
473
|
const delay = this._reconnectDelay;
|
|
451
474
|
this._reconnectDelay = Math.min(this._reconnectDelay * 2, MAX_RECONNECT_DELAY);
|
|
@@ -1436,6 +1459,7 @@ var ClawOpsAgent = class {
|
|
|
1436
1459
|
}
|
|
1437
1460
|
/** Connect to the ClawOps platform and start listening for calls. */
|
|
1438
1461
|
async connect() {
|
|
1462
|
+
if (this._controlWs) return;
|
|
1439
1463
|
if (!this._apiKey) {
|
|
1440
1464
|
throw new AgentError("API key is required. Set CLAWOPS_API_KEY or pass apiKey option.");
|
|
1441
1465
|
}
|
|
@@ -1750,20 +1774,13 @@ var ClawOpsAgent = class {
|
|
|
1750
1774
|
}
|
|
1751
1775
|
};
|
|
1752
1776
|
|
|
1753
|
-
// src/agent/pipeline/
|
|
1754
|
-
var
|
|
1755
|
-
var HANG_UP_TOOL = {
|
|
1756
|
-
type: "function",
|
|
1777
|
+
// src/agent/pipeline/builtin-tool-schemas.ts
|
|
1778
|
+
var HANG_UP = {
|
|
1757
1779
|
name: "hang_up",
|
|
1758
1780
|
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
1759
|
-
parameters: {
|
|
1760
|
-
type: "object",
|
|
1761
|
-
properties: {},
|
|
1762
|
-
required: []
|
|
1763
|
-
}
|
|
1781
|
+
parameters: { type: "object", properties: {} }
|
|
1764
1782
|
};
|
|
1765
|
-
var
|
|
1766
|
-
type: "function",
|
|
1783
|
+
var COLLECT_DTMF = {
|
|
1767
1784
|
name: "collect_dtmf",
|
|
1768
1785
|
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
|
|
1769
1786
|
parameters: {
|
|
@@ -1776,8 +1793,7 @@ var COLLECT_DTMF_TOOL = {
|
|
|
1776
1793
|
required: ["max_digits"]
|
|
1777
1794
|
}
|
|
1778
1795
|
};
|
|
1779
|
-
var
|
|
1780
|
-
type: "function",
|
|
1796
|
+
var SEND_DTMF = {
|
|
1781
1797
|
name: "send_dtmf",
|
|
1782
1798
|
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
1783
1799
|
parameters: {
|
|
@@ -1785,199 +1801,530 @@ var SEND_DTMF_TOOL = {
|
|
|
1785
1801
|
properties: {
|
|
1786
1802
|
digits: {
|
|
1787
1803
|
type: "string",
|
|
1788
|
-
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30."
|
|
1804
|
+
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30. \uC608: '1', '1234#', '1w2'"
|
|
1789
1805
|
}
|
|
1790
1806
|
},
|
|
1791
1807
|
required: ["digits"]
|
|
1792
1808
|
}
|
|
1793
1809
|
};
|
|
1794
|
-
var
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1810
|
+
var TOOL_MAP = /* @__PURE__ */ new Map([
|
|
1811
|
+
["hang_up" /* HANG_UP */, HANG_UP],
|
|
1812
|
+
["collect_dtmf" /* COLLECT_DTMF */, COLLECT_DTMF],
|
|
1813
|
+
["send_dtmf" /* SEND_DTMF */, SEND_DTMF]
|
|
1814
|
+
]);
|
|
1815
|
+
var BUILTIN_TOOL_NAMES = new Set(
|
|
1816
|
+
Array.from(TOOL_MAP.values()).map((s) => s.name)
|
|
1817
|
+
);
|
|
1818
|
+
function toChatCompletions(schema) {
|
|
1819
|
+
return {
|
|
1820
|
+
type: "function",
|
|
1821
|
+
function: {
|
|
1822
|
+
name: schema.name,
|
|
1823
|
+
description: schema.description,
|
|
1824
|
+
parameters: schema.parameters
|
|
1825
|
+
}
|
|
1826
|
+
};
|
|
1827
|
+
}
|
|
1828
|
+
function toRealtime(schema) {
|
|
1829
|
+
return {
|
|
1830
|
+
type: "function",
|
|
1831
|
+
name: schema.name,
|
|
1832
|
+
description: schema.description,
|
|
1833
|
+
parameters: schema.parameters
|
|
1834
|
+
};
|
|
1835
|
+
}
|
|
1836
|
+
function toGemini(schema) {
|
|
1837
|
+
return {
|
|
1838
|
+
name: schema.name,
|
|
1839
|
+
description: schema.description,
|
|
1840
|
+
parameters: schema.parameters
|
|
1841
|
+
};
|
|
1842
|
+
}
|
|
1843
|
+
var CONVERTERS = {
|
|
1844
|
+
chat: toChatCompletions,
|
|
1845
|
+
realtime: toRealtime,
|
|
1846
|
+
gemini: toGemini
|
|
1847
|
+
};
|
|
1848
|
+
function getBuiltinToolSchemas(builtinTools, fmt) {
|
|
1849
|
+
const converter = CONVERTERS[fmt];
|
|
1850
|
+
const result = [];
|
|
1851
|
+
for (const [toolEnum, schema] of TOOL_MAP) {
|
|
1852
|
+
if (builtinTools === null || builtinTools.has(toolEnum)) {
|
|
1853
|
+
result.push(converter(schema));
|
|
1854
|
+
}
|
|
1806
1855
|
}
|
|
1807
|
-
|
|
1808
|
-
|
|
1856
|
+
return result;
|
|
1857
|
+
}
|
|
1858
|
+
function isBuiltinTool(name) {
|
|
1859
|
+
return BUILTIN_TOOL_NAMES.has(name);
|
|
1860
|
+
}
|
|
1861
|
+
async function executeBuiltinTool(funcName, args, call) {
|
|
1862
|
+
if (funcName === "hang_up") {
|
|
1863
|
+
await call.hangup();
|
|
1864
|
+
return "";
|
|
1809
1865
|
}
|
|
1810
|
-
|
|
1811
|
-
|
|
1866
|
+
if (funcName === "collect_dtmf") {
|
|
1867
|
+
try {
|
|
1868
|
+
const result = await call.collectDtmf({
|
|
1869
|
+
maxDigits: args["max_digits"] ?? 4,
|
|
1870
|
+
finishOnKey: args["finish_on_key"] ?? "#",
|
|
1871
|
+
timeout: args["timeout"] ?? 5
|
|
1872
|
+
});
|
|
1873
|
+
return result || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)";
|
|
1874
|
+
} catch (e) {
|
|
1875
|
+
return `Error: ${e}`;
|
|
1876
|
+
}
|
|
1877
|
+
}
|
|
1878
|
+
if (funcName === "send_dtmf") {
|
|
1879
|
+
try {
|
|
1880
|
+
await call.sendDtmfSequence(args["digits"] ?? "");
|
|
1881
|
+
return "sent";
|
|
1882
|
+
} catch (e) {
|
|
1883
|
+
return `Error: ${e}`;
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
return null;
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
// src/agent/pipeline/pipeline-session.ts
|
|
1890
|
+
var PipelineSession = class {
|
|
1891
|
+
_stt;
|
|
1892
|
+
_llm;
|
|
1893
|
+
_tts;
|
|
1894
|
+
_systemPrompt;
|
|
1895
|
+
_greeting;
|
|
1896
|
+
_language;
|
|
1897
|
+
_temperature;
|
|
1898
|
+
_maxTokens;
|
|
1899
|
+
_sampleRate;
|
|
1900
|
+
_interruptOnSpeech;
|
|
1901
|
+
_callSession = null;
|
|
1812
1902
|
_tools = null;
|
|
1813
1903
|
_recorder = null;
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
this.
|
|
1825
|
-
this._systemPrompt = options.systemPrompt ?? "";
|
|
1826
|
-
this._model = options.model ?? "gpt-realtime-1.5";
|
|
1827
|
-
this._voice = options.voice ?? "marin";
|
|
1828
|
-
this._language = options.language ?? "ko";
|
|
1829
|
-
this._eagerness = options.eagerness ?? "high";
|
|
1904
|
+
_conversation = [];
|
|
1905
|
+
_audioBuffer = [];
|
|
1906
|
+
_running = false;
|
|
1907
|
+
_speaking = false;
|
|
1908
|
+
_builtinTools = null;
|
|
1909
|
+
_log = NOOP_LOGGER;
|
|
1910
|
+
constructor(options) {
|
|
1911
|
+
this._stt = options.stt;
|
|
1912
|
+
this._llm = options.llm;
|
|
1913
|
+
this._tts = options.tts;
|
|
1914
|
+
this._systemPrompt = options.systemPrompt;
|
|
1830
1915
|
this._greeting = options.greeting ?? true;
|
|
1916
|
+
this._language = options.language ?? "ko";
|
|
1917
|
+
this._temperature = options.temperature;
|
|
1918
|
+
this._maxTokens = options.maxTokens;
|
|
1919
|
+
this._sampleRate = options.sampleRate ?? 8e3;
|
|
1920
|
+
this._interruptOnSpeech = options.interruptOnSpeech ?? true;
|
|
1921
|
+
if (options.toolRegistry) this._tools = options.toolRegistry;
|
|
1922
|
+
if (options.recorder) this._recorder = options.recorder;
|
|
1831
1923
|
}
|
|
1832
|
-
/** Inject per-call ToolRegistry. */
|
|
1833
1924
|
setToolRegistry(registry) {
|
|
1834
1925
|
this._tools = registry;
|
|
1835
1926
|
}
|
|
1836
|
-
/** Inject per-call AudioRecorder. */
|
|
1837
1927
|
setRecorder(recorder) {
|
|
1838
1928
|
this._recorder = recorder;
|
|
1839
1929
|
}
|
|
1840
|
-
|
|
1841
|
-
this.
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
this.
|
|
1845
|
-
this.
|
|
1846
|
-
|
|
1847
|
-
this._audioRemainder = Buffer.alloc(0);
|
|
1848
|
-
if (!this._apiKey) {
|
|
1849
|
-
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey option.");
|
|
1930
|
+
setBuiltinTools(tools) {
|
|
1931
|
+
this._builtinTools = tools;
|
|
1932
|
+
}
|
|
1933
|
+
setLogger(logger) {
|
|
1934
|
+
this._log = logger;
|
|
1935
|
+
if ("setLogger" in this._stt && typeof this._stt.setLogger === "function") {
|
|
1936
|
+
this._stt.setLogger(logger);
|
|
1850
1937
|
}
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
this._send({ type: "response.create" });
|
|
1866
|
-
}
|
|
1867
|
-
resolve();
|
|
1868
|
-
});
|
|
1869
|
-
ws.on("message", (data) => {
|
|
1870
|
-
try {
|
|
1871
|
-
const msg = JSON.parse(data.toString());
|
|
1872
|
-
this._handleMessage(msg);
|
|
1873
|
-
} catch {
|
|
1874
|
-
}
|
|
1875
|
-
});
|
|
1876
|
-
ws.on("close", () => {
|
|
1877
|
-
this._closed = true;
|
|
1938
|
+
if ("setLogger" in this._tts && typeof this._tts.setLogger === "function") {
|
|
1939
|
+
this._tts.setLogger(logger);
|
|
1940
|
+
}
|
|
1941
|
+
}
|
|
1942
|
+
async start(callSession, tools) {
|
|
1943
|
+
this._callSession = callSession;
|
|
1944
|
+
this._tools = tools ?? null;
|
|
1945
|
+
this._running = true;
|
|
1946
|
+
this._log.info("PipelineSession started");
|
|
1947
|
+
this._conversation = [];
|
|
1948
|
+
if (this._systemPrompt) {
|
|
1949
|
+
this._conversation.push({
|
|
1950
|
+
role: "system",
|
|
1951
|
+
content: this._systemPrompt
|
|
1878
1952
|
});
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
}
|
|
1883
|
-
this._log.error({ err }, "OpenAI Realtime WS error");
|
|
1953
|
+
}
|
|
1954
|
+
if (this._greeting) {
|
|
1955
|
+
this._generateGreeting().catch((err) => {
|
|
1956
|
+
this._log.error({ err }, "Greeting error");
|
|
1884
1957
|
});
|
|
1958
|
+
}
|
|
1959
|
+
this._runSttLoop().catch((err) => {
|
|
1960
|
+
this._log.error({ err }, "STT loop error");
|
|
1885
1961
|
});
|
|
1886
1962
|
}
|
|
1887
|
-
async feedDtmf(digits) {
|
|
1888
|
-
await this._waitForResponseDone();
|
|
1889
|
-
this._send({
|
|
1890
|
-
type: "conversation.item.create",
|
|
1891
|
-
item: {
|
|
1892
|
-
type: "message",
|
|
1893
|
-
role: "user",
|
|
1894
|
-
content: [{ type: "input_text", text: `[DTMF \uC785\uB825: ${digits}]` }]
|
|
1895
|
-
}
|
|
1896
|
-
});
|
|
1897
|
-
this._send({ type: "response.create" });
|
|
1898
|
-
}
|
|
1899
1963
|
feedAudio(audio) {
|
|
1900
|
-
if (this.
|
|
1901
|
-
this.
|
|
1902
|
-
type: "input_audio_buffer.append",
|
|
1903
|
-
audio: audio.toString("base64")
|
|
1904
|
-
});
|
|
1905
|
-
}
|
|
1906
|
-
}
|
|
1907
|
-
async stop() {
|
|
1908
|
-
this._closed = true;
|
|
1909
|
-
if (this._ws) {
|
|
1910
|
-
this._ws.close();
|
|
1911
|
-
this._ws = null;
|
|
1964
|
+
if (this._running) {
|
|
1965
|
+
this._audioBuffer.push(audio);
|
|
1912
1966
|
}
|
|
1913
1967
|
}
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
toolSchemas.push(HANG_UP_TOOL);
|
|
1919
|
-
if (!this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */))
|
|
1920
|
-
toolSchemas.push(COLLECT_DTMF_TOOL);
|
|
1921
|
-
if (!this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */))
|
|
1922
|
-
toolSchemas.push(SEND_DTMF_TOOL);
|
|
1923
|
-
this._send({
|
|
1924
|
-
type: "session.update",
|
|
1925
|
-
session: {
|
|
1926
|
-
modalities: ["text", "audio"],
|
|
1927
|
-
voice: this._voice,
|
|
1928
|
-
instructions: this._systemPrompt,
|
|
1929
|
-
input_audio_format: "g711_ulaw",
|
|
1930
|
-
output_audio_format: "g711_ulaw",
|
|
1931
|
-
input_audio_transcription: {
|
|
1932
|
-
model: "whisper-1",
|
|
1933
|
-
language: this._language
|
|
1934
|
-
},
|
|
1935
|
-
input_audio_noise_reduction: { type: "far_field" },
|
|
1936
|
-
turn_detection: {
|
|
1937
|
-
type: "semantic_vad",
|
|
1938
|
-
interrupt_response: true,
|
|
1939
|
-
eagerness: this._eagerness
|
|
1940
|
-
},
|
|
1941
|
-
tools: toolSchemas
|
|
1942
|
-
}
|
|
1968
|
+
async feedDtmf(digits) {
|
|
1969
|
+
this._conversation.push({
|
|
1970
|
+
role: "user",
|
|
1971
|
+
content: `[DTMF \uC785\uB825: ${digits}]`
|
|
1943
1972
|
});
|
|
1973
|
+
await this._respond();
|
|
1944
1974
|
}
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
}
|
|
1961
|
-
this._sentAudioChunks++;
|
|
1962
|
-
this._audioRemainder = Buffer.alloc(0);
|
|
1975
|
+
async stop() {
|
|
1976
|
+
this._running = false;
|
|
1977
|
+
this._log.info("PipelineSession stopped");
|
|
1978
|
+
this._audioBuffer = [];
|
|
1979
|
+
}
|
|
1980
|
+
async _runSttLoop() {
|
|
1981
|
+
const audioStream = this._createAudioStream();
|
|
1982
|
+
for await (const event of this._stt.transcribe(audioStream, {
|
|
1983
|
+
sampleRate: this._sampleRate
|
|
1984
|
+
})) {
|
|
1985
|
+
if (!this._running) break;
|
|
1986
|
+
if (event.type === "interim" && this._speaking && this._interruptOnSpeech) {
|
|
1987
|
+
this._speaking = false;
|
|
1988
|
+
if (this._callSession) {
|
|
1989
|
+
this._callSession.clearAudio();
|
|
1963
1990
|
}
|
|
1964
|
-
|
|
1965
|
-
}
|
|
1966
|
-
case "input_audio_buffer.speech_started": {
|
|
1967
|
-
this._handleTruncation();
|
|
1968
|
-
break;
|
|
1991
|
+
this._log.info('Barge-in: "%s"', event.transcript.substring(0, 30));
|
|
1969
1992
|
}
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
this._handleToolCall(item);
|
|
1974
|
-
}
|
|
1975
|
-
break;
|
|
1993
|
+
if (event.type === "final" && event.transcript.trim()) {
|
|
1994
|
+
this._log.info("STT: %s", event.transcript);
|
|
1995
|
+
await this._handleUserSpeech(event.transcript);
|
|
1976
1996
|
}
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1997
|
+
}
|
|
1998
|
+
}
|
|
1999
|
+
async *_createAudioStream() {
|
|
2000
|
+
while (this._running) {
|
|
2001
|
+
if (this._audioBuffer.length > 0) {
|
|
2002
|
+
const ulaw = this._audioBuffer.shift();
|
|
2003
|
+
const pcm8k = ulawToPcm16(ulaw);
|
|
2004
|
+
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
2005
|
+
yield pcm16k;
|
|
2006
|
+
} else {
|
|
2007
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
2008
|
+
}
|
|
2009
|
+
}
|
|
2010
|
+
}
|
|
2011
|
+
async _generateGreeting() {
|
|
2012
|
+
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
2013
|
+
await this._respond();
|
|
2014
|
+
}
|
|
2015
|
+
async _handleUserSpeech(transcript) {
|
|
2016
|
+
this._conversation.push({ role: "user", content: transcript });
|
|
2017
|
+
await this._respond();
|
|
2018
|
+
}
|
|
2019
|
+
_buildEffectiveTools() {
|
|
2020
|
+
const builtinSchemas = getBuiltinToolSchemas(this._builtinTools, "chat");
|
|
2021
|
+
const dtmfSchemas = builtinSchemas.filter((s) => {
|
|
2022
|
+
const name = s["function"]?.["name"];
|
|
2023
|
+
return name === "collect_dtmf" || name === "send_dtmf";
|
|
2024
|
+
});
|
|
2025
|
+
if (dtmfSchemas.length === 0) return this._tools ?? void 0;
|
|
2026
|
+
const base = this._tools ? this._tools.fork() : new ToolRegistry();
|
|
2027
|
+
for (const schema of dtmfSchemas) {
|
|
2028
|
+
const fn = schema["function"];
|
|
2029
|
+
const params = fn["parameters"];
|
|
2030
|
+
base.register({
|
|
2031
|
+
name: fn["name"],
|
|
2032
|
+
description: fn["description"],
|
|
2033
|
+
parameters: params["properties"] ?? {},
|
|
2034
|
+
required: params["required"] ?? [],
|
|
2035
|
+
handler: async () => ""
|
|
2036
|
+
});
|
|
2037
|
+
}
|
|
2038
|
+
return base;
|
|
2039
|
+
}
|
|
2040
|
+
async _respond() {
|
|
2041
|
+
let fullResponse = "";
|
|
2042
|
+
const textChunks = [];
|
|
2043
|
+
const effectiveTools = this._buildEffectiveTools();
|
|
2044
|
+
const llmStream = this._llm.generate(this._conversation, {
|
|
2045
|
+
tools: effectiveTools,
|
|
2046
|
+
temperature: this._temperature,
|
|
2047
|
+
maxTokens: this._maxTokens
|
|
2048
|
+
});
|
|
2049
|
+
for await (const chunk of llmStream) {
|
|
2050
|
+
if (!this._running) break;
|
|
2051
|
+
if (chunk.type === "text" && chunk.text) {
|
|
2052
|
+
textChunks.push(chunk.text);
|
|
2053
|
+
fullResponse += chunk.text;
|
|
2054
|
+
} else if (chunk.type === "tool_call" && chunk.toolCall) {
|
|
2055
|
+
await this._handleToolCall(chunk);
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
if (fullResponse.trim()) {
|
|
2059
|
+
this._log.info("Assistant: %s", fullResponse.substring(0, 100));
|
|
2060
|
+
this._conversation.push({ role: "assistant", content: fullResponse });
|
|
2061
|
+
await this._synthesizeAndSend(fullResponse);
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
async _handleToolCall(chunk) {
|
|
2065
|
+
if (!chunk.toolCall) return;
|
|
2066
|
+
const { id, name, arguments: argsStr } = chunk.toolCall;
|
|
2067
|
+
try {
|
|
2068
|
+
const args = JSON.parse(argsStr);
|
|
2069
|
+
if (BUILTIN_TOOL_NAMES.has(name) && this._callSession) {
|
|
2070
|
+
const result2 = await executeBuiltinTool(name, args, this._callSession);
|
|
2071
|
+
if (result2 !== null) {
|
|
2072
|
+
if (name === "hang_up") return;
|
|
2073
|
+
this._conversation.push({ role: "tool", content: result2, tool_call_id: id, name });
|
|
2074
|
+
await this._respond();
|
|
2075
|
+
return;
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
if (!this._tools) return;
|
|
2079
|
+
const result = await this._tools.call(name, args);
|
|
2080
|
+
this._conversation.push({
|
|
2081
|
+
role: "assistant",
|
|
2082
|
+
content: ""
|
|
2083
|
+
// Tool call info stored in the message flow
|
|
2084
|
+
});
|
|
2085
|
+
this._conversation.push({
|
|
2086
|
+
role: "tool",
|
|
2087
|
+
content: typeof result === "string" ? result : JSON.stringify(result),
|
|
2088
|
+
tool_call_id: id,
|
|
2089
|
+
name
|
|
2090
|
+
});
|
|
2091
|
+
const effectiveTools = this._buildEffectiveTools();
|
|
2092
|
+
let followUpText = "";
|
|
2093
|
+
const followUpStream = this._llm.generate(this._conversation, {
|
|
2094
|
+
tools: effectiveTools,
|
|
2095
|
+
temperature: this._temperature,
|
|
2096
|
+
maxTokens: this._maxTokens
|
|
2097
|
+
});
|
|
2098
|
+
for await (const followChunk of followUpStream) {
|
|
2099
|
+
if (!this._running) break;
|
|
2100
|
+
if (followChunk.type === "text" && followChunk.text) {
|
|
2101
|
+
followUpText += followChunk.text;
|
|
2102
|
+
}
|
|
2103
|
+
}
|
|
2104
|
+
if (followUpText.trim()) {
|
|
2105
|
+
this._conversation.push({ role: "assistant", content: followUpText });
|
|
2106
|
+
await this._synthesizeAndSend(followUpText);
|
|
2107
|
+
}
|
|
2108
|
+
} catch (err) {
|
|
2109
|
+
this._log.error({ err }, "Tool call failed: %s", name);
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
async _synthesizeAndSend(text) {
|
|
2113
|
+
if (!this._callSession || !this._running) return;
|
|
2114
|
+
this._speaking = true;
|
|
2115
|
+
try {
|
|
2116
|
+
for await (const audioChunk of this._tts.synthesize(text, {
|
|
2117
|
+
sampleRate: this._sampleRate
|
|
2118
|
+
})) {
|
|
2119
|
+
if (!this._running || !this._speaking) break;
|
|
2120
|
+
if (this._recorder) {
|
|
2121
|
+
const pcm8k2 = this._sampleRate !== 8e3 ? resamplePcm16(audioChunk, this._sampleRate, 8e3) : audioChunk;
|
|
2122
|
+
this._recorder.writeOutbound(pcm8k2);
|
|
2123
|
+
}
|
|
2124
|
+
const pcm8k = resamplePcm16(audioChunk, this._sampleRate, 8e3);
|
|
2125
|
+
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2126
|
+
for (let off = 0; off < ulaw.length; off += 160) {
|
|
2127
|
+
let chunk = ulaw.subarray(off, off + 160);
|
|
2128
|
+
if (chunk.length < 160) {
|
|
2129
|
+
chunk = Buffer.concat([chunk, Buffer.alloc(160 - chunk.length, 255)]);
|
|
2130
|
+
}
|
|
2131
|
+
this._callSession.sendAudio(chunk);
|
|
2132
|
+
}
|
|
2133
|
+
}
|
|
2134
|
+
} catch (err) {
|
|
2135
|
+
this._log.error({ err }, "TTS error");
|
|
2136
|
+
} finally {
|
|
2137
|
+
this._speaking = false;
|
|
2138
|
+
}
|
|
2139
|
+
}
|
|
2140
|
+
};
|
|
2141
|
+
|
|
2142
|
+
// src/agent/pipeline/realtime/openai-realtime.ts
|
|
2143
|
+
var OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime?model=";
|
|
2144
|
+
var OpenAIRealtime = class {
|
|
2145
|
+
_apiKey;
|
|
2146
|
+
_systemPrompt;
|
|
2147
|
+
_model;
|
|
2148
|
+
_voice;
|
|
2149
|
+
_language;
|
|
2150
|
+
_turnDetection;
|
|
2151
|
+
_greeting;
|
|
2152
|
+
_log = NOOP_LOGGER;
|
|
2153
|
+
_builtinTools = null;
|
|
2154
|
+
setLogger(logger) {
|
|
2155
|
+
this._log = logger;
|
|
2156
|
+
}
|
|
2157
|
+
setBuiltinTools(tools) {
|
|
2158
|
+
this._builtinTools = tools;
|
|
2159
|
+
}
|
|
2160
|
+
_ws = null;
|
|
2161
|
+
_call = null;
|
|
2162
|
+
_tools = null;
|
|
2163
|
+
_recorder = null;
|
|
2164
|
+
_closed = false;
|
|
2165
|
+
// PlaybackState — 현재 재생 중인 응답 상태
|
|
2166
|
+
_playback = null;
|
|
2167
|
+
_latestMediaTs = 0;
|
|
2168
|
+
// Pending tool call tracking — 인터럽트 시 취소용
|
|
2169
|
+
_pendingToolCalls = /* @__PURE__ */ new Map();
|
|
2170
|
+
// Response state tracking — prevent sending response.create while one is active
|
|
2171
|
+
_responseInProgress = false;
|
|
2172
|
+
_onResponseDone = null;
|
|
2173
|
+
constructor(options = {}) {
|
|
2174
|
+
this._apiKey = options.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
|
|
2175
|
+
this._systemPrompt = options.systemPrompt ?? "";
|
|
2176
|
+
this._model = options.model ?? "gpt-realtime-1.5";
|
|
2177
|
+
this._voice = options.voice ?? "marin";
|
|
2178
|
+
this._language = options.language ?? "ko";
|
|
2179
|
+
this._turnDetection = options.turnDetection !== void 0 ? options.turnDetection : { type: "semantic_vad", eagerness: "medium", interrupt_response: true };
|
|
2180
|
+
this._greeting = options.greeting ?? true;
|
|
2181
|
+
}
|
|
2182
|
+
/** Inject per-call ToolRegistry. */
|
|
2183
|
+
setToolRegistry(registry) {
|
|
2184
|
+
this._tools = registry;
|
|
2185
|
+
}
|
|
2186
|
+
/** Inject per-call AudioRecorder. */
|
|
2187
|
+
setRecorder(recorder) {
|
|
2188
|
+
this._recorder = recorder;
|
|
2189
|
+
}
|
|
2190
|
+
async start(callSession, tools) {
|
|
2191
|
+
this._call = callSession;
|
|
2192
|
+
if (tools) this._tools = tools;
|
|
2193
|
+
this._closed = false;
|
|
2194
|
+
this._playback = null;
|
|
2195
|
+
this._latestMediaTs = 0;
|
|
2196
|
+
if (!this._apiKey) {
|
|
2197
|
+
throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey option.");
|
|
2198
|
+
}
|
|
2199
|
+
const { WebSocket } = await import('ws');
|
|
2200
|
+
const url = `${OPENAI_REALTIME_URL}${this._model}`;
|
|
2201
|
+
this._ws = new WebSocket(url, {
|
|
2202
|
+
headers: {
|
|
2203
|
+
Authorization: `Bearer ${this._apiKey}`,
|
|
2204
|
+
"OpenAI-Beta": "realtime=v1"
|
|
2205
|
+
}
|
|
2206
|
+
});
|
|
2207
|
+
return new Promise((resolve, reject) => {
|
|
2208
|
+
const ws = this._ws;
|
|
2209
|
+
ws.on("open", () => {
|
|
2210
|
+
this._sendSessionUpdate();
|
|
2211
|
+
this._log.info("OpenAI Realtime connected");
|
|
2212
|
+
if (this._greeting) {
|
|
2213
|
+
this._send({ type: "response.create" });
|
|
2214
|
+
}
|
|
2215
|
+
resolve();
|
|
2216
|
+
});
|
|
2217
|
+
ws.on("message", (data) => {
|
|
2218
|
+
try {
|
|
2219
|
+
const msg = JSON.parse(data.toString());
|
|
2220
|
+
this._handleMessage(msg);
|
|
2221
|
+
} catch {
|
|
2222
|
+
}
|
|
2223
|
+
});
|
|
2224
|
+
ws.on("close", () => {
|
|
2225
|
+
this._closed = true;
|
|
2226
|
+
});
|
|
2227
|
+
ws.on("error", (err) => {
|
|
2228
|
+
if (!this._ws) {
|
|
2229
|
+
reject(err);
|
|
2230
|
+
}
|
|
2231
|
+
this._log.error({ err }, "OpenAI Realtime WS error");
|
|
2232
|
+
});
|
|
2233
|
+
});
|
|
2234
|
+
}
|
|
2235
|
+
async feedDtmf(digits) {
|
|
2236
|
+
await this._waitForResponseDone();
|
|
2237
|
+
this._send({
|
|
2238
|
+
type: "conversation.item.create",
|
|
2239
|
+
item: {
|
|
2240
|
+
type: "message",
|
|
2241
|
+
role: "user",
|
|
2242
|
+
content: [{ type: "input_text", text: `[DTMF \uC785\uB825: ${digits}]` }]
|
|
2243
|
+
}
|
|
2244
|
+
});
|
|
2245
|
+
this._send({ type: "response.create" });
|
|
2246
|
+
}
|
|
2247
|
+
feedAudio(audio) {
|
|
2248
|
+
this._latestMediaTs = Date.now();
|
|
2249
|
+
if (this._ws && this._ws.readyState === 1 && !this._closed) {
|
|
2250
|
+
this._send({
|
|
2251
|
+
type: "input_audio_buffer.append",
|
|
2252
|
+
audio: audio.toString("base64")
|
|
2253
|
+
});
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
async stop() {
|
|
2257
|
+
this._closed = true;
|
|
2258
|
+
for (const [, controller] of this._pendingToolCalls) {
|
|
2259
|
+
controller.abort();
|
|
2260
|
+
}
|
|
2261
|
+
this._pendingToolCalls.clear();
|
|
2262
|
+
if (this._ws) {
|
|
2263
|
+
this._ws.close();
|
|
2264
|
+
this._ws = null;
|
|
2265
|
+
}
|
|
2266
|
+
}
|
|
2267
|
+
_sendSessionUpdate() {
|
|
2268
|
+
if (!this._ws || this._ws.readyState !== 1) return;
|
|
2269
|
+
const toolSchemas = this._tools ? this._tools.toOpenAITools().map((t) => ({ type: "function", ...t.function })) : [];
|
|
2270
|
+
toolSchemas.push(...getBuiltinToolSchemas(this._builtinTools, "realtime"));
|
|
2271
|
+
this._send({
|
|
2272
|
+
type: "session.update",
|
|
2273
|
+
session: {
|
|
2274
|
+
modalities: ["text", "audio"],
|
|
2275
|
+
voice: this._voice,
|
|
2276
|
+
instructions: this._systemPrompt,
|
|
2277
|
+
input_audio_format: "g711_ulaw",
|
|
2278
|
+
output_audio_format: "g711_ulaw",
|
|
2279
|
+
input_audio_transcription: {
|
|
2280
|
+
model: "whisper-1",
|
|
2281
|
+
language: this._language
|
|
2282
|
+
},
|
|
2283
|
+
input_audio_noise_reduction: { type: "far_field" },
|
|
2284
|
+
turn_detection: this._turnDetection,
|
|
2285
|
+
tools: toolSchemas
|
|
2286
|
+
}
|
|
2287
|
+
});
|
|
2288
|
+
}
|
|
2289
|
+
_handleMessage(msg) {
|
|
2290
|
+
const type = msg["type"];
|
|
2291
|
+
switch (type) {
|
|
2292
|
+
case "response.audio.delta": {
|
|
2293
|
+
this._handleAudioDelta(msg);
|
|
2294
|
+
break;
|
|
2295
|
+
}
|
|
2296
|
+
case "response.audio.done": {
|
|
2297
|
+
if (this._playback) {
|
|
2298
|
+
this._playback.generating = false;
|
|
2299
|
+
if (this._playback.audioRemainder.length > 0) {
|
|
2300
|
+
const padded = Buffer.concat([
|
|
2301
|
+
this._playback.audioRemainder,
|
|
2302
|
+
Buffer.alloc(160 - this._playback.audioRemainder.length, 255)
|
|
2303
|
+
]);
|
|
2304
|
+
if (this._call) {
|
|
2305
|
+
this._call.sendAudio(padded);
|
|
2306
|
+
}
|
|
2307
|
+
this._playback.sentChunks++;
|
|
2308
|
+
this._playback.audioRemainder = Buffer.alloc(0);
|
|
2309
|
+
}
|
|
2310
|
+
}
|
|
2311
|
+
break;
|
|
2312
|
+
}
|
|
2313
|
+
case "input_audio_buffer.speech_started": {
|
|
2314
|
+
this._handleTruncation();
|
|
2315
|
+
break;
|
|
2316
|
+
}
|
|
2317
|
+
case "response.output_item.done": {
|
|
2318
|
+
const item = msg["item"];
|
|
2319
|
+
if (item && item["type"] === "function_call") {
|
|
2320
|
+
this._handleToolCall(item);
|
|
2321
|
+
}
|
|
2322
|
+
break;
|
|
2323
|
+
}
|
|
2324
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
2325
|
+
if (this._call) {
|
|
2326
|
+
this._call._emit("transcript", "user", msg["transcript"] ?? "");
|
|
2327
|
+
}
|
|
1981
2328
|
break;
|
|
1982
2329
|
}
|
|
1983
2330
|
case "response.audio_transcript.done": {
|
|
@@ -2006,128 +2353,121 @@ var OpenAIRealtime = class {
|
|
|
2006
2353
|
}
|
|
2007
2354
|
}
|
|
2008
2355
|
_handleAudioDelta(msg) {
|
|
2009
|
-
if (this.
|
|
2010
|
-
this.
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2356
|
+
if (this._playback === null) {
|
|
2357
|
+
this._playback = {
|
|
2358
|
+
itemId: msg["item_id"] || "",
|
|
2359
|
+
startTs: this._latestMediaTs || Date.now(),
|
|
2360
|
+
sentChunks: 0,
|
|
2361
|
+
generating: true,
|
|
2362
|
+
audioRemainder: Buffer.alloc(0)
|
|
2363
|
+
};
|
|
2364
|
+
} else if (msg["item_id"]) {
|
|
2365
|
+
this._playback.itemId = msg["item_id"];
|
|
2015
2366
|
}
|
|
2367
|
+
const pb = this._playback;
|
|
2016
2368
|
const ulaw = Buffer.from(msg["delta"], "base64");
|
|
2017
2369
|
if (this._recorder) {
|
|
2018
2370
|
this._recorder.writeOutbound(ulawToPcm16(ulaw));
|
|
2019
2371
|
}
|
|
2020
|
-
const combined = Buffer.concat([
|
|
2372
|
+
const combined = Buffer.concat([pb.audioRemainder, ulaw]);
|
|
2021
2373
|
const chunkSize = 160;
|
|
2022
2374
|
const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
|
|
2023
2375
|
for (let off = 0; off < fullEnd; off += chunkSize) {
|
|
2024
2376
|
if (this._call) {
|
|
2025
2377
|
this._call.sendAudio(combined.subarray(off, off + chunkSize));
|
|
2026
2378
|
}
|
|
2027
|
-
|
|
2379
|
+
pb.sentChunks++;
|
|
2028
2380
|
}
|
|
2029
|
-
|
|
2381
|
+
pb.audioRemainder = combined.subarray(fullEnd);
|
|
2030
2382
|
}
|
|
2031
2383
|
_handleTruncation() {
|
|
2032
|
-
|
|
2033
|
-
|
|
2384
|
+
for (const [, controller] of this._pendingToolCalls) {
|
|
2385
|
+
controller.abort();
|
|
2034
2386
|
}
|
|
2035
|
-
|
|
2036
|
-
this._send({
|
|
2037
|
-
type: "conversation.item.truncate",
|
|
2038
|
-
item_id: this._lastAssistantItem,
|
|
2039
|
-
content_index: 0,
|
|
2040
|
-
audio_end_ms: audioEndMs
|
|
2041
|
-
});
|
|
2387
|
+
this._pendingToolCalls.clear();
|
|
2042
2388
|
if (this._call) {
|
|
2043
2389
|
this._call.clearAudio();
|
|
2044
2390
|
}
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2391
|
+
const pb = this._playback;
|
|
2392
|
+
if (pb === null) {
|
|
2393
|
+
return;
|
|
2394
|
+
}
|
|
2395
|
+
const playedMs = Math.max(0, (this._latestMediaTs || Date.now()) - pb.startTs);
|
|
2396
|
+
this._log.info(
|
|
2397
|
+
"[Interrupt] item=%s played=%dms total=%dms",
|
|
2398
|
+
pb.itemId,
|
|
2399
|
+
playedMs,
|
|
2400
|
+
pb.sentChunks * 20
|
|
2401
|
+
);
|
|
2402
|
+
this._playback = null;
|
|
2049
2403
|
}
|
|
2050
2404
|
async _handleToolCall(item) {
|
|
2051
2405
|
const funcName = item["name"];
|
|
2052
2406
|
const callId = item["call_id"];
|
|
2053
2407
|
this._log.info("Tool call: %s", funcName);
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2408
|
+
const controller = new AbortController();
|
|
2409
|
+
this._pendingToolCalls.set(callId, controller);
|
|
2410
|
+
try {
|
|
2411
|
+
if (BUILTIN_TOOL_NAMES.has(funcName) && this._call) {
|
|
2412
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2413
|
+
const result2 = await executeBuiltinTool(funcName, args, this._call);
|
|
2414
|
+
if (result2 !== null) {
|
|
2415
|
+
if (funcName === "hang_up") return;
|
|
2416
|
+
if (controller.signal.aborted) return;
|
|
2417
|
+
await this._waitForResponseDone();
|
|
2418
|
+
this._send({
|
|
2419
|
+
type: "conversation.item.create",
|
|
2420
|
+
item: {
|
|
2421
|
+
type: "function_call_output",
|
|
2422
|
+
call_id: callId,
|
|
2423
|
+
output: result2
|
|
2424
|
+
}
|
|
2069
2425
|
});
|
|
2070
|
-
|
|
2071
|
-
|
|
2426
|
+
this._send({ type: "response.create" });
|
|
2427
|
+
return;
|
|
2072
2428
|
}
|
|
2073
|
-
await this._waitForResponseDone();
|
|
2074
|
-
this._send({
|
|
2075
|
-
type: "conversation.item.create",
|
|
2076
|
-
item: {
|
|
2077
|
-
type: "function_call_output",
|
|
2078
|
-
call_id: callId,
|
|
2079
|
-
output: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)"
|
|
2080
|
-
}
|
|
2081
|
-
});
|
|
2082
|
-
this._send({ type: "response.create" });
|
|
2083
2429
|
}
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
if (funcName === "send_dtmf") {
|
|
2087
|
-
if (this._call) {
|
|
2088
|
-
let result2;
|
|
2089
|
-
try {
|
|
2090
|
-
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2091
|
-
await this._call.sendDtmfSequence(args["digits"] ?? "");
|
|
2092
|
-
result2 = "sent";
|
|
2093
|
-
} catch (err) {
|
|
2094
|
-
result2 = `Error: ${err}`;
|
|
2095
|
-
}
|
|
2430
|
+
if (!this._tools || !this._tools.has(funcName)) {
|
|
2431
|
+
this._log.error("Unknown tool: %s", funcName);
|
|
2096
2432
|
await this._waitForResponseDone();
|
|
2097
2433
|
this._send({
|
|
2098
2434
|
type: "conversation.item.create",
|
|
2099
2435
|
item: {
|
|
2100
2436
|
type: "function_call_output",
|
|
2101
2437
|
call_id: callId,
|
|
2102
|
-
output:
|
|
2438
|
+
output: JSON.stringify({ error: `Unknown tool: ${funcName}` })
|
|
2103
2439
|
}
|
|
2104
2440
|
});
|
|
2105
2441
|
this._send({ type: "response.create" });
|
|
2442
|
+
return;
|
|
2106
2443
|
}
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
try {
|
|
2115
|
-
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2116
|
-
result = await this._tools.call(funcName, args);
|
|
2117
|
-
} catch (err) {
|
|
2118
|
-
this._log.error({ err }, "Tool call failed: %s", funcName);
|
|
2119
|
-
result = `Error: ${err}`;
|
|
2120
|
-
}
|
|
2121
|
-
await this._waitForResponseDone();
|
|
2122
|
-
this._send({
|
|
2123
|
-
type: "conversation.item.create",
|
|
2124
|
-
item: {
|
|
2125
|
-
type: "function_call_output",
|
|
2126
|
-
call_id: callId,
|
|
2127
|
-
output: typeof result === "string" ? result : JSON.stringify(result)
|
|
2444
|
+
let result;
|
|
2445
|
+
try {
|
|
2446
|
+
const args = JSON.parse(item["arguments"] ?? "{}");
|
|
2447
|
+
result = await this._tools.call(funcName, args);
|
|
2448
|
+
} catch (err) {
|
|
2449
|
+
this._log.error({ err }, "Tool call failed: %s", funcName);
|
|
2450
|
+
result = `Error: ${err}`;
|
|
2128
2451
|
}
|
|
2129
|
-
|
|
2130
|
-
|
|
2452
|
+
if (controller.signal.aborted) {
|
|
2453
|
+
this._log.info("Tool call cancelled (user interrupted): %s", funcName);
|
|
2454
|
+
return;
|
|
2455
|
+
}
|
|
2456
|
+
const resultStr = typeof result === "string" ? result : JSON.stringify(result);
|
|
2457
|
+
await this._waitForResponseDone();
|
|
2458
|
+
this._send({
|
|
2459
|
+
type: "conversation.item.create",
|
|
2460
|
+
item: {
|
|
2461
|
+
type: "function_call_output",
|
|
2462
|
+
call_id: callId,
|
|
2463
|
+
output: resultStr
|
|
2464
|
+
}
|
|
2465
|
+
});
|
|
2466
|
+
this._log.info("[ToolResult] %s call_id=%s len=%d", funcName, callId, resultStr.length);
|
|
2467
|
+
this._send({ type: "response.create" });
|
|
2468
|
+
} finally {
|
|
2469
|
+
this._pendingToolCalls.delete(callId);
|
|
2470
|
+
}
|
|
2131
2471
|
}
|
|
2132
2472
|
_waitForResponseDone() {
|
|
2133
2473
|
if (!this._responseInProgress) return Promise.resolve();
|
|
@@ -2142,39 +2482,7 @@ var OpenAIRealtime = class {
|
|
|
2142
2482
|
}
|
|
2143
2483
|
};
|
|
2144
2484
|
|
|
2145
|
-
// src/agent/pipeline/gemini-realtime.ts
|
|
2146
|
-
var HANG_UP_TOOL2 = {
|
|
2147
|
-
name: "hang_up",
|
|
2148
|
-
description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
|
|
2149
|
-
parameters: { type: "object", properties: {} }
|
|
2150
|
-
};
|
|
2151
|
-
var COLLECT_DTMF_TOOL2 = {
|
|
2152
|
-
name: "collect_dtmf",
|
|
2153
|
-
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
|
|
2154
|
-
parameters: {
|
|
2155
|
-
type: "object",
|
|
2156
|
-
properties: {
|
|
2157
|
-
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
2158
|
-
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
2159
|
-
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
2160
|
-
},
|
|
2161
|
-
required: ["max_digits"]
|
|
2162
|
-
}
|
|
2163
|
-
};
|
|
2164
|
-
var SEND_DTMF_TOOL2 = {
|
|
2165
|
-
name: "send_dtmf",
|
|
2166
|
-
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
2167
|
-
parameters: {
|
|
2168
|
-
type: "object",
|
|
2169
|
-
properties: {
|
|
2170
|
-
digits: {
|
|
2171
|
-
type: "string",
|
|
2172
|
-
description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30."
|
|
2173
|
-
}
|
|
2174
|
-
},
|
|
2175
|
-
required: ["digits"]
|
|
2176
|
-
}
|
|
2177
|
-
};
|
|
2485
|
+
// src/agent/pipeline/realtime/gemini-realtime.ts
|
|
2178
2486
|
function resolveRef(ref, defs) {
|
|
2179
2487
|
const parts = ref.replace(/^#\//, "").split("/");
|
|
2180
2488
|
let result = defs;
|
|
@@ -2384,9 +2692,7 @@ var GeminiRealtime = class {
|
|
|
2384
2692
|
t.function.parameters ?? { type: "object", properties: {} }
|
|
2385
2693
|
)
|
|
2386
2694
|
})) : [];
|
|
2387
|
-
|
|
2388
|
-
if (!this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */)) toolDefs.push(COLLECT_DTMF_TOOL2);
|
|
2389
|
-
if (!this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */)) toolDefs.push(SEND_DTMF_TOOL2);
|
|
2695
|
+
toolDefs.push(...getBuiltinToolSchemas(this._builtinTools, "gemini"));
|
|
2390
2696
|
return toolDefs;
|
|
2391
2697
|
}
|
|
2392
2698
|
_handleMessage(msg) {
|
|
@@ -2474,51 +2780,17 @@ var GeminiRealtime = class {
|
|
|
2474
2780
|
const fcId = fc.id ?? "";
|
|
2475
2781
|
const args = fc.args ?? {};
|
|
2476
2782
|
this._log.info({ tool: name, args }, "Tool call: %s", name);
|
|
2477
|
-
if (name
|
|
2478
|
-
|
|
2479
|
-
if (
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
}
|
|
2484
|
-
if (name === "collect_dtmf") {
|
|
2485
|
-
if (this._call) {
|
|
2486
|
-
let result;
|
|
2487
|
-
try {
|
|
2488
|
-
this._log.info({ maxDigits: args["max_digits"] ?? 4, timeout: args["timeout"] ?? 5 }, "collect_dtmf: waiting for digits");
|
|
2489
|
-
result = await this._call.collectDtmf({
|
|
2490
|
-
maxDigits: args["max_digits"] ?? 4,
|
|
2491
|
-
finishOnKey: args["finish_on_key"] ?? "#",
|
|
2492
|
-
timeout: args["timeout"] ?? 5
|
|
2493
|
-
});
|
|
2494
|
-
this._log.info("DTMF collected: %s", result || "(empty)");
|
|
2495
|
-
} catch (err) {
|
|
2496
|
-
this._log.error({ err }, "collect_dtmf error");
|
|
2497
|
-
result = `Error: ${err}`;
|
|
2498
|
-
}
|
|
2499
|
-
responses.push({
|
|
2500
|
-
id: fcId,
|
|
2501
|
-
name,
|
|
2502
|
-
response: { result: result || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)" }
|
|
2503
|
-
});
|
|
2504
|
-
}
|
|
2505
|
-
continue;
|
|
2506
|
-
}
|
|
2507
|
-
if (name === "send_dtmf") {
|
|
2508
|
-
if (this._call) {
|
|
2509
|
-
let result;
|
|
2510
|
-
try {
|
|
2511
|
-
this._log.info('send_dtmf: digits="%s"', args["digits"] ?? "");
|
|
2512
|
-
await this._call.sendDtmfSequence(args["digits"] ?? "");
|
|
2513
|
-
result = "sent";
|
|
2514
|
-
this._log.info("send_dtmf: sent");
|
|
2515
|
-
} catch (err) {
|
|
2516
|
-
this._log.error({ err }, "send_dtmf error");
|
|
2517
|
-
result = `Error: ${err}`;
|
|
2783
|
+
if (BUILTIN_TOOL_NAMES.has(name) && this._call) {
|
|
2784
|
+
const result = await executeBuiltinTool(name, args, this._call);
|
|
2785
|
+
if (result !== null) {
|
|
2786
|
+
if (name === "hang_up") {
|
|
2787
|
+
this._log.info("hang_up: ending call");
|
|
2788
|
+
return;
|
|
2518
2789
|
}
|
|
2790
|
+
this._log.info("Builtin tool result: %s -> %s", name, result);
|
|
2519
2791
|
responses.push({ id: fcId, name, response: { result } });
|
|
2792
|
+
continue;
|
|
2520
2793
|
}
|
|
2521
|
-
continue;
|
|
2522
2794
|
}
|
|
2523
2795
|
if (!this._tools || !this._tools.has(name)) {
|
|
2524
2796
|
this._log.error("Unknown tool: %s", name);
|
|
@@ -2553,306 +2825,7 @@ var GeminiRealtime = class {
|
|
|
2553
2825
|
}
|
|
2554
2826
|
};
|
|
2555
2827
|
|
|
2556
|
-
// src/agent/pipeline/
|
|
2557
|
-
var COLLECT_DTMF_TOOL3 = {
|
|
2558
|
-
function: {
|
|
2559
|
-
description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4.",
|
|
2560
|
-
parameters: {
|
|
2561
|
-
properties: {
|
|
2562
|
-
max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
|
|
2563
|
-
finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
|
|
2564
|
-
timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
|
|
2565
|
-
},
|
|
2566
|
-
required: ["max_digits"]
|
|
2567
|
-
}
|
|
2568
|
-
}
|
|
2569
|
-
};
|
|
2570
|
-
var SEND_DTMF_TOOL3 = {
|
|
2571
|
-
function: {
|
|
2572
|
-
description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
|
|
2573
|
-
parameters: {
|
|
2574
|
-
properties: {
|
|
2575
|
-
digits: { type: "string", description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30." }
|
|
2576
|
-
},
|
|
2577
|
-
required: ["digits"]
|
|
2578
|
-
}
|
|
2579
|
-
}
|
|
2580
|
-
};
|
|
2581
|
-
var PipelineSession = class {
|
|
2582
|
-
_stt;
|
|
2583
|
-
_llm;
|
|
2584
|
-
_tts;
|
|
2585
|
-
_systemPrompt;
|
|
2586
|
-
_greeting;
|
|
2587
|
-
_language;
|
|
2588
|
-
_temperature;
|
|
2589
|
-
_maxTokens;
|
|
2590
|
-
_sampleRate;
|
|
2591
|
-
_interruptOnSpeech;
|
|
2592
|
-
_callSession = null;
|
|
2593
|
-
_tools = null;
|
|
2594
|
-
_recorder = null;
|
|
2595
|
-
_conversation = [];
|
|
2596
|
-
_audioBuffer = [];
|
|
2597
|
-
_running = false;
|
|
2598
|
-
_speaking = false;
|
|
2599
|
-
_builtinTools = null;
|
|
2600
|
-
_log = NOOP_LOGGER;
|
|
2601
|
-
constructor(options) {
|
|
2602
|
-
this._stt = options.stt;
|
|
2603
|
-
this._llm = options.llm;
|
|
2604
|
-
this._tts = options.tts;
|
|
2605
|
-
this._systemPrompt = options.systemPrompt;
|
|
2606
|
-
this._greeting = options.greeting ?? true;
|
|
2607
|
-
this._language = options.language ?? "ko";
|
|
2608
|
-
this._temperature = options.temperature;
|
|
2609
|
-
this._maxTokens = options.maxTokens;
|
|
2610
|
-
this._sampleRate = options.sampleRate ?? 8e3;
|
|
2611
|
-
this._interruptOnSpeech = options.interruptOnSpeech ?? true;
|
|
2612
|
-
if (options.toolRegistry) this._tools = options.toolRegistry;
|
|
2613
|
-
if (options.recorder) this._recorder = options.recorder;
|
|
2614
|
-
}
|
|
2615
|
-
setToolRegistry(registry) {
|
|
2616
|
-
this._tools = registry;
|
|
2617
|
-
}
|
|
2618
|
-
setRecorder(recorder) {
|
|
2619
|
-
this._recorder = recorder;
|
|
2620
|
-
}
|
|
2621
|
-
setBuiltinTools(tools) {
|
|
2622
|
-
this._builtinTools = tools;
|
|
2623
|
-
}
|
|
2624
|
-
setLogger(logger) {
|
|
2625
|
-
this._log = logger;
|
|
2626
|
-
if ("setLogger" in this._stt && typeof this._stt.setLogger === "function") {
|
|
2627
|
-
this._stt.setLogger(logger);
|
|
2628
|
-
}
|
|
2629
|
-
if ("setLogger" in this._tts && typeof this._tts.setLogger === "function") {
|
|
2630
|
-
this._tts.setLogger(logger);
|
|
2631
|
-
}
|
|
2632
|
-
}
|
|
2633
|
-
async start(callSession, tools) {
|
|
2634
|
-
this._callSession = callSession;
|
|
2635
|
-
this._tools = tools ?? null;
|
|
2636
|
-
this._running = true;
|
|
2637
|
-
this._log.info("PipelineSession started");
|
|
2638
|
-
this._conversation = [];
|
|
2639
|
-
if (this._systemPrompt) {
|
|
2640
|
-
this._conversation.push({
|
|
2641
|
-
role: "system",
|
|
2642
|
-
content: this._systemPrompt
|
|
2643
|
-
});
|
|
2644
|
-
}
|
|
2645
|
-
if (this._greeting) {
|
|
2646
|
-
this._generateGreeting().catch((err) => {
|
|
2647
|
-
this._log.error({ err }, "Greeting error");
|
|
2648
|
-
});
|
|
2649
|
-
}
|
|
2650
|
-
this._runSttLoop().catch((err) => {
|
|
2651
|
-
this._log.error({ err }, "STT loop error");
|
|
2652
|
-
});
|
|
2653
|
-
}
|
|
2654
|
-
feedAudio(audio) {
|
|
2655
|
-
if (this._running) {
|
|
2656
|
-
this._audioBuffer.push(audio);
|
|
2657
|
-
}
|
|
2658
|
-
}
|
|
2659
|
-
async feedDtmf(digits) {
|
|
2660
|
-
this._conversation.push({
|
|
2661
|
-
role: "user",
|
|
2662
|
-
content: `[DTMF \uC785\uB825: ${digits}]`
|
|
2663
|
-
});
|
|
2664
|
-
await this._respond();
|
|
2665
|
-
}
|
|
2666
|
-
async stop() {
|
|
2667
|
-
this._running = false;
|
|
2668
|
-
this._log.info("PipelineSession stopped");
|
|
2669
|
-
this._audioBuffer = [];
|
|
2670
|
-
}
|
|
2671
|
-
async _runSttLoop() {
|
|
2672
|
-
const audioStream = this._createAudioStream();
|
|
2673
|
-
for await (const event of this._stt.transcribe(audioStream, {
|
|
2674
|
-
sampleRate: this._sampleRate
|
|
2675
|
-
})) {
|
|
2676
|
-
if (!this._running) break;
|
|
2677
|
-
if (event.type === "interim" && this._speaking && this._interruptOnSpeech) {
|
|
2678
|
-
this._speaking = false;
|
|
2679
|
-
if (this._callSession) {
|
|
2680
|
-
this._callSession.clearAudio();
|
|
2681
|
-
}
|
|
2682
|
-
this._log.info('Barge-in: "%s"', event.transcript.substring(0, 30));
|
|
2683
|
-
}
|
|
2684
|
-
if (event.type === "final" && event.transcript.trim()) {
|
|
2685
|
-
this._log.info("STT: %s", event.transcript);
|
|
2686
|
-
await this._handleUserSpeech(event.transcript);
|
|
2687
|
-
}
|
|
2688
|
-
}
|
|
2689
|
-
}
|
|
2690
|
-
async *_createAudioStream() {
|
|
2691
|
-
while (this._running) {
|
|
2692
|
-
if (this._audioBuffer.length > 0) {
|
|
2693
|
-
const ulaw = this._audioBuffer.shift();
|
|
2694
|
-
const pcm8k = ulawToPcm16(ulaw);
|
|
2695
|
-
const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
|
|
2696
|
-
yield pcm16k;
|
|
2697
|
-
} else {
|
|
2698
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
2699
|
-
}
|
|
2700
|
-
}
|
|
2701
|
-
}
|
|
2702
|
-
async _generateGreeting() {
|
|
2703
|
-
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
2704
|
-
await this._respond();
|
|
2705
|
-
}
|
|
2706
|
-
async _handleUserSpeech(transcript) {
|
|
2707
|
-
this._conversation.push({ role: "user", content: transcript });
|
|
2708
|
-
await this._respond();
|
|
2709
|
-
}
|
|
2710
|
-
_buildEffectiveTools() {
|
|
2711
|
-
const includeCollectDtmf = !this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */);
|
|
2712
|
-
const includeSendDtmf = !this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */);
|
|
2713
|
-
if (!includeCollectDtmf && !includeSendDtmf) return this._tools ?? void 0;
|
|
2714
|
-
const base = this._tools ? this._tools.fork() : new ToolRegistry();
|
|
2715
|
-
if (includeCollectDtmf) {
|
|
2716
|
-
base.register({
|
|
2717
|
-
name: "collect_dtmf",
|
|
2718
|
-
description: COLLECT_DTMF_TOOL3.function.description,
|
|
2719
|
-
parameters: COLLECT_DTMF_TOOL3.function.parameters.properties,
|
|
2720
|
-
required: COLLECT_DTMF_TOOL3.function.parameters.required,
|
|
2721
|
-
handler: async () => ""
|
|
2722
|
-
});
|
|
2723
|
-
}
|
|
2724
|
-
if (includeSendDtmf) {
|
|
2725
|
-
base.register({
|
|
2726
|
-
name: "send_dtmf",
|
|
2727
|
-
description: SEND_DTMF_TOOL3.function.description,
|
|
2728
|
-
parameters: SEND_DTMF_TOOL3.function.parameters.properties,
|
|
2729
|
-
required: SEND_DTMF_TOOL3.function.parameters.required,
|
|
2730
|
-
handler: async () => ""
|
|
2731
|
-
});
|
|
2732
|
-
}
|
|
2733
|
-
return base;
|
|
2734
|
-
}
|
|
2735
|
-
async _respond() {
|
|
2736
|
-
let fullResponse = "";
|
|
2737
|
-
const textChunks = [];
|
|
2738
|
-
const effectiveTools = this._buildEffectiveTools();
|
|
2739
|
-
const llmStream = this._llm.generate(this._conversation, {
|
|
2740
|
-
tools: effectiveTools,
|
|
2741
|
-
temperature: this._temperature,
|
|
2742
|
-
maxTokens: this._maxTokens
|
|
2743
|
-
});
|
|
2744
|
-
for await (const chunk of llmStream) {
|
|
2745
|
-
if (!this._running) break;
|
|
2746
|
-
if (chunk.type === "text" && chunk.text) {
|
|
2747
|
-
textChunks.push(chunk.text);
|
|
2748
|
-
fullResponse += chunk.text;
|
|
2749
|
-
} else if (chunk.type === "tool_call" && chunk.toolCall) {
|
|
2750
|
-
await this._handleToolCall(chunk);
|
|
2751
|
-
}
|
|
2752
|
-
}
|
|
2753
|
-
if (fullResponse.trim()) {
|
|
2754
|
-
this._log.info("Assistant: %s", fullResponse.substring(0, 100));
|
|
2755
|
-
this._conversation.push({ role: "assistant", content: fullResponse });
|
|
2756
|
-
await this._synthesizeAndSend(fullResponse);
|
|
2757
|
-
}
|
|
2758
|
-
}
|
|
2759
|
-
async _handleToolCall(chunk) {
|
|
2760
|
-
if (!chunk.toolCall) return;
|
|
2761
|
-
const { id, name, arguments: argsStr } = chunk.toolCall;
|
|
2762
|
-
try {
|
|
2763
|
-
const args = JSON.parse(argsStr);
|
|
2764
|
-
if (name === "collect_dtmf" && this._callSession) {
|
|
2765
|
-
let result2;
|
|
2766
|
-
try {
|
|
2767
|
-
result2 = await this._callSession.collectDtmf({
|
|
2768
|
-
maxDigits: args["max_digits"] ?? 4,
|
|
2769
|
-
finishOnKey: args["finish_on_key"] ?? "#",
|
|
2770
|
-
timeout: args["timeout"] ?? 5
|
|
2771
|
-
});
|
|
2772
|
-
} catch (err) {
|
|
2773
|
-
result2 = `Error: ${err}`;
|
|
2774
|
-
}
|
|
2775
|
-
this._conversation.push({ role: "tool", content: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)", tool_call_id: id, name });
|
|
2776
|
-
await this._respond();
|
|
2777
|
-
return;
|
|
2778
|
-
}
|
|
2779
|
-
if (name === "send_dtmf" && this._callSession) {
|
|
2780
|
-
let result2;
|
|
2781
|
-
try {
|
|
2782
|
-
await this._callSession.sendDtmfSequence(args["digits"] ?? "");
|
|
2783
|
-
result2 = "sent";
|
|
2784
|
-
} catch (err) {
|
|
2785
|
-
result2 = `Error: ${err}`;
|
|
2786
|
-
}
|
|
2787
|
-
this._conversation.push({ role: "tool", content: result2, tool_call_id: id, name });
|
|
2788
|
-
await this._respond();
|
|
2789
|
-
return;
|
|
2790
|
-
}
|
|
2791
|
-
if (!this._tools) return;
|
|
2792
|
-
const result = await this._tools.call(name, args);
|
|
2793
|
-
this._conversation.push({
|
|
2794
|
-
role: "assistant",
|
|
2795
|
-
content: ""
|
|
2796
|
-
// Tool call info stored in the message flow
|
|
2797
|
-
});
|
|
2798
|
-
this._conversation.push({
|
|
2799
|
-
role: "tool",
|
|
2800
|
-
content: typeof result === "string" ? result : JSON.stringify(result),
|
|
2801
|
-
tool_call_id: id,
|
|
2802
|
-
name
|
|
2803
|
-
});
|
|
2804
|
-
const effectiveTools = this._buildEffectiveTools();
|
|
2805
|
-
let followUpText = "";
|
|
2806
|
-
const followUpStream = this._llm.generate(this._conversation, {
|
|
2807
|
-
tools: effectiveTools,
|
|
2808
|
-
temperature: this._temperature,
|
|
2809
|
-
maxTokens: this._maxTokens
|
|
2810
|
-
});
|
|
2811
|
-
for await (const followChunk of followUpStream) {
|
|
2812
|
-
if (!this._running) break;
|
|
2813
|
-
if (followChunk.type === "text" && followChunk.text) {
|
|
2814
|
-
followUpText += followChunk.text;
|
|
2815
|
-
}
|
|
2816
|
-
}
|
|
2817
|
-
if (followUpText.trim()) {
|
|
2818
|
-
this._conversation.push({ role: "assistant", content: followUpText });
|
|
2819
|
-
await this._synthesizeAndSend(followUpText);
|
|
2820
|
-
}
|
|
2821
|
-
} catch (err) {
|
|
2822
|
-
this._log.error({ err }, "Tool call failed: %s", name);
|
|
2823
|
-
}
|
|
2824
|
-
}
|
|
2825
|
-
async _synthesizeAndSend(text) {
|
|
2826
|
-
if (!this._callSession || !this._running) return;
|
|
2827
|
-
this._speaking = true;
|
|
2828
|
-
try {
|
|
2829
|
-
for await (const audioChunk of this._tts.synthesize(text, {
|
|
2830
|
-
sampleRate: this._sampleRate
|
|
2831
|
-
})) {
|
|
2832
|
-
if (!this._running || !this._speaking) break;
|
|
2833
|
-
if (this._recorder) {
|
|
2834
|
-
const pcm8k2 = this._sampleRate !== 8e3 ? resamplePcm16(audioChunk, this._sampleRate, 8e3) : audioChunk;
|
|
2835
|
-
this._recorder.writeOutbound(pcm8k2);
|
|
2836
|
-
}
|
|
2837
|
-
const pcm8k = resamplePcm16(audioChunk, this._sampleRate, 8e3);
|
|
2838
|
-
const ulaw = pcm16ToUlaw(pcm8k);
|
|
2839
|
-
for (let off = 0; off < ulaw.length; off += 160) {
|
|
2840
|
-
let chunk = ulaw.subarray(off, off + 160);
|
|
2841
|
-
if (chunk.length < 160) {
|
|
2842
|
-
chunk = Buffer.concat([chunk, Buffer.alloc(160 - chunk.length, 255)]);
|
|
2843
|
-
}
|
|
2844
|
-
this._callSession.sendAudio(chunk);
|
|
2845
|
-
}
|
|
2846
|
-
}
|
|
2847
|
-
} catch (err) {
|
|
2848
|
-
this._log.error({ err }, "TTS error");
|
|
2849
|
-
} finally {
|
|
2850
|
-
this._speaking = false;
|
|
2851
|
-
}
|
|
2852
|
-
}
|
|
2853
|
-
};
|
|
2854
|
-
|
|
2855
|
-
// src/agent/pipeline/deepgram-stt.ts
|
|
2828
|
+
// src/agent/pipeline/stt/deepgram-stt.ts
|
|
2856
2829
|
var DeepgramSTT = class {
|
|
2857
2830
|
_options;
|
|
2858
2831
|
_log = NOOP_LOGGER;
|
|
@@ -2973,7 +2946,7 @@ var DeepgramSTT = class {
|
|
|
2973
2946
|
}
|
|
2974
2947
|
};
|
|
2975
2948
|
|
|
2976
|
-
// src/agent/pipeline/elevenlabs-tts.ts
|
|
2949
|
+
// src/agent/pipeline/tts/elevenlabs-tts.ts
|
|
2977
2950
|
var ElevenLabsTTS = class {
|
|
2978
2951
|
_options;
|
|
2979
2952
|
_log = NOOP_LOGGER;
|
|
@@ -3126,7 +3099,7 @@ var ElevenLabsTTS = class {
|
|
|
3126
3099
|
}
|
|
3127
3100
|
};
|
|
3128
3101
|
|
|
3129
|
-
// src/agent/pipeline/openai-llm.ts
|
|
3102
|
+
// src/agent/pipeline/llm/openai-llm.ts
|
|
3130
3103
|
var OpenAILLM = class {
|
|
3131
3104
|
_options;
|
|
3132
3105
|
constructor(options = {}) {
|
|
@@ -3212,7 +3185,7 @@ var OpenAILLM = class {
|
|
|
3212
3185
|
}
|
|
3213
3186
|
};
|
|
3214
3187
|
|
|
3215
|
-
// src/agent/pipeline/anthropic-llm.ts
|
|
3188
|
+
// src/agent/pipeline/llm/anthropic-llm.ts
|
|
3216
3189
|
var AnthropicLLM = class {
|
|
3217
3190
|
_options;
|
|
3218
3191
|
constructor(options = {}) {
|
|
@@ -3317,7 +3290,7 @@ var AnthropicLLM = class {
|
|
|
3317
3290
|
}
|
|
3318
3291
|
};
|
|
3319
3292
|
|
|
3320
|
-
// src/agent/pipeline/gemini-llm.ts
|
|
3293
|
+
// src/agent/pipeline/llm/gemini-llm.ts
|
|
3321
3294
|
var GeminiLLM = class {
|
|
3322
3295
|
_options;
|
|
3323
3296
|
constructor(options = {}) {
|
|
@@ -3401,7 +3374,7 @@ var GeminiLLM = class {
|
|
|
3401
3374
|
}
|
|
3402
3375
|
};
|
|
3403
3376
|
|
|
3404
|
-
// src/agent/pipeline/openai-compat-llm.ts
|
|
3377
|
+
// src/agent/pipeline/llm/openai-compat-llm.ts
|
|
3405
3378
|
var OpenAICompatLLM = class {
|
|
3406
3379
|
_options;
|
|
3407
3380
|
constructor(options) {
|
|
@@ -3484,7 +3457,7 @@ var OpenAICompatLLM = class {
|
|
|
3484
3457
|
}
|
|
3485
3458
|
};
|
|
3486
3459
|
|
|
3487
|
-
// src/agent/pipeline/ollama-llm.ts
|
|
3460
|
+
// src/agent/pipeline/llm/ollama-llm.ts
|
|
3488
3461
|
var OllamaLLM = class {
|
|
3489
3462
|
_inner;
|
|
3490
3463
|
constructor(options = {}) {
|
|
@@ -3502,7 +3475,7 @@ var OllamaLLM = class {
|
|
|
3502
3475
|
}
|
|
3503
3476
|
};
|
|
3504
3477
|
|
|
3505
|
-
// src/agent/pipeline/mistral-llm.ts
|
|
3478
|
+
// src/agent/pipeline/llm/mistral-llm.ts
|
|
3506
3479
|
var MistralLLM = class {
|
|
3507
3480
|
_inner;
|
|
3508
3481
|
constructor(options = {}) {
|
|
@@ -3519,7 +3492,7 @@ var MistralLLM = class {
|
|
|
3519
3492
|
}
|
|
3520
3493
|
};
|
|
3521
3494
|
|
|
3522
|
-
// src/agent/pipeline/groq-llm.ts
|
|
3495
|
+
// src/agent/pipeline/llm/groq-llm.ts
|
|
3523
3496
|
var GroqLLM = class {
|
|
3524
3497
|
_inner;
|
|
3525
3498
|
constructor(options = {}) {
|
|
@@ -3536,7 +3509,7 @@ var GroqLLM = class {
|
|
|
3536
3509
|
}
|
|
3537
3510
|
};
|
|
3538
3511
|
|
|
3539
|
-
// src/agent/pipeline/perplexity-llm.ts
|
|
3512
|
+
// src/agent/pipeline/llm/perplexity-llm.ts
|
|
3540
3513
|
var PerplexityLLM = class {
|
|
3541
3514
|
_inner;
|
|
3542
3515
|
constructor(options = {}) {
|
|
@@ -3553,7 +3526,7 @@ var PerplexityLLM = class {
|
|
|
3553
3526
|
}
|
|
3554
3527
|
};
|
|
3555
3528
|
|
|
3556
|
-
// src/agent/pipeline/together-llm.ts
|
|
3529
|
+
// src/agent/pipeline/llm/together-llm.ts
|
|
3557
3530
|
var TogetherLLM = class {
|
|
3558
3531
|
_inner;
|
|
3559
3532
|
constructor(options = {}) {
|
|
@@ -3570,7 +3543,7 @@ var TogetherLLM = class {
|
|
|
3570
3543
|
}
|
|
3571
3544
|
};
|
|
3572
3545
|
|
|
3573
|
-
// src/agent/pipeline/fireworks-llm.ts
|
|
3546
|
+
// src/agent/pipeline/llm/fireworks-llm.ts
|
|
3574
3547
|
var FireworksLLM = class {
|
|
3575
3548
|
_inner;
|
|
3576
3549
|
constructor(options = {}) {
|
|
@@ -3587,7 +3560,7 @@ var FireworksLLM = class {
|
|
|
3587
3560
|
}
|
|
3588
3561
|
};
|
|
3589
3562
|
|
|
3590
|
-
// src/agent/pipeline/deepseek-llm.ts
|
|
3563
|
+
// src/agent/pipeline/llm/deepseek-llm.ts
|
|
3591
3564
|
var DeepSeekLLM = class {
|
|
3592
3565
|
_inner;
|
|
3593
3566
|
constructor(options = {}) {
|
|
@@ -3604,7 +3577,7 @@ var DeepSeekLLM = class {
|
|
|
3604
3577
|
}
|
|
3605
3578
|
};
|
|
3606
3579
|
|
|
3607
|
-
// src/agent/pipeline/xai-llm.ts
|
|
3580
|
+
// src/agent/pipeline/llm/xai-llm.ts
|
|
3608
3581
|
var XaiLLM = class {
|
|
3609
3582
|
_inner;
|
|
3610
3583
|
constructor(options = {}) {
|
|
@@ -3640,6 +3613,6 @@ function mcpServerHTTP(options) {
|
|
|
3640
3613
|
};
|
|
3641
3614
|
}
|
|
3642
3615
|
|
|
3643
|
-
export { AnthropicLLM, AudioRecorder, BuiltinTool, CallSession, ClawOpsAgent, DECODE_TABLE, DeepSeekLLM, DeepgramSTT, ElevenLabsTTS, FireworksLLM, GeminiLLM, GeminiRealtime, GroqLLM, MCPClient, MistralLLM, OllamaLLM, OpenAICompatLLM, OpenAILLM, OpenAIRealtime, PerplexityLLM, PipelineSession, TogetherLLM, ToolRegistry, XaiLLM, createAgentLogger, createPipelineLogger, functionTool, getTracingConfig, mcpServerHTTP, mcpServerStdio, pcm16ToUlaw, resamplePcm16, resetTracingConfig, setTracingConfig, ulawToPcm16, zodToToolParams };
|
|
3616
|
+
export { AnthropicLLM, AudioRecorder, BUILTIN_TOOL_NAMES, BuiltinTool, CallSession, ClawOpsAgent, DECODE_TABLE, DeepSeekLLM, DeepgramSTT, ElevenLabsTTS, FireworksLLM, GeminiLLM, GeminiRealtime, GroqLLM, MCPClient, MistralLLM, OllamaLLM, OpenAICompatLLM, OpenAILLM, OpenAIRealtime, PerplexityLLM, PipelineSession, TogetherLLM, ToolRegistry, XaiLLM, createAgentLogger, createPipelineLogger, executeBuiltinTool, functionTool, getBuiltinToolSchemas, getTracingConfig, isBuiltinTool, mcpServerHTTP, mcpServerStdio, pcm16ToUlaw, resamplePcm16, resetTracingConfig, setTracingConfig, ulawToPcm16, zodToToolParams };
|
|
3644
3617
|
//# sourceMappingURL=index.js.map
|
|
3645
3618
|
//# sourceMappingURL=index.js.map
|