@teamlearners/clawops 0.5.5 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,10 @@
1
1
  'use strict';
2
2
 
3
- var chunk6IQN5RQD_cjs = require('../chunk-6IQN5RQD.cjs');
3
+ var chunkR742GPVL_cjs = require('../chunk-R742GPVL.cjs');
4
4
  var pino = require('pino');
5
5
  var fs = require('fs');
6
6
  var path = require('path');
7
+ var os = require('os');
7
8
 
8
9
  function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
9
10
 
@@ -28,6 +29,7 @@ function _interopNamespace(e) {
28
29
  var pino__default = /*#__PURE__*/_interopDefault(pino);
29
30
  var fs__namespace = /*#__PURE__*/_interopNamespace(fs);
30
31
  var path__namespace = /*#__PURE__*/_interopNamespace(path);
32
+ var os__default = /*#__PURE__*/_interopDefault(os);
31
33
 
32
34
  // src/agent/audio.ts
33
35
  var _BIAS = 132;
@@ -998,6 +1000,36 @@ var AudioRecorder = class {
998
1000
  }
999
1001
  }
1000
1002
  };
1003
+ var MAX_ERRORS = 20;
1004
+ var MAX_ERROR_MESSAGE_LENGTH = 200;
1005
+ function getSdkInfo() {
1006
+ return {
1007
+ name: "clawops-node",
1008
+ version: chunkR742GPVL_cjs.VERSION,
1009
+ runtime: `node/${process.versions.node}`,
1010
+ os: `${process.platform}/${os__default.default.arch()}`
1011
+ };
1012
+ }
1013
+ function createCallMetrics() {
1014
+ return {
1015
+ firstResponseMs: null,
1016
+ turnCount: 0,
1017
+ toolCallCount: 0,
1018
+ toolErrorCount: 0,
1019
+ bargeInCount: 0,
1020
+ endReason: null,
1021
+ errors: []
1022
+ };
1023
+ }
1024
+ function addMetricError(metrics, err) {
1025
+ metrics.toolErrorCount++;
1026
+ if (metrics.errors.length < MAX_ERRORS) {
1027
+ metrics.errors.push({
1028
+ type: err.name || "Error",
1029
+ message: (err.message || "").slice(0, MAX_ERROR_MESSAGE_LENGTH)
1030
+ });
1031
+ }
1032
+ }
1001
1033
 
1002
1034
  // src/agent/session.ts
1003
1035
  var CallSession = class {
@@ -1023,6 +1055,8 @@ var CallSession = class {
1023
1055
  _handlers = /* @__PURE__ */ new Map();
1024
1056
  _endedPromise;
1025
1057
  _resolveEnded;
1058
+ _metrics = createCallMetrics();
1059
+ _firstResponseSent = false;
1026
1060
  constructor(options) {
1027
1061
  this.callId = options.callId;
1028
1062
  this.fromNumber = options.fromNumber;
@@ -1045,6 +1079,30 @@ var CallSession = class {
1045
1079
  get duration() {
1046
1080
  return (Date.now() - this.startTime.getTime()) / 1e3;
1047
1081
  }
1082
+ get metrics() {
1083
+ return this._metrics;
1084
+ }
1085
+ recordFirstResponse() {
1086
+ if (!this._firstResponseSent) {
1087
+ this._firstResponseSent = true;
1088
+ this._metrics.firstResponseMs = Date.now() - this.startTime.getTime();
1089
+ }
1090
+ }
1091
+ recordTurn() {
1092
+ this._metrics.turnCount++;
1093
+ }
1094
+ recordToolCall() {
1095
+ this._metrics.toolCallCount++;
1096
+ }
1097
+ recordToolError(err) {
1098
+ addMetricError(this._metrics, err);
1099
+ }
1100
+ recordBargeIn() {
1101
+ this._metrics.bargeInCount++;
1102
+ }
1103
+ recordEndReason(reason) {
1104
+ this._metrics.endReason = reason;
1105
+ }
1048
1106
  /** Bind transport functions (called internally by the agent). */
1049
1107
  _bindTransport(send, clear, hangup, sendDtmf, isConnected) {
1050
1108
  this._sendAudioFn = send;
@@ -1426,7 +1484,7 @@ var ClawOpsAgent = class {
1426
1484
  constructor(options) {
1427
1485
  this._apiKey = options.apiKey ?? process.env["CLAWOPS_API_KEY"] ?? "";
1428
1486
  this._accountId = options.accountId ?? process.env["CLAWOPS_ACCOUNT_ID"] ?? "";
1429
- this._baseUrl = options.baseUrl ?? chunk6IQN5RQD_cjs.DEFAULT_BASE_URL;
1487
+ this._baseUrl = options.baseUrl ?? chunkR742GPVL_cjs.DEFAULT_BASE_URL;
1430
1488
  this._fromNumber = options.from;
1431
1489
  this._session = options.session;
1432
1490
  this._recording = options.recording ?? false;
@@ -1451,7 +1509,7 @@ var ClawOpsAgent = class {
1451
1509
  tool(nameOrTool, description, parameters, handler) {
1452
1510
  if (typeof nameOrTool === "string") {
1453
1511
  if (!description || !parameters || !handler) {
1454
- throw new chunk6IQN5RQD_cjs.AgentError(
1512
+ throw new chunkR742GPVL_cjs.AgentError(
1455
1513
  "tool(name, description, parameters, handler) requires all arguments."
1456
1514
  );
1457
1515
  }
@@ -1485,11 +1543,12 @@ var ClawOpsAgent = class {
1485
1543
  }
1486
1544
  /** Connect to the ClawOps platform and start listening for calls. */
1487
1545
  async connect() {
1546
+ if (this._controlWs) return;
1488
1547
  if (!this._apiKey) {
1489
- throw new chunk6IQN5RQD_cjs.AgentError("API key is required. Set CLAWOPS_API_KEY or pass apiKey option.");
1548
+ throw new chunkR742GPVL_cjs.AgentError("API key is required. Set CLAWOPS_API_KEY or pass apiKey option.");
1490
1549
  }
1491
1550
  if (!this._accountId) {
1492
- throw new chunk6IQN5RQD_cjs.AgentError(
1551
+ throw new chunkR742GPVL_cjs.AgentError(
1493
1552
  "Account ID is required. Set CLAWOPS_ACCOUNT_ID or pass accountId option."
1494
1553
  );
1495
1554
  }
@@ -1508,8 +1567,12 @@ var ClawOpsAgent = class {
1508
1567
  try {
1509
1568
  await this._controlWs.connect();
1510
1569
  await this._controlWs.waitConnected();
1570
+ try {
1571
+ this._controlWs.send({ event: "agent.hello", sdk: getSdkInfo() });
1572
+ } catch {
1573
+ }
1511
1574
  } catch (err) {
1512
- throw new chunk6IQN5RQD_cjs.AgentConnectionError(
1575
+ throw new chunkR742GPVL_cjs.AgentConnectionError(
1513
1576
  `Failed to connect to ClawOps: ${err instanceof Error ? err.message : String(err)}`
1514
1577
  );
1515
1578
  }
@@ -1560,7 +1623,7 @@ var ClawOpsAgent = class {
1560
1623
  });
1561
1624
  if (resp.status !== 201) {
1562
1625
  const error = await resp.json();
1563
- throw new chunk6IQN5RQD_cjs.AgentError(`\uBC1C\uC2E0 \uC2E4\uD328 (${resp.status}): ${error["error"] ?? ""}`);
1626
+ throw new chunkR742GPVL_cjs.AgentError(`\uBC1C\uC2E0 \uC2E4\uD328 (${resp.status}): ${error["error"] ?? ""}`);
1564
1627
  }
1565
1628
  const data = await resp.json();
1566
1629
  const callSession = new CallSession({
@@ -1721,9 +1784,11 @@ var ClawOpsAgent = class {
1721
1784
  session._bindTransport(
1722
1785
  (audio) => {
1723
1786
  mediaWs.sendAudio(audio.toString("base64"));
1787
+ session.recordFirstResponse();
1724
1788
  },
1725
1789
  () => {
1726
1790
  mediaWs.sendClear();
1791
+ session.recordBargeIn();
1727
1792
  },
1728
1793
  async () => {
1729
1794
  await mediaWs.flush();
@@ -1774,10 +1839,22 @@ var ClawOpsAgent = class {
1774
1839
  await mediaWs.connect(mediaWsUrl, this._apiKey);
1775
1840
  this._log.info("Media stream started: %s", session.callId);
1776
1841
  await sessionHandler.start(session, sessionTools);
1842
+ const telemetry = sessionHandler.getTelemetry?.() ?? null;
1843
+ if (telemetry) {
1844
+ telemetry.toolCount = sessionTools?.size ?? 0;
1845
+ telemetry.mcpServerCount = this._mcpServers?.length ?? 0;
1846
+ telemetry.builtinTools = this._builtinTools ? [...this._builtinTools].map((t) => t.toString()) : [];
1847
+ telemetry.recordingEnabled = this._recording;
1848
+ try {
1849
+ this._controlWs.send({ event: "call.telemetry", callId: session.callId, telemetry });
1850
+ } catch {
1851
+ }
1852
+ }
1777
1853
  await session.wait();
1778
1854
  await sessionHandler.stop();
1779
1855
  } catch (err) {
1780
1856
  this._log.error({ err }, "Call session error: %s", session.callId);
1857
+ session.recordEndReason("error");
1781
1858
  } finally {
1782
1859
  if (mcpClients.length > 0) {
1783
1860
  sessionTools.clearMcpTools();
@@ -1789,6 +1866,13 @@ var ClawOpsAgent = class {
1789
1866
  if (recorder) {
1790
1867
  recorder.stop();
1791
1868
  }
1869
+ if (!session.metrics.endReason) {
1870
+ session.recordEndReason(session.status === "ended" ? "user_hangup" : "agent_hangup");
1871
+ }
1872
+ try {
1873
+ this._controlWs?.send({ event: "call.metrics", callId: session.callId, metrics: session.metrics });
1874
+ } catch {
1875
+ }
1792
1876
  session._emit("call_end");
1793
1877
  session._markEnded();
1794
1878
  this._activeSessions.delete(session.callId);
@@ -1799,20 +1883,13 @@ var ClawOpsAgent = class {
1799
1883
  }
1800
1884
  };
1801
1885
 
1802
- // src/agent/pipeline/openai-realtime.ts
1803
- var OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime?model=";
1804
- var HANG_UP_TOOL = {
1805
- type: "function",
1886
+ // src/agent/pipeline/builtin-tool-schemas.ts
1887
+ var HANG_UP = {
1806
1888
  name: "hang_up",
1807
1889
  description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
1808
- parameters: {
1809
- type: "object",
1810
- properties: {},
1811
- required: []
1812
- }
1890
+ parameters: { type: "object", properties: {} }
1813
1891
  };
1814
- var COLLECT_DTMF_TOOL = {
1815
- type: "function",
1892
+ var COLLECT_DTMF = {
1816
1893
  name: "collect_dtmf",
1817
1894
  description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
1818
1895
  parameters: {
@@ -1825,8 +1902,7 @@ var COLLECT_DTMF_TOOL = {
1825
1902
  required: ["max_digits"]
1826
1903
  }
1827
1904
  };
1828
- var SEND_DTMF_TOOL = {
1829
- type: "function",
1905
+ var SEND_DTMF = {
1830
1906
  name: "send_dtmf",
1831
1907
  description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
1832
1908
  parameters: {
@@ -1834,492 +1910,419 @@ var SEND_DTMF_TOOL = {
1834
1910
  properties: {
1835
1911
  digits: {
1836
1912
  type: "string",
1837
- description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30."
1913
+ description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30. \uC608: '1', '1234#', '1w2'"
1838
1914
  }
1839
1915
  },
1840
1916
  required: ["digits"]
1841
1917
  }
1842
1918
  };
1843
- var OpenAIRealtime = class {
1844
- _apiKey;
1845
- _systemPrompt;
1846
- _model;
1847
- _voice;
1848
- _language;
1849
- _eagerness;
1850
- _greeting;
1851
- _log = NOOP_LOGGER;
1852
- _builtinTools = null;
1853
- setLogger(logger) {
1854
- this._log = logger;
1919
+ var TOOL_MAP = /* @__PURE__ */ new Map([
1920
+ ["hang_up" /* HANG_UP */, HANG_UP],
1921
+ ["collect_dtmf" /* COLLECT_DTMF */, COLLECT_DTMF],
1922
+ ["send_dtmf" /* SEND_DTMF */, SEND_DTMF]
1923
+ ]);
1924
+ var BUILTIN_TOOL_NAMES = new Set(
1925
+ Array.from(TOOL_MAP.values()).map((s) => s.name)
1926
+ );
1927
+ function toChatCompletions(schema) {
1928
+ return {
1929
+ type: "function",
1930
+ function: {
1931
+ name: schema.name,
1932
+ description: schema.description,
1933
+ parameters: schema.parameters
1934
+ }
1935
+ };
1936
+ }
1937
+ function toRealtime(schema) {
1938
+ return {
1939
+ type: "function",
1940
+ name: schema.name,
1941
+ description: schema.description,
1942
+ parameters: schema.parameters
1943
+ };
1944
+ }
1945
+ function toGemini(schema) {
1946
+ return {
1947
+ name: schema.name,
1948
+ description: schema.description,
1949
+ parameters: schema.parameters
1950
+ };
1951
+ }
1952
+ var CONVERTERS = {
1953
+ chat: toChatCompletions,
1954
+ realtime: toRealtime,
1955
+ gemini: toGemini
1956
+ };
1957
+ function getBuiltinToolSchemas(builtinTools, fmt) {
1958
+ const converter = CONVERTERS[fmt];
1959
+ const result = [];
1960
+ for (const [toolEnum, schema] of TOOL_MAP) {
1961
+ if (builtinTools === null || builtinTools.has(toolEnum)) {
1962
+ result.push(converter(schema));
1963
+ }
1855
1964
  }
1856
- setBuiltinTools(tools) {
1857
- this._builtinTools = tools;
1965
+ return result;
1966
+ }
1967
+ function isBuiltinTool(name) {
1968
+ return BUILTIN_TOOL_NAMES.has(name);
1969
+ }
1970
+ async function executeBuiltinTool(funcName, args, call) {
1971
+ if (funcName === "hang_up") {
1972
+ await call.hangup();
1973
+ return "";
1858
1974
  }
1859
- _ws = null;
1860
- _call = null;
1975
+ if (funcName === "collect_dtmf") {
1976
+ try {
1977
+ const result = await call.collectDtmf({
1978
+ maxDigits: args["max_digits"] ?? 4,
1979
+ finishOnKey: args["finish_on_key"] ?? "#",
1980
+ timeout: args["timeout"] ?? 5
1981
+ });
1982
+ return result || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)";
1983
+ } catch (e) {
1984
+ return `Error: ${e}`;
1985
+ }
1986
+ }
1987
+ if (funcName === "send_dtmf") {
1988
+ try {
1989
+ await call.sendDtmfSequence(args["digits"] ?? "");
1990
+ return "sent";
1991
+ } catch (e) {
1992
+ return `Error: ${e}`;
1993
+ }
1994
+ }
1995
+ return null;
1996
+ }
1997
+
1998
+ // src/agent/pipeline/pipeline-session.ts
1999
+ var PipelineSession = class {
2000
+ _stt;
2001
+ _llm;
2002
+ _tts;
2003
+ _systemPrompt;
2004
+ _greeting;
2005
+ _language;
2006
+ _temperature;
2007
+ _maxTokens;
2008
+ _sampleRate;
2009
+ _interruptOnSpeech;
2010
+ _callSession = null;
1861
2011
  _tools = null;
1862
2012
  _recorder = null;
1863
- _closed = false;
1864
- // Truncation / barge-in tracking (matching Python SDK)
1865
- _lastAssistantItem = null;
1866
- _responseStartTs = null;
1867
- _sentAudioChunks = 0;
1868
- _audioRemainder = Buffer.alloc(0);
1869
- // Response state tracking — prevent sending response.create while one is active
1870
- _responseInProgress = false;
1871
- _onResponseDone = null;
1872
- constructor(options = {}) {
1873
- this._apiKey = options.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
1874
- this._systemPrompt = options.systemPrompt ?? "";
1875
- this._model = options.model ?? "gpt-realtime-1.5";
1876
- this._voice = options.voice ?? "marin";
1877
- this._language = options.language ?? "ko";
1878
- this._eagerness = options.eagerness ?? "high";
2013
+ _conversation = [];
2014
+ _audioBuffer = [];
2015
+ _running = false;
2016
+ _speaking = false;
2017
+ _builtinTools = null;
2018
+ _log = NOOP_LOGGER;
2019
+ constructor(options) {
2020
+ this._stt = options.stt;
2021
+ this._llm = options.llm;
2022
+ this._tts = options.tts;
2023
+ this._systemPrompt = options.systemPrompt;
1879
2024
  this._greeting = options.greeting ?? true;
2025
+ this._language = options.language ?? "ko";
2026
+ this._temperature = options.temperature;
2027
+ this._maxTokens = options.maxTokens;
2028
+ this._sampleRate = options.sampleRate ?? 8e3;
2029
+ this._interruptOnSpeech = options.interruptOnSpeech ?? true;
2030
+ if (options.toolRegistry) this._tools = options.toolRegistry;
2031
+ if (options.recorder) this._recorder = options.recorder;
1880
2032
  }
1881
- /** Inject per-call ToolRegistry. */
1882
2033
  setToolRegistry(registry) {
1883
2034
  this._tools = registry;
1884
2035
  }
1885
- /** Inject per-call AudioRecorder. */
1886
2036
  setRecorder(recorder) {
1887
2037
  this._recorder = recorder;
1888
2038
  }
1889
- async start(callSession, tools) {
1890
- this._call = callSession;
1891
- if (tools) this._tools = tools;
1892
- this._closed = false;
1893
- this._lastAssistantItem = null;
1894
- this._responseStartTs = null;
1895
- this._sentAudioChunks = 0;
1896
- this._audioRemainder = Buffer.alloc(0);
1897
- if (!this._apiKey) {
1898
- throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey option.");
2039
+ setBuiltinTools(tools) {
2040
+ this._builtinTools = tools;
2041
+ }
2042
+ getTelemetry() {
2043
+ const llm = this._llm;
2044
+ const stt = this._stt;
2045
+ const tts = this._tts;
2046
+ return {
2047
+ sessionType: "pipeline",
2048
+ llm: llm.provider && llm.model ? { provider: llm.provider, model: llm.model } : null,
2049
+ stt: stt.provider && stt.model ? { provider: stt.provider, model: stt.model } : null,
2050
+ tts: tts.provider && tts.model ? { provider: tts.provider, model: tts.model } : null,
2051
+ voice: tts.voiceId ?? null,
2052
+ language: this._language,
2053
+ greetingEnabled: this._greeting,
2054
+ recordingEnabled: !!this._recorder,
2055
+ toolCount: this._tools?.size ?? 0,
2056
+ mcpServerCount: 0,
2057
+ builtinTools: []
2058
+ };
2059
+ }
2060
+ setLogger(logger) {
2061
+ this._log = logger;
2062
+ if ("setLogger" in this._stt && typeof this._stt.setLogger === "function") {
2063
+ this._stt.setLogger(logger);
1899
2064
  }
1900
- const { WebSocket } = await import('ws');
1901
- const url = `${OPENAI_REALTIME_URL}${this._model}`;
1902
- this._ws = new WebSocket(url, {
1903
- headers: {
1904
- Authorization: `Bearer ${this._apiKey}`,
1905
- "OpenAI-Beta": "realtime=v1"
1906
- }
1907
- });
1908
- return new Promise((resolve, reject) => {
1909
- const ws = this._ws;
1910
- ws.on("open", () => {
1911
- this._sendSessionUpdate();
1912
- this._log.info("OpenAI Realtime connected");
1913
- if (this._greeting) {
1914
- this._send({ type: "response.create" });
1915
- }
1916
- resolve();
1917
- });
1918
- ws.on("message", (data) => {
1919
- try {
1920
- const msg = JSON.parse(data.toString());
1921
- this._handleMessage(msg);
1922
- } catch {
1923
- }
1924
- });
1925
- ws.on("close", () => {
1926
- this._closed = true;
2065
+ if ("setLogger" in this._tts && typeof this._tts.setLogger === "function") {
2066
+ this._tts.setLogger(logger);
2067
+ }
2068
+ }
2069
+ async start(callSession, tools) {
2070
+ this._callSession = callSession;
2071
+ this._tools = tools ?? null;
2072
+ this._running = true;
2073
+ this._log.info("PipelineSession started");
2074
+ this._conversation = [];
2075
+ if (this._systemPrompt) {
2076
+ this._conversation.push({
2077
+ role: "system",
2078
+ content: this._systemPrompt
1927
2079
  });
1928
- ws.on("error", (err) => {
1929
- if (!this._ws) {
1930
- reject(err);
1931
- }
1932
- this._log.error({ err }, "OpenAI Realtime WS error");
2080
+ }
2081
+ if (this._greeting) {
2082
+ this._generateGreeting().catch((err) => {
2083
+ this._log.error({ err }, "Greeting error");
1933
2084
  });
2085
+ }
2086
+ this._runSttLoop().catch((err) => {
2087
+ this._log.error({ err }, "STT loop error");
1934
2088
  });
1935
2089
  }
1936
- async feedDtmf(digits) {
1937
- await this._waitForResponseDone();
1938
- this._send({
1939
- type: "conversation.item.create",
1940
- item: {
1941
- type: "message",
1942
- role: "user",
1943
- content: [{ type: "input_text", text: `[DTMF \uC785\uB825: ${digits}]` }]
1944
- }
1945
- });
1946
- this._send({ type: "response.create" });
1947
- }
1948
2090
  feedAudio(audio) {
1949
- if (this._ws && this._ws.readyState === 1 && !this._closed) {
1950
- this._send({
1951
- type: "input_audio_buffer.append",
1952
- audio: audio.toString("base64")
1953
- });
2091
+ if (this._running) {
2092
+ this._audioBuffer.push(audio);
1954
2093
  }
1955
2094
  }
2095
+ async feedDtmf(digits) {
2096
+ this._conversation.push({
2097
+ role: "user",
2098
+ content: `[DTMF \uC785\uB825: ${digits}]`
2099
+ });
2100
+ await this._respond();
2101
+ }
1956
2102
  async stop() {
1957
- this._closed = true;
1958
- if (this._ws) {
1959
- this._ws.close();
1960
- this._ws = null;
1961
- }
2103
+ this._running = false;
2104
+ this._log.info("PipelineSession stopped");
2105
+ this._audioBuffer = [];
1962
2106
  }
1963
- _sendSessionUpdate() {
1964
- if (!this._ws || this._ws.readyState !== 1) return;
1965
- const toolSchemas = this._tools ? this._tools.toOpenAITools().map((t) => ({ type: "function", ...t.function })) : [];
1966
- if (!this._builtinTools || this._builtinTools.has("hang_up" /* HANG_UP */))
1967
- toolSchemas.push(HANG_UP_TOOL);
1968
- if (!this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */))
1969
- toolSchemas.push(COLLECT_DTMF_TOOL);
1970
- if (!this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */))
1971
- toolSchemas.push(SEND_DTMF_TOOL);
1972
- this._send({
1973
- type: "session.update",
1974
- session: {
1975
- modalities: ["text", "audio"],
1976
- voice: this._voice,
1977
- instructions: this._systemPrompt,
1978
- input_audio_format: "g711_ulaw",
1979
- output_audio_format: "g711_ulaw",
1980
- input_audio_transcription: {
1981
- model: "whisper-1",
1982
- language: this._language
1983
- },
1984
- input_audio_noise_reduction: { type: "far_field" },
1985
- turn_detection: {
1986
- type: "semantic_vad",
1987
- interrupt_response: true,
1988
- eagerness: this._eagerness
1989
- },
1990
- tools: toolSchemas
2107
+ async _runSttLoop() {
2108
+ const audioStream = this._createAudioStream();
2109
+ for await (const event of this._stt.transcribe(audioStream, {
2110
+ sampleRate: this._sampleRate
2111
+ })) {
2112
+ if (!this._running) break;
2113
+ if (event.type === "interim" && this._speaking && this._interruptOnSpeech) {
2114
+ this._speaking = false;
2115
+ if (this._callSession) {
2116
+ this._callSession.clearAudio();
2117
+ }
2118
+ this._log.info('Barge-in: "%s"', event.transcript.substring(0, 30));
1991
2119
  }
1992
- });
2120
+ if (event.type === "final" && event.transcript.trim()) {
2121
+ this._log.info("STT: %s", event.transcript);
2122
+ await this._handleUserSpeech(event.transcript);
2123
+ }
2124
+ }
1993
2125
  }
1994
- _handleMessage(msg) {
1995
- const type = msg["type"];
1996
- switch (type) {
1997
- case "response.audio.delta": {
1998
- this._handleAudioDelta(msg);
1999
- break;
2000
- }
2001
- case "response.audio.done": {
2002
- if (this._audioRemainder.length > 0) {
2003
- const padded = Buffer.concat([
2004
- this._audioRemainder,
2005
- Buffer.alloc(160 - this._audioRemainder.length, 255)
2006
- ]);
2007
- if (this._call) {
2008
- this._call.sendAudio(padded);
2009
- }
2010
- this._sentAudioChunks++;
2011
- this._audioRemainder = Buffer.alloc(0);
2012
- }
2013
- break;
2014
- }
2015
- case "input_audio_buffer.speech_started": {
2016
- this._handleTruncation();
2017
- break;
2018
- }
2019
- case "response.output_item.done": {
2020
- const item = msg["item"];
2021
- if (item && item["type"] === "function_call") {
2022
- this._handleToolCall(item);
2023
- }
2024
- break;
2025
- }
2026
- case "conversation.item.input_audio_transcription.completed": {
2027
- if (this._call) {
2028
- this._call._emit("transcript", "user", msg["transcript"] ?? "");
2029
- }
2030
- break;
2031
- }
2032
- case "response.audio_transcript.done": {
2033
- if (this._call) {
2034
- this._call._emit("transcript", "assistant", msg["transcript"] ?? "");
2035
- }
2036
- break;
2037
- }
2038
- case "response.created": {
2039
- this._responseInProgress = true;
2040
- break;
2041
- }
2042
- case "response.done": {
2043
- this._responseInProgress = false;
2044
- if (this._onResponseDone) {
2045
- const cb = this._onResponseDone;
2046
- this._onResponseDone = null;
2047
- cb();
2048
- }
2049
- break;
2050
- }
2051
- case "error": {
2052
- this._log.error({ apiError: msg["error"] }, "OpenAI error");
2053
- break;
2126
+ async *_createAudioStream() {
2127
+ while (this._running) {
2128
+ if (this._audioBuffer.length > 0) {
2129
+ const ulaw = this._audioBuffer.shift();
2130
+ const pcm8k = ulawToPcm16(ulaw);
2131
+ const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
2132
+ yield pcm16k;
2133
+ } else {
2134
+ await new Promise((resolve) => setTimeout(resolve, 20));
2054
2135
  }
2055
2136
  }
2056
2137
  }
2057
- _handleAudioDelta(msg) {
2058
- if (this._responseStartTs === null) {
2059
- this._responseStartTs = Date.now();
2060
- this._sentAudioChunks = 0;
2061
- }
2062
- if (msg["item_id"]) {
2063
- this._lastAssistantItem = msg["item_id"];
2064
- }
2065
- const ulaw = Buffer.from(msg["delta"], "base64");
2066
- if (this._recorder) {
2067
- this._recorder.writeOutbound(ulawToPcm16(ulaw));
2068
- }
2069
- const combined = Buffer.concat([this._audioRemainder, ulaw]);
2070
- const chunkSize = 160;
2071
- const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
2072
- for (let off = 0; off < fullEnd; off += chunkSize) {
2073
- if (this._call) {
2074
- this._call.sendAudio(combined.subarray(off, off + chunkSize));
2075
- }
2076
- this._sentAudioChunks++;
2077
- }
2078
- this._audioRemainder = combined.subarray(fullEnd);
2138
+ async _generateGreeting() {
2139
+ await new Promise((resolve) => setTimeout(resolve, 500));
2140
+ await this._respond();
2079
2141
  }
2080
- _handleTruncation() {
2081
- if (!this._lastAssistantItem || this._responseStartTs === null) {
2082
- return;
2083
- }
2084
- const audioEndMs = Math.max(0, this._sentAudioChunks * 20);
2085
- this._send({
2086
- type: "conversation.item.truncate",
2087
- item_id: this._lastAssistantItem,
2088
- content_index: 0,
2089
- audio_end_ms: audioEndMs
2142
+ async _handleUserSpeech(transcript) {
2143
+ this._conversation.push({ role: "user", content: transcript });
2144
+ await this._respond();
2145
+ }
2146
+ _buildEffectiveTools() {
2147
+ const builtinSchemas = getBuiltinToolSchemas(this._builtinTools, "chat");
2148
+ const dtmfSchemas = builtinSchemas.filter((s) => {
2149
+ const name = s["function"]?.["name"];
2150
+ return name === "collect_dtmf" || name === "send_dtmf";
2090
2151
  });
2091
- if (this._call) {
2092
- this._call.clearAudio();
2152
+ if (dtmfSchemas.length === 0) return this._tools ?? void 0;
2153
+ const base = this._tools ? this._tools.fork() : new ToolRegistry();
2154
+ for (const schema of dtmfSchemas) {
2155
+ const fn = schema["function"];
2156
+ const params = fn["parameters"];
2157
+ base.register({
2158
+ name: fn["name"],
2159
+ description: fn["description"],
2160
+ parameters: params["properties"] ?? {},
2161
+ required: params["required"] ?? [],
2162
+ handler: async () => ""
2163
+ });
2093
2164
  }
2094
- this._lastAssistantItem = null;
2095
- this._responseStartTs = null;
2096
- this._sentAudioChunks = 0;
2097
- this._audioRemainder = Buffer.alloc(0);
2165
+ return base;
2098
2166
  }
2099
- async _handleToolCall(item) {
2100
- const funcName = item["name"];
2101
- const callId = item["call_id"];
2102
- this._log.info("Tool call: %s", funcName);
2103
- if (funcName === "hang_up") {
2104
- if (this._call) {
2105
- await this._call.hangup();
2167
+ async _respond() {
2168
+ let fullResponse = "";
2169
+ const textChunks = [];
2170
+ const effectiveTools = this._buildEffectiveTools();
2171
+ const llmStream = this._llm.generate(this._conversation, {
2172
+ tools: effectiveTools,
2173
+ temperature: this._temperature,
2174
+ maxTokens: this._maxTokens
2175
+ });
2176
+ for await (const chunk of llmStream) {
2177
+ if (!this._running) break;
2178
+ if (chunk.type === "text" && chunk.text) {
2179
+ textChunks.push(chunk.text);
2180
+ fullResponse += chunk.text;
2181
+ } else if (chunk.type === "tool_call" && chunk.toolCall) {
2182
+ await this._handleToolCall(chunk);
2106
2183
  }
2107
- return;
2108
2184
  }
2109
- if (funcName === "collect_dtmf") {
2110
- if (this._call) {
2111
- let result2;
2112
- try {
2113
- const args = JSON.parse(item["arguments"] ?? "{}");
2114
- result2 = await this._call.collectDtmf({
2115
- maxDigits: args["max_digits"] ?? 4,
2116
- finishOnKey: args["finish_on_key"] ?? "#",
2117
- timeout: args["timeout"] ?? 5
2118
- });
2119
- } catch (err) {
2120
- result2 = `Error: ${err}`;
2185
+ if (fullResponse.trim()) {
2186
+ this._log.info("Assistant: %s", fullResponse.substring(0, 100));
2187
+ this._conversation.push({ role: "assistant", content: fullResponse });
2188
+ await this._synthesizeAndSend(fullResponse);
2189
+ }
2190
+ }
2191
+ async _handleToolCall(chunk) {
2192
+ if (!chunk.toolCall) return;
2193
+ const { id, name, arguments: argsStr } = chunk.toolCall;
2194
+ try {
2195
+ const args = JSON.parse(argsStr);
2196
+ if (BUILTIN_TOOL_NAMES.has(name) && this._callSession) {
2197
+ const result2 = await executeBuiltinTool(name, args, this._callSession);
2198
+ if (result2 !== null) {
2199
+ if (name === "hang_up") return;
2200
+ this._conversation.push({ role: "tool", content: result2, tool_call_id: id, name });
2201
+ await this._respond();
2202
+ return;
2121
2203
  }
2122
- await this._waitForResponseDone();
2123
- this._send({
2124
- type: "conversation.item.create",
2125
- item: {
2126
- type: "function_call_output",
2127
- call_id: callId,
2128
- output: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)"
2129
- }
2130
- });
2131
- this._send({ type: "response.create" });
2132
2204
  }
2133
- return;
2134
- }
2135
- if (funcName === "send_dtmf") {
2136
- if (this._call) {
2137
- let result2;
2138
- try {
2139
- const args = JSON.parse(item["arguments"] ?? "{}");
2140
- await this._call.sendDtmfSequence(args["digits"] ?? "");
2141
- result2 = "sent";
2142
- } catch (err) {
2143
- result2 = `Error: ${err}`;
2205
+ if (!this._tools) return;
2206
+ this._callSession?.recordToolCall();
2207
+ const result = await this._tools.call(name, args);
2208
+ this._conversation.push({
2209
+ role: "assistant",
2210
+ content: ""
2211
+ // Tool call info stored in the message flow
2212
+ });
2213
+ this._conversation.push({
2214
+ role: "tool",
2215
+ content: typeof result === "string" ? result : JSON.stringify(result),
2216
+ tool_call_id: id,
2217
+ name
2218
+ });
2219
+ const effectiveTools = this._buildEffectiveTools();
2220
+ let followUpText = "";
2221
+ const followUpStream = this._llm.generate(this._conversation, {
2222
+ tools: effectiveTools,
2223
+ temperature: this._temperature,
2224
+ maxTokens: this._maxTokens
2225
+ });
2226
+ for await (const followChunk of followUpStream) {
2227
+ if (!this._running) break;
2228
+ if (followChunk.type === "text" && followChunk.text) {
2229
+ followUpText += followChunk.text;
2144
2230
  }
2145
- await this._waitForResponseDone();
2146
- this._send({
2147
- type: "conversation.item.create",
2148
- item: {
2149
- type: "function_call_output",
2150
- call_id: callId,
2151
- output: result2
2152
- }
2153
- });
2154
- this._send({ type: "response.create" });
2155
2231
  }
2156
- return;
2157
- }
2158
- if (!this._tools || !this._tools.has(funcName)) {
2159
- this._log.error("Unknown tool: %s", funcName);
2160
- return;
2161
- }
2162
- let result;
2163
- try {
2164
- const args = JSON.parse(item["arguments"] ?? "{}");
2165
- result = await this._tools.call(funcName, args);
2166
- } catch (err) {
2167
- this._log.error({ err }, "Tool call failed: %s", funcName);
2168
- result = `Error: ${err}`;
2169
- }
2170
- await this._waitForResponseDone();
2171
- this._send({
2172
- type: "conversation.item.create",
2173
- item: {
2174
- type: "function_call_output",
2175
- call_id: callId,
2176
- output: typeof result === "string" ? result : JSON.stringify(result)
2232
+ if (followUpText.trim()) {
2233
+ this._conversation.push({ role: "assistant", content: followUpText });
2234
+ await this._synthesizeAndSend(followUpText);
2177
2235
  }
2178
- });
2179
- this._send({ type: "response.create" });
2180
- }
2181
- _waitForResponseDone() {
2182
- if (!this._responseInProgress) return Promise.resolve();
2183
- return new Promise((resolve) => {
2184
- this._onResponseDone = resolve;
2185
- });
2186
- }
2187
- _send(data) {
2188
- if (this._ws && this._ws.readyState === 1 && !this._closed) {
2189
- this._ws.send(JSON.stringify(data));
2190
- }
2191
- }
2192
- };
2193
-
2194
- // src/agent/pipeline/gemini-realtime.ts
2195
- var HANG_UP_TOOL2 = {
2196
- name: "hang_up",
2197
- description: "End the phone call. Use when the conversation is finished or the caller says goodbye.",
2198
- parameters: { type: "object", properties: {} }
2199
- };
2200
- var COLLECT_DTMF_TOOL2 = {
2201
- name: "collect_dtmf",
2202
- description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4. \uBC18\uB4DC\uC2DC \uC0AC\uC6A9\uC790\uC5D0\uAC8C \uBB34\uC5C7\uC744 \uC785\uB825\uD574\uC57C \uD558\uB294\uC9C0 \uC548\uB0B4\uD55C \uD6C4 \uD638\uCD9C\uD558\uC138\uC694.",
2203
- parameters: {
2204
- type: "object",
2205
- properties: {
2206
- max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
2207
- finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
2208
- timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
2209
- },
2210
- required: ["max_digits"]
2211
- }
2212
- };
2213
- var SEND_DTMF_TOOL2 = {
2214
- name: "send_dtmf",
2215
- description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
2216
- parameters: {
2217
- type: "object",
2218
- properties: {
2219
- digits: {
2220
- type: "string",
2221
- description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30."
2236
+ } catch (err) {
2237
+ this._log.error({ err }, "Tool call failed: %s", name);
2238
+ if (err instanceof Error) {
2239
+ this._callSession?.recordToolError(err);
2222
2240
  }
2223
- },
2224
- required: ["digits"]
2225
- }
2226
- };
2227
- function resolveRef(ref, defs) {
2228
- const parts = ref.replace(/^#\//, "").split("/");
2229
- let result = defs;
2230
- for (const part of parts) {
2231
- if (result && typeof result === "object" && !Array.isArray(result)) {
2232
- result = result[part];
2233
- } else {
2234
- return {};
2235
2241
  }
2236
2242
  }
2237
- return typeof result === "object" && result !== null && !Array.isArray(result) ? result : {};
2238
- }
2239
- function sanitizeSchemaForGemini(schema, defs, depth = 0) {
2240
- if (depth > 15) return { type: "object", properties: {} };
2241
- if (!schema || typeof schema !== "object") return { type: "object", properties: {} };
2242
- if (defs === void 0) {
2243
- defs = schema["$defs"] ?? schema["definitions"] ?? {};
2244
- }
2245
- if (typeof schema["$ref"] === "string") {
2246
- const resolved = resolveRef(schema["$ref"], { $defs: defs, definitions: defs });
2247
- if (resolved && Object.keys(resolved).length > 0) {
2248
- return sanitizeSchemaForGemini(resolved, defs, depth + 1);
2249
- }
2250
- return { type: "object", properties: {} };
2251
- }
2252
- for (const comboKey of ["oneOf", "anyOf", "allOf"]) {
2253
- const variants = schema[comboKey];
2254
- if (Array.isArray(variants) && variants.length > 0) {
2255
- for (const v of variants) {
2256
- if (v && typeof v === "object") {
2257
- const resolved = sanitizeSchemaForGemini(v, defs, depth + 1);
2258
- if (resolved["type"] === "object" && resolved["properties"]) {
2259
- return resolved;
2243
+ async _synthesizeAndSend(text) {
2244
+ if (!this._callSession || !this._running) return;
2245
+ this._speaking = true;
2246
+ try {
2247
+ for await (const audioChunk of this._tts.synthesize(text, {
2248
+ sampleRate: this._sampleRate
2249
+ })) {
2250
+ if (!this._running || !this._speaking) break;
2251
+ if (this._recorder) {
2252
+ const pcm8k2 = this._sampleRate !== 8e3 ? resamplePcm16(audioChunk, this._sampleRate, 8e3) : audioChunk;
2253
+ this._recorder.writeOutbound(pcm8k2);
2254
+ }
2255
+ const pcm8k = resamplePcm16(audioChunk, this._sampleRate, 8e3);
2256
+ const ulaw = pcm16ToUlaw(pcm8k);
2257
+ for (let off = 0; off < ulaw.length; off += 160) {
2258
+ let chunk = ulaw.subarray(off, off + 160);
2259
+ if (chunk.length < 160) {
2260
+ chunk = Buffer.concat([chunk, Buffer.alloc(160 - chunk.length, 255)]);
2260
2261
  }
2262
+ this._callSession.sendAudio(chunk);
2261
2263
  }
2262
2264
  }
2263
- const first = variants[0];
2264
- if (first && typeof first === "object") {
2265
- return sanitizeSchemaForGemini(first, defs, depth + 1);
2266
- }
2267
- }
2268
- }
2269
- const result = {};
2270
- let schemaType = schema["type"];
2271
- if (Array.isArray(schemaType)) {
2272
- const nonNull = schemaType.filter((t) => t !== "null");
2273
- schemaType = nonNull[0] ?? "string";
2274
- }
2275
- if (schemaType) result["type"] = schemaType;
2276
- if (schema["description"]) result["description"] = schema["description"];
2277
- if (schema["enum"]) result["enum"] = schema["enum"];
2278
- if (schema["required"]) result["required"] = schema["required"];
2279
- if (schema["properties"] && typeof schema["properties"] === "object") {
2280
- const props = {};
2281
- for (const [key, val] of Object.entries(schema["properties"])) {
2282
- if (val && typeof val === "object") {
2283
- props[key] = sanitizeSchemaForGemini(val, defs, depth + 1);
2284
- }
2265
+ } catch (err) {
2266
+ this._log.error({ err }, "TTS error");
2267
+ } finally {
2268
+ this._speaking = false;
2285
2269
  }
2286
- result["properties"] = props;
2287
- }
2288
- if (schema["items"] && typeof schema["items"] === "object" && !Array.isArray(schema["items"])) {
2289
- result["items"] = sanitizeSchemaForGemini(
2290
- schema["items"],
2291
- defs,
2292
- depth + 1
2293
- );
2294
2270
  }
2295
- if (!result["type"] && result["properties"]) result["type"] = "object";
2296
- if (result["type"] === "object" && !result["properties"]) result["properties"] = {};
2297
- return result;
2298
- }
2299
- var GeminiRealtime = class {
2271
+ };
2272
+
2273
+ // src/agent/pipeline/realtime/openai-realtime.ts
2274
+ var OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime?model=";
2275
+ var OpenAIRealtime = class {
2300
2276
  _apiKey;
2301
2277
  _systemPrompt;
2302
2278
  _model;
2303
2279
  _voice;
2304
2280
  _language;
2281
+ _turnDetection;
2305
2282
  _greeting;
2306
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
2307
- _session = null;
2283
+ _log = NOOP_LOGGER;
2284
+ _builtinTools = null;
2285
+ setLogger(logger) {
2286
+ this._log = logger;
2287
+ }
2288
+ setBuiltinTools(tools) {
2289
+ this._builtinTools = tools;
2290
+ }
2291
+ getTelemetry() {
2292
+ return {
2293
+ sessionType: "openai_realtime",
2294
+ llm: { provider: "openai", model: this._model },
2295
+ stt: null,
2296
+ tts: null,
2297
+ voice: this._voice,
2298
+ language: this._language,
2299
+ greetingEnabled: this._greeting,
2300
+ recordingEnabled: !!this._recorder,
2301
+ toolCount: this._tools?.size ?? 0,
2302
+ mcpServerCount: 0,
2303
+ builtinTools: []
2304
+ };
2305
+ }
2306
+ _ws = null;
2308
2307
  _call = null;
2309
2308
  _tools = null;
2310
2309
  _recorder = null;
2311
2310
  _closed = false;
2312
- _sentAudioChunks = 0;
2313
- _audioRemainder = Buffer.alloc(0);
2314
- _builtinTools = null;
2315
- _toolCallInProgress = false;
2316
- _log = NOOP_LOGGER;
2311
+ // PlaybackState — 현재 재생 중인 응답 상태
2312
+ _playback = null;
2313
+ _latestMediaTs = 0;
2314
+ // Pending tool call tracking — 인터럽트 시 취소용
2315
+ _pendingToolCalls = /* @__PURE__ */ new Map();
2316
+ // Response state tracking — prevent sending response.create while one is active
2317
+ _responseInProgress = false;
2318
+ _onResponseDone = null;
2317
2319
  constructor(options = {}) {
2318
- this._apiKey = options.apiKey ?? process.env["GOOGLE_API_KEY"] ?? "";
2320
+ this._apiKey = options.apiKey ?? process.env["OPENAI_API_KEY"] ?? "";
2319
2321
  this._systemPrompt = options.systemPrompt ?? "";
2320
- this._model = options.model ?? "gemini-2.5-flash-native-audio-preview-12-2025";
2321
- this._voice = options.voice ?? "Kore";
2322
+ this._model = options.model ?? "gpt-realtime-1.5";
2323
+ this._voice = options.voice ?? "marin";
2322
2324
  this._language = options.language ?? "ko";
2325
+ this._turnDetection = options.turnDetection !== void 0 ? options.turnDetection : { type: "semantic_vad", eagerness: "medium", interrupt_response: true };
2323
2326
  this._greeting = options.greeting ?? true;
2324
2327
  }
2325
2328
  /** Inject per-call ToolRegistry. */
@@ -2330,340 +2333,409 @@ var GeminiRealtime = class {
2330
2333
  setRecorder(recorder) {
2331
2334
  this._recorder = recorder;
2332
2335
  }
2333
- setBuiltinTools(tools) {
2334
- this._builtinTools = tools;
2335
- }
2336
- setLogger(logger) {
2337
- this._log = logger;
2338
- }
2339
2336
  async start(callSession, tools) {
2340
2337
  this._call = callSession;
2341
2338
  if (tools) this._tools = tools;
2342
2339
  this._closed = false;
2343
- this._sentAudioChunks = 0;
2344
- this._audioRemainder = Buffer.alloc(0);
2340
+ this._playback = null;
2341
+ this._latestMediaTs = 0;
2345
2342
  if (!this._apiKey) {
2346
- throw new Error("Google API key is required. Set GOOGLE_API_KEY or pass apiKey option.");
2347
- }
2348
- const { GoogleGenAI } = await import('@google/genai/node');
2349
- const client = new GoogleGenAI({ apiKey: this._apiKey });
2350
- const config = {
2351
- responseModalities: ["AUDIO"],
2352
- speechConfig: {
2353
- voiceConfig: {
2354
- prebuiltVoiceConfig: {
2355
- voiceName: this._voice
2356
- }
2357
- }
2358
- },
2359
- inputAudioTranscription: {},
2360
- outputAudioTranscription: {}
2361
- };
2362
- if (this._systemPrompt) {
2363
- config["systemInstruction"] = this._systemPrompt;
2364
- }
2365
- const toolSchemas = this._buildToolSchemas();
2366
- if (toolSchemas.length > 0) {
2367
- config["tools"] = [{ functionDeclarations: toolSchemas }];
2343
+ throw new Error("OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey option.");
2368
2344
  }
2369
- this._session = await client.live.connect({
2370
- model: this._model,
2371
- config,
2372
- callbacks: {
2373
- onmessage: (msg) => this._handleMessage(msg),
2374
- onerror: (err) => {
2375
- this._log.error({ err }, "Gemini SDK error");
2376
- },
2377
- onclose: (ev) => {
2378
- this._log.info({ code: ev?.code ?? "unknown" }, "Gemini connection closed");
2379
- this._closed = true;
2380
- }
2345
+ const { WebSocket } = await import('ws');
2346
+ const url = `${OPENAI_REALTIME_URL}${this._model}`;
2347
+ this._ws = new WebSocket(url, {
2348
+ headers: {
2349
+ Authorization: `Bearer ${this._apiKey}`,
2350
+ "OpenAI-Beta": "realtime=v1"
2381
2351
  }
2382
2352
  });
2383
- if (this._greeting) {
2384
- this._session.sendClientContent({
2385
- turns: [
2386
- {
2387
- role: "user",
2388
- parts: [{ text: "\uC778\uC0AC\uD574 \uC8FC\uC138\uC694." }]
2389
- }
2390
- ],
2391
- turnComplete: true
2353
+ return new Promise((resolve, reject) => {
2354
+ const ws = this._ws;
2355
+ ws.on("open", () => {
2356
+ this._sendSessionUpdate();
2357
+ this._log.info("OpenAI Realtime connected");
2358
+ if (this._greeting) {
2359
+ this._send({ type: "response.create" });
2360
+ }
2361
+ resolve();
2392
2362
  });
2393
- }
2394
- }
2395
- feedAudio(audio) {
2396
- if (this._session && !this._closed && !this._toolCallInProgress) {
2397
- const pcm8k = ulawToPcm16(audio);
2398
- if (this._recorder) {
2399
- this._recorder.writeInbound(pcm8k);
2400
- }
2401
- const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
2402
- this._session.sendRealtimeInput({
2403
- audio: {
2404
- data: Buffer.from(pcm16k).toString("base64"),
2405
- mimeType: "audio/pcm;rate=16000"
2363
+ ws.on("message", (data) => {
2364
+ try {
2365
+ const msg = JSON.parse(data.toString());
2366
+ this._handleMessage(msg);
2367
+ } catch {
2368
+ }
2369
+ });
2370
+ ws.on("close", () => {
2371
+ this._closed = true;
2372
+ });
2373
+ ws.on("error", (err) => {
2374
+ if (!this._ws) {
2375
+ reject(err);
2406
2376
  }
2377
+ this._log.error({ err }, "OpenAI Realtime WS error");
2407
2378
  });
2408
- }
2379
+ });
2409
2380
  }
2410
2381
  async feedDtmf(digits) {
2411
- if (this._session) {
2412
- this._session.sendClientContent({
2413
- turns: [{ role: "user", parts: [{ text: `[DTMF \uC785\uB825: ${digits}]` }] }],
2414
- turnComplete: true
2382
+ await this._waitForResponseDone();
2383
+ this._send({
2384
+ type: "conversation.item.create",
2385
+ item: {
2386
+ type: "message",
2387
+ role: "user",
2388
+ content: [{ type: "input_text", text: `[DTMF \uC785\uB825: ${digits}]` }]
2389
+ }
2390
+ });
2391
+ this._send({ type: "response.create" });
2392
+ }
2393
+ feedAudio(audio) {
2394
+ this._latestMediaTs = Date.now();
2395
+ if (this._ws && this._ws.readyState === 1 && !this._closed) {
2396
+ this._send({
2397
+ type: "input_audio_buffer.append",
2398
+ audio: audio.toString("base64")
2415
2399
  });
2416
2400
  }
2417
2401
  }
2418
2402
  async stop() {
2419
2403
  this._closed = true;
2420
- if (this._session) {
2421
- try {
2422
- this._session.close();
2423
- } catch {
2424
- }
2425
- this._session = null;
2404
+ for (const [, controller] of this._pendingToolCalls) {
2405
+ controller.abort();
2406
+ }
2407
+ this._pendingToolCalls.clear();
2408
+ if (this._ws) {
2409
+ this._ws.close();
2410
+ this._ws = null;
2426
2411
  }
2427
2412
  }
2428
- _buildToolSchemas() {
2429
- const toolDefs = this._tools ? this._tools.toOpenAITools().map((t) => ({
2430
- name: t.function.name,
2431
- description: t.function.description,
2432
- parameters: sanitizeSchemaForGemini(
2433
- t.function.parameters ?? { type: "object", properties: {} }
2434
- )
2435
- })) : [];
2436
- if (!this._builtinTools || this._builtinTools.has("hang_up" /* HANG_UP */)) toolDefs.push(HANG_UP_TOOL2);
2437
- if (!this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */)) toolDefs.push(COLLECT_DTMF_TOOL2);
2438
- if (!this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */)) toolDefs.push(SEND_DTMF_TOOL2);
2439
- return toolDefs;
2413
+ _sendSessionUpdate() {
2414
+ if (!this._ws || this._ws.readyState !== 1) return;
2415
+ const toolSchemas = this._tools ? this._tools.toOpenAITools().map((t) => ({ type: "function", ...t.function })) : [];
2416
+ toolSchemas.push(...getBuiltinToolSchemas(this._builtinTools, "realtime"));
2417
+ this._send({
2418
+ type: "session.update",
2419
+ session: {
2420
+ modalities: ["text", "audio"],
2421
+ voice: this._voice,
2422
+ instructions: this._systemPrompt,
2423
+ input_audio_format: "g711_ulaw",
2424
+ output_audio_format: "g711_ulaw",
2425
+ input_audio_transcription: {
2426
+ model: "whisper-1",
2427
+ language: this._language
2428
+ },
2429
+ input_audio_noise_reduction: { type: "far_field" },
2430
+ turn_detection: this._turnDetection,
2431
+ tools: toolSchemas
2432
+ }
2433
+ });
2440
2434
  }
2441
2435
  _handleMessage(msg) {
2442
- if (!this._call) return;
2443
- const serverContent = msg.serverContent;
2444
- if (serverContent) {
2445
- const modelTurn = serverContent.modelTurn;
2446
- if (modelTurn) {
2447
- for (const part of modelTurn.parts ?? []) {
2448
- const inlineData = part.inlineData;
2449
- if (inlineData?.data) {
2450
- const mimeType = inlineData.mimeType ?? "";
2451
- if (mimeType.includes("audio")) {
2452
- this._handleAudioData(inlineData.data);
2436
+ const type = msg["type"];
2437
+ switch (type) {
2438
+ case "response.audio.delta": {
2439
+ this._handleAudioDelta(msg);
2440
+ break;
2441
+ }
2442
+ case "response.audio.done": {
2443
+ if (this._playback) {
2444
+ this._playback.generating = false;
2445
+ if (this._playback.audioRemainder.length > 0) {
2446
+ const padded = Buffer.concat([
2447
+ this._playback.audioRemainder,
2448
+ Buffer.alloc(160 - this._playback.audioRemainder.length, 255)
2449
+ ]);
2450
+ if (this._call) {
2451
+ this._call.sendAudio(padded);
2453
2452
  }
2453
+ this._playback.sentChunks++;
2454
+ this._playback.audioRemainder = Buffer.alloc(0);
2454
2455
  }
2455
2456
  }
2457
+ break;
2456
2458
  }
2457
- if (serverContent.turnComplete) {
2458
- this._log.debug("Turn complete");
2459
- this._flushAudioRemainder();
2459
+ case "input_audio_buffer.speech_started": {
2460
+ this._handleTruncation();
2461
+ break;
2460
2462
  }
2461
- if (serverContent.interrupted) {
2462
- this._log.info("Barge-in detected");
2463
+ case "response.output_item.done": {
2464
+ const item = msg["item"];
2465
+ if (item && item["type"] === "function_call") {
2466
+ this._handleToolCall(item);
2467
+ }
2468
+ break;
2469
+ }
2470
+ case "conversation.item.input_audio_transcription.completed": {
2463
2471
  if (this._call) {
2464
- this._call.clearAudio();
2472
+ this._call._emit("transcript", "user", msg["transcript"] ?? "");
2465
2473
  }
2466
- this._sentAudioChunks = 0;
2467
- this._audioRemainder = Buffer.alloc(0);
2474
+ break;
2468
2475
  }
2469
- const inputText = serverContent.inputTranscription?.text;
2470
- if (inputText && this._call) {
2471
- this._log.info("User: %s", inputText);
2472
- this._call._emit("transcript", "user", inputText);
2476
+ case "response.audio_transcript.done": {
2477
+ if (this._call) {
2478
+ this._call._emit("transcript", "assistant", msg["transcript"] ?? "");
2479
+ }
2480
+ break;
2473
2481
  }
2474
- const outputText = serverContent.outputTranscription?.text;
2475
- if (outputText && this._call) {
2476
- this._log.info("Assistant: %s", outputText);
2477
- this._call._emit("transcript", "assistant", outputText);
2482
+ case "response.created": {
2483
+ this._responseInProgress = true;
2484
+ break;
2485
+ }
2486
+ case "response.done": {
2487
+ this._responseInProgress = false;
2488
+ if (this._onResponseDone) {
2489
+ const cb = this._onResponseDone;
2490
+ this._onResponseDone = null;
2491
+ cb();
2492
+ }
2493
+ break;
2494
+ }
2495
+ case "error": {
2496
+ this._log.error({ apiError: msg["error"] }, "OpenAI error");
2497
+ break;
2478
2498
  }
2479
- }
2480
- if (msg.toolCall) {
2481
- this._handleToolCall(msg.toolCall);
2482
- }
2483
- const toolCancellation = msg["toolCallCancellation"];
2484
- if (toolCancellation) {
2485
- this._log.info({ ids: toolCancellation.ids }, "Tool call cancelled");
2486
2499
  }
2487
2500
  }
2488
- _handleAudioData(b64Data) {
2489
- if (!this._call) return;
2490
- const pcm24k = Buffer.from(b64Data, "base64");
2501
+ _handleAudioDelta(msg) {
2502
+ if (this._playback === null) {
2503
+ this._playback = {
2504
+ itemId: msg["item_id"] || "",
2505
+ startTs: this._latestMediaTs || Date.now(),
2506
+ sentChunks: 0,
2507
+ generating: true,
2508
+ audioRemainder: Buffer.alloc(0)
2509
+ };
2510
+ } else if (msg["item_id"]) {
2511
+ this._playback.itemId = msg["item_id"];
2512
+ }
2513
+ const pb = this._playback;
2514
+ const ulaw = Buffer.from(msg["delta"], "base64");
2491
2515
  if (this._recorder) {
2492
- this._recorder.writeOutbound(resamplePcm16(pcm24k, 24e3, 8e3));
2516
+ this._recorder.writeOutbound(ulawToPcm16(ulaw));
2493
2517
  }
2494
- const pcm8k = resamplePcm16(pcm24k, 24e3, 8e3);
2495
- const ulaw = pcm16ToUlaw(pcm8k);
2496
- const combined = Buffer.concat([this._audioRemainder, ulaw]);
2518
+ const combined = Buffer.concat([pb.audioRemainder, ulaw]);
2497
2519
  const chunkSize = 160;
2498
2520
  const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
2499
- if (fullEnd > 0) {
2500
- this._call.sendAudio(combined.subarray(0, fullEnd));
2501
- this._sentAudioChunks += fullEnd / chunkSize;
2521
+ for (let off = 0; off < fullEnd; off += chunkSize) {
2522
+ if (this._call) {
2523
+ this._call.sendAudio(combined.subarray(off, off + chunkSize));
2524
+ }
2525
+ pb.sentChunks++;
2502
2526
  }
2503
- this._audioRemainder = combined.subarray(fullEnd);
2527
+ pb.audioRemainder = combined.subarray(fullEnd);
2504
2528
  }
2505
- _flushAudioRemainder() {
2506
- if (this._audioRemainder.length > 0 && this._call) {
2507
- const padded = Buffer.concat([
2508
- this._audioRemainder,
2509
- Buffer.alloc(160 - this._audioRemainder.length, 255)
2510
- ]);
2511
- this._call.sendAudio(padded);
2512
- this._sentAudioChunks++;
2513
- this._audioRemainder = Buffer.alloc(0);
2529
+ _handleTruncation() {
2530
+ for (const [, controller] of this._pendingToolCalls) {
2531
+ controller.abort();
2532
+ }
2533
+ this._pendingToolCalls.clear();
2534
+ if (this._call) {
2535
+ this._call.clearAudio();
2536
+ }
2537
+ const pb = this._playback;
2538
+ if (pb === null) {
2539
+ return;
2514
2540
  }
2541
+ const playedMs = Math.max(0, (this._latestMediaTs || Date.now()) - pb.startTs);
2542
+ this._log.info(
2543
+ "[Interrupt] item=%s played=%dms total=%dms",
2544
+ pb.itemId,
2545
+ playedMs,
2546
+ pb.sentChunks * 20
2547
+ );
2548
+ this._playback = null;
2515
2549
  }
2516
- async _handleToolCall(toolCall) {
2517
- const functionCalls = toolCall.functionCalls;
2518
- if (!functionCalls) return;
2519
- this._toolCallInProgress = true;
2520
- const responses = [];
2521
- for (const fc of functionCalls) {
2522
- const name = fc.name ?? "";
2523
- const fcId = fc.id ?? "";
2524
- const args = fc.args ?? {};
2525
- this._log.info({ tool: name, args }, "Tool call: %s", name);
2526
- if (name === "hang_up") {
2527
- this._log.info("hang_up: ending call");
2528
- if (this._call) {
2529
- await this._call.hangup();
2550
+ async _handleToolCall(item) {
2551
+ const funcName = item["name"];
2552
+ const callId = item["call_id"];
2553
+ this._log.info("Tool call: %s", funcName);
2554
+ const controller = new AbortController();
2555
+ this._pendingToolCalls.set(callId, controller);
2556
+ try {
2557
+ if (BUILTIN_TOOL_NAMES.has(funcName) && this._call) {
2558
+ const args = JSON.parse(item["arguments"] ?? "{}");
2559
+ const result2 = await executeBuiltinTool(funcName, args, this._call);
2560
+ if (result2 !== null) {
2561
+ if (funcName === "hang_up") return;
2562
+ if (controller.signal.aborted) return;
2563
+ await this._waitForResponseDone();
2564
+ this._send({
2565
+ type: "conversation.item.create",
2566
+ item: {
2567
+ type: "function_call_output",
2568
+ call_id: callId,
2569
+ output: result2
2570
+ }
2571
+ });
2572
+ this._send({ type: "response.create" });
2573
+ return;
2530
2574
  }
2531
- return;
2532
2575
  }
2533
- if (name === "collect_dtmf") {
2534
- if (this._call) {
2535
- let result;
2536
- try {
2537
- this._log.info({ maxDigits: args["max_digits"] ?? 4, timeout: args["timeout"] ?? 5 }, "collect_dtmf: waiting for digits");
2538
- result = await this._call.collectDtmf({
2539
- maxDigits: args["max_digits"] ?? 4,
2540
- finishOnKey: args["finish_on_key"] ?? "#",
2541
- timeout: args["timeout"] ?? 5
2542
- });
2543
- this._log.info("DTMF collected: %s", result || "(empty)");
2544
- } catch (err) {
2545
- this._log.error({ err }, "collect_dtmf error");
2546
- result = `Error: ${err}`;
2576
+ if (!this._tools || !this._tools.has(funcName)) {
2577
+ this._log.error("Unknown tool: %s", funcName);
2578
+ await this._waitForResponseDone();
2579
+ this._send({
2580
+ type: "conversation.item.create",
2581
+ item: {
2582
+ type: "function_call_output",
2583
+ call_id: callId,
2584
+ output: JSON.stringify({ error: `Unknown tool: ${funcName}` })
2547
2585
  }
2548
- responses.push({
2549
- id: fcId,
2550
- name,
2551
- response: { result: result || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)" }
2552
- });
2586
+ });
2587
+ this._send({ type: "response.create" });
2588
+ return;
2589
+ }
2590
+ let result;
2591
+ try {
2592
+ const args = JSON.parse(item["arguments"] ?? "{}");
2593
+ this._call?.recordToolCall();
2594
+ result = await this._tools.call(funcName, args);
2595
+ } catch (err) {
2596
+ this._log.error({ err }, "Tool call failed: %s", funcName);
2597
+ if (err instanceof Error) {
2598
+ this._call?.recordToolError(err);
2553
2599
  }
2554
- continue;
2600
+ result = `Error: ${err}`;
2555
2601
  }
2556
- if (name === "send_dtmf") {
2557
- if (this._call) {
2558
- let result;
2559
- try {
2560
- this._log.info('send_dtmf: digits="%s"', args["digits"] ?? "");
2561
- await this._call.sendDtmfSequence(args["digits"] ?? "");
2562
- result = "sent";
2563
- this._log.info("send_dtmf: sent");
2564
- } catch (err) {
2565
- this._log.error({ err }, "send_dtmf error");
2566
- result = `Error: ${err}`;
2602
+ if (controller.signal.aborted) {
2603
+ this._log.info("Tool call cancelled (user interrupted): %s", funcName);
2604
+ return;
2605
+ }
2606
+ const resultStr = typeof result === "string" ? result : JSON.stringify(result);
2607
+ await this._waitForResponseDone();
2608
+ this._send({
2609
+ type: "conversation.item.create",
2610
+ item: {
2611
+ type: "function_call_output",
2612
+ call_id: callId,
2613
+ output: resultStr
2614
+ }
2615
+ });
2616
+ this._log.info("[ToolResult] %s call_id=%s len=%d", funcName, callId, resultStr.length);
2617
+ this._send({ type: "response.create" });
2618
+ } finally {
2619
+ this._pendingToolCalls.delete(callId);
2620
+ }
2621
+ }
2622
+ _waitForResponseDone() {
2623
+ if (!this._responseInProgress) return Promise.resolve();
2624
+ return new Promise((resolve) => {
2625
+ this._onResponseDone = resolve;
2626
+ });
2627
+ }
2628
+ _send(data) {
2629
+ if (this._ws && this._ws.readyState === 1 && !this._closed) {
2630
+ this._ws.send(JSON.stringify(data));
2631
+ }
2632
+ }
2633
+ };
2634
+
2635
+ // src/agent/pipeline/realtime/gemini-realtime.ts
2636
+ function resolveRef(ref, defs) {
2637
+ const parts = ref.replace(/^#\//, "").split("/");
2638
+ let result = defs;
2639
+ for (const part of parts) {
2640
+ if (result && typeof result === "object" && !Array.isArray(result)) {
2641
+ result = result[part];
2642
+ } else {
2643
+ return {};
2644
+ }
2645
+ }
2646
+ return typeof result === "object" && result !== null && !Array.isArray(result) ? result : {};
2647
+ }
2648
+ function sanitizeSchemaForGemini(schema, defs, depth = 0) {
2649
+ if (depth > 15) return { type: "object", properties: {} };
2650
+ if (!schema || typeof schema !== "object") return { type: "object", properties: {} };
2651
+ if (defs === void 0) {
2652
+ defs = schema["$defs"] ?? schema["definitions"] ?? {};
2653
+ }
2654
+ if (typeof schema["$ref"] === "string") {
2655
+ const resolved = resolveRef(schema["$ref"], { $defs: defs, definitions: defs });
2656
+ if (resolved && Object.keys(resolved).length > 0) {
2657
+ return sanitizeSchemaForGemini(resolved, defs, depth + 1);
2658
+ }
2659
+ return { type: "object", properties: {} };
2660
+ }
2661
+ for (const comboKey of ["oneOf", "anyOf", "allOf"]) {
2662
+ const variants = schema[comboKey];
2663
+ if (Array.isArray(variants) && variants.length > 0) {
2664
+ for (const v of variants) {
2665
+ if (v && typeof v === "object") {
2666
+ const resolved = sanitizeSchemaForGemini(v, defs, depth + 1);
2667
+ if (resolved["type"] === "object" && resolved["properties"]) {
2668
+ return resolved;
2567
2669
  }
2568
- responses.push({ id: fcId, name, response: { result } });
2569
2670
  }
2570
- continue;
2571
- }
2572
- if (!this._tools || !this._tools.has(name)) {
2573
- this._log.error("Unknown tool: %s", name);
2574
- responses.push({ id: fcId, name, response: { error: `Unknown tool: ${name}` } });
2575
- continue;
2576
2671
  }
2577
- try {
2578
- const result = await this._tools.call(name, args);
2579
- const resultStr = typeof result === "string" ? result : JSON.stringify(result);
2580
- this._log.info("Tool result: %s -> %s", name, resultStr.substring(0, 200));
2581
- responses.push({
2582
- id: fcId,
2583
- name,
2584
- response: { result: resultStr }
2585
- });
2586
- } catch (err) {
2587
- this._log.error({ err }, "Tool call failed: %s", name);
2588
- responses.push({
2589
- id: fcId,
2590
- name,
2591
- response: { error: String(err) }
2592
- });
2672
+ const first = variants[0];
2673
+ if (first && typeof first === "object") {
2674
+ return sanitizeSchemaForGemini(first, defs, depth + 1);
2593
2675
  }
2594
2676
  }
2595
- if (responses.length > 0 && this._session) {
2596
- this._log.debug("Sending %d tool response(s)", responses.length);
2597
- this._session.sendToolResponse({
2598
- functionResponses: responses
2599
- });
2600
- }
2601
- this._toolCallInProgress = false;
2602
2677
  }
2603
- };
2604
-
2605
- // src/agent/pipeline/pipeline-session.ts
2606
- var COLLECT_DTMF_TOOL3 = {
2607
- function: {
2608
- description: "\uC0AC\uC6A9\uC790\uB85C\uBD80\uD130 DTMF(\uC804\uD654 \uD0A4\uD328\uB4DC) \uC785\uB825\uC744 \uC218\uC9D1\uD569\uB2C8\uB2E4.",
2609
- parameters: {
2610
- properties: {
2611
- max_digits: { type: "integer", description: "\uC218\uC9D1\uD560 \uCD5C\uB300 \uC790\uB9BF\uC218" },
2612
- finish_on_key: { type: "string", description: "\uC785\uB825 \uC885\uB8CC \uD0A4 (\uAE30\uBCF8: #)" },
2613
- timeout: { type: "integer", description: "\uC785\uB825 \uB300\uAE30 \uC2DC\uAC04(\uCD08, \uAE30\uBCF8: 5)" }
2614
- },
2615
- required: ["max_digits"]
2616
- }
2678
+ const result = {};
2679
+ let schemaType = schema["type"];
2680
+ if (Array.isArray(schemaType)) {
2681
+ const nonNull = schemaType.filter((t) => t !== "null");
2682
+ schemaType = nonNull[0] ?? "string";
2617
2683
  }
2618
- };
2619
- var SEND_DTMF_TOOL3 = {
2620
- function: {
2621
- description: "DTMF \uC2E0\uD638\uB97C \uC804\uC1A1\uD569\uB2C8\uB2E4. ARS \uBA54\uB274 \uD0D0\uC0C9\uC774\uB098 \uB0B4\uC120\uBC88\uD638 \uC785\uB825 \uC2DC \uC0AC\uC6A9\uD569\uB2C8\uB2E4.",
2622
- parameters: {
2623
- properties: {
2624
- digits: { type: "string", description: "\uC804\uC1A1\uD560 \uBC88\uD638 (0-9, *, #). 'w'\uB294 500ms \uB300\uAE30, 'W'\uB294 1000ms \uB300\uAE30." }
2625
- },
2626
- required: ["digits"]
2684
+ if (schemaType) result["type"] = schemaType;
2685
+ if (schema["description"]) result["description"] = schema["description"];
2686
+ if (schema["enum"]) result["enum"] = schema["enum"];
2687
+ if (schema["required"]) result["required"] = schema["required"];
2688
+ if (schema["properties"] && typeof schema["properties"] === "object") {
2689
+ const props = {};
2690
+ for (const [key, val] of Object.entries(schema["properties"])) {
2691
+ if (val && typeof val === "object") {
2692
+ props[key] = sanitizeSchemaForGemini(val, defs, depth + 1);
2693
+ }
2627
2694
  }
2695
+ result["properties"] = props;
2628
2696
  }
2629
- };
2630
- var PipelineSession = class {
2631
- _stt;
2632
- _llm;
2633
- _tts;
2697
+ if (schema["items"] && typeof schema["items"] === "object" && !Array.isArray(schema["items"])) {
2698
+ result["items"] = sanitizeSchemaForGemini(
2699
+ schema["items"],
2700
+ defs,
2701
+ depth + 1
2702
+ );
2703
+ }
2704
+ if (!result["type"] && result["properties"]) result["type"] = "object";
2705
+ if (result["type"] === "object" && !result["properties"]) result["properties"] = {};
2706
+ return result;
2707
+ }
2708
+ var GeminiRealtime = class {
2709
+ _apiKey;
2634
2710
  _systemPrompt;
2635
- _greeting;
2711
+ _model;
2712
+ _voice;
2636
2713
  _language;
2637
- _temperature;
2638
- _maxTokens;
2639
- _sampleRate;
2640
- _interruptOnSpeech;
2641
- _callSession = null;
2714
+ _greeting;
2715
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
2716
+ _session = null;
2717
+ _call = null;
2642
2718
  _tools = null;
2643
2719
  _recorder = null;
2644
- _conversation = [];
2645
- _audioBuffer = [];
2646
- _running = false;
2647
- _speaking = false;
2720
+ _closed = false;
2721
+ _sentAudioChunks = 0;
2722
+ _audioRemainder = Buffer.alloc(0);
2648
2723
  _builtinTools = null;
2724
+ _toolCallInProgress = false;
2649
2725
  _log = NOOP_LOGGER;
2650
- constructor(options) {
2651
- this._stt = options.stt;
2652
- this._llm = options.llm;
2653
- this._tts = options.tts;
2654
- this._systemPrompt = options.systemPrompt;
2655
- this._greeting = options.greeting ?? true;
2726
+ constructor(options = {}) {
2727
+ this._apiKey = options.apiKey ?? process.env["GOOGLE_API_KEY"] ?? "";
2728
+ this._systemPrompt = options.systemPrompt ?? "";
2729
+ this._model = options.model ?? "gemini-2.5-flash-native-audio-preview-12-2025";
2730
+ this._voice = options.voice ?? "Kore";
2656
2731
  this._language = options.language ?? "ko";
2657
- this._temperature = options.temperature;
2658
- this._maxTokens = options.maxTokens;
2659
- this._sampleRate = options.sampleRate ?? 8e3;
2660
- this._interruptOnSpeech = options.interruptOnSpeech ?? true;
2661
- if (options.toolRegistry) this._tools = options.toolRegistry;
2662
- if (options.recorder) this._recorder = options.recorder;
2732
+ this._greeting = options.greeting ?? true;
2663
2733
  }
2734
+ /** Inject per-call ToolRegistry. */
2664
2735
  setToolRegistry(registry) {
2665
2736
  this._tools = registry;
2666
2737
  }
2738
+ /** Inject per-call AudioRecorder. */
2667
2739
  setRecorder(recorder) {
2668
2740
  this._recorder = recorder;
2669
2741
  }
@@ -2672,239 +2744,266 @@ var PipelineSession = class {
2672
2744
  }
2673
2745
  setLogger(logger) {
2674
2746
  this._log = logger;
2675
- if ("setLogger" in this._stt && typeof this._stt.setLogger === "function") {
2676
- this._stt.setLogger(logger);
2677
- }
2678
- if ("setLogger" in this._tts && typeof this._tts.setLogger === "function") {
2679
- this._tts.setLogger(logger);
2680
- }
2747
+ }
2748
+ getTelemetry() {
2749
+ return {
2750
+ sessionType: "gemini_realtime",
2751
+ llm: { provider: "gemini", model: this._model },
2752
+ stt: null,
2753
+ tts: null,
2754
+ voice: this._voice,
2755
+ language: this._language,
2756
+ greetingEnabled: this._greeting,
2757
+ recordingEnabled: !!this._recorder,
2758
+ toolCount: this._tools?.size ?? 0,
2759
+ mcpServerCount: 0,
2760
+ builtinTools: []
2761
+ };
2681
2762
  }
2682
2763
  async start(callSession, tools) {
2683
- this._callSession = callSession;
2684
- this._tools = tools ?? null;
2685
- this._running = true;
2686
- this._log.info("PipelineSession started");
2687
- this._conversation = [];
2688
- if (this._systemPrompt) {
2689
- this._conversation.push({
2690
- role: "system",
2691
- content: this._systemPrompt
2692
- });
2764
+ this._call = callSession;
2765
+ if (tools) this._tools = tools;
2766
+ this._closed = false;
2767
+ this._sentAudioChunks = 0;
2768
+ this._audioRemainder = Buffer.alloc(0);
2769
+ if (!this._apiKey) {
2770
+ throw new Error("Google API key is required. Set GOOGLE_API_KEY or pass apiKey option.");
2693
2771
  }
2694
- if (this._greeting) {
2695
- this._generateGreeting().catch((err) => {
2696
- this._log.error({ err }, "Greeting error");
2697
- });
2772
+ const { GoogleGenAI } = await import('@google/genai/node');
2773
+ const client = new GoogleGenAI({ apiKey: this._apiKey });
2774
+ const config = {
2775
+ responseModalities: ["AUDIO"],
2776
+ speechConfig: {
2777
+ voiceConfig: {
2778
+ prebuiltVoiceConfig: {
2779
+ voiceName: this._voice
2780
+ }
2781
+ }
2782
+ },
2783
+ inputAudioTranscription: {},
2784
+ outputAudioTranscription: {}
2785
+ };
2786
+ if (this._systemPrompt) {
2787
+ config["systemInstruction"] = this._systemPrompt;
2698
2788
  }
2699
- this._runSttLoop().catch((err) => {
2700
- this._log.error({ err }, "STT loop error");
2701
- });
2702
- }
2703
- feedAudio(audio) {
2704
- if (this._running) {
2705
- this._audioBuffer.push(audio);
2789
+ const toolSchemas = this._buildToolSchemas();
2790
+ if (toolSchemas.length > 0) {
2791
+ config["tools"] = [{ functionDeclarations: toolSchemas }];
2706
2792
  }
2707
- }
2708
- async feedDtmf(digits) {
2709
- this._conversation.push({
2710
- role: "user",
2711
- content: `[DTMF \uC785\uB825: ${digits}]`
2712
- });
2713
- await this._respond();
2714
- }
2715
- async stop() {
2716
- this._running = false;
2717
- this._log.info("PipelineSession stopped");
2718
- this._audioBuffer = [];
2719
- }
2720
- async _runSttLoop() {
2721
- const audioStream = this._createAudioStream();
2722
- for await (const event of this._stt.transcribe(audioStream, {
2723
- sampleRate: this._sampleRate
2724
- })) {
2725
- if (!this._running) break;
2726
- if (event.type === "interim" && this._speaking && this._interruptOnSpeech) {
2727
- this._speaking = false;
2728
- if (this._callSession) {
2729
- this._callSession.clearAudio();
2793
+ this._session = await client.live.connect({
2794
+ model: this._model,
2795
+ config,
2796
+ callbacks: {
2797
+ onmessage: (msg) => this._handleMessage(msg),
2798
+ onerror: (err) => {
2799
+ this._log.error({ err }, "Gemini SDK error");
2800
+ },
2801
+ onclose: (ev) => {
2802
+ this._log.info({ code: ev?.code ?? "unknown" }, "Gemini connection closed");
2803
+ this._closed = true;
2730
2804
  }
2731
- this._log.info('Barge-in: "%s"', event.transcript.substring(0, 30));
2732
- }
2733
- if (event.type === "final" && event.transcript.trim()) {
2734
- this._log.info("STT: %s", event.transcript);
2735
- await this._handleUserSpeech(event.transcript);
2736
2805
  }
2806
+ });
2807
+ if (this._greeting) {
2808
+ this._session.sendClientContent({
2809
+ turns: [
2810
+ {
2811
+ role: "user",
2812
+ parts: [{ text: "\uC778\uC0AC\uD574 \uC8FC\uC138\uC694." }]
2813
+ }
2814
+ ],
2815
+ turnComplete: true
2816
+ });
2737
2817
  }
2738
2818
  }
2739
- async *_createAudioStream() {
2740
- while (this._running) {
2741
- if (this._audioBuffer.length > 0) {
2742
- const ulaw = this._audioBuffer.shift();
2743
- const pcm8k = ulawToPcm16(ulaw);
2744
- const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
2745
- yield pcm16k;
2746
- } else {
2747
- await new Promise((resolve) => setTimeout(resolve, 20));
2819
+ feedAudio(audio) {
2820
+ if (this._session && !this._closed && !this._toolCallInProgress) {
2821
+ const pcm8k = ulawToPcm16(audio);
2822
+ if (this._recorder) {
2823
+ this._recorder.writeInbound(pcm8k);
2748
2824
  }
2749
- }
2750
- }
2751
- async _generateGreeting() {
2752
- await new Promise((resolve) => setTimeout(resolve, 500));
2753
- await this._respond();
2754
- }
2755
- async _handleUserSpeech(transcript) {
2756
- this._conversation.push({ role: "user", content: transcript });
2757
- await this._respond();
2758
- }
2759
- _buildEffectiveTools() {
2760
- const includeCollectDtmf = !this._builtinTools || this._builtinTools.has("collect_dtmf" /* COLLECT_DTMF */);
2761
- const includeSendDtmf = !this._builtinTools || this._builtinTools.has("send_dtmf" /* SEND_DTMF */);
2762
- if (!includeCollectDtmf && !includeSendDtmf) return this._tools ?? void 0;
2763
- const base = this._tools ? this._tools.fork() : new ToolRegistry();
2764
- if (includeCollectDtmf) {
2765
- base.register({
2766
- name: "collect_dtmf",
2767
- description: COLLECT_DTMF_TOOL3.function.description,
2768
- parameters: COLLECT_DTMF_TOOL3.function.parameters.properties,
2769
- required: COLLECT_DTMF_TOOL3.function.parameters.required,
2770
- handler: async () => ""
2825
+ const pcm16k = resamplePcm16(pcm8k, 8e3, 16e3);
2826
+ this._session.sendRealtimeInput({
2827
+ audio: {
2828
+ data: Buffer.from(pcm16k).toString("base64"),
2829
+ mimeType: "audio/pcm;rate=16000"
2830
+ }
2771
2831
  });
2772
2832
  }
2773
- if (includeSendDtmf) {
2774
- base.register({
2775
- name: "send_dtmf",
2776
- description: SEND_DTMF_TOOL3.function.description,
2777
- parameters: SEND_DTMF_TOOL3.function.parameters.properties,
2778
- required: SEND_DTMF_TOOL3.function.parameters.required,
2779
- handler: async () => ""
2833
+ }
2834
+ async feedDtmf(digits) {
2835
+ if (this._session) {
2836
+ this._session.sendClientContent({
2837
+ turns: [{ role: "user", parts: [{ text: `[DTMF \uC785\uB825: ${digits}]` }] }],
2838
+ turnComplete: true
2780
2839
  });
2781
2840
  }
2782
- return base;
2783
2841
  }
2784
- async _respond() {
2785
- let fullResponse = "";
2786
- const textChunks = [];
2787
- const effectiveTools = this._buildEffectiveTools();
2788
- const llmStream = this._llm.generate(this._conversation, {
2789
- tools: effectiveTools,
2790
- temperature: this._temperature,
2791
- maxTokens: this._maxTokens
2792
- });
2793
- for await (const chunk of llmStream) {
2794
- if (!this._running) break;
2795
- if (chunk.type === "text" && chunk.text) {
2796
- textChunks.push(chunk.text);
2797
- fullResponse += chunk.text;
2798
- } else if (chunk.type === "tool_call" && chunk.toolCall) {
2799
- await this._handleToolCall(chunk);
2842
+ async stop() {
2843
+ this._closed = true;
2844
+ if (this._session) {
2845
+ try {
2846
+ this._session.close();
2847
+ } catch {
2800
2848
  }
2801
- }
2802
- if (fullResponse.trim()) {
2803
- this._log.info("Assistant: %s", fullResponse.substring(0, 100));
2804
- this._conversation.push({ role: "assistant", content: fullResponse });
2805
- await this._synthesizeAndSend(fullResponse);
2849
+ this._session = null;
2806
2850
  }
2807
2851
  }
2808
- async _handleToolCall(chunk) {
2809
- if (!chunk.toolCall) return;
2810
- const { id, name, arguments: argsStr } = chunk.toolCall;
2811
- try {
2812
- const args = JSON.parse(argsStr);
2813
- if (name === "collect_dtmf" && this._callSession) {
2814
- let result2;
2815
- try {
2816
- result2 = await this._callSession.collectDtmf({
2817
- maxDigits: args["max_digits"] ?? 4,
2818
- finishOnKey: args["finish_on_key"] ?? "#",
2819
- timeout: args["timeout"] ?? 5
2820
- });
2821
- } catch (err) {
2822
- result2 = `Error: ${err}`;
2852
+ _buildToolSchemas() {
2853
+ const toolDefs = this._tools ? this._tools.toOpenAITools().map((t) => ({
2854
+ name: t.function.name,
2855
+ description: t.function.description,
2856
+ parameters: sanitizeSchemaForGemini(
2857
+ t.function.parameters ?? { type: "object", properties: {} }
2858
+ )
2859
+ })) : [];
2860
+ toolDefs.push(...getBuiltinToolSchemas(this._builtinTools, "gemini"));
2861
+ return toolDefs;
2862
+ }
2863
+ _handleMessage(msg) {
2864
+ if (!this._call) return;
2865
+ const serverContent = msg.serverContent;
2866
+ if (serverContent) {
2867
+ const modelTurn = serverContent.modelTurn;
2868
+ if (modelTurn) {
2869
+ for (const part of modelTurn.parts ?? []) {
2870
+ const inlineData = part.inlineData;
2871
+ if (inlineData?.data) {
2872
+ const mimeType = inlineData.mimeType ?? "";
2873
+ if (mimeType.includes("audio")) {
2874
+ this._handleAudioData(inlineData.data);
2875
+ }
2876
+ }
2823
2877
  }
2824
- this._conversation.push({ role: "tool", content: result2 || "(\uD0C0\uC784\uC544\uC6C3 - \uC785\uB825 \uC5C6\uC74C)", tool_call_id: id, name });
2825
- await this._respond();
2826
- return;
2827
2878
  }
2828
- if (name === "send_dtmf" && this._callSession) {
2829
- let result2;
2830
- try {
2831
- await this._callSession.sendDtmfSequence(args["digits"] ?? "");
2832
- result2 = "sent";
2833
- } catch (err) {
2834
- result2 = `Error: ${err}`;
2835
- }
2836
- this._conversation.push({ role: "tool", content: result2, tool_call_id: id, name });
2837
- await this._respond();
2838
- return;
2879
+ if (serverContent.turnComplete) {
2880
+ this._log.debug("Turn complete");
2881
+ this._flushAudioRemainder();
2839
2882
  }
2840
- if (!this._tools) return;
2841
- const result = await this._tools.call(name, args);
2842
- this._conversation.push({
2843
- role: "assistant",
2844
- content: ""
2845
- // Tool call info stored in the message flow
2846
- });
2847
- this._conversation.push({
2848
- role: "tool",
2849
- content: typeof result === "string" ? result : JSON.stringify(result),
2850
- tool_call_id: id,
2851
- name
2852
- });
2853
- const effectiveTools = this._buildEffectiveTools();
2854
- let followUpText = "";
2855
- const followUpStream = this._llm.generate(this._conversation, {
2856
- tools: effectiveTools,
2857
- temperature: this._temperature,
2858
- maxTokens: this._maxTokens
2859
- });
2860
- for await (const followChunk of followUpStream) {
2861
- if (!this._running) break;
2862
- if (followChunk.type === "text" && followChunk.text) {
2863
- followUpText += followChunk.text;
2883
+ if (serverContent.interrupted) {
2884
+ this._log.info("Barge-in detected");
2885
+ if (this._call) {
2886
+ this._call.clearAudio();
2864
2887
  }
2888
+ this._sentAudioChunks = 0;
2889
+ this._audioRemainder = Buffer.alloc(0);
2865
2890
  }
2866
- if (followUpText.trim()) {
2867
- this._conversation.push({ role: "assistant", content: followUpText });
2868
- await this._synthesizeAndSend(followUpText);
2891
+ const inputText = serverContent.inputTranscription?.text;
2892
+ if (inputText && this._call) {
2893
+ this._log.info("User: %s", inputText);
2894
+ this._call._emit("transcript", "user", inputText);
2869
2895
  }
2870
- } catch (err) {
2871
- this._log.error({ err }, "Tool call failed: %s", name);
2896
+ const outputText = serverContent.outputTranscription?.text;
2897
+ if (outputText && this._call) {
2898
+ this._log.info("Assistant: %s", outputText);
2899
+ this._call._emit("transcript", "assistant", outputText);
2900
+ }
2901
+ }
2902
+ if (msg.toolCall) {
2903
+ this._handleToolCall(msg.toolCall);
2904
+ }
2905
+ const toolCancellation = msg["toolCallCancellation"];
2906
+ if (toolCancellation) {
2907
+ this._log.info({ ids: toolCancellation.ids }, "Tool call cancelled");
2872
2908
  }
2873
2909
  }
2874
- async _synthesizeAndSend(text) {
2875
- if (!this._callSession || !this._running) return;
2876
- this._speaking = true;
2877
- try {
2878
- for await (const audioChunk of this._tts.synthesize(text, {
2879
- sampleRate: this._sampleRate
2880
- })) {
2881
- if (!this._running || !this._speaking) break;
2882
- if (this._recorder) {
2883
- const pcm8k2 = this._sampleRate !== 8e3 ? resamplePcm16(audioChunk, this._sampleRate, 8e3) : audioChunk;
2884
- this._recorder.writeOutbound(pcm8k2);
2885
- }
2886
- const pcm8k = resamplePcm16(audioChunk, this._sampleRate, 8e3);
2887
- const ulaw = pcm16ToUlaw(pcm8k);
2888
- for (let off = 0; off < ulaw.length; off += 160) {
2889
- let chunk = ulaw.subarray(off, off + 160);
2890
- if (chunk.length < 160) {
2891
- chunk = Buffer.concat([chunk, Buffer.alloc(160 - chunk.length, 255)]);
2910
+ _handleAudioData(b64Data) {
2911
+ if (!this._call) return;
2912
+ const pcm24k = Buffer.from(b64Data, "base64");
2913
+ if (this._recorder) {
2914
+ this._recorder.writeOutbound(resamplePcm16(pcm24k, 24e3, 8e3));
2915
+ }
2916
+ const pcm8k = resamplePcm16(pcm24k, 24e3, 8e3);
2917
+ const ulaw = pcm16ToUlaw(pcm8k);
2918
+ const combined = Buffer.concat([this._audioRemainder, ulaw]);
2919
+ const chunkSize = 160;
2920
+ const fullEnd = Math.floor(combined.length / chunkSize) * chunkSize;
2921
+ if (fullEnd > 0) {
2922
+ this._call.sendAudio(combined.subarray(0, fullEnd));
2923
+ this._sentAudioChunks += fullEnd / chunkSize;
2924
+ }
2925
+ this._audioRemainder = combined.subarray(fullEnd);
2926
+ }
2927
+ _flushAudioRemainder() {
2928
+ if (this._audioRemainder.length > 0 && this._call) {
2929
+ const padded = Buffer.concat([
2930
+ this._audioRemainder,
2931
+ Buffer.alloc(160 - this._audioRemainder.length, 255)
2932
+ ]);
2933
+ this._call.sendAudio(padded);
2934
+ this._sentAudioChunks++;
2935
+ this._audioRemainder = Buffer.alloc(0);
2936
+ }
2937
+ }
2938
+ async _handleToolCall(toolCall) {
2939
+ const functionCalls = toolCall.functionCalls;
2940
+ if (!functionCalls) return;
2941
+ this._toolCallInProgress = true;
2942
+ const responses = [];
2943
+ for (const fc of functionCalls) {
2944
+ const name = fc.name ?? "";
2945
+ const fcId = fc.id ?? "";
2946
+ const args = fc.args ?? {};
2947
+ this._log.info({ tool: name, args }, "Tool call: %s", name);
2948
+ if (BUILTIN_TOOL_NAMES.has(name) && this._call) {
2949
+ const result = await executeBuiltinTool(name, args, this._call);
2950
+ if (result !== null) {
2951
+ if (name === "hang_up") {
2952
+ this._log.info("hang_up: ending call");
2953
+ return;
2892
2954
  }
2893
- this._callSession.sendAudio(chunk);
2955
+ this._log.info("Builtin tool result: %s -> %s", name, result);
2956
+ responses.push({ id: fcId, name, response: { result } });
2957
+ continue;
2894
2958
  }
2895
2959
  }
2896
- } catch (err) {
2897
- this._log.error({ err }, "TTS error");
2898
- } finally {
2899
- this._speaking = false;
2960
+ if (!this._tools || !this._tools.has(name)) {
2961
+ this._log.error("Unknown tool: %s", name);
2962
+ responses.push({ id: fcId, name, response: { error: `Unknown tool: ${name}` } });
2963
+ continue;
2964
+ }
2965
+ try {
2966
+ this._call?.recordToolCall();
2967
+ const result = await this._tools.call(name, args);
2968
+ const resultStr = typeof result === "string" ? result : JSON.stringify(result);
2969
+ this._log.info("Tool result: %s -> %s", name, resultStr.substring(0, 200));
2970
+ responses.push({
2971
+ id: fcId,
2972
+ name,
2973
+ response: { result: resultStr }
2974
+ });
2975
+ } catch (err) {
2976
+ this._log.error({ err }, "Tool call failed: %s", name);
2977
+ if (err instanceof Error) {
2978
+ this._call?.recordToolError(err);
2979
+ }
2980
+ responses.push({
2981
+ id: fcId,
2982
+ name,
2983
+ response: { error: String(err) }
2984
+ });
2985
+ }
2986
+ }
2987
+ if (responses.length > 0 && this._session) {
2988
+ this._log.debug("Sending %d tool response(s)", responses.length);
2989
+ this._session.sendToolResponse({
2990
+ functionResponses: responses
2991
+ });
2900
2992
  }
2993
+ this._toolCallInProgress = false;
2901
2994
  }
2902
2995
  };
2903
2996
 
2904
- // src/agent/pipeline/deepgram-stt.ts
2997
+ // src/agent/pipeline/stt/deepgram-stt.ts
2905
2998
  var DeepgramSTT = class {
2906
2999
  _options;
2907
3000
  _log = NOOP_LOGGER;
3001
+ get provider() {
3002
+ return "deepgram";
3003
+ }
3004
+ get model() {
3005
+ return this._options.model ?? "nova-3";
3006
+ }
2908
3007
  setLogger(logger) {
2909
3008
  this._log = logger;
2910
3009
  }
@@ -3022,10 +3121,19 @@ var DeepgramSTT = class {
3022
3121
  }
3023
3122
  };
3024
3123
 
3025
- // src/agent/pipeline/elevenlabs-tts.ts
3124
+ // src/agent/pipeline/tts/elevenlabs-tts.ts
3026
3125
  var ElevenLabsTTS = class {
3027
3126
  _options;
3028
3127
  _log = NOOP_LOGGER;
3128
+ get provider() {
3129
+ return "elevenlabs";
3130
+ }
3131
+ get model() {
3132
+ return this._options.model ?? "eleven_flash_v2_5";
3133
+ }
3134
+ get voiceId() {
3135
+ return this._options.voiceId ?? "EXAVITQu4vr4xnSDxMaL";
3136
+ }
3029
3137
  setLogger(logger) {
3030
3138
  this._log = logger;
3031
3139
  }
@@ -3175,9 +3283,15 @@ var ElevenLabsTTS = class {
3175
3283
  }
3176
3284
  };
3177
3285
 
3178
- // src/agent/pipeline/openai-llm.ts
3286
+ // src/agent/pipeline/llm/openai-llm.ts
3179
3287
  var OpenAILLM = class {
3180
3288
  _options;
3289
+ get provider() {
3290
+ return "openai";
3291
+ }
3292
+ get model() {
3293
+ return this._options.model;
3294
+ }
3181
3295
  constructor(options = {}) {
3182
3296
  this._options = {
3183
3297
  model: "gpt-4o-mini",
@@ -3261,9 +3375,15 @@ var OpenAILLM = class {
3261
3375
  }
3262
3376
  };
3263
3377
 
3264
- // src/agent/pipeline/anthropic-llm.ts
3378
+ // src/agent/pipeline/llm/anthropic-llm.ts
3265
3379
  var AnthropicLLM = class {
3266
3380
  _options;
3381
+ get provider() {
3382
+ return "anthropic";
3383
+ }
3384
+ get model() {
3385
+ return this._options.model;
3386
+ }
3267
3387
  constructor(options = {}) {
3268
3388
  this._options = {
3269
3389
  model: "claude-sonnet-4-6",
@@ -3366,9 +3486,15 @@ var AnthropicLLM = class {
3366
3486
  }
3367
3487
  };
3368
3488
 
3369
- // src/agent/pipeline/gemini-llm.ts
3489
+ // src/agent/pipeline/llm/gemini-llm.ts
3370
3490
  var GeminiLLM = class {
3371
3491
  _options;
3492
+ get provider() {
3493
+ return "gemini";
3494
+ }
3495
+ get model() {
3496
+ return this._options.model;
3497
+ }
3372
3498
  constructor(options = {}) {
3373
3499
  this._options = {
3374
3500
  model: "gemini-2.5-flash",
@@ -3450,9 +3576,15 @@ var GeminiLLM = class {
3450
3576
  }
3451
3577
  };
3452
3578
 
3453
- // src/agent/pipeline/openai-compat-llm.ts
3579
+ // src/agent/pipeline/llm/openai-compat-llm.ts
3454
3580
  var OpenAICompatLLM = class {
3455
3581
  _options;
3582
+ get provider() {
3583
+ return "openai_compatible";
3584
+ }
3585
+ get model() {
3586
+ return this._options.model;
3587
+ }
3456
3588
  constructor(options) {
3457
3589
  this._options = options;
3458
3590
  }
@@ -3533,9 +3665,15 @@ var OpenAICompatLLM = class {
3533
3665
  }
3534
3666
  };
3535
3667
 
3536
- // src/agent/pipeline/ollama-llm.ts
3668
+ // src/agent/pipeline/llm/ollama-llm.ts
3537
3669
  var OllamaLLM = class {
3538
3670
  _inner;
3671
+ get provider() {
3672
+ return "ollama";
3673
+ }
3674
+ get model() {
3675
+ return this._inner.model;
3676
+ }
3539
3677
  constructor(options = {}) {
3540
3678
  const baseUrl = options.baseUrl ?? process.env["OLLAMA_BASE_URL"] ?? "http://localhost:11434/v1";
3541
3679
  this._inner = new OpenAICompatLLM({
@@ -3551,9 +3689,15 @@ var OllamaLLM = class {
3551
3689
  }
3552
3690
  };
3553
3691
 
3554
- // src/agent/pipeline/mistral-llm.ts
3692
+ // src/agent/pipeline/llm/mistral-llm.ts
3555
3693
  var MistralLLM = class {
3556
3694
  _inner;
3695
+ get provider() {
3696
+ return "mistral";
3697
+ }
3698
+ get model() {
3699
+ return this._inner.model;
3700
+ }
3557
3701
  constructor(options = {}) {
3558
3702
  this._inner = new OpenAICompatLLM({
3559
3703
  apiKey: options.apiKey ?? process.env["MISTRAL_API_KEY"],
@@ -3568,9 +3712,15 @@ var MistralLLM = class {
3568
3712
  }
3569
3713
  };
3570
3714
 
3571
- // src/agent/pipeline/groq-llm.ts
3715
+ // src/agent/pipeline/llm/groq-llm.ts
3572
3716
  var GroqLLM = class {
3573
3717
  _inner;
3718
+ get provider() {
3719
+ return "groq";
3720
+ }
3721
+ get model() {
3722
+ return this._inner.model;
3723
+ }
3574
3724
  constructor(options = {}) {
3575
3725
  this._inner = new OpenAICompatLLM({
3576
3726
  apiKey: options.apiKey ?? process.env["GROQ_API_KEY"],
@@ -3585,9 +3735,15 @@ var GroqLLM = class {
3585
3735
  }
3586
3736
  };
3587
3737
 
3588
- // src/agent/pipeline/perplexity-llm.ts
3738
+ // src/agent/pipeline/llm/perplexity-llm.ts
3589
3739
  var PerplexityLLM = class {
3590
3740
  _inner;
3741
+ get provider() {
3742
+ return "perplexity";
3743
+ }
3744
+ get model() {
3745
+ return this._inner.model;
3746
+ }
3591
3747
  constructor(options = {}) {
3592
3748
  this._inner = new OpenAICompatLLM({
3593
3749
  apiKey: options.apiKey ?? process.env["PERPLEXITY_API_KEY"],
@@ -3602,9 +3758,15 @@ var PerplexityLLM = class {
3602
3758
  }
3603
3759
  };
3604
3760
 
3605
- // src/agent/pipeline/together-llm.ts
3761
+ // src/agent/pipeline/llm/together-llm.ts
3606
3762
  var TogetherLLM = class {
3607
3763
  _inner;
3764
+ get provider() {
3765
+ return "together";
3766
+ }
3767
+ get model() {
3768
+ return this._inner.model;
3769
+ }
3608
3770
  constructor(options = {}) {
3609
3771
  this._inner = new OpenAICompatLLM({
3610
3772
  apiKey: options.apiKey ?? process.env["TOGETHER_API_KEY"],
@@ -3619,9 +3781,15 @@ var TogetherLLM = class {
3619
3781
  }
3620
3782
  };
3621
3783
 
3622
- // src/agent/pipeline/fireworks-llm.ts
3784
+ // src/agent/pipeline/llm/fireworks-llm.ts
3623
3785
  var FireworksLLM = class {
3624
3786
  _inner;
3787
+ get provider() {
3788
+ return "fireworks";
3789
+ }
3790
+ get model() {
3791
+ return this._inner.model;
3792
+ }
3625
3793
  constructor(options = {}) {
3626
3794
  this._inner = new OpenAICompatLLM({
3627
3795
  apiKey: options.apiKey ?? process.env["FIREWORKS_API_KEY"],
@@ -3636,9 +3804,15 @@ var FireworksLLM = class {
3636
3804
  }
3637
3805
  };
3638
3806
 
3639
- // src/agent/pipeline/deepseek-llm.ts
3807
+ // src/agent/pipeline/llm/deepseek-llm.ts
3640
3808
  var DeepSeekLLM = class {
3641
3809
  _inner;
3810
+ get provider() {
3811
+ return "deepseek";
3812
+ }
3813
+ get model() {
3814
+ return this._inner.model;
3815
+ }
3642
3816
  constructor(options = {}) {
3643
3817
  this._inner = new OpenAICompatLLM({
3644
3818
  apiKey: options.apiKey ?? process.env["DEEPSEEK_API_KEY"],
@@ -3653,9 +3827,15 @@ var DeepSeekLLM = class {
3653
3827
  }
3654
3828
  };
3655
3829
 
3656
- // src/agent/pipeline/xai-llm.ts
3830
+ // src/agent/pipeline/llm/xai-llm.ts
3657
3831
  var XaiLLM = class {
3658
3832
  _inner;
3833
+ get provider() {
3834
+ return "xai";
3835
+ }
3836
+ get model() {
3837
+ return this._inner.model;
3838
+ }
3659
3839
  constructor(options = {}) {
3660
3840
  this._inner = new OpenAICompatLLM({
3661
3841
  apiKey: options.apiKey ?? process.env["XAI_API_KEY"],
@@ -3691,6 +3871,7 @@ function mcpServerHTTP(options) {
3691
3871
 
3692
3872
  exports.AnthropicLLM = AnthropicLLM;
3693
3873
  exports.AudioRecorder = AudioRecorder;
3874
+ exports.BUILTIN_TOOL_NAMES = BUILTIN_TOOL_NAMES;
3694
3875
  exports.BuiltinTool = BuiltinTool;
3695
3876
  exports.CallSession = CallSession;
3696
3877
  exports.ClawOpsAgent = ClawOpsAgent;
@@ -3715,8 +3896,11 @@ exports.ToolRegistry = ToolRegistry;
3715
3896
  exports.XaiLLM = XaiLLM;
3716
3897
  exports.createAgentLogger = createAgentLogger;
3717
3898
  exports.createPipelineLogger = createPipelineLogger;
3899
+ exports.executeBuiltinTool = executeBuiltinTool;
3718
3900
  exports.functionTool = functionTool;
3901
+ exports.getBuiltinToolSchemas = getBuiltinToolSchemas;
3719
3902
  exports.getTracingConfig = getTracingConfig;
3903
+ exports.isBuiltinTool = isBuiltinTool;
3720
3904
  exports.mcpServerHTTP = mcpServerHTTP;
3721
3905
  exports.mcpServerStdio = mcpServerStdio;
3722
3906
  exports.pcm16ToUlaw = pcm16ToUlaw;