@livekit/agents 1.0.24 → 1.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/inference/llm.cjs +1 -2
- package/dist/inference/llm.cjs.map +1 -1
- package/dist/inference/llm.d.ts.map +1 -1
- package/dist/inference/llm.js +1 -2
- package/dist/inference/llm.js.map +1 -1
- package/dist/inference/stt.cjs +1 -1
- package/dist/inference/stt.cjs.map +1 -1
- package/dist/inference/stt.d.ts.map +1 -1
- package/dist/inference/stt.js +1 -1
- package/dist/inference/stt.js.map +1 -1
- package/dist/inference/tts.cjs +4 -4
- package/dist/inference/tts.cjs.map +1 -1
- package/dist/inference/tts.d.cts +0 -1
- package/dist/inference/tts.d.ts +0 -1
- package/dist/inference/tts.d.ts.map +1 -1
- package/dist/inference/tts.js +4 -4
- package/dist/inference/tts.js.map +1 -1
- package/dist/ipc/job_proc_lazy_main.cjs +1 -1
- package/dist/ipc/job_proc_lazy_main.cjs.map +1 -1
- package/dist/ipc/job_proc_lazy_main.js +1 -1
- package/dist/ipc/job_proc_lazy_main.js.map +1 -1
- package/dist/job.cjs +29 -2
- package/dist/job.cjs.map +1 -1
- package/dist/job.d.cts +6 -0
- package/dist/job.d.ts +6 -0
- package/dist/job.d.ts.map +1 -1
- package/dist/job.js +19 -2
- package/dist/job.js.map +1 -1
- package/dist/llm/llm.cjs +2 -1
- package/dist/llm/llm.cjs.map +1 -1
- package/dist/llm/llm.d.cts +1 -1
- package/dist/llm/llm.d.ts +1 -1
- package/dist/llm/llm.d.ts.map +1 -1
- package/dist/llm/llm.js +2 -1
- package/dist/llm/llm.js.map +1 -1
- package/dist/stream/deferred_stream.cjs +12 -4
- package/dist/stream/deferred_stream.cjs.map +1 -1
- package/dist/stream/deferred_stream.d.cts +6 -1
- package/dist/stream/deferred_stream.d.ts +6 -1
- package/dist/stream/deferred_stream.d.ts.map +1 -1
- package/dist/stream/deferred_stream.js +12 -4
- package/dist/stream/deferred_stream.js.map +1 -1
- package/dist/stream/deferred_stream.test.cjs +2 -2
- package/dist/stream/deferred_stream.test.cjs.map +1 -1
- package/dist/stream/deferred_stream.test.js +2 -2
- package/dist/stream/deferred_stream.test.js.map +1 -1
- package/dist/stt/stream_adapter.cjs +15 -8
- package/dist/stt/stream_adapter.cjs.map +1 -1
- package/dist/stt/stream_adapter.d.cts +7 -3
- package/dist/stt/stream_adapter.d.ts +7 -3
- package/dist/stt/stream_adapter.d.ts.map +1 -1
- package/dist/stt/stream_adapter.js +15 -8
- package/dist/stt/stream_adapter.js.map +1 -1
- package/dist/stt/stt.cjs +8 -3
- package/dist/stt/stt.cjs.map +1 -1
- package/dist/stt/stt.d.cts +9 -3
- package/dist/stt/stt.d.ts +9 -3
- package/dist/stt/stt.d.ts.map +1 -1
- package/dist/stt/stt.js +9 -4
- package/dist/stt/stt.js.map +1 -1
- package/dist/telemetry/traces.cjs +23 -2
- package/dist/telemetry/traces.cjs.map +1 -1
- package/dist/telemetry/traces.d.ts.map +1 -1
- package/dist/telemetry/traces.js +23 -2
- package/dist/telemetry/traces.js.map +1 -1
- package/dist/tts/stream_adapter.cjs +10 -7
- package/dist/tts/stream_adapter.cjs.map +1 -1
- package/dist/tts/stream_adapter.d.cts +6 -3
- package/dist/tts/stream_adapter.d.ts +6 -3
- package/dist/tts/stream_adapter.d.ts.map +1 -1
- package/dist/tts/stream_adapter.js +10 -7
- package/dist/tts/stream_adapter.js.map +1 -1
- package/dist/tts/tts.cjs +27 -16
- package/dist/tts/tts.cjs.map +1 -1
- package/dist/tts/tts.d.cts +12 -5
- package/dist/tts/tts.d.ts +12 -5
- package/dist/tts/tts.d.ts.map +1 -1
- package/dist/tts/tts.js +28 -17
- package/dist/tts/tts.js.map +1 -1
- package/dist/types.cjs +21 -32
- package/dist/types.cjs.map +1 -1
- package/dist/types.d.cts +41 -10
- package/dist/types.d.ts +41 -10
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +18 -30
- package/dist/types.js.map +1 -1
- package/dist/voice/agent.cjs +54 -19
- package/dist/voice/agent.cjs.map +1 -1
- package/dist/voice/agent.d.ts.map +1 -1
- package/dist/voice/agent.js +54 -19
- package/dist/voice/agent.js.map +1 -1
- package/dist/voice/agent_activity.cjs +0 -3
- package/dist/voice/agent_activity.cjs.map +1 -1
- package/dist/voice/agent_activity.d.ts.map +1 -1
- package/dist/voice/agent_activity.js +0 -3
- package/dist/voice/agent_activity.js.map +1 -1
- package/dist/voice/agent_session.cjs +107 -27
- package/dist/voice/agent_session.cjs.map +1 -1
- package/dist/voice/agent_session.d.cts +16 -2
- package/dist/voice/agent_session.d.ts +16 -2
- package/dist/voice/agent_session.d.ts.map +1 -1
- package/dist/voice/agent_session.js +110 -27
- package/dist/voice/agent_session.js.map +1 -1
- package/dist/voice/events.cjs.map +1 -1
- package/dist/voice/events.d.cts +4 -4
- package/dist/voice/events.d.ts +4 -4
- package/dist/voice/events.d.ts.map +1 -1
- package/dist/voice/events.js.map +1 -1
- package/dist/voice/generation.cjs +6 -7
- package/dist/voice/generation.cjs.map +1 -1
- package/dist/voice/generation.d.ts.map +1 -1
- package/dist/voice/generation.js +7 -8
- package/dist/voice/generation.js.map +1 -1
- package/dist/voice/io.cjs +16 -0
- package/dist/voice/io.cjs.map +1 -1
- package/dist/voice/io.d.cts +8 -0
- package/dist/voice/io.d.ts +8 -0
- package/dist/voice/io.d.ts.map +1 -1
- package/dist/voice/io.js +16 -0
- package/dist/voice/io.js.map +1 -1
- package/dist/voice/recorder_io/index.cjs +23 -0
- package/dist/voice/recorder_io/index.cjs.map +1 -0
- package/dist/voice/recorder_io/index.d.cts +2 -0
- package/dist/voice/recorder_io/index.d.ts +2 -0
- package/dist/voice/recorder_io/index.d.ts.map +1 -0
- package/dist/voice/recorder_io/index.js +2 -0
- package/dist/voice/recorder_io/index.js.map +1 -0
- package/dist/voice/recorder_io/recorder_io.cjs +542 -0
- package/dist/voice/recorder_io/recorder_io.cjs.map +1 -0
- package/dist/voice/recorder_io/recorder_io.d.cts +100 -0
- package/dist/voice/recorder_io/recorder_io.d.ts +100 -0
- package/dist/voice/recorder_io/recorder_io.d.ts.map +1 -0
- package/dist/voice/recorder_io/recorder_io.js +508 -0
- package/dist/voice/recorder_io/recorder_io.js.map +1 -0
- package/dist/voice/report.cjs +7 -2
- package/dist/voice/report.cjs.map +1 -1
- package/dist/voice/report.d.cts +11 -1
- package/dist/voice/report.d.ts +11 -1
- package/dist/voice/report.d.ts.map +1 -1
- package/dist/voice/report.js +7 -2
- package/dist/voice/report.js.map +1 -1
- package/dist/voice/room_io/_input.cjs +2 -1
- package/dist/voice/room_io/_input.cjs.map +1 -1
- package/dist/voice/room_io/_input.d.ts.map +1 -1
- package/dist/voice/room_io/_input.js +2 -1
- package/dist/voice/room_io/_input.js.map +1 -1
- package/dist/voice/room_io/_output.cjs +8 -7
- package/dist/voice/room_io/_output.cjs.map +1 -1
- package/dist/voice/room_io/_output.d.cts +2 -1
- package/dist/voice/room_io/_output.d.ts +2 -1
- package/dist/voice/room_io/_output.d.ts.map +1 -1
- package/dist/voice/room_io/_output.js +8 -7
- package/dist/voice/room_io/_output.js.map +1 -1
- package/dist/worker.cjs +4 -3
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.js +4 -3
- package/dist/worker.js.map +1 -1
- package/package.json +1 -1
- package/src/inference/llm.ts +0 -1
- package/src/inference/stt.ts +1 -2
- package/src/inference/tts.ts +5 -4
- package/src/ipc/job_proc_lazy_main.ts +1 -1
- package/src/job.ts +21 -2
- package/src/llm/llm.ts +2 -2
- package/src/stream/deferred_stream.test.ts +3 -3
- package/src/stream/deferred_stream.ts +22 -5
- package/src/stt/stream_adapter.ts +18 -8
- package/src/stt/stt.ts +19 -6
- package/src/telemetry/traces.ts +25 -3
- package/src/tts/stream_adapter.ts +15 -7
- package/src/tts/tts.ts +46 -21
- package/src/types.ts +57 -33
- package/src/voice/agent.ts +59 -19
- package/src/voice/agent_activity.ts +0 -3
- package/src/voice/agent_session.ts +142 -35
- package/src/voice/events.ts +6 -3
- package/src/voice/generation.ts +10 -8
- package/src/voice/io.ts +19 -0
- package/src/voice/recorder_io/index.ts +4 -0
- package/src/voice/recorder_io/recorder_io.ts +690 -0
- package/src/voice/report.ts +20 -3
- package/src/voice/room_io/_input.ts +2 -1
- package/src/voice/room_io/_output.ts +10 -7
- package/src/worker.ts +1 -1
package/dist/types.js
CHANGED
|
@@ -1,35 +1,23 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
this.maxRetry = options.maxRetry ?? 3;
|
|
10
|
-
this.retryIntervalMs = options.retryIntervalMs ?? 2e3;
|
|
11
|
-
this.timeoutMs = options.timeoutMs ?? 1e4;
|
|
12
|
-
if (this.maxRetry < 0) {
|
|
13
|
-
throw new Error("maxRetry must be greater than or equal to 0");
|
|
14
|
-
}
|
|
15
|
-
if (this.retryIntervalMs < 0) {
|
|
16
|
-
throw new Error("retryIntervalMs must be greater than or equal to 0");
|
|
17
|
-
}
|
|
18
|
-
if (this.timeoutMs < 0) {
|
|
19
|
-
throw new Error("timeoutMs must be greater than or equal to 0");
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
/** @internal */
|
|
23
|
-
_intervalForRetry(numRetries) {
|
|
24
|
-
if (numRetries === 0) {
|
|
25
|
-
return 0.1;
|
|
26
|
-
}
|
|
27
|
-
return this.retryIntervalMs;
|
|
1
|
+
const DEFAULT_API_CONNECT_OPTIONS = {
|
|
2
|
+
maxRetry: 3,
|
|
3
|
+
retryIntervalMs: 2e3,
|
|
4
|
+
timeoutMs: 1e4
|
|
5
|
+
};
|
|
6
|
+
function intervalForRetry(connOptions, numRetries) {
|
|
7
|
+
if (numRetries === 0) {
|
|
8
|
+
return 0.1;
|
|
28
9
|
}
|
|
10
|
+
return connOptions.retryIntervalMs;
|
|
29
11
|
}
|
|
30
|
-
const
|
|
12
|
+
const DEFAULT_SESSION_CONNECT_OPTIONS = {
|
|
13
|
+
sttConnOptions: DEFAULT_API_CONNECT_OPTIONS,
|
|
14
|
+
llmConnOptions: DEFAULT_API_CONNECT_OPTIONS,
|
|
15
|
+
ttsConnOptions: DEFAULT_API_CONNECT_OPTIONS,
|
|
16
|
+
maxUnrecoverableErrors: 3
|
|
17
|
+
};
|
|
31
18
|
export {
|
|
32
|
-
|
|
33
|
-
|
|
19
|
+
DEFAULT_API_CONNECT_OPTIONS,
|
|
20
|
+
DEFAULT_SESSION_CONNECT_OPTIONS,
|
|
21
|
+
intervalForRetry
|
|
34
22
|
};
|
|
35
23
|
//# sourceMappingURL=types.js.map
|
package/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/types.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nexport
|
|
1
|
+
{"version":3,"sources":["../src/types.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Connection options for API calls, controlling retry and timeout behavior.\n */\nexport interface APIConnectOptions {\n /** Maximum number of retries to connect to the API. Default: 3 */\n maxRetry: number;\n /** Interval between retries to connect to the API in milliseconds. Default: 2000 */\n retryIntervalMs: number;\n /** Timeout for connecting to the API in milliseconds. Default: 10000 */\n timeoutMs: number;\n}\n\nexport const DEFAULT_API_CONNECT_OPTIONS: APIConnectOptions = {\n maxRetry: 3,\n retryIntervalMs: 2000,\n timeoutMs: 10000,\n};\n\n/**\n * Return the interval for the given number of retries.\n * The first retry is immediate, and then uses specified retryIntervalMs.\n * @internal\n */\nexport function intervalForRetry(connOptions: APIConnectOptions, numRetries: number): number {\n if (numRetries === 0) {\n return 0.1;\n }\n return connOptions.retryIntervalMs;\n}\n\n/**\n * Connection options for the agent session, controlling retry and timeout behavior\n * for STT, LLM, and TTS connections.\n */\nexport interface SessionConnectOptions {\n /** Connection options for speech-to-text. */\n sttConnOptions?: Partial<APIConnectOptions>;\n /** Connection options for the language model. */\n llmConnOptions?: Partial<APIConnectOptions>;\n /** Connection options for text-to-speech. */\n ttsConnOptions?: Partial<APIConnectOptions>;\n /** Maximum number of consecutive unrecoverable errors from LLM or TTS before closing the session. Default: 3 */\n maxUnrecoverableErrors?: number;\n}\n\n/**\n * Resolved session connect options with all values populated.\n * @internal\n */\nexport interface ResolvedSessionConnectOptions {\n sttConnOptions: APIConnectOptions;\n llmConnOptions: APIConnectOptions;\n ttsConnOptions: APIConnectOptions;\n maxUnrecoverableErrors: number;\n}\n\nexport const DEFAULT_SESSION_CONNECT_OPTIONS: ResolvedSessionConnectOptions = {\n sttConnOptions: DEFAULT_API_CONNECT_OPTIONS,\n llmConnOptions: DEFAULT_API_CONNECT_OPTIONS,\n ttsConnOptions: DEFAULT_API_CONNECT_OPTIONS,\n maxUnrecoverableErrors: 3,\n};\n"],"mappings":"AAgBO,MAAM,8BAAiD;AAAA,EAC5D,UAAU;AAAA,EACV,iBAAiB;AAAA,EACjB,WAAW;AACb;AAOO,SAAS,iBAAiB,aAAgC,YAA4B;AAC3F,MAAI,eAAe,GAAG;AACpB,WAAO;AAAA,EACT;AACA,SAAO,YAAY;AACrB;AA4BO,MAAM,kCAAiE;AAAA,EAC5E,gBAAgB;AAAA,EAChB,gBAAgB;AAAA,EAChB,gBAAgB;AAAA,EAChB,wBAAwB;AAC1B;","names":[]}
|
package/dist/voice/agent.cjs
CHANGED
|
@@ -177,25 +177,37 @@ class Agent {
|
|
|
177
177
|
}
|
|
178
178
|
let wrapped_stt = activity.stt;
|
|
179
179
|
if (!wrapped_stt.capabilities.streaming) {
|
|
180
|
-
|
|
180
|
+
const vad = agent.vad || activity.vad;
|
|
181
|
+
if (!vad) {
|
|
181
182
|
throw new Error(
|
|
182
183
|
"STT does not support streaming, add a VAD to the AgentTask/VoiceAgent to enable streaming"
|
|
183
184
|
);
|
|
184
185
|
}
|
|
185
|
-
wrapped_stt = new import_stt.StreamAdapter(wrapped_stt,
|
|
186
|
+
wrapped_stt = new import_stt.StreamAdapter(wrapped_stt, vad);
|
|
186
187
|
}
|
|
187
|
-
const
|
|
188
|
+
const connOptions = activity.agentSession.connOptions.sttConnOptions;
|
|
189
|
+
const stream = wrapped_stt.stream({ connOptions });
|
|
188
190
|
stream.updateInputStream(audio);
|
|
191
|
+
let cleaned = false;
|
|
192
|
+
const cleanup = () => {
|
|
193
|
+
if (cleaned) return;
|
|
194
|
+
cleaned = true;
|
|
195
|
+
stream.detachInputStream();
|
|
196
|
+
stream.close();
|
|
197
|
+
};
|
|
189
198
|
return new import_web.ReadableStream({
|
|
190
199
|
async start(controller) {
|
|
191
|
-
|
|
192
|
-
|
|
200
|
+
try {
|
|
201
|
+
for await (const event of stream) {
|
|
202
|
+
controller.enqueue(event);
|
|
203
|
+
}
|
|
204
|
+
controller.close();
|
|
205
|
+
} finally {
|
|
206
|
+
cleanup();
|
|
193
207
|
}
|
|
194
|
-
controller.close();
|
|
195
208
|
},
|
|
196
209
|
cancel() {
|
|
197
|
-
|
|
198
|
-
stream.close();
|
|
210
|
+
cleanup();
|
|
199
211
|
}
|
|
200
212
|
});
|
|
201
213
|
},
|
|
@@ -210,21 +222,33 @@ class Agent {
|
|
|
210
222
|
);
|
|
211
223
|
}
|
|
212
224
|
const { toolChoice } = modelSettings;
|
|
225
|
+
const connOptions = activity.agentSession.connOptions.llmConnOptions;
|
|
213
226
|
const stream = activity.llm.chat({
|
|
214
227
|
chatCtx,
|
|
215
228
|
toolCtx,
|
|
216
229
|
toolChoice,
|
|
230
|
+
connOptions,
|
|
217
231
|
parallelToolCalls: true
|
|
218
232
|
});
|
|
233
|
+
let cleaned = false;
|
|
234
|
+
const cleanup = () => {
|
|
235
|
+
if (cleaned) return;
|
|
236
|
+
cleaned = true;
|
|
237
|
+
stream.close();
|
|
238
|
+
};
|
|
219
239
|
return new import_web.ReadableStream({
|
|
220
240
|
async start(controller) {
|
|
221
|
-
|
|
222
|
-
|
|
241
|
+
try {
|
|
242
|
+
for await (const chunk of stream) {
|
|
243
|
+
controller.enqueue(chunk);
|
|
244
|
+
}
|
|
245
|
+
controller.close();
|
|
246
|
+
} finally {
|
|
247
|
+
cleanup();
|
|
223
248
|
}
|
|
224
|
-
controller.close();
|
|
225
249
|
},
|
|
226
250
|
cancel() {
|
|
227
|
-
|
|
251
|
+
cleanup();
|
|
228
252
|
}
|
|
229
253
|
});
|
|
230
254
|
},
|
|
@@ -237,20 +261,31 @@ class Agent {
|
|
|
237
261
|
if (!activity.tts.capabilities.streaming) {
|
|
238
262
|
wrapped_tts = new import_tts.StreamAdapter(wrapped_tts, new import_basic.SentenceTokenizer());
|
|
239
263
|
}
|
|
240
|
-
const
|
|
264
|
+
const connOptions = activity.agentSession.connOptions.ttsConnOptions;
|
|
265
|
+
const stream = wrapped_tts.stream({ connOptions });
|
|
241
266
|
stream.updateInputStream(text);
|
|
267
|
+
let cleaned = false;
|
|
268
|
+
const cleanup = () => {
|
|
269
|
+
if (cleaned) return;
|
|
270
|
+
cleaned = true;
|
|
271
|
+
stream.close();
|
|
272
|
+
};
|
|
242
273
|
return new import_web.ReadableStream({
|
|
243
274
|
async start(controller) {
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
275
|
+
try {
|
|
276
|
+
for await (const chunk of stream) {
|
|
277
|
+
if (chunk === import_tts.SynthesizeStream.END_OF_STREAM) {
|
|
278
|
+
break;
|
|
279
|
+
}
|
|
280
|
+
controller.enqueue(chunk.frame);
|
|
247
281
|
}
|
|
248
|
-
controller.
|
|
282
|
+
controller.close();
|
|
283
|
+
} finally {
|
|
284
|
+
cleanup();
|
|
249
285
|
}
|
|
250
|
-
controller.close();
|
|
251
286
|
},
|
|
252
287
|
cancel() {
|
|
253
|
-
|
|
288
|
+
cleanup();
|
|
254
289
|
}
|
|
255
290
|
});
|
|
256
291
|
},
|
package/dist/voice/agent.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/voice/agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AsyncLocalStorage } from 'node:async_hooks';\nimport { ReadableStream } from 'node:stream/web';\nimport {\n LLM as InferenceLLM,\n STT as InferenceSTT,\n TTS as InferenceTTS,\n type LLMModels,\n type STTModelString,\n type TTSModelString,\n} from '../inference/index.js';\nimport { ReadonlyChatContext } from '../llm/chat_context.js';\nimport type { ChatMessage, FunctionCall, RealtimeModel } from '../llm/index.js';\nimport {\n type ChatChunk,\n ChatContext,\n LLM,\n type ToolChoice,\n type ToolContext,\n} from '../llm/index.js';\nimport type { STT, SpeechEvent } from '../stt/index.js';\nimport { StreamAdapter as STTStreamAdapter } from '../stt/index.js';\nimport { SentenceTokenizer as BasicSentenceTokenizer } from '../tokenize/basic/index.js';\nimport type { TTS } from '../tts/index.js';\nimport { SynthesizeStream, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';\nimport type { VAD } from '../vad.js';\nimport type { AgentActivity } from './agent_activity.js';\nimport type { AgentSession, TurnDetectionMode } from './agent_session.js';\n\nexport const asyncLocalStorage = new AsyncLocalStorage<{ functionCall?: FunctionCall }>();\nexport const STOP_RESPONSE_SYMBOL = Symbol('StopResponse');\n\nexport class StopResponse extends Error {\n constructor() {\n super();\n this.name = 'StopResponse';\n\n Object.defineProperty(this, STOP_RESPONSE_SYMBOL, {\n value: true,\n });\n }\n}\n\nexport function isStopResponse(value: unknown): value is StopResponse {\n return (\n value !== undefined &&\n value !== null &&\n typeof value === 'object' &&\n STOP_RESPONSE_SYMBOL in value\n );\n}\n\nexport interface ModelSettings {\n /** The tool choice to use when calling the LLM. */\n toolChoice?: ToolChoice;\n}\n\nexport interface AgentOptions<UserData> {\n id?: string;\n instructions: string;\n chatCtx?: ChatContext;\n tools?: ToolContext<UserData>;\n turnDetection?: TurnDetectionMode;\n stt?: STT | STTModelString;\n vad?: VAD;\n llm?: LLM | RealtimeModel | LLMModels;\n tts?: TTS | TTSModelString;\n allowInterruptions?: boolean;\n minConsecutiveSpeechDelay?: number;\n}\n\nexport class Agent<UserData = any> {\n private _id: string;\n private turnDetection?: TurnDetectionMode;\n private _stt?: STT;\n private _vad?: VAD;\n private _llm?: LLM | RealtimeModel;\n private _tts?: TTS;\n\n /** @internal */\n _agentActivity?: AgentActivity;\n\n /** @internal */\n _chatCtx: ChatContext;\n\n /** @internal */\n _instructions: string;\n\n /** @internal */\n _tools?: ToolContext<UserData>;\n\n constructor({\n id,\n instructions,\n chatCtx,\n tools,\n turnDetection,\n stt,\n vad,\n llm,\n tts,\n }: AgentOptions<UserData>) {\n if (id) {\n this._id = id;\n } else {\n // Convert class name to snake_case\n const className = this.constructor.name;\n if (className === 'Agent') {\n this._id = 'default_agent';\n } else {\n this._id = className\n .replace(/([A-Z])/g, '_$1')\n .toLowerCase()\n .replace(/^_/, '');\n }\n }\n\n this._instructions = instructions;\n this._tools = { ...tools };\n this._chatCtx = chatCtx\n ? chatCtx.copy({\n toolCtx: this._tools,\n })\n : ChatContext.empty();\n\n this.turnDetection = turnDetection;\n this._vad = vad;\n\n if (typeof stt === 'string') {\n this._stt = InferenceSTT.fromModelString(stt);\n } else {\n this._stt = stt;\n }\n\n if (typeof llm === 'string') {\n this._llm = InferenceLLM.fromModelString(llm);\n } else {\n this._llm = llm;\n }\n\n if (typeof tts === 'string') {\n this._tts = InferenceTTS.fromModelString(tts);\n } else {\n this._tts = tts;\n }\n\n this._agentActivity = undefined;\n }\n\n get vad(): VAD | undefined {\n return this._vad;\n }\n\n get stt(): STT | undefined {\n return this._stt;\n }\n\n get llm(): LLM | RealtimeModel | undefined {\n return this._llm;\n }\n\n get tts(): TTS | undefined {\n return this._tts;\n }\n\n get chatCtx(): ReadonlyChatContext {\n return new ReadonlyChatContext(this._chatCtx.items);\n }\n\n get id(): string {\n return this._id;\n }\n\n get instructions(): string {\n return this._instructions;\n }\n\n get toolCtx(): ToolContext<UserData> {\n return { ...this._tools };\n }\n\n get session(): AgentSession<UserData> {\n return this.getActivityOrThrow().agentSession as AgentSession<UserData>;\n }\n\n async onEnter(): Promise<void> {}\n\n async onExit(): Promise<void> {}\n\n async transcriptionNode(\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<string> | null> {\n return Agent.default.transcriptionNode(this, text, modelSettings);\n }\n\n async onUserTurnCompleted(_chatCtx: ChatContext, _newMessage: ChatMessage): Promise<void> {}\n\n async sttNode(\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<SpeechEvent | string> | null> {\n return Agent.default.sttNode(this, audio, modelSettings);\n }\n\n async llmNode(\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<ChatChunk | string> | null> {\n return Agent.default.llmNode(this, chatCtx, toolCtx, modelSettings);\n }\n\n async ttsNode(\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return Agent.default.ttsNode(this, text, modelSettings);\n }\n\n async realtimeAudioOutputNode(\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return Agent.default.realtimeAudioOutputNode(this, audio, modelSettings);\n }\n\n // realtime_audio_output_node\n\n getActivityOrThrow(): AgentActivity {\n if (!this._agentActivity) {\n throw new Error('Agent activity not found');\n }\n return this._agentActivity;\n }\n\n async updateChatCtx(chatCtx: ChatContext): Promise<void> {\n if (!this._agentActivity) {\n this._chatCtx = chatCtx.copy({ toolCtx: this.toolCtx });\n return;\n }\n\n this._agentActivity.updateChatCtx(chatCtx);\n }\n\n static default = {\n async sttNode(\n agent: Agent,\n audio: ReadableStream<AudioFrame>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<SpeechEvent | string> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.stt) {\n throw new Error('sttNode called but no STT node is available');\n }\n\n let wrapped_stt = activity.stt;\n\n if (!wrapped_stt.capabilities.streaming) {\n if (!agent.vad) {\n throw new Error(\n 'STT does not support streaming, add a VAD to the AgentTask/VoiceAgent to enable streaming',\n );\n }\n wrapped_stt = new STTStreamAdapter(wrapped_stt, agent.vad);\n }\n\n const stream = wrapped_stt.stream();\n stream.updateInputStream(audio);\n\n return new ReadableStream({\n async start(controller) {\n for await (const event of stream) {\n controller.enqueue(event);\n }\n controller.close();\n },\n cancel() {\n stream.detachInputStream();\n stream.close();\n },\n });\n },\n\n async llmNode(\n agent: Agent,\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<ChatChunk | string> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.llm) {\n throw new Error('llmNode called but no LLM node is available');\n }\n\n if (!(activity.llm instanceof LLM)) {\n throw new Error(\n 'llmNode should only be used with LLM (non-multimodal/realtime APIs) nodes',\n );\n }\n\n // TODO(brian): make parallelToolCalls configurable\n const { toolChoice } = modelSettings;\n\n const stream = activity.llm.chat({\n chatCtx,\n toolCtx,\n toolChoice,\n parallelToolCalls: true,\n });\n return new ReadableStream({\n async start(controller) {\n for await (const chunk of stream) {\n controller.enqueue(chunk);\n }\n controller.close();\n },\n cancel() {\n stream.close();\n },\n });\n },\n\n async ttsNode(\n agent: Agent,\n text: ReadableStream<string>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.tts) {\n throw new Error('ttsNode called but no TTS node is available');\n }\n\n let wrapped_tts = activity.tts;\n\n if (!activity.tts.capabilities.streaming) {\n wrapped_tts = new TTSStreamAdapter(wrapped_tts, new BasicSentenceTokenizer());\n }\n\n const stream = wrapped_tts.stream();\n stream.updateInputStream(text);\n\n return new ReadableStream({\n async start(controller) {\n for await (const chunk of stream) {\n if (chunk === SynthesizeStream.END_OF_STREAM) {\n break;\n }\n controller.enqueue(chunk.frame);\n }\n controller.close();\n },\n cancel() {\n stream.close();\n },\n });\n },\n\n async transcriptionNode(\n agent: Agent,\n text: ReadableStream<string>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<string> | null> {\n return text;\n },\n\n async realtimeAudioOutputNode(\n _agent: Agent,\n audio: ReadableStream<AudioFrame>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return audio;\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,8BAAkC;AAClC,iBAA+B;AAC/B,uBAOO;AACP,0BAAoC;AAEpC,iBAMO;AAEP,iBAAkD;AAClD,mBAA4D;AAE5D,iBAAoE;AAK7D,MAAM,oBAAoB,IAAI,0CAAmD;AACjF,MAAM,uBAAuB,OAAO,cAAc;AAElD,MAAM,qBAAqB,MAAM;AAAA,EACtC,cAAc;AACZ,UAAM;AACN,SAAK,OAAO;AAEZ,WAAO,eAAe,MAAM,sBAAsB;AAAA,MAChD,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AACF;AAEO,SAAS,eAAe,OAAuC;AACpE,SACE,UAAU,UACV,UAAU,QACV,OAAO,UAAU,YACjB,wBAAwB;AAE5B;AAqBO,MAAM,MAAsB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGR;AAAA;AAAA,EAGA;AAAA;AAAA,EAGA;AAAA;AAAA,EAGA;AAAA,EAEA,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAA2B;AACzB,QAAI,IAAI;AACN,WAAK,MAAM;AAAA,IACb,OAAO;AAEL,YAAM,YAAY,KAAK,YAAY;AACnC,UAAI,cAAc,SAAS;AACzB,aAAK,MAAM;AAAA,MACb,OAAO;AACL,aAAK,MAAM,UACR,QAAQ,YAAY,KAAK,EACzB,YAAY,EACZ,QAAQ,MAAM,EAAE;AAAA,MACrB;AAAA,IACF;AAEA,SAAK,gBAAgB;AACrB,SAAK,SAAS,EAAE,GAAG,MAAM;AACzB,SAAK,WAAW,UACZ,QAAQ,KAAK;AAAA,MACX,SAAS,KAAK;AAAA,IAChB,CAAC,IACD,uBAAY,MAAM;AAEtB,SAAK,gBAAgB;AACrB,SAAK,OAAO;AAEZ,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,iBAAAA,IAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,iBAAAC,IAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,iBAAAC,IAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuC;AACzC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAA+B;AACjC,WAAO,IAAI,wCAAoB,KAAK,SAAS,KAAK;AAAA,EACpD;AAAA,EAEA,IAAI,KAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAiC;AACnC,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,UAAkC;AACpC,WAAO,KAAK,mBAAmB,EAAE;AAAA,EACnC;AAAA,EAEA,MAAM,UAAyB;AAAA,EAAC;AAAA,EAEhC,MAAM,SAAwB;AAAA,EAAC;AAAA,EAE/B,MAAM,kBACJ,MACA,eACwC;AACxC,WAAO,MAAM,QAAQ,kBAAkB,MAAM,MAAM,aAAa;AAAA,EAClE;AAAA,EAEA,MAAM,oBAAoB,UAAuB,aAAyC;AAAA,EAAC;AAAA,EAE3F,MAAM,QACJ,OACA,eACsD;AACtD,WAAO,MAAM,QAAQ,QAAQ,MAAM,OAAO,aAAa;AAAA,EACzD;AAAA,EAEA,MAAM,QACJ,SACA,SACA,eACoD;AACpD,WAAO,MAAM,QAAQ,QAAQ,MAAM,SAAS,SAAS,aAAa;AAAA,EACpE;AAAA,EAEA,MAAM,QACJ,MACA,eAC4C;AAC5C,WAAO,MAAM,QAAQ,QAAQ,MAAM,MAAM,aAAa;AAAA,EACxD;AAAA,EAEA,MAAM,wBACJ,OACA,eAC4C;AAC5C,WAAO,MAAM,QAAQ,wBAAwB,MAAM,OAAO,aAAa;AAAA,EACzE;AAAA;AAAA,EAIA,qBAAoC;AAClC,QAAI,CAAC,KAAK,gBAAgB;AACxB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,cAAc,SAAqC;AACvD,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,WAAW,QAAQ,KAAK,EAAE,SAAS,KAAK,QAAQ,CAAC;AACtD;AAAA,IACF;AAEA,SAAK,eAAe,cAAc,OAAO;AAAA,EAC3C;AAAA,EAEA,OAAO,UAAU;AAAA,IACf,MAAM,QACJ,OACA,OACA,gBACsD;AACtD,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,cAAc,SAAS;AAE3B,UAAI,CAAC,YAAY,aAAa,WAAW;AACvC,YAAI,CAAC,MAAM,KAAK;AACd,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA,sBAAc,IAAI,WAAAC,cAAiB,aAAa,MAAM,GAAG;AAAA,MAC3D;AAEA,YAAM,SAAS,YAAY,OAAO;AAClC,aAAO,kBAAkB,KAAK;AAE9B,aAAO,IAAI,0BAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,2BAAiB,SAAS,QAAQ;AAChC,uBAAW,QAAQ,KAAK;AAAA,UAC1B;AACA,qBAAW,MAAM;AAAA,QACnB;AAAA,QACA,SAAS;AACP,iBAAO,kBAAkB;AACzB,iBAAO,MAAM;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,QACJ,OACA,SACA,SACA,eACoD;AACpD,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,EAAE,SAAS,eAAe,iBAAM;AAClC,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAGA,YAAM,EAAE,WAAW,IAAI;AAEvB,YAAM,SAAS,SAAS,IAAI,KAAK;AAAA,QAC/B;AAAA,QACA;AAAA,QACA;AAAA,QACA,mBAAmB;AAAA,MACrB,CAAC;AACD,aAAO,IAAI,0BAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,2BAAiB,SAAS,QAAQ;AAChC,uBAAW,QAAQ,KAAK;AAAA,UAC1B;AACA,qBAAW,MAAM;AAAA,QACnB;AAAA,QACA,SAAS;AACP,iBAAO,MAAM;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,QACJ,OACA,MACA,gBAC4C;AAC5C,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,cAAc,SAAS;AAE3B,UAAI,CAAC,SAAS,IAAI,aAAa,WAAW;AACxC,sBAAc,IAAI,WAAAC,cAAiB,aAAa,IAAI,aAAAC,kBAAuB,CAAC;AAAA,MAC9E;AAEA,YAAM,SAAS,YAAY,OAAO;AAClC,aAAO,kBAAkB,IAAI;AAE7B,aAAO,IAAI,0BAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,2BAAiB,SAAS,QAAQ;AAChC,gBAAI,UAAU,4BAAiB,eAAe;AAC5C;AAAA,YACF;AACA,uBAAW,QAAQ,MAAM,KAAK;AAAA,UAChC;AACA,qBAAW,MAAM;AAAA,QACnB;AAAA,QACA,SAAS;AACP,iBAAO,MAAM;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,kBACJ,OACA,MACA,gBACwC;AACxC,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,wBACJ,QACA,OACA,gBAC4C;AAC5C,aAAO;AAAA,IACT;AAAA,EACF;AACF;","names":["InferenceSTT","InferenceLLM","InferenceTTS","STTStreamAdapter","TTSStreamAdapter","BasicSentenceTokenizer"]}
|
|
1
|
+
{"version":3,"sources":["../../src/voice/agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AsyncLocalStorage } from 'node:async_hooks';\nimport { ReadableStream } from 'node:stream/web';\nimport {\n LLM as InferenceLLM,\n STT as InferenceSTT,\n TTS as InferenceTTS,\n type LLMModels,\n type STTModelString,\n type TTSModelString,\n} from '../inference/index.js';\nimport { ReadonlyChatContext } from '../llm/chat_context.js';\nimport type { ChatMessage, FunctionCall, RealtimeModel } from '../llm/index.js';\nimport {\n type ChatChunk,\n ChatContext,\n LLM,\n type ToolChoice,\n type ToolContext,\n} from '../llm/index.js';\nimport type { STT, SpeechEvent } from '../stt/index.js';\nimport { StreamAdapter as STTStreamAdapter } from '../stt/index.js';\nimport { SentenceTokenizer as BasicSentenceTokenizer } from '../tokenize/basic/index.js';\nimport type { TTS } from '../tts/index.js';\nimport { SynthesizeStream, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';\nimport type { VAD } from '../vad.js';\nimport type { AgentActivity } from './agent_activity.js';\nimport type { AgentSession, TurnDetectionMode } from './agent_session.js';\n\nexport const asyncLocalStorage = new AsyncLocalStorage<{ functionCall?: FunctionCall }>();\nexport const STOP_RESPONSE_SYMBOL = Symbol('StopResponse');\n\nexport class StopResponse extends Error {\n constructor() {\n super();\n this.name = 'StopResponse';\n\n Object.defineProperty(this, STOP_RESPONSE_SYMBOL, {\n value: true,\n });\n }\n}\n\nexport function isStopResponse(value: unknown): value is StopResponse {\n return (\n value !== undefined &&\n value !== null &&\n typeof value === 'object' &&\n STOP_RESPONSE_SYMBOL in value\n );\n}\n\nexport interface ModelSettings {\n /** The tool choice to use when calling the LLM. */\n toolChoice?: ToolChoice;\n}\n\nexport interface AgentOptions<UserData> {\n id?: string;\n instructions: string;\n chatCtx?: ChatContext;\n tools?: ToolContext<UserData>;\n turnDetection?: TurnDetectionMode;\n stt?: STT | STTModelString;\n vad?: VAD;\n llm?: LLM | RealtimeModel | LLMModels;\n tts?: TTS | TTSModelString;\n allowInterruptions?: boolean;\n minConsecutiveSpeechDelay?: number;\n}\n\nexport class Agent<UserData = any> {\n private _id: string;\n private turnDetection?: TurnDetectionMode;\n private _stt?: STT;\n private _vad?: VAD;\n private _llm?: LLM | RealtimeModel;\n private _tts?: TTS;\n\n /** @internal */\n _agentActivity?: AgentActivity;\n\n /** @internal */\n _chatCtx: ChatContext;\n\n /** @internal */\n _instructions: string;\n\n /** @internal */\n _tools?: ToolContext<UserData>;\n\n constructor({\n id,\n instructions,\n chatCtx,\n tools,\n turnDetection,\n stt,\n vad,\n llm,\n tts,\n }: AgentOptions<UserData>) {\n if (id) {\n this._id = id;\n } else {\n // Convert class name to snake_case\n const className = this.constructor.name;\n if (className === 'Agent') {\n this._id = 'default_agent';\n } else {\n this._id = className\n .replace(/([A-Z])/g, '_$1')\n .toLowerCase()\n .replace(/^_/, '');\n }\n }\n\n this._instructions = instructions;\n this._tools = { ...tools };\n this._chatCtx = chatCtx\n ? chatCtx.copy({\n toolCtx: this._tools,\n })\n : ChatContext.empty();\n\n this.turnDetection = turnDetection;\n this._vad = vad;\n\n if (typeof stt === 'string') {\n this._stt = InferenceSTT.fromModelString(stt);\n } else {\n this._stt = stt;\n }\n\n if (typeof llm === 'string') {\n this._llm = InferenceLLM.fromModelString(llm);\n } else {\n this._llm = llm;\n }\n\n if (typeof tts === 'string') {\n this._tts = InferenceTTS.fromModelString(tts);\n } else {\n this._tts = tts;\n }\n\n this._agentActivity = undefined;\n }\n\n get vad(): VAD | undefined {\n return this._vad;\n }\n\n get stt(): STT | undefined {\n return this._stt;\n }\n\n get llm(): LLM | RealtimeModel | undefined {\n return this._llm;\n }\n\n get tts(): TTS | undefined {\n return this._tts;\n }\n\n get chatCtx(): ReadonlyChatContext {\n return new ReadonlyChatContext(this._chatCtx.items);\n }\n\n get id(): string {\n return this._id;\n }\n\n get instructions(): string {\n return this._instructions;\n }\n\n get toolCtx(): ToolContext<UserData> {\n return { ...this._tools };\n }\n\n get session(): AgentSession<UserData> {\n return this.getActivityOrThrow().agentSession as AgentSession<UserData>;\n }\n\n async onEnter(): Promise<void> {}\n\n async onExit(): Promise<void> {}\n\n async transcriptionNode(\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<string> | null> {\n return Agent.default.transcriptionNode(this, text, modelSettings);\n }\n\n async onUserTurnCompleted(_chatCtx: ChatContext, _newMessage: ChatMessage): Promise<void> {}\n\n async sttNode(\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<SpeechEvent | string> | null> {\n return Agent.default.sttNode(this, audio, modelSettings);\n }\n\n async llmNode(\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<ChatChunk | string> | null> {\n return Agent.default.llmNode(this, chatCtx, toolCtx, modelSettings);\n }\n\n async ttsNode(\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return Agent.default.ttsNode(this, text, modelSettings);\n }\n\n async realtimeAudioOutputNode(\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return Agent.default.realtimeAudioOutputNode(this, audio, modelSettings);\n }\n\n // realtime_audio_output_node\n\n getActivityOrThrow(): AgentActivity {\n if (!this._agentActivity) {\n throw new Error('Agent activity not found');\n }\n return this._agentActivity;\n }\n\n async updateChatCtx(chatCtx: ChatContext): Promise<void> {\n if (!this._agentActivity) {\n this._chatCtx = chatCtx.copy({ toolCtx: this.toolCtx });\n return;\n }\n\n this._agentActivity.updateChatCtx(chatCtx);\n }\n\n static default = {\n async sttNode(\n agent: Agent,\n audio: ReadableStream<AudioFrame>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<SpeechEvent | string> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.stt) {\n throw new Error('sttNode called but no STT node is available');\n }\n\n let wrapped_stt = activity.stt;\n\n if (!wrapped_stt.capabilities.streaming) {\n const vad = agent.vad || activity.vad;\n if (!vad) {\n throw new Error(\n 'STT does not support streaming, add a VAD to the AgentTask/VoiceAgent to enable streaming',\n );\n }\n wrapped_stt = new STTStreamAdapter(wrapped_stt, vad);\n }\n\n const connOptions = activity.agentSession.connOptions.sttConnOptions;\n const stream = wrapped_stt.stream({ connOptions });\n stream.updateInputStream(audio);\n\n let cleaned = false;\n const cleanup = () => {\n if (cleaned) return;\n cleaned = true;\n stream.detachInputStream();\n stream.close();\n };\n\n return new ReadableStream({\n async start(controller) {\n try {\n for await (const event of stream) {\n controller.enqueue(event);\n }\n controller.close();\n } finally {\n // Always clean up the STT stream, whether it ends naturally or is cancelled\n cleanup();\n }\n },\n cancel() {\n cleanup();\n },\n });\n },\n\n async llmNode(\n agent: Agent,\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<ChatChunk | string> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.llm) {\n throw new Error('llmNode called but no LLM node is available');\n }\n\n if (!(activity.llm instanceof LLM)) {\n throw new Error(\n 'llmNode should only be used with LLM (non-multimodal/realtime APIs) nodes',\n );\n }\n\n // TODO(brian): make parallelToolCalls configurable\n const { toolChoice } = modelSettings;\n const connOptions = activity.agentSession.connOptions.llmConnOptions;\n\n const stream = activity.llm.chat({\n chatCtx,\n toolCtx,\n toolChoice,\n connOptions,\n parallelToolCalls: true,\n });\n\n let cleaned = false;\n const cleanup = () => {\n if (cleaned) return;\n cleaned = true;\n stream.close();\n };\n\n return new ReadableStream({\n async start(controller) {\n try {\n for await (const chunk of stream) {\n controller.enqueue(chunk);\n }\n controller.close();\n } finally {\n cleanup();\n }\n },\n cancel() {\n cleanup();\n },\n });\n },\n\n async ttsNode(\n agent: Agent,\n text: ReadableStream<string>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.tts) {\n throw new Error('ttsNode called but no TTS node is available');\n }\n\n let wrapped_tts = activity.tts;\n\n if (!activity.tts.capabilities.streaming) {\n wrapped_tts = new TTSStreamAdapter(wrapped_tts, new BasicSentenceTokenizer());\n }\n\n const connOptions = activity.agentSession.connOptions.ttsConnOptions;\n const stream = wrapped_tts.stream({ connOptions });\n stream.updateInputStream(text);\n\n let cleaned = false;\n const cleanup = () => {\n if (cleaned) return;\n cleaned = true;\n stream.close();\n };\n\n return new ReadableStream({\n async start(controller) {\n try {\n for await (const chunk of stream) {\n if (chunk === SynthesizeStream.END_OF_STREAM) {\n break;\n }\n controller.enqueue(chunk.frame);\n }\n controller.close();\n } finally {\n cleanup();\n }\n },\n cancel() {\n cleanup();\n },\n });\n },\n\n async transcriptionNode(\n agent: Agent,\n text: ReadableStream<string>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<string> | null> {\n return text;\n },\n\n async realtimeAudioOutputNode(\n _agent: Agent,\n audio: ReadableStream<AudioFrame>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return audio;\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,8BAAkC;AAClC,iBAA+B;AAC/B,uBAOO;AACP,0BAAoC;AAEpC,iBAMO;AAEP,iBAAkD;AAClD,mBAA4D;AAE5D,iBAAoE;AAK7D,MAAM,oBAAoB,IAAI,0CAAmD;AACjF,MAAM,uBAAuB,OAAO,cAAc;AAElD,MAAM,qBAAqB,MAAM;AAAA,EACtC,cAAc;AACZ,UAAM;AACN,SAAK,OAAO;AAEZ,WAAO,eAAe,MAAM,sBAAsB;AAAA,MAChD,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AACF;AAEO,SAAS,eAAe,OAAuC;AACpE,SACE,UAAU,UACV,UAAU,QACV,OAAO,UAAU,YACjB,wBAAwB;AAE5B;AAqBO,MAAM,MAAsB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGR;AAAA;AAAA,EAGA;AAAA;AAAA,EAGA;AAAA;AAAA,EAGA;AAAA,EAEA,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAA2B;AACzB,QAAI,IAAI;AACN,WAAK,MAAM;AAAA,IACb,OAAO;AAEL,YAAM,YAAY,KAAK,YAAY;AACnC,UAAI,cAAc,SAAS;AACzB,aAAK,MAAM;AAAA,MACb,OAAO;AACL,aAAK,MAAM,UACR,QAAQ,YAAY,KAAK,EACzB,YAAY,EACZ,QAAQ,MAAM,EAAE;AAAA,MACrB;AAAA,IACF;AAEA,SAAK,gBAAgB;AACrB,SAAK,SAAS,EAAE,GAAG,MAAM;AACzB,SAAK,WAAW,UACZ,QAAQ,KAAK;AAAA,MACX,SAAS,KAAK;AAAA,IAChB,CAAC,IACD,uBAAY,MAAM;AAEtB,SAAK,gBAAgB;AACrB,SAAK,OAAO;AAEZ,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,iBAAAA,IAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,iBAAAC,IAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,iBAAAC,IAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuC;AACzC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAA+B;AACjC,WAAO,IAAI,wCAAoB,KAAK,SAAS,KAAK;AAAA,EACpD;AAAA,EAEA,IAAI,KAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAiC;AACnC,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,UAAkC;AACpC,WAAO,KAAK,mBAAmB,EAAE;AAAA,EACnC;AAAA,EAEA,MAAM,UAAyB;AAAA,EAAC;AAAA,EAEhC,MAAM,SAAwB;AAAA,EAAC;AAAA,EAE/B,MAAM,kBACJ,MACA,eACwC;AACxC,WAAO,MAAM,QAAQ,kBAAkB,MAAM,MAAM,aAAa;AAAA,EAClE;AAAA,EAEA,MAAM,oBAAoB,UAAuB,aAAyC;AAAA,EAAC;AAAA,EAE3F,MAAM,QACJ,OACA,eACsD;AACtD,WAAO,MAAM,QAAQ,QAAQ,MAAM,OAAO,aAAa;AAAA,EACzD;AAAA,EAEA,MAAM,QACJ,SACA,SACA,eACoD;AACpD,WAAO,MAAM,QAAQ,QAAQ,MAAM,SAAS,SAAS,aAAa;AAAA,EACpE;AAAA,EAEA,MAAM,QACJ,MACA,eAC4C;AAC5C,WAAO,MAAM,QAAQ,QAAQ,MAAM,MAAM,aAAa;AAAA,EACxD;AAAA,EAEA,MAAM,wBACJ,OACA,eAC4C;AAC5C,WAAO,MAAM,QAAQ,wBAAwB,MAAM,OAAO,aAAa;AAAA,EACzE;AAAA;AAAA,EAIA,qBAAoC;AAClC,QAAI,CAAC,KAAK,gBAAgB;AACxB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,cAAc,SAAqC;AACvD,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,WAAW,QAAQ,KAAK,EAAE,SAAS,KAAK,QAAQ,CAAC;AACtD;AAAA,IACF;AAEA,SAAK,eAAe,cAAc,OAAO;AAAA,EAC3C;AAAA,EAEA,OAAO,UAAU;AAAA,IACf,MAAM,QACJ,OACA,OACA,gBACsD;AACtD,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,cAAc,SAAS;AAE3B,UAAI,CAAC,YAAY,aAAa,WAAW;AACvC,cAAM,MAAM,MAAM,OAAO,SAAS;AAClC,YAAI,CAAC,KAAK;AACR,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA,sBAAc,IAAI,WAAAC,cAAiB,aAAa,GAAG;AAAA,MACrD;AAEA,YAAM,cAAc,SAAS,aAAa,YAAY;AACtD,YAAM,SAAS,YAAY,OAAO,EAAE,YAAY,CAAC;AACjD,aAAO,kBAAkB,KAAK;AAE9B,UAAI,UAAU;AACd,YAAM,UAAU,MAAM;AACpB,YAAI,QAAS;AACb,kBAAU;AACV,eAAO,kBAAkB;AACzB,eAAO,MAAM;AAAA,MACf;AAEA,aAAO,IAAI,0BAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,cAAI;AACF,6BAAiB,SAAS,QAAQ;AAChC,yBAAW,QAAQ,KAAK;AAAA,YAC1B;AACA,uBAAW,MAAM;AAAA,UACnB,UAAE;AAEA,oBAAQ;AAAA,UACV;AAAA,QACF;AAAA,QACA,SAAS;AACP,kBAAQ;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,QACJ,OACA,SACA,SACA,eACoD;AACpD,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,EAAE,SAAS,eAAe,iBAAM;AAClC,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAGA,YAAM,EAAE,WAAW,IAAI;AACvB,YAAM,cAAc,SAAS,aAAa,YAAY;AAEtD,YAAM,SAAS,SAAS,IAAI,KAAK;AAAA,QAC/B;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,mBAAmB;AAAA,MACrB,CAAC;AAED,UAAI,UAAU;AACd,YAAM,UAAU,MAAM;AACpB,YAAI,QAAS;AACb,kBAAU;AACV,eAAO,MAAM;AAAA,MACf;AAEA,aAAO,IAAI,0BAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,cAAI;AACF,6BAAiB,SAAS,QAAQ;AAChC,yBAAW,QAAQ,KAAK;AAAA,YAC1B;AACA,uBAAW,MAAM;AAAA,UACnB,UAAE;AACA,oBAAQ;AAAA,UACV;AAAA,QACF;AAAA,QACA,SAAS;AACP,kBAAQ;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,QACJ,OACA,MACA,gBAC4C;AAC5C,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,cAAc,SAAS;AAE3B,UAAI,CAAC,SAAS,IAAI,aAAa,WAAW;AACxC,sBAAc,IAAI,WAAAC,cAAiB,aAAa,IAAI,aAAAC,kBAAuB,CAAC;AAAA,MAC9E;AAEA,YAAM,cAAc,SAAS,aAAa,YAAY;AACtD,YAAM,SAAS,YAAY,OAAO,EAAE,YAAY,CAAC;AACjD,aAAO,kBAAkB,IAAI;AAE7B,UAAI,UAAU;AACd,YAAM,UAAU,MAAM;AACpB,YAAI,QAAS;AACb,kBAAU;AACV,eAAO,MAAM;AAAA,MACf;AAEA,aAAO,IAAI,0BAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,cAAI;AACF,6BAAiB,SAAS,QAAQ;AAChC,kBAAI,UAAU,4BAAiB,eAAe;AAC5C;AAAA,cACF;AACA,yBAAW,QAAQ,MAAM,KAAK;AAAA,YAChC;AACA,uBAAW,MAAM;AAAA,UACnB,UAAE;AACA,oBAAQ;AAAA,UACV;AAAA,QACF;AAAA,QACA,SAAS;AACP,kBAAQ;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,kBACJ,OACA,MACA,gBACwC;AACxC,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,wBACJ,QACA,OACA,gBAC4C;AAC5C,aAAO;AAAA,IACT;AAAA,EACF;AACF;","names":["InferenceSTT","InferenceLLM","InferenceTTS","STTStreamAdapter","TTSStreamAdapter","BasicSentenceTokenizer"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../src/voice/agent.ts"],"names":[],"mappings":";;AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChF,OAAO,EACL,KAAK,SAAS,EACd,WAAW,EACX,GAAG,EACH,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGxD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE1E,eAAO,MAAM,iBAAiB;;EAA2D,CAAC;AAC1F,eAAO,MAAM,oBAAoB,eAAyB,CAAC;AAE3D,qBAAa,YAAa,SAAQ,KAAK;;CAStC;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAOpE;AAED,MAAM,WAAW,aAAa;IAC5B,mDAAmD;IACnD,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,YAAY,CAAC,QAAQ;IACpC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,KAAK,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9B,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,GAAG,SAAS,CAAC;IACtC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,yBAAyB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,qBAAa,KAAK,CAAC,QAAQ,GAAG,GAAG;IAC/B,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,IAAI,CAAC,CAAM;IACnB,OAAO,CAAC,IAAI,CAAC,CAAM;IACnB,OAAO,CAAC,IAAI,CAAC,CAAsB;IACnC,OAAO,CAAC,IAAI,CAAC,CAAM;IAEnB,gBAAgB;IAChB,cAAc,CAAC,EAAE,aAAa,CAAC;IAE/B,gBAAgB;IAChB,QAAQ,EAAE,WAAW,CAAC;IAEtB,gBAAgB;IAChB,aAAa,EAAE,MAAM,CAAC;IAEtB,gBAAgB;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAEnB,EACV,EAAE,EACF,YAAY,EACZ,OAAO,EACP,KAAK,EACL,aAAa,EACb,GAAG,EACH,GAAG,EACH,GAAG,EACH,GAAG,GACJ,EAAE,YAAY,CAAC,QAAQ,CAAC;IAgDzB,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,OAAO,IAAI,mBAAmB,CAEjC;IAED,IAAI,EAAE,IAAI,MAAM,CAEf;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,OAAO,IAAI,WAAW,CAAC,QAAQ,CAAC,CAEnC;IAED,IAAI,OAAO,IAAI,YAAY,CAAC,QAAQ,CAAC,CAEpC;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAExB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvB,iBAAiB,CACrB,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,EAC5B,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAInC,mBAAmB,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnF,OAAO,CACX,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,WAAW,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;IAIjD,OAAO,CACX,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,WAAW,EACpB,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;IAI/C,OAAO,CACX,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,EAC5B,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAIvC,uBAAuB,CAC3B,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAM7C,kBAAkB,IAAI,aAAa;IAO7B,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IASxD,MAAM,CAAC,OAAO;uBAEH,KAAK,SACL,eAAe,UAAU,CAAC,kBACjB,aAAa,GAC5B,QAAQ,eAAe,WAAW,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../src/voice/agent.ts"],"names":[],"mappings":";;AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChF,OAAO,EACL,KAAK,SAAS,EACd,WAAW,EACX,GAAG,EACH,KAAK,UAAU,EACf,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGxD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE1E,eAAO,MAAM,iBAAiB;;EAA2D,CAAC;AAC1F,eAAO,MAAM,oBAAoB,eAAyB,CAAC;AAE3D,qBAAa,YAAa,SAAQ,KAAK;;CAStC;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAOpE;AAED,MAAM,WAAW,aAAa;IAC5B,mDAAmD;IACnD,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,YAAY,CAAC,QAAQ;IACpC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,KAAK,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9B,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,GAAG,SAAS,CAAC;IACtC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,yBAAyB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,qBAAa,KAAK,CAAC,QAAQ,GAAG,GAAG;IAC/B,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,IAAI,CAAC,CAAM;IACnB,OAAO,CAAC,IAAI,CAAC,CAAM;IACnB,OAAO,CAAC,IAAI,CAAC,CAAsB;IACnC,OAAO,CAAC,IAAI,CAAC,CAAM;IAEnB,gBAAgB;IAChB,cAAc,CAAC,EAAE,aAAa,CAAC;IAE/B,gBAAgB;IAChB,QAAQ,EAAE,WAAW,CAAC;IAEtB,gBAAgB;IAChB,aAAa,EAAE,MAAM,CAAC;IAEtB,gBAAgB;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAEnB,EACV,EAAE,EACF,YAAY,EACZ,OAAO,EACP,KAAK,EACL,aAAa,EACb,GAAG,EACH,GAAG,EACH,GAAG,EACH,GAAG,GACJ,EAAE,YAAY,CAAC,QAAQ,CAAC;IAgDzB,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,OAAO,IAAI,mBAAmB,CAEjC;IAED,IAAI,EAAE,IAAI,MAAM,CAEf;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,OAAO,IAAI,WAAW,CAAC,QAAQ,CAAC,CAEnC;IAED,IAAI,OAAO,IAAI,YAAY,CAAC,QAAQ,CAAC,CAEpC;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAExB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvB,iBAAiB,CACrB,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,EAC5B,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IAInC,mBAAmB,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnF,OAAO,CACX,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,WAAW,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;IAIjD,OAAO,CACX,OAAO,EAAE,WAAW,EACpB,OAAO,EAAE,WAAW,EACpB,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;IAI/C,OAAO,CACX,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,EAC5B,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAIvC,uBAAuB,CAC3B,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,EACjC,aAAa,EAAE,aAAa,GAC3B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAM7C,kBAAkB,IAAI,aAAa;IAO7B,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IASxD,MAAM,CAAC,OAAO;uBAEH,KAAK,SACL,eAAe,UAAU,CAAC,kBACjB,aAAa,GAC5B,QAAQ,eAAe,WAAW,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;uBAiD9C,KAAK,WACH,WAAW,WACX,WAAW,iBACL,aAAa,GAC3B,QAAQ,eAAe,SAAS,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC;uBAiD5C,KAAK,QACN,eAAe,MAAM,CAAC,kBACZ,aAAa,GAC5B,QAAQ,eAAe,UAAU,CAAC,GAAG,IAAI,CAAC;iCA4CpC,KAAK,QACN,eAAe,MAAM,CAAC,kBACZ,aAAa,GAC5B,QAAQ,eAAe,MAAM,CAAC,GAAG,IAAI,CAAC;wCAK/B,KAAK,SACN,eAAe,UAAU,CAAC,kBACjB,aAAa,GAC5B,QAAQ,eAAe,UAAU,CAAC,GAAG,IAAI,CAAC;MAG7C;CACH"}
|
package/dist/voice/agent.js
CHANGED
|
@@ -157,25 +157,37 @@ class Agent {
|
|
|
157
157
|
}
|
|
158
158
|
let wrapped_stt = activity.stt;
|
|
159
159
|
if (!wrapped_stt.capabilities.streaming) {
|
|
160
|
-
|
|
160
|
+
const vad = agent.vad || activity.vad;
|
|
161
|
+
if (!vad) {
|
|
161
162
|
throw new Error(
|
|
162
163
|
"STT does not support streaming, add a VAD to the AgentTask/VoiceAgent to enable streaming"
|
|
163
164
|
);
|
|
164
165
|
}
|
|
165
|
-
wrapped_stt = new STTStreamAdapter(wrapped_stt,
|
|
166
|
+
wrapped_stt = new STTStreamAdapter(wrapped_stt, vad);
|
|
166
167
|
}
|
|
167
|
-
const
|
|
168
|
+
const connOptions = activity.agentSession.connOptions.sttConnOptions;
|
|
169
|
+
const stream = wrapped_stt.stream({ connOptions });
|
|
168
170
|
stream.updateInputStream(audio);
|
|
171
|
+
let cleaned = false;
|
|
172
|
+
const cleanup = () => {
|
|
173
|
+
if (cleaned) return;
|
|
174
|
+
cleaned = true;
|
|
175
|
+
stream.detachInputStream();
|
|
176
|
+
stream.close();
|
|
177
|
+
};
|
|
169
178
|
return new ReadableStream({
|
|
170
179
|
async start(controller) {
|
|
171
|
-
|
|
172
|
-
|
|
180
|
+
try {
|
|
181
|
+
for await (const event of stream) {
|
|
182
|
+
controller.enqueue(event);
|
|
183
|
+
}
|
|
184
|
+
controller.close();
|
|
185
|
+
} finally {
|
|
186
|
+
cleanup();
|
|
173
187
|
}
|
|
174
|
-
controller.close();
|
|
175
188
|
},
|
|
176
189
|
cancel() {
|
|
177
|
-
|
|
178
|
-
stream.close();
|
|
190
|
+
cleanup();
|
|
179
191
|
}
|
|
180
192
|
});
|
|
181
193
|
},
|
|
@@ -190,21 +202,33 @@ class Agent {
|
|
|
190
202
|
);
|
|
191
203
|
}
|
|
192
204
|
const { toolChoice } = modelSettings;
|
|
205
|
+
const connOptions = activity.agentSession.connOptions.llmConnOptions;
|
|
193
206
|
const stream = activity.llm.chat({
|
|
194
207
|
chatCtx,
|
|
195
208
|
toolCtx,
|
|
196
209
|
toolChoice,
|
|
210
|
+
connOptions,
|
|
197
211
|
parallelToolCalls: true
|
|
198
212
|
});
|
|
213
|
+
let cleaned = false;
|
|
214
|
+
const cleanup = () => {
|
|
215
|
+
if (cleaned) return;
|
|
216
|
+
cleaned = true;
|
|
217
|
+
stream.close();
|
|
218
|
+
};
|
|
199
219
|
return new ReadableStream({
|
|
200
220
|
async start(controller) {
|
|
201
|
-
|
|
202
|
-
|
|
221
|
+
try {
|
|
222
|
+
for await (const chunk of stream) {
|
|
223
|
+
controller.enqueue(chunk);
|
|
224
|
+
}
|
|
225
|
+
controller.close();
|
|
226
|
+
} finally {
|
|
227
|
+
cleanup();
|
|
203
228
|
}
|
|
204
|
-
controller.close();
|
|
205
229
|
},
|
|
206
230
|
cancel() {
|
|
207
|
-
|
|
231
|
+
cleanup();
|
|
208
232
|
}
|
|
209
233
|
});
|
|
210
234
|
},
|
|
@@ -217,20 +241,31 @@ class Agent {
|
|
|
217
241
|
if (!activity.tts.capabilities.streaming) {
|
|
218
242
|
wrapped_tts = new TTSStreamAdapter(wrapped_tts, new BasicSentenceTokenizer());
|
|
219
243
|
}
|
|
220
|
-
const
|
|
244
|
+
const connOptions = activity.agentSession.connOptions.ttsConnOptions;
|
|
245
|
+
const stream = wrapped_tts.stream({ connOptions });
|
|
221
246
|
stream.updateInputStream(text);
|
|
247
|
+
let cleaned = false;
|
|
248
|
+
const cleanup = () => {
|
|
249
|
+
if (cleaned) return;
|
|
250
|
+
cleaned = true;
|
|
251
|
+
stream.close();
|
|
252
|
+
};
|
|
222
253
|
return new ReadableStream({
|
|
223
254
|
async start(controller) {
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
255
|
+
try {
|
|
256
|
+
for await (const chunk of stream) {
|
|
257
|
+
if (chunk === SynthesizeStream.END_OF_STREAM) {
|
|
258
|
+
break;
|
|
259
|
+
}
|
|
260
|
+
controller.enqueue(chunk.frame);
|
|
227
261
|
}
|
|
228
|
-
controller.
|
|
262
|
+
controller.close();
|
|
263
|
+
} finally {
|
|
264
|
+
cleanup();
|
|
229
265
|
}
|
|
230
|
-
controller.close();
|
|
231
266
|
},
|
|
232
267
|
cancel() {
|
|
233
|
-
|
|
268
|
+
cleanup();
|
|
234
269
|
}
|
|
235
270
|
});
|
|
236
271
|
},
|
package/dist/voice/agent.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/voice/agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AsyncLocalStorage } from 'node:async_hooks';\nimport { ReadableStream } from 'node:stream/web';\nimport {\n LLM as InferenceLLM,\n STT as InferenceSTT,\n TTS as InferenceTTS,\n type LLMModels,\n type STTModelString,\n type TTSModelString,\n} from '../inference/index.js';\nimport { ReadonlyChatContext } from '../llm/chat_context.js';\nimport type { ChatMessage, FunctionCall, RealtimeModel } from '../llm/index.js';\nimport {\n type ChatChunk,\n ChatContext,\n LLM,\n type ToolChoice,\n type ToolContext,\n} from '../llm/index.js';\nimport type { STT, SpeechEvent } from '../stt/index.js';\nimport { StreamAdapter as STTStreamAdapter } from '../stt/index.js';\nimport { SentenceTokenizer as BasicSentenceTokenizer } from '../tokenize/basic/index.js';\nimport type { TTS } from '../tts/index.js';\nimport { SynthesizeStream, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';\nimport type { VAD } from '../vad.js';\nimport type { AgentActivity } from './agent_activity.js';\nimport type { AgentSession, TurnDetectionMode } from './agent_session.js';\n\nexport const asyncLocalStorage = new AsyncLocalStorage<{ functionCall?: FunctionCall }>();\nexport const STOP_RESPONSE_SYMBOL = Symbol('StopResponse');\n\nexport class StopResponse extends Error {\n constructor() {\n super();\n this.name = 'StopResponse';\n\n Object.defineProperty(this, STOP_RESPONSE_SYMBOL, {\n value: true,\n });\n }\n}\n\nexport function isStopResponse(value: unknown): value is StopResponse {\n return (\n value !== undefined &&\n value !== null &&\n typeof value === 'object' &&\n STOP_RESPONSE_SYMBOL in value\n );\n}\n\nexport interface ModelSettings {\n /** The tool choice to use when calling the LLM. */\n toolChoice?: ToolChoice;\n}\n\nexport interface AgentOptions<UserData> {\n id?: string;\n instructions: string;\n chatCtx?: ChatContext;\n tools?: ToolContext<UserData>;\n turnDetection?: TurnDetectionMode;\n stt?: STT | STTModelString;\n vad?: VAD;\n llm?: LLM | RealtimeModel | LLMModels;\n tts?: TTS | TTSModelString;\n allowInterruptions?: boolean;\n minConsecutiveSpeechDelay?: number;\n}\n\nexport class Agent<UserData = any> {\n private _id: string;\n private turnDetection?: TurnDetectionMode;\n private _stt?: STT;\n private _vad?: VAD;\n private _llm?: LLM | RealtimeModel;\n private _tts?: TTS;\n\n /** @internal */\n _agentActivity?: AgentActivity;\n\n /** @internal */\n _chatCtx: ChatContext;\n\n /** @internal */\n _instructions: string;\n\n /** @internal */\n _tools?: ToolContext<UserData>;\n\n constructor({\n id,\n instructions,\n chatCtx,\n tools,\n turnDetection,\n stt,\n vad,\n llm,\n tts,\n }: AgentOptions<UserData>) {\n if (id) {\n this._id = id;\n } else {\n // Convert class name to snake_case\n const className = this.constructor.name;\n if (className === 'Agent') {\n this._id = 'default_agent';\n } else {\n this._id = className\n .replace(/([A-Z])/g, '_$1')\n .toLowerCase()\n .replace(/^_/, '');\n }\n }\n\n this._instructions = instructions;\n this._tools = { ...tools };\n this._chatCtx = chatCtx\n ? chatCtx.copy({\n toolCtx: this._tools,\n })\n : ChatContext.empty();\n\n this.turnDetection = turnDetection;\n this._vad = vad;\n\n if (typeof stt === 'string') {\n this._stt = InferenceSTT.fromModelString(stt);\n } else {\n this._stt = stt;\n }\n\n if (typeof llm === 'string') {\n this._llm = InferenceLLM.fromModelString(llm);\n } else {\n this._llm = llm;\n }\n\n if (typeof tts === 'string') {\n this._tts = InferenceTTS.fromModelString(tts);\n } else {\n this._tts = tts;\n }\n\n this._agentActivity = undefined;\n }\n\n get vad(): VAD | undefined {\n return this._vad;\n }\n\n get stt(): STT | undefined {\n return this._stt;\n }\n\n get llm(): LLM | RealtimeModel | undefined {\n return this._llm;\n }\n\n get tts(): TTS | undefined {\n return this._tts;\n }\n\n get chatCtx(): ReadonlyChatContext {\n return new ReadonlyChatContext(this._chatCtx.items);\n }\n\n get id(): string {\n return this._id;\n }\n\n get instructions(): string {\n return this._instructions;\n }\n\n get toolCtx(): ToolContext<UserData> {\n return { ...this._tools };\n }\n\n get session(): AgentSession<UserData> {\n return this.getActivityOrThrow().agentSession as AgentSession<UserData>;\n }\n\n async onEnter(): Promise<void> {}\n\n async onExit(): Promise<void> {}\n\n async transcriptionNode(\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<string> | null> {\n return Agent.default.transcriptionNode(this, text, modelSettings);\n }\n\n async onUserTurnCompleted(_chatCtx: ChatContext, _newMessage: ChatMessage): Promise<void> {}\n\n async sttNode(\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<SpeechEvent | string> | null> {\n return Agent.default.sttNode(this, audio, modelSettings);\n }\n\n async llmNode(\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<ChatChunk | string> | null> {\n return Agent.default.llmNode(this, chatCtx, toolCtx, modelSettings);\n }\n\n async ttsNode(\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return Agent.default.ttsNode(this, text, modelSettings);\n }\n\n async realtimeAudioOutputNode(\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return Agent.default.realtimeAudioOutputNode(this, audio, modelSettings);\n }\n\n // realtime_audio_output_node\n\n getActivityOrThrow(): AgentActivity {\n if (!this._agentActivity) {\n throw new Error('Agent activity not found');\n }\n return this._agentActivity;\n }\n\n async updateChatCtx(chatCtx: ChatContext): Promise<void> {\n if (!this._agentActivity) {\n this._chatCtx = chatCtx.copy({ toolCtx: this.toolCtx });\n return;\n }\n\n this._agentActivity.updateChatCtx(chatCtx);\n }\n\n static default = {\n async sttNode(\n agent: Agent,\n audio: ReadableStream<AudioFrame>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<SpeechEvent | string> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.stt) {\n throw new Error('sttNode called but no STT node is available');\n }\n\n let wrapped_stt = activity.stt;\n\n if (!wrapped_stt.capabilities.streaming) {\n if (!agent.vad) {\n throw new Error(\n 'STT does not support streaming, add a VAD to the AgentTask/VoiceAgent to enable streaming',\n );\n }\n wrapped_stt = new STTStreamAdapter(wrapped_stt, agent.vad);\n }\n\n const stream = wrapped_stt.stream();\n stream.updateInputStream(audio);\n\n return new ReadableStream({\n async start(controller) {\n for await (const event of stream) {\n controller.enqueue(event);\n }\n controller.close();\n },\n cancel() {\n stream.detachInputStream();\n stream.close();\n },\n });\n },\n\n async llmNode(\n agent: Agent,\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<ChatChunk | string> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.llm) {\n throw new Error('llmNode called but no LLM node is available');\n }\n\n if (!(activity.llm instanceof LLM)) {\n throw new Error(\n 'llmNode should only be used with LLM (non-multimodal/realtime APIs) nodes',\n );\n }\n\n // TODO(brian): make parallelToolCalls configurable\n const { toolChoice } = modelSettings;\n\n const stream = activity.llm.chat({\n chatCtx,\n toolCtx,\n toolChoice,\n parallelToolCalls: true,\n });\n return new ReadableStream({\n async start(controller) {\n for await (const chunk of stream) {\n controller.enqueue(chunk);\n }\n controller.close();\n },\n cancel() {\n stream.close();\n },\n });\n },\n\n async ttsNode(\n agent: Agent,\n text: ReadableStream<string>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.tts) {\n throw new Error('ttsNode called but no TTS node is available');\n }\n\n let wrapped_tts = activity.tts;\n\n if (!activity.tts.capabilities.streaming) {\n wrapped_tts = new TTSStreamAdapter(wrapped_tts, new BasicSentenceTokenizer());\n }\n\n const stream = wrapped_tts.stream();\n stream.updateInputStream(text);\n\n return new ReadableStream({\n async start(controller) {\n for await (const chunk of stream) {\n if (chunk === SynthesizeStream.END_OF_STREAM) {\n break;\n }\n controller.enqueue(chunk.frame);\n }\n controller.close();\n },\n cancel() {\n stream.close();\n },\n });\n },\n\n async transcriptionNode(\n agent: Agent,\n text: ReadableStream<string>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<string> | null> {\n return text;\n },\n\n async realtimeAudioOutputNode(\n _agent: Agent,\n audio: ReadableStream<AudioFrame>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return audio;\n },\n };\n}\n"],"mappings":"AAIA,SAAS,yBAAyB;AAClC,SAAS,sBAAsB;AAC/B;AAAA,EACE,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,OAIF;AACP,SAAS,2BAA2B;AAEpC;AAAA,EAEE;AAAA,EACA;AAAA,OAGK;AAEP,SAAS,iBAAiB,wBAAwB;AAClD,SAAS,qBAAqB,8BAA8B;AAE5D,SAAS,kBAAkB,iBAAiB,wBAAwB;AAK7D,MAAM,oBAAoB,IAAI,kBAAmD;AACjF,MAAM,uBAAuB,OAAO,cAAc;AAElD,MAAM,qBAAqB,MAAM;AAAA,EACtC,cAAc;AACZ,UAAM;AACN,SAAK,OAAO;AAEZ,WAAO,eAAe,MAAM,sBAAsB;AAAA,MAChD,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AACF;AAEO,SAAS,eAAe,OAAuC;AACpE,SACE,UAAU,UACV,UAAU,QACV,OAAO,UAAU,YACjB,wBAAwB;AAE5B;AAqBO,MAAM,MAAsB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGR;AAAA;AAAA,EAGA;AAAA;AAAA,EAGA;AAAA;AAAA,EAGA;AAAA,EAEA,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAA2B;AACzB,QAAI,IAAI;AACN,WAAK,MAAM;AAAA,IACb,OAAO;AAEL,YAAM,YAAY,KAAK,YAAY;AACnC,UAAI,cAAc,SAAS;AACzB,aAAK,MAAM;AAAA,MACb,OAAO;AACL,aAAK,MAAM,UACR,QAAQ,YAAY,KAAK,EACzB,YAAY,EACZ,QAAQ,MAAM,EAAE;AAAA,MACrB;AAAA,IACF;AAEA,SAAK,gBAAgB;AACrB,SAAK,SAAS,EAAE,GAAG,MAAM;AACzB,SAAK,WAAW,UACZ,QAAQ,KAAK;AAAA,MACX,SAAS,KAAK;AAAA,IAChB,CAAC,IACD,YAAY,MAAM;AAEtB,SAAK,gBAAgB;AACrB,SAAK,OAAO;AAEZ,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,aAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,aAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,aAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuC;AACzC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAA+B;AACjC,WAAO,IAAI,oBAAoB,KAAK,SAAS,KAAK;AAAA,EACpD;AAAA,EAEA,IAAI,KAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAiC;AACnC,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,UAAkC;AACpC,WAAO,KAAK,mBAAmB,EAAE;AAAA,EACnC;AAAA,EAEA,MAAM,UAAyB;AAAA,EAAC;AAAA,EAEhC,MAAM,SAAwB;AAAA,EAAC;AAAA,EAE/B,MAAM,kBACJ,MACA,eACwC;AACxC,WAAO,MAAM,QAAQ,kBAAkB,MAAM,MAAM,aAAa;AAAA,EAClE;AAAA,EAEA,MAAM,oBAAoB,UAAuB,aAAyC;AAAA,EAAC;AAAA,EAE3F,MAAM,QACJ,OACA,eACsD;AACtD,WAAO,MAAM,QAAQ,QAAQ,MAAM,OAAO,aAAa;AAAA,EACzD;AAAA,EAEA,MAAM,QACJ,SACA,SACA,eACoD;AACpD,WAAO,MAAM,QAAQ,QAAQ,MAAM,SAAS,SAAS,aAAa;AAAA,EACpE;AAAA,EAEA,MAAM,QACJ,MACA,eAC4C;AAC5C,WAAO,MAAM,QAAQ,QAAQ,MAAM,MAAM,aAAa;AAAA,EACxD;AAAA,EAEA,MAAM,wBACJ,OACA,eAC4C;AAC5C,WAAO,MAAM,QAAQ,wBAAwB,MAAM,OAAO,aAAa;AAAA,EACzE;AAAA;AAAA,EAIA,qBAAoC;AAClC,QAAI,CAAC,KAAK,gBAAgB;AACxB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,cAAc,SAAqC;AACvD,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,WAAW,QAAQ,KAAK,EAAE,SAAS,KAAK,QAAQ,CAAC;AACtD;AAAA,IACF;AAEA,SAAK,eAAe,cAAc,OAAO;AAAA,EAC3C;AAAA,EAEA,OAAO,UAAU;AAAA,IACf,MAAM,QACJ,OACA,OACA,gBACsD;AACtD,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,cAAc,SAAS;AAE3B,UAAI,CAAC,YAAY,aAAa,WAAW;AACvC,YAAI,CAAC,MAAM,KAAK;AACd,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA,sBAAc,IAAI,iBAAiB,aAAa,MAAM,GAAG;AAAA,MAC3D;AAEA,YAAM,SAAS,YAAY,OAAO;AAClC,aAAO,kBAAkB,KAAK;AAE9B,aAAO,IAAI,eAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,2BAAiB,SAAS,QAAQ;AAChC,uBAAW,QAAQ,KAAK;AAAA,UAC1B;AACA,qBAAW,MAAM;AAAA,QACnB;AAAA,QACA,SAAS;AACP,iBAAO,kBAAkB;AACzB,iBAAO,MAAM;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,QACJ,OACA,SACA,SACA,eACoD;AACpD,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,EAAE,SAAS,eAAe,MAAM;AAClC,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAGA,YAAM,EAAE,WAAW,IAAI;AAEvB,YAAM,SAAS,SAAS,IAAI,KAAK;AAAA,QAC/B;AAAA,QACA;AAAA,QACA;AAAA,QACA,mBAAmB;AAAA,MACrB,CAAC;AACD,aAAO,IAAI,eAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,2BAAiB,SAAS,QAAQ;AAChC,uBAAW,QAAQ,KAAK;AAAA,UAC1B;AACA,qBAAW,MAAM;AAAA,QACnB;AAAA,QACA,SAAS;AACP,iBAAO,MAAM;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,QACJ,OACA,MACA,gBAC4C;AAC5C,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,cAAc,SAAS;AAE3B,UAAI,CAAC,SAAS,IAAI,aAAa,WAAW;AACxC,sBAAc,IAAI,iBAAiB,aAAa,IAAI,uBAAuB,CAAC;AAAA,MAC9E;AAEA,YAAM,SAAS,YAAY,OAAO;AAClC,aAAO,kBAAkB,IAAI;AAE7B,aAAO,IAAI,eAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,2BAAiB,SAAS,QAAQ;AAChC,gBAAI,UAAU,iBAAiB,eAAe;AAC5C;AAAA,YACF;AACA,uBAAW,QAAQ,MAAM,KAAK;AAAA,UAChC;AACA,qBAAW,MAAM;AAAA,QACnB;AAAA,QACA,SAAS;AACP,iBAAO,MAAM;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,kBACJ,OACA,MACA,gBACwC;AACxC,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,wBACJ,QACA,OACA,gBAC4C;AAC5C,aAAO;AAAA,IACT;AAAA,EACF;AACF;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/voice/agent.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AsyncLocalStorage } from 'node:async_hooks';\nimport { ReadableStream } from 'node:stream/web';\nimport {\n LLM as InferenceLLM,\n STT as InferenceSTT,\n TTS as InferenceTTS,\n type LLMModels,\n type STTModelString,\n type TTSModelString,\n} from '../inference/index.js';\nimport { ReadonlyChatContext } from '../llm/chat_context.js';\nimport type { ChatMessage, FunctionCall, RealtimeModel } from '../llm/index.js';\nimport {\n type ChatChunk,\n ChatContext,\n LLM,\n type ToolChoice,\n type ToolContext,\n} from '../llm/index.js';\nimport type { STT, SpeechEvent } from '../stt/index.js';\nimport { StreamAdapter as STTStreamAdapter } from '../stt/index.js';\nimport { SentenceTokenizer as BasicSentenceTokenizer } from '../tokenize/basic/index.js';\nimport type { TTS } from '../tts/index.js';\nimport { SynthesizeStream, StreamAdapter as TTSStreamAdapter } from '../tts/index.js';\nimport type { VAD } from '../vad.js';\nimport type { AgentActivity } from './agent_activity.js';\nimport type { AgentSession, TurnDetectionMode } from './agent_session.js';\n\nexport const asyncLocalStorage = new AsyncLocalStorage<{ functionCall?: FunctionCall }>();\nexport const STOP_RESPONSE_SYMBOL = Symbol('StopResponse');\n\nexport class StopResponse extends Error {\n constructor() {\n super();\n this.name = 'StopResponse';\n\n Object.defineProperty(this, STOP_RESPONSE_SYMBOL, {\n value: true,\n });\n }\n}\n\nexport function isStopResponse(value: unknown): value is StopResponse {\n return (\n value !== undefined &&\n value !== null &&\n typeof value === 'object' &&\n STOP_RESPONSE_SYMBOL in value\n );\n}\n\nexport interface ModelSettings {\n /** The tool choice to use when calling the LLM. */\n toolChoice?: ToolChoice;\n}\n\nexport interface AgentOptions<UserData> {\n id?: string;\n instructions: string;\n chatCtx?: ChatContext;\n tools?: ToolContext<UserData>;\n turnDetection?: TurnDetectionMode;\n stt?: STT | STTModelString;\n vad?: VAD;\n llm?: LLM | RealtimeModel | LLMModels;\n tts?: TTS | TTSModelString;\n allowInterruptions?: boolean;\n minConsecutiveSpeechDelay?: number;\n}\n\nexport class Agent<UserData = any> {\n private _id: string;\n private turnDetection?: TurnDetectionMode;\n private _stt?: STT;\n private _vad?: VAD;\n private _llm?: LLM | RealtimeModel;\n private _tts?: TTS;\n\n /** @internal */\n _agentActivity?: AgentActivity;\n\n /** @internal */\n _chatCtx: ChatContext;\n\n /** @internal */\n _instructions: string;\n\n /** @internal */\n _tools?: ToolContext<UserData>;\n\n constructor({\n id,\n instructions,\n chatCtx,\n tools,\n turnDetection,\n stt,\n vad,\n llm,\n tts,\n }: AgentOptions<UserData>) {\n if (id) {\n this._id = id;\n } else {\n // Convert class name to snake_case\n const className = this.constructor.name;\n if (className === 'Agent') {\n this._id = 'default_agent';\n } else {\n this._id = className\n .replace(/([A-Z])/g, '_$1')\n .toLowerCase()\n .replace(/^_/, '');\n }\n }\n\n this._instructions = instructions;\n this._tools = { ...tools };\n this._chatCtx = chatCtx\n ? chatCtx.copy({\n toolCtx: this._tools,\n })\n : ChatContext.empty();\n\n this.turnDetection = turnDetection;\n this._vad = vad;\n\n if (typeof stt === 'string') {\n this._stt = InferenceSTT.fromModelString(stt);\n } else {\n this._stt = stt;\n }\n\n if (typeof llm === 'string') {\n this._llm = InferenceLLM.fromModelString(llm);\n } else {\n this._llm = llm;\n }\n\n if (typeof tts === 'string') {\n this._tts = InferenceTTS.fromModelString(tts);\n } else {\n this._tts = tts;\n }\n\n this._agentActivity = undefined;\n }\n\n get vad(): VAD | undefined {\n return this._vad;\n }\n\n get stt(): STT | undefined {\n return this._stt;\n }\n\n get llm(): LLM | RealtimeModel | undefined {\n return this._llm;\n }\n\n get tts(): TTS | undefined {\n return this._tts;\n }\n\n get chatCtx(): ReadonlyChatContext {\n return new ReadonlyChatContext(this._chatCtx.items);\n }\n\n get id(): string {\n return this._id;\n }\n\n get instructions(): string {\n return this._instructions;\n }\n\n get toolCtx(): ToolContext<UserData> {\n return { ...this._tools };\n }\n\n get session(): AgentSession<UserData> {\n return this.getActivityOrThrow().agentSession as AgentSession<UserData>;\n }\n\n async onEnter(): Promise<void> {}\n\n async onExit(): Promise<void> {}\n\n async transcriptionNode(\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<string> | null> {\n return Agent.default.transcriptionNode(this, text, modelSettings);\n }\n\n async onUserTurnCompleted(_chatCtx: ChatContext, _newMessage: ChatMessage): Promise<void> {}\n\n async sttNode(\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<SpeechEvent | string> | null> {\n return Agent.default.sttNode(this, audio, modelSettings);\n }\n\n async llmNode(\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<ChatChunk | string> | null> {\n return Agent.default.llmNode(this, chatCtx, toolCtx, modelSettings);\n }\n\n async ttsNode(\n text: ReadableStream<string>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return Agent.default.ttsNode(this, text, modelSettings);\n }\n\n async realtimeAudioOutputNode(\n audio: ReadableStream<AudioFrame>,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return Agent.default.realtimeAudioOutputNode(this, audio, modelSettings);\n }\n\n // realtime_audio_output_node\n\n getActivityOrThrow(): AgentActivity {\n if (!this._agentActivity) {\n throw new Error('Agent activity not found');\n }\n return this._agentActivity;\n }\n\n async updateChatCtx(chatCtx: ChatContext): Promise<void> {\n if (!this._agentActivity) {\n this._chatCtx = chatCtx.copy({ toolCtx: this.toolCtx });\n return;\n }\n\n this._agentActivity.updateChatCtx(chatCtx);\n }\n\n static default = {\n async sttNode(\n agent: Agent,\n audio: ReadableStream<AudioFrame>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<SpeechEvent | string> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.stt) {\n throw new Error('sttNode called but no STT node is available');\n }\n\n let wrapped_stt = activity.stt;\n\n if (!wrapped_stt.capabilities.streaming) {\n const vad = agent.vad || activity.vad;\n if (!vad) {\n throw new Error(\n 'STT does not support streaming, add a VAD to the AgentTask/VoiceAgent to enable streaming',\n );\n }\n wrapped_stt = new STTStreamAdapter(wrapped_stt, vad);\n }\n\n const connOptions = activity.agentSession.connOptions.sttConnOptions;\n const stream = wrapped_stt.stream({ connOptions });\n stream.updateInputStream(audio);\n\n let cleaned = false;\n const cleanup = () => {\n if (cleaned) return;\n cleaned = true;\n stream.detachInputStream();\n stream.close();\n };\n\n return new ReadableStream({\n async start(controller) {\n try {\n for await (const event of stream) {\n controller.enqueue(event);\n }\n controller.close();\n } finally {\n // Always clean up the STT stream, whether it ends naturally or is cancelled\n cleanup();\n }\n },\n cancel() {\n cleanup();\n },\n });\n },\n\n async llmNode(\n agent: Agent,\n chatCtx: ChatContext,\n toolCtx: ToolContext,\n modelSettings: ModelSettings,\n ): Promise<ReadableStream<ChatChunk | string> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.llm) {\n throw new Error('llmNode called but no LLM node is available');\n }\n\n if (!(activity.llm instanceof LLM)) {\n throw new Error(\n 'llmNode should only be used with LLM (non-multimodal/realtime APIs) nodes',\n );\n }\n\n // TODO(brian): make parallelToolCalls configurable\n const { toolChoice } = modelSettings;\n const connOptions = activity.agentSession.connOptions.llmConnOptions;\n\n const stream = activity.llm.chat({\n chatCtx,\n toolCtx,\n toolChoice,\n connOptions,\n parallelToolCalls: true,\n });\n\n let cleaned = false;\n const cleanup = () => {\n if (cleaned) return;\n cleaned = true;\n stream.close();\n };\n\n return new ReadableStream({\n async start(controller) {\n try {\n for await (const chunk of stream) {\n controller.enqueue(chunk);\n }\n controller.close();\n } finally {\n cleanup();\n }\n },\n cancel() {\n cleanup();\n },\n });\n },\n\n async ttsNode(\n agent: Agent,\n text: ReadableStream<string>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n const activity = agent.getActivityOrThrow();\n if (!activity.tts) {\n throw new Error('ttsNode called but no TTS node is available');\n }\n\n let wrapped_tts = activity.tts;\n\n if (!activity.tts.capabilities.streaming) {\n wrapped_tts = new TTSStreamAdapter(wrapped_tts, new BasicSentenceTokenizer());\n }\n\n const connOptions = activity.agentSession.connOptions.ttsConnOptions;\n const stream = wrapped_tts.stream({ connOptions });\n stream.updateInputStream(text);\n\n let cleaned = false;\n const cleanup = () => {\n if (cleaned) return;\n cleaned = true;\n stream.close();\n };\n\n return new ReadableStream({\n async start(controller) {\n try {\n for await (const chunk of stream) {\n if (chunk === SynthesizeStream.END_OF_STREAM) {\n break;\n }\n controller.enqueue(chunk.frame);\n }\n controller.close();\n } finally {\n cleanup();\n }\n },\n cancel() {\n cleanup();\n },\n });\n },\n\n async transcriptionNode(\n agent: Agent,\n text: ReadableStream<string>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<string> | null> {\n return text;\n },\n\n async realtimeAudioOutputNode(\n _agent: Agent,\n audio: ReadableStream<AudioFrame>,\n _modelSettings: ModelSettings,\n ): Promise<ReadableStream<AudioFrame> | null> {\n return audio;\n },\n };\n}\n"],"mappings":"AAIA,SAAS,yBAAyB;AAClC,SAAS,sBAAsB;AAC/B;AAAA,EACE,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,OAIF;AACP,SAAS,2BAA2B;AAEpC;AAAA,EAEE;AAAA,EACA;AAAA,OAGK;AAEP,SAAS,iBAAiB,wBAAwB;AAClD,SAAS,qBAAqB,8BAA8B;AAE5D,SAAS,kBAAkB,iBAAiB,wBAAwB;AAK7D,MAAM,oBAAoB,IAAI,kBAAmD;AACjF,MAAM,uBAAuB,OAAO,cAAc;AAElD,MAAM,qBAAqB,MAAM;AAAA,EACtC,cAAc;AACZ,UAAM;AACN,SAAK,OAAO;AAEZ,WAAO,eAAe,MAAM,sBAAsB;AAAA,MAChD,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AACF;AAEO,SAAS,eAAe,OAAuC;AACpE,SACE,UAAU,UACV,UAAU,QACV,OAAO,UAAU,YACjB,wBAAwB;AAE5B;AAqBO,MAAM,MAAsB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGR;AAAA;AAAA,EAGA;AAAA;AAAA,EAGA;AAAA;AAAA,EAGA;AAAA,EAEA,YAAY;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAA2B;AACzB,QAAI,IAAI;AACN,WAAK,MAAM;AAAA,IACb,OAAO;AAEL,YAAM,YAAY,KAAK,YAAY;AACnC,UAAI,cAAc,SAAS;AACzB,aAAK,MAAM;AAAA,MACb,OAAO;AACL,aAAK,MAAM,UACR,QAAQ,YAAY,KAAK,EACzB,YAAY,EACZ,QAAQ,MAAM,EAAE;AAAA,MACrB;AAAA,IACF;AAEA,SAAK,gBAAgB;AACrB,SAAK,SAAS,EAAE,GAAG,MAAM;AACzB,SAAK,WAAW,UACZ,QAAQ,KAAK;AAAA,MACX,SAAS,KAAK;AAAA,IAChB,CAAC,IACD,YAAY,MAAM;AAEtB,SAAK,gBAAgB;AACrB,SAAK,OAAO;AAEZ,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,aAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,aAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,OAAO,aAAa,gBAAgB,GAAG;AAAA,IAC9C,OAAO;AACL,WAAK,OAAO;AAAA,IACd;AAEA,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuC;AACzC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,MAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAA+B;AACjC,WAAO,IAAI,oBAAoB,KAAK,SAAS,KAAK;AAAA,EACpD;AAAA,EAEA,IAAI,KAAa;AACf,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAiC;AACnC,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,IAAI,UAAkC;AACpC,WAAO,KAAK,mBAAmB,EAAE;AAAA,EACnC;AAAA,EAEA,MAAM,UAAyB;AAAA,EAAC;AAAA,EAEhC,MAAM,SAAwB;AAAA,EAAC;AAAA,EAE/B,MAAM,kBACJ,MACA,eACwC;AACxC,WAAO,MAAM,QAAQ,kBAAkB,MAAM,MAAM,aAAa;AAAA,EAClE;AAAA,EAEA,MAAM,oBAAoB,UAAuB,aAAyC;AAAA,EAAC;AAAA,EAE3F,MAAM,QACJ,OACA,eACsD;AACtD,WAAO,MAAM,QAAQ,QAAQ,MAAM,OAAO,aAAa;AAAA,EACzD;AAAA,EAEA,MAAM,QACJ,SACA,SACA,eACoD;AACpD,WAAO,MAAM,QAAQ,QAAQ,MAAM,SAAS,SAAS,aAAa;AAAA,EACpE;AAAA,EAEA,MAAM,QACJ,MACA,eAC4C;AAC5C,WAAO,MAAM,QAAQ,QAAQ,MAAM,MAAM,aAAa;AAAA,EACxD;AAAA,EAEA,MAAM,wBACJ,OACA,eAC4C;AAC5C,WAAO,MAAM,QAAQ,wBAAwB,MAAM,OAAO,aAAa;AAAA,EACzE;AAAA;AAAA,EAIA,qBAAoC;AAClC,QAAI,CAAC,KAAK,gBAAgB;AACxB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,cAAc,SAAqC;AACvD,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,WAAW,QAAQ,KAAK,EAAE,SAAS,KAAK,QAAQ,CAAC;AACtD;AAAA,IACF;AAEA,SAAK,eAAe,cAAc,OAAO;AAAA,EAC3C;AAAA,EAEA,OAAO,UAAU;AAAA,IACf,MAAM,QACJ,OACA,OACA,gBACsD;AACtD,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,cAAc,SAAS;AAE3B,UAAI,CAAC,YAAY,aAAa,WAAW;AACvC,cAAM,MAAM,MAAM,OAAO,SAAS;AAClC,YAAI,CAAC,KAAK;AACR,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AACA,sBAAc,IAAI,iBAAiB,aAAa,GAAG;AAAA,MACrD;AAEA,YAAM,cAAc,SAAS,aAAa,YAAY;AACtD,YAAM,SAAS,YAAY,OAAO,EAAE,YAAY,CAAC;AACjD,aAAO,kBAAkB,KAAK;AAE9B,UAAI,UAAU;AACd,YAAM,UAAU,MAAM;AACpB,YAAI,QAAS;AACb,kBAAU;AACV,eAAO,kBAAkB;AACzB,eAAO,MAAM;AAAA,MACf;AAEA,aAAO,IAAI,eAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,cAAI;AACF,6BAAiB,SAAS,QAAQ;AAChC,yBAAW,QAAQ,KAAK;AAAA,YAC1B;AACA,uBAAW,MAAM;AAAA,UACnB,UAAE;AAEA,oBAAQ;AAAA,UACV;AAAA,QACF;AAAA,QACA,SAAS;AACP,kBAAQ;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,QACJ,OACA,SACA,SACA,eACoD;AACpD,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,EAAE,SAAS,eAAe,MAAM;AAClC,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAGA,YAAM,EAAE,WAAW,IAAI;AACvB,YAAM,cAAc,SAAS,aAAa,YAAY;AAEtD,YAAM,SAAS,SAAS,IAAI,KAAK;AAAA,QAC/B;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,mBAAmB;AAAA,MACrB,CAAC;AAED,UAAI,UAAU;AACd,YAAM,UAAU,MAAM;AACpB,YAAI,QAAS;AACb,kBAAU;AACV,eAAO,MAAM;AAAA,MACf;AAEA,aAAO,IAAI,eAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,cAAI;AACF,6BAAiB,SAAS,QAAQ;AAChC,yBAAW,QAAQ,KAAK;AAAA,YAC1B;AACA,uBAAW,MAAM;AAAA,UACnB,UAAE;AACA,oBAAQ;AAAA,UACV;AAAA,QACF;AAAA,QACA,SAAS;AACP,kBAAQ;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,QACJ,OACA,MACA,gBAC4C;AAC5C,YAAM,WAAW,MAAM,mBAAmB;AAC1C,UAAI,CAAC,SAAS,KAAK;AACjB,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAEA,UAAI,cAAc,SAAS;AAE3B,UAAI,CAAC,SAAS,IAAI,aAAa,WAAW;AACxC,sBAAc,IAAI,iBAAiB,aAAa,IAAI,uBAAuB,CAAC;AAAA,MAC9E;AAEA,YAAM,cAAc,SAAS,aAAa,YAAY;AACtD,YAAM,SAAS,YAAY,OAAO,EAAE,YAAY,CAAC;AACjD,aAAO,kBAAkB,IAAI;AAE7B,UAAI,UAAU;AACd,YAAM,UAAU,MAAM;AACpB,YAAI,QAAS;AACb,kBAAU;AACV,eAAO,MAAM;AAAA,MACf;AAEA,aAAO,IAAI,eAAe;AAAA,QACxB,MAAM,MAAM,YAAY;AACtB,cAAI;AACF,6BAAiB,SAAS,QAAQ;AAChC,kBAAI,UAAU,iBAAiB,eAAe;AAC5C;AAAA,cACF;AACA,yBAAW,QAAQ,MAAM,KAAK;AAAA,YAChC;AACA,uBAAW,MAAM;AAAA,UACnB,UAAE;AACA,oBAAQ;AAAA,UACV;AAAA,QACF;AAAA,QACA,SAAS;AACP,kBAAQ;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,kBACJ,OACA,MACA,gBACwC;AACxC,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,wBACJ,QACA,OACA,gBAC4C;AAC5C,aAAO;AAAA,IACT;AAAA,EACF;AACF;","names":[]}
|
|
@@ -1687,15 +1687,12 @@ ${instructions}` : instructions,
|
|
|
1687
1687
|
}
|
|
1688
1688
|
if (this.stt instanceof import_stt.STT) {
|
|
1689
1689
|
this.stt.off("metrics_collected", this.onMetricsCollected);
|
|
1690
|
-
await this.stt.close();
|
|
1691
1690
|
}
|
|
1692
1691
|
if (this.tts instanceof import_tts.TTS) {
|
|
1693
1692
|
this.tts.off("metrics_collected", this.onMetricsCollected);
|
|
1694
|
-
await this.tts.close();
|
|
1695
1693
|
}
|
|
1696
1694
|
if (this.vad instanceof import_vad.VAD) {
|
|
1697
1695
|
this.vad.off("metrics_collected", this.onMetricsCollected);
|
|
1698
|
-
await this.vad.close();
|
|
1699
1696
|
}
|
|
1700
1697
|
this.detachAudioInput();
|
|
1701
1698
|
await ((_a = this.realtimeSession) == null ? void 0 : _a.close());
|