@livekit/agents 1.0.5 → 1.0.7

package/dist/voice/agent_session.d.cts

@@ -2,6 +2,7 @@
 import type { AudioFrame, Room } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import type { ReadableStream } from 'node:stream/web';
+import { type LLMModels, type STTModelString, type TTSModelString } from '../inference/index.js';
 import { ChatContext, ChatMessage } from '../llm/chat_context.js';
 import type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
@@ -39,10 +40,10 @@ export type AgentSessionCallbacks = {
 };
 export type AgentSessionOptions<UserData = UnknownUserData> = {
     turnDetection?: TurnDetectionMode;
-    stt?: STT;
+    stt?: STT | STTModelString;
     vad?: VAD;
-    llm?: LLM | RealtimeModel;
-    tts?: TTS;
+    llm?: LLM | RealtimeModel | LLMModels;
+    tts?: TTS | TTSModelString;
     userData?: UserData;
     voiceOptions?: Partial<VoiceOptions>;
 };

package/dist/voice/agent_session.d.ts

@@ -2,6 +2,7 @@
 import type { AudioFrame, Room } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import type { ReadableStream } from 'node:stream/web';
+import { type LLMModels, type STTModelString, type TTSModelString } from '../inference/index.js';
 import { ChatContext, ChatMessage } from '../llm/chat_context.js';
 import type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
@@ -39,10 +40,10 @@ export type AgentSessionCallbacks = {
 };
 export type AgentSessionOptions<UserData = UnknownUserData> = {
     turnDetection?: TurnDetectionMode;
-    stt?: STT;
+    stt?: STT | STTModelString;
     vad?: VAD;
-    llm?: LLM | RealtimeModel;
-    tts?: TTS;
+    llm?: LLM | RealtimeModel | LLMModels;
+    tts?: TTS | TTSModelString;
     userData?: UserData;
     voiceOptions?: Partial<VoiceOptions>;
 };

package/dist/voice/agent_session.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent_session.d.ts","sourceRoot":"","sources":["../../src/voice/agent_session.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EACL,sBAAsB,EACtB,KAAK,UAAU,EACf,KAAK,sBAAsB,EAC3B,KAAK,UAAU,EACf,WAAW,EACX,KAAK,0BAA0B,EAC/B,KAAK,UAAU,EACf,KAAK,0BAA0B,EAC/B,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,SAAS,EACd,KAAK,qBAAqB,EAK3B,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAU,KAAK,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,YAAY;IAC3B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,6BAA6B,EAAE,OAAO,CAAC;IACvC,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;CACtB;AAYD,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,cAAc,GAAG,QAAQ,GAAG,aAAa,CAAC;AAE1F,MAAM,MAAM,qBAAqB,GAAG;IAClC,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAAE,yBAAyB,KAAK,IAAI,CAAC;IACvF,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,EAAE,CAAC,EAAE,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACjF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACzE,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;IACzD,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;CAC1D,CAAC;AAEF,MAAM,MAAM,mBAAmB,CAAC,QAAQ,GAAG,eAAe,IAAI;IAC5D,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,CAAC;IAC1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;CACtC,CAAC;2CAImC,aAAa,qBAAqB,CAAC;AAFxE,qBAAa,YAAY,CACvB,QAAQ,GAAG,eAAe,CAC1B,SAAQ,iBAA+D;IACvE,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,CAAC;IAC1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAElC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAE/B,OAAO,CAAC,KAAK,CAAC,CAAQ;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAgB;IACjC,OAAO,CAAC,YAAY,CAAC,CAAgB;IACrC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,SAAS,CAA0B;IAE3C,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IAEvB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,WAAW,CAA8B;IAEjD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,WAAW,CAA8B;gBAErC,IAAI,EAAE,mBAAmB,CAAC,QAAQ,CAAC;IA6B/C,IAAI,KAAK,IAAI,UAAU,CAEtB;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,IAAI,QAAQ,IAAI,QAAQ,CAMvB;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAEK,KAAK,CAAC,EACV,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,aAAa,GACd,EAAE;QACD,KAAK,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,IAAI,CAAC;QACX,YAAY,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACzC,aAAa,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAC5C,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDjB,WAAW,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAQ/B,cAAc;IAQd,aAAa;IAOb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAQf,SAAS;IAOT,aAAa,CAAC,OAAO,CAAC,EAAE;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,kBAAkB,CAAC,EAAE,OAAO,CAAC;KAC9B,GAAG,YAAY;YAsBF,cAAc;IAmB5B,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,UAAU,IAAI,UAAU,CAE3B;IAED,IAAI,YAAY,IAAI,KAAK,CAMxB;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,gBAAgB;IAChB,UAAU,CAAC,EACT,MAAM,EACN,KAAa,EACb,KAAY,GACb,EAAE;QACD,MAAM,EAAE,WAAW,CAAC;QACpB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,KAAK,CAAC,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;KACpE,GAAG,IAAI;IAOR,gBAAgB;IAChB,QAAQ,CAAC,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI;IAc1E,gBAAgB;IAChB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI;IAK/C,gBAAgB;IAChB,iBAAiB,CAAC,KAAK,EAAE,UAAU;IAanC,gBAAgB;IAChB,gBAAgB,CAAC,KAAK,EAAE,SAAS;IAcjC,OAAO,CAAC,mBAAmB;IAU3B,OAAO,CAAC,oBAAoB;IAE5B,OAAO,CAAC,mBAAmB;YAEb,SAAS;CA6CxB"}
+{"version":3,"file":"agent_session.d.ts","sourceRoot":"","sources":["../../src/voice/agent_session.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAIL,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,cAAc,EACpB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAClE,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EACL,sBAAsB,EACtB,KAAK,UAAU,EACf,KAAK,sBAAsB,EAC3B,KAAK,UAAU,EACf,WAAW,EACX,KAAK,0BAA0B,EAC/B,KAAK,UAAU,EACf,KAAK,0BAA0B,EAC/B,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,yBAAyB,EAC9B,KAAK,SAAS,EACd,KAAK,qBAAqB,EAK3B,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAU,KAAK,gBAAgB,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,YAAY;IAC3B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,6BAA6B,EAAE,OAAO,CAAC;IACvC,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;CACtB;AAYD,MAAM,MAAM,iBAAiB,GAAG,KAAK,GAAG,KAAK,GAAG,cAAc,GAAG,QAAQ,GAAG,aAAa,CAAC;AAE1F,MAAM,MAAM,qBAAqB,GAAG;IAClC,CAAC,sBAAsB,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAAE,yBAAyB,KAAK,IAAI,CAAC;IACvF,CAAC,sBAAsB,CAAC,iBAAiB,CAAC,EAAE,CAAC,EAAE,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACjF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,qBAAqB,CAAC,EAAE,CAAC,EAAE,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACzF,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,EAAE,CAAC,EAAE,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,CAAC,sBAAsB,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACzE,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;IACzD,CAAC,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,UAAU,KAAK,IAAI,CAAC;CAC1D,CAAC;AAEF,MAAM,MAAM,mBAAmB,CAAC,QAAQ,GAAG,eAAe,IAAI;IAC5D,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAClC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,GAAG,SAAS,CAAC;IACtC,GAAG,CAAC,EAAE,GAAG,GAAG,cAAc,CAAC;IAC3B,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;CACtC,CAAC;2CAImC,aAAa,qBAAqB,CAAC;AAFxE,qBAAa,YAAY,CACvB,QAAQ,GAAG,eAAe,CAC1B,SAAQ,iBAA+D;IACvE,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,CAAC;IAC1B,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAElC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAE/B,OAAO,CAAC,KAAK,CAAC,CAAQ;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAgB;IACjC,OAAO,CAAC,YAAY,CAAC,CAAgB;IACrC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,SAAS,CAA0B;IAE3C,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IAEvB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,WAAW,CAA8B;IAEjD,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,OAAO,CAAc;IAE7B,OAAO,CAAC,WAAW,CAA8B;gBAErC,IAAI,EAAE,mBAAmB,CAAC,QAAQ,CAAC;IA6C/C,IAAI,KAAK,IAAI,UAAU,CAEtB;IAED,IAAI,MAAM,IAAI,WAAW,CAExB;IAED,IAAI,QAAQ,IAAI,QAAQ,CAMvB;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAE3B;IAEK,KAAK,CAAC,EACV,KAAK,EACL,IAAI,EACJ,YAAY,EACZ,aAAa,GACd,EAAE;QACD,KAAK,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,IAAI,CAAC;QACX,YAAY,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;QACzC,aAAa,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAC5C,GAAG,OAAO,CAAC,IAAI,CAAC;IAyDjB,WAAW,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAQ/B,cAAc;IAQd,aAAa;IAOb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAQf,SAAS;IAOT,aAAa,CAAC,OAAO,CAAC,EAAE;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,kBAAkB,CAAC,EAAE,OAAO,CAAC;KAC9B,GAAG,YAAY;YAsBF,cAAc;IAmB5B,IAAI,OAAO,IAAI,WAAW,CAEzB;IAED,IAAI,UAAU,IAAI,UAAU,CAE3B;IAED,IAAI,YAAY,IAAI,KAAK,CAMxB;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,gBAAgB;IAChB,UAAU,CAAC,EACT,MAAM,EACN,KAAa,EACb,KAAY,GACb,EAAE;QACD,MAAM,EAAE,WAAW,CAAC;QACpB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,KAAK,CAAC,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;KACpE,GAAG,IAAI;IAOR,gBAAgB;IAChB,QAAQ,CAAC,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI;IAc1E,gBAAgB;IAChB,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI;IAK/C,gBAAgB;IAChB,iBAAiB,CAAC,KAAK,EAAE,UAAU;IAanC,gBAAgB;IAChB,gBAAgB,CAAC,KAAK,EAAE,SAAS;IAcjC,OAAO,CAAC,mBAAmB;IAU3B,OAAO,CAAC,oBAAoB;IAE5B,OAAO,CAAC,mBAAmB;YAEb,SAAS;CA6CxB"}

package/dist/voice/agent_session.js

@@ -1,4 +1,9 @@
 import { EventEmitter } from "node:events";
+import {
+  LLM as InferenceLLM,
+  STT as InferenceSTT,
+  TTS as InferenceTTS
+} from "../inference/index.js";
 import { getJobContext } from "../job.js";
 import { ChatContext, ChatMessage } from "../llm/chat_context.js";
 import { log } from "../log.js";
@@ -54,9 +59,21 @@ class AgentSession extends EventEmitter {
       voiceOptions = defaultVoiceOptions
     } = opts;
     this.vad = vad;
-    this.stt = stt;
-    this.llm = llm;
-    this.tts = tts;
+    if (typeof stt === "string") {
+      this.stt = InferenceSTT.fromModelString(stt);
+    } else {
+      this.stt = stt;
+    }
+    if (typeof llm === "string") {
+      this.llm = InferenceLLM.fromModelString(llm);
+    } else {
+      this.llm = llm;
+    }
+    if (typeof tts === "string") {
+      this.tts = InferenceTTS.fromModelString(tts);
+    } else {
+      this.tts = tts;
+    }
     this.turnDetection = turnDetection;
     this._userData = userData;
     this._input = new AgentInput(this.onAudioInputChanged);

package/dist/voice/agent_session.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/agent_session.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, Room } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { getJobContext } from '../job.js';\nimport { ChatContext, ChatMessage } from '../llm/chat_context.js';\nimport type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport { log } from '../log.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS, TTSError } from '../tts/tts.js';\nimport type { VAD } from '../vad.js';\nimport type { Agent } from './agent.js';\nimport { AgentActivity } from './agent_activity.js';\nimport type { _TurnDetector } from './audio_recognition.js';\nimport {\n  AgentSessionEventTypes,\n  type AgentState,\n  type AgentStateChangedEvent,\n  type CloseEvent,\n  CloseReason,\n  type ConversationItemAddedEvent,\n  type ErrorEvent,\n  type FunctionToolsExecutedEvent,\n  type MetricsCollectedEvent,\n  type SpeechCreatedEvent,\n  type UserInputTranscribedEvent,\n  type UserState,\n  type UserStateChangedEvent,\n  createAgentStateChangedEvent,\n  createCloseEvent,\n  createConversationItemAddedEvent,\n  createUserStateChangedEvent,\n} from './events.js';\nimport { AgentInput, AgentOutput } from './io.js';\nimport { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';\nimport type { UnknownUserData } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport interface VoiceOptions {\n  allowInterruptions: boolean;\n  discardAudioIfUninterruptible: boolean;\n  minInterruptionDuration: number;\n  minInterruptionWords: number;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n  maxToolSteps: number;\n}\n\nconst defaultVoiceOptions: VoiceOptions = {\n  allowInterruptions: true,\n  discardAudioIfUninterruptible: true,\n  minInterruptionDuration: 500,\n  minInterruptionWords: 0,\n  minEndpointingDelay: 500,\n  maxEndpointingDelay: 6000,\n  maxToolSteps: 3,\n} as const;\n\nexport type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;\n\nexport type AgentSessionCallbacks = {\n  [AgentSessionEventTypes.UserInputTranscribed]: (ev: UserInputTranscribedEvent) => void;\n  [AgentSessionEventTypes.AgentStateChanged]: (ev: AgentStateChangedEvent) => void;\n  [AgentSessionEventTypes.UserStateChanged]: (ev: UserStateChangedEvent) => void;\n  [AgentSessionEventTypes.ConversationItemAdded]: (ev: ConversationItemAddedEvent) => void;\n  [AgentSessionEventTypes.FunctionToolsExecuted]: (ev: FunctionToolsExecutedEvent) => void;\n  [AgentSessionEventTypes.MetricsCollected]: (ev: MetricsCollectedEvent) => void;\n  [AgentSessionEventTypes.SpeechCreated]: (ev: SpeechCreatedEvent) => void;\n  [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void;\n  [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void;\n};\n\nexport type AgentSessionOptions<UserData = UnknownUserData> = {\n  turnDetection?: TurnDetectionMode;\n  stt?: STT;\n  vad?: VAD;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  userData?: UserData;\n  voiceOptions?: Partial<VoiceOptions>;\n};\n\nexport class AgentSession<\n  UserData = UnknownUserData,\n> extends (EventEmitter as new () => TypedEmitter<AgentSessionCallbacks>) {\n  vad?: VAD;\n  stt?: STT;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  turnDetection?: TurnDetectionMode;\n\n  readonly options: VoiceOptions;\n\n  private agent?: Agent;\n  private activity?: AgentActivity;\n  private nextActivity?: AgentActivity;\n  private started = false;\n  private userState: UserState = 'listening';\n\n  private roomIO?: RoomIO;\n  private logger = log();\n\n  private _chatCtx: ChatContext;\n  private _userData: UserData | undefined;\n  private _agentState: AgentState = 'initializing';\n\n  private _input: AgentInput;\n  private _output: AgentOutput;\n\n  private closingTask: Promise<void> | null = null;\n\n  constructor(opts: AgentSessionOptions<UserData>) {\n    super();\n\n    const {\n      vad,\n      stt,\n      llm,\n      tts,\n      turnDetection,\n      userData,\n      voiceOptions = defaultVoiceOptions,\n    } = opts;\n\n    this.vad = vad;\n    this.stt = stt;\n    this.llm = llm;\n    this.tts = tts;\n    this.turnDetection = turnDetection;\n    this._userData = userData;\n\n    // configurable IO\n    this._input = new AgentInput(this.onAudioInputChanged);\n    this._output = new AgentOutput(this.onAudioOutputChanged, this.onTextOutputChanged);\n\n    // This is the \"global\" chat context, it holds the entire conversation history\n    this._chatCtx = ChatContext.empty();\n    this.options = { ...defaultVoiceOptions, ...voiceOptions };\n  }\n\n  get input(): AgentInput {\n    return this._input;\n  }\n\n  get output(): AgentOutput {\n    return this._output;\n  }\n\n  get userData(): UserData {\n    if (this._userData === undefined) {\n      throw new Error('Voice agent userData is not set');\n    }\n\n    return this._userData;\n  }\n\n  get history(): ChatContext {\n    return this._chatCtx;\n  }\n\n  set userData(value: UserData) {\n    this._userData = value;\n  }\n\n  async start({\n    agent,\n    room,\n    inputOptions,\n    outputOptions,\n  }: {\n    agent: Agent;\n    room: Room;\n    inputOptions?: Partial<RoomInputOptions>;\n    outputOptions?: Partial<RoomOutputOptions>;\n  }): Promise<void> {\n    if (this.started) {\n      return;\n    }\n\n    this.agent = agent;\n    this._updateAgentState('initializing');\n\n    const tasks: Promise<void>[] = [];\n    // Check for existing input/output configuration and warn if needed\n    if (this.input.audio && inputOptions?.audioEnabled !== false) {\n      this.logger.warn('RoomIO audio input is enabled but input.audio is already set, ignoring..');\n    }\n\n    if (this.output.audio && outputOptions?.audioEnabled !== false) {\n      this.logger.warn(\n        'RoomIO audio output is enabled but output.audio is already set, ignoring..',\n      );\n    }\n\n    if (this.output.transcription && outputOptions?.transcriptionEnabled !== false) {\n      this.logger.warn(\n        'RoomIO transcription output is enabled but output.transcription is already set, ignoring..',\n      );\n    }\n\n    this.roomIO = new RoomIO({\n      agentSession: this,\n      room,\n      inputOptions,\n      outputOptions,\n    });\n    this.roomIO.start();\n\n    const ctx = getJobContext();\n    if (ctx && ctx.room === room && !room.isConnected) {\n      this.logger.debug('Auto-connecting to room via job context');\n      tasks.push(ctx.connect());\n    }\n    // TODO(AJS-265): add shutdown callback to job context\n    tasks.push(this.updateActivity(this.agent));\n\n    await Promise.allSettled(tasks);\n\n    // Log used IO configuration\n    this.logger.debug(\n      `using audio io: ${this.input.audio ? '`' + this.input.audio.constructor.name + '`' : '(none)'} -> \\`AgentSession\\` -> ${this.output.audio ? '`' + this.output.audio.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug(\n      `using transcript io: \\`AgentSession\\` -> ${this.output.transcription ? '`' + this.output.transcription.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.started = true;\n    this._updateAgentState('listening');\n  }\n\n  updateAgent(agent: Agent): void {\n    this.agent = agent;\n\n    if (this.started) {\n      this.updateActivity(agent);\n    }\n  }\n\n  commitUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    this.activity.commitUserTurn();\n  }\n\n  clearUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    this.activity.clearUserTurn();\n  }\n\n  say(\n    text: string | ReadableStream<string>,\n    options?: {\n      audio?: ReadableStream<AudioFrame>;\n      allowInterruptions?: boolean;\n      addToChatCtx?: boolean;\n    },\n  ): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.activity.say(text, options);\n  }\n\n  interrupt() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    return this.activity.interrupt();\n  }\n\n  generateReply(options?: {\n    userInput?: string;\n    instructions?: string;\n    toolChoice?: ToolChoice;\n    allowInterruptions?: boolean;\n  }): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    const userMessage = options?.userInput\n      ? new ChatMessage({\n          role: 'user',\n          content: options.userInput,\n        })\n      : undefined;\n\n    if (this.activity.draining) {\n      if (!this.nextActivity) {\n        throw new Error('AgentSession is closing, cannot use generateReply()');\n      }\n      return this.nextActivity.generateReply({ userMessage, ...options });\n    }\n\n    return this.activity.generateReply({ userMessage, ...options });\n  }\n\n  private async updateActivity(agent: Agent): Promise<void> {\n    // TODO(AJS-129): add lock to agent activity core lifecycle\n    this.nextActivity = new AgentActivity(agent, this);\n\n    if (this.activity) {\n      await this.activity.drain();\n      await this.activity.close();\n    }\n\n    this.activity = this.nextActivity;\n    this.nextActivity = undefined;\n\n    await this.activity.start();\n\n    if (this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  get chatCtx(): ChatContext {\n    return this._chatCtx.copy();\n  }\n\n  get agentState(): AgentState {\n    return this._agentState;\n  }\n\n  get currentAgent(): Agent {\n    if (!this.agent) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.agent;\n  }\n\n  async close(): Promise<void> {\n    await this.closeImpl(CloseReason.USER_INITIATED);\n  }\n\n  /** @internal */\n  _closeSoon({\n    reason,\n    drain = false,\n    error = null,\n  }: {\n    reason: CloseReason;\n    drain?: boolean;\n    error?: RealtimeModelError | STTError | TTSError | LLMError | null;\n  }): void {\n    if (this.closingTask) {\n      return;\n    }\n    this.closeImpl(reason, error, drain);\n  }\n\n  /** @internal */\n  _onError(error: RealtimeModelError | STTError | TTSError | LLMError): void {\n    if (this.closingTask || error.recoverable) {\n      return;\n    }\n\n    this.logger.error(error, 'AgentSession is closing due to unrecoverable error');\n\n    this.closingTask = (async () => {\n      await this.closeImpl(CloseReason.ERROR, error);\n    })().then(() => {\n      this.closingTask = null;\n    });\n  }\n\n  /** @internal */\n  _conversationItemAdded(item: ChatMessage): void {\n    this._chatCtx.insert(item);\n    this.emit(AgentSessionEventTypes.ConversationItemAdded, createConversationItemAddedEvent(item));\n  }\n\n  /** @internal */\n  _updateAgentState(state: AgentState) {\n    if (this._agentState === state) {\n      return;\n    }\n\n    const oldState = this._agentState;\n    this._agentState = state;\n    this.emit(\n      AgentSessionEventTypes.AgentStateChanged,\n      createAgentStateChangedEvent(oldState, state),\n    );\n  }\n\n  /** @internal */\n  _updateUserState(state: UserState) {\n    if (this.userState === state) {\n      return;\n    }\n\n    const oldState = this.userState;\n    this.userState = state;\n    this.emit(\n      AgentSessionEventTypes.UserStateChanged,\n      createUserStateChangedEvent(oldState, state),\n    );\n  }\n\n  // -- User changed input/output streams/sinks --\n  private onAudioInputChanged(): void {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity && this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  private onAudioOutputChanged(): void {}\n\n  private onTextOutputChanged(): void {}\n\n  private async closeImpl(\n    reason: CloseReason,\n    error: RealtimeModelError | LLMError | TTSError | STTError | null = null,\n    drain: boolean = false,\n  ): Promise<void> {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity) {\n      if (!drain) {\n        try {\n          this.activity.interrupt();\n        } catch (error) {\n          // uninterruptible speech [copied from python]\n          // TODO(shubhra): force interrupt or wait for it to finish?\n          // it might be an audio played from the error callback\n        }\n      }\n      await this.activity.drain();\n      // wait any uninterruptible speech to finish\n      await this.activity.currentSpeech?.waitForPlayout();\n      this.activity.detachAudioInput();\n    }\n\n    // detach the inputs and outputs\n    this.input.audio = null;\n    this.output.audio = null;\n    this.output.transcription = null;\n\n    await this.roomIO?.close();\n    this.roomIO = undefined;\n\n    await this.activity?.close();\n    this.activity = undefined;\n\n    this.started = false;\n\n    this.emit(AgentSessionEventTypes.Close, createCloseEvent(reason, error));\n\n    this.userState = 'listening';\n    this._agentState = 'initializing';\n\n    this.logger.info({ reason, error }, 'AgentSession closed');\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,qBAAqB;AAC9B,SAAS,aAAa,mBAAmB;AAGzC,SAAS,WAAW;AAMpB,SAAS,qBAAqB;AAE9B;AAAA,EACE;AAAA,EAIA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,YAAY,mBAAmB;AACxC,SAAS,cAA6D;AActE,MAAM,sBAAoC;AAAA,EACxC,oBAAoB;AAAA,EACpB,+BAA+B;AAAA,EAC/B,yBAAyB;AAAA,EACzB,sBAAsB;AAAA,EACtB,qBAAqB;AAAA,EACrB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AA0BO,MAAM,qBAEF,aAA+D;AAAA,EACxE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAES;AAAA,EAED;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV,YAAuB;AAAA,EAEvB;AAAA,EACA,SAAS,IAAI;AAAA,EAEb;AAAA,EACA;AAAA,EACA,cAA0B;AAAA,EAE1B;AAAA,EACA;AAAA,EAEA,cAAoC;AAAA,EAE5C,YAAY,MAAqC;AAC/C,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IACjB,IAAI;AAEJ,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,MAAM;AACX,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAGjB,SAAK,SAAS,IAAI,WAAW,KAAK,mBAAmB;AACrD,SAAK,UAAU,IAAI,YAAY,KAAK,sBAAsB,KAAK,mBAAmB;AAGlF,SAAK,WAAW,YAAY,MAAM;AAClC,SAAK,UAAU,EAAE,GAAG,qBAAqB,GAAG,aAAa;AAAA,EAC3D;AAAA,EAEA,IAAI,QAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,WAAqB;AACvB,QAAI,KAAK,cAAc,QAAW;AAChC,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,OAAiB;AAC5B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,MAAM;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKkB;AAChB,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AAEA,SAAK,QAAQ;AACb,SAAK,kBAAkB,cAAc;AAErC,UAAM,QAAyB,CAAC;AAEhC,QAAI,KAAK,MAAM,UAAS,6CAAc,kBAAiB,OAAO;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAAA,IAC7F;AAEA,QAAI,KAAK,OAAO,UAAS,+CAAe,kBAAiB,OAAO;AAC9D,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,kBAAiB,+CAAe,0BAAyB,OAAO;AAC9E,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,cAAc;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,SAAK,OAAO,MAAM;AAElB,UAAM,MAAM,cAAc;AAC1B,QAAI,OAAO,IAAI,SAAS,QAAQ,CAAC,KAAK,aAAa;AACjD,WAAK,OAAO,MAAM,yCAAyC;AAC3D,YAAM,KAAK,IAAI,QAAQ,CAAC;AAAA,IAC1B;AAEA,UAAM,KAAK,KAAK,eAAe,KAAK,KAAK,CAAC;AAE1C,UAAM,QAAQ,WAAW,KAAK;AAG9B,SAAK,OAAO;AAAA,MACV,mBAAmB,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,YAAY,OAAO,MAAM,QAAQ,2BAA2B,KAAK,OAAO,QAAQ,MAAM,KAAK,OAAO,MAAM,YAAY,OAAO,MAAM,QAAQ;AAAA,IACxM;AAEA,SAAK,OAAO;AAAA,MACV,4CAA4C,KAAK,OAAO,gBAAgB,MAAM,KAAK,OAAO,cAAc,YAAY,OAAO,MAAM,QAAQ;AAAA,IAC3I;AAEA,SAAK,UAAU;AACf,SAAK,kBAAkB,WAAW;AAAA,EACpC;AAAA,EAEA,YAAY,OAAoB;AAC9B,SAAK,QAAQ;AAEb,QAAI,KAAK,SAAS;AAChB,WAAK,eAAe,KAAK;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,iBAAiB;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,SAAK,SAAS,eAAe;AAAA,EAC/B;AAAA,EAEA,gBAAgB;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,SAAK,SAAS,cAAc;AAAA,EAC9B;AAAA,EAEA,IACE,MACA,SAKc;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK,SAAS,IAAI,MAAM,OAAO;AAAA,EACxC;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,WAAO,KAAK,SAAS,UAAU;AAAA,EACjC;AAAA,EAEA,cAAc,SAKG;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,UAAM,eAAc,mCAAS,aACzB,IAAI,YAAY;AAAA,MACd,MAAM;AAAA,MACN,SAAS,QAAQ;AAAA,IACnB,CAAC,IACD;AAEJ,QAAI,KAAK,SAAS,UAAU;AAC1B,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AACA,aAAO,KAAK,aAAa,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,IACpE;AAEA,WAAO,KAAK,SAAS,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,eAAe,OAA6B;AAExD,SAAK,eAAe,IAAI,cAAc,OAAO,IAAI;AAEjD,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,SAAS,MAAM;AAC1B,YAAM,KAAK,SAAS,MAAM;AAAA,IAC5B;AAEA,SAAK,WAAW,KAAK;AACrB,SAAK,eAAe;AAEpB,UAAM,KAAK,SAAS,MAAM;AAE1B,QAAI,KAAK,OAAO,OAAO;AACrB,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,aAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAsB;AACxB,QAAI,CAAC,KAAK,OAAO;AACf,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,UAAU,YAAY,cAAc;AAAA,EACjD;AAAA;AAAA,EAGA,WAAW;AAAA,IACT;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV,GAIS;AACP,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AACA,SAAK,UAAU,QAAQ,OAAO,KAAK;AAAA,EACrC;AAAA;AAAA,EAGA,SAAS,OAAkE;AACzE,QAAI,KAAK,eAAe,MAAM,aAAa;AACzC;AAAA,IACF;AAEA,SAAK,OAAO,MAAM,OAAO,oDAAoD;AAE7E,SAAK,eAAe,YAAY;AAC9B,YAAM,KAAK,UAAU,YAAY,OAAO,KAAK;AAAA,IAC/C,GAAG,EAAE,KAAK,MAAM;AACd,WAAK,cAAc;AAAA,IACrB,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,uBAAuB,MAAyB;AAC9C,SAAK,SAAS,OAAO,IAAI;AACzB,SAAK,KAAK,uBAAuB,uBAAuB,iCAAiC,IAAI,CAAC;AAAA,EAChG;AAAA;AAAA,EAGA,kBAAkB,OAAmB;AACnC,QAAI,KAAK,gBAAgB,OAAO;AAC9B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,cAAc;AACnB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,6BAA6B,UAAU,KAAK;AAAA,IAC9C;AAAA,EACF;AAAA;AAAA,EAGA,iBAAiB,OAAkB;AACjC,QAAI,KAAK,cAAc,OAAO;AAC5B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,YAAY;AACjB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,4BAA4B,UAAU,KAAK;AAAA,IAC7C;AAAA,EACF;AAAA;AAAA,EAGQ,sBAA4B;AAClC,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,YAAY,KAAK,OAAO,OAAO;AACtC,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,uBAA6B;AAAA,EAAC;AAAA,EAE9B,sBAA4B;AAAA,EAAC;AAAA,EAErC,MAAc,UACZ,QACA,QAAoE,MACpE,QAAiB,OACF;AA/anB;AAgbI,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,UAAU;AACjB,UAAI,CAAC,OAAO;AACV,YAAI;AACF,eAAK,SAAS,UAAU;AAAA,QAC1B,SAASA,QAAO;AAAA,QAIhB;AAAA,MACF;AACA,YAAM,KAAK,SAAS,MAAM;AAE1B,cAAM,UAAK,SAAS,kBAAd,mBAA6B;AACnC,WAAK,SAAS,iBAAiB;AAAA,IACjC;AAGA,SAAK,MAAM,QAAQ;AACnB,SAAK,OAAO,QAAQ;AACpB,SAAK,OAAO,gBAAgB;AAE5B,YAAM,UAAK,WAAL,mBAAa;AACnB,SAAK,SAAS;AAEd,YAAM,UAAK,aAAL,mBAAe;AACrB,SAAK,WAAW;AAEhB,SAAK,UAAU;AAEf,SAAK,KAAK,uBAAuB,OAAO,iBAAiB,QAAQ,KAAK,CAAC;AAEvE,SAAK,YAAY;AACjB,SAAK,cAAc;AAEnB,SAAK,OAAO,KAAK,EAAE,QAAQ,MAAM,GAAG,qBAAqB;AAAA,EAC3D;AACF;","names":["error"]}
+{"version":3,"sources":["../../src/voice/agent_session.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame, Room } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport {\n  LLM as InferenceLLM,\n  STT as InferenceSTT,\n  TTS as InferenceTTS,\n  type LLMModels,\n  type STTModelString,\n  type TTSModelString,\n} from '../inference/index.js';\nimport { getJobContext } from '../job.js';\nimport { ChatContext, ChatMessage } from '../llm/chat_context.js';\nimport type { LLM, RealtimeModel, RealtimeModelError, ToolChoice } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport { log } from '../log.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS, TTSError } from '../tts/tts.js';\nimport type { VAD } from '../vad.js';\nimport type { Agent } from './agent.js';\nimport { AgentActivity } from './agent_activity.js';\nimport type { _TurnDetector } from './audio_recognition.js';\nimport {\n  AgentSessionEventTypes,\n  type AgentState,\n  type AgentStateChangedEvent,\n  type CloseEvent,\n  CloseReason,\n  type ConversationItemAddedEvent,\n  type ErrorEvent,\n  type FunctionToolsExecutedEvent,\n  type MetricsCollectedEvent,\n  type SpeechCreatedEvent,\n  type UserInputTranscribedEvent,\n  type UserState,\n  type UserStateChangedEvent,\n  createAgentStateChangedEvent,\n  createCloseEvent,\n  createConversationItemAddedEvent,\n  createUserStateChangedEvent,\n} from './events.js';\nimport { AgentInput, AgentOutput } from './io.js';\nimport { RoomIO, type RoomInputOptions, type RoomOutputOptions } from './room_io/index.js';\nimport type { UnknownUserData } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport interface VoiceOptions {\n  allowInterruptions: boolean;\n  discardAudioIfUninterruptible: boolean;\n  minInterruptionDuration: number;\n  minInterruptionWords: number;\n  minEndpointingDelay: number;\n  maxEndpointingDelay: number;\n  maxToolSteps: number;\n}\n\nconst defaultVoiceOptions: VoiceOptions = {\n  allowInterruptions: true,\n  discardAudioIfUninterruptible: true,\n  minInterruptionDuration: 500,\n  minInterruptionWords: 0,\n  minEndpointingDelay: 500,\n  maxEndpointingDelay: 6000,\n  maxToolSteps: 3,\n} as const;\n\nexport type TurnDetectionMode = 'stt' | 'vad' | 'realtime_llm' | 'manual' | _TurnDetector;\n\nexport type AgentSessionCallbacks = {\n  [AgentSessionEventTypes.UserInputTranscribed]: (ev: UserInputTranscribedEvent) => void;\n  [AgentSessionEventTypes.AgentStateChanged]: (ev: AgentStateChangedEvent) => void;\n  [AgentSessionEventTypes.UserStateChanged]: (ev: UserStateChangedEvent) => void;\n  [AgentSessionEventTypes.ConversationItemAdded]: (ev: ConversationItemAddedEvent) => void;\n  [AgentSessionEventTypes.FunctionToolsExecuted]: (ev: FunctionToolsExecutedEvent) => void;\n  [AgentSessionEventTypes.MetricsCollected]: (ev: MetricsCollectedEvent) => void;\n  [AgentSessionEventTypes.SpeechCreated]: (ev: SpeechCreatedEvent) => void;\n  [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void;\n  [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void;\n};\n\nexport type AgentSessionOptions<UserData = UnknownUserData> = {\n  turnDetection?: TurnDetectionMode;\n  stt?: STT | STTModelString;\n  vad?: VAD;\n  llm?: LLM | RealtimeModel | LLMModels;\n  tts?: TTS | TTSModelString;\n  userData?: UserData;\n  voiceOptions?: Partial<VoiceOptions>;\n};\n\nexport class AgentSession<\n  UserData = UnknownUserData,\n> extends (EventEmitter as new () => TypedEmitter<AgentSessionCallbacks>) {\n  vad?: VAD;\n  stt?: STT;\n  llm?: LLM | RealtimeModel;\n  tts?: TTS;\n  turnDetection?: TurnDetectionMode;\n\n  readonly options: VoiceOptions;\n\n  private agent?: Agent;\n  private activity?: AgentActivity;\n  private nextActivity?: AgentActivity;\n  private started = false;\n  private userState: UserState = 'listening';\n\n  private roomIO?: RoomIO;\n  private logger = log();\n\n  private _chatCtx: ChatContext;\n  private _userData: UserData | undefined;\n  private _agentState: AgentState = 'initializing';\n\n  private _input: AgentInput;\n  private _output: AgentOutput;\n\n  private closingTask: Promise<void> | null = null;\n\n  constructor(opts: AgentSessionOptions<UserData>) {\n    super();\n\n    const {\n      vad,\n      stt,\n      llm,\n      tts,\n      turnDetection,\n      userData,\n      voiceOptions = defaultVoiceOptions,\n    } = opts;\n\n    this.vad = vad;\n\n    if (typeof stt === 'string') {\n      this.stt = InferenceSTT.fromModelString(stt);\n    } else {\n      this.stt = stt;\n    }\n\n    if (typeof llm === 'string') {\n      this.llm = InferenceLLM.fromModelString(llm);\n    } else {\n      this.llm = llm;\n    }\n\n    if (typeof tts === 'string') {\n      this.tts = InferenceTTS.fromModelString(tts);\n    } else {\n      this.tts = tts;\n    }\n\n    this.turnDetection = turnDetection;\n    this._userData = userData;\n\n    // configurable IO\n    this._input = new AgentInput(this.onAudioInputChanged);\n    this._output = new AgentOutput(this.onAudioOutputChanged, this.onTextOutputChanged);\n\n    // This is the \"global\" chat context, it holds the entire conversation history\n    this._chatCtx = ChatContext.empty();\n    this.options = { ...defaultVoiceOptions, ...voiceOptions };\n  }\n\n  get input(): AgentInput {\n    return this._input;\n  }\n\n  get output(): AgentOutput {\n    return this._output;\n  }\n\n  get userData(): UserData {\n    if (this._userData === undefined) {\n      throw new Error('Voice agent userData is not set');\n    }\n\n    return this._userData;\n  }\n\n  get history(): ChatContext {\n    return this._chatCtx;\n  }\n\n  set userData(value: UserData) {\n    this._userData = value;\n  }\n\n  async start({\n    agent,\n    room,\n    inputOptions,\n    outputOptions,\n  }: {\n    agent: Agent;\n    room: Room;\n    inputOptions?: Partial<RoomInputOptions>;\n    outputOptions?: Partial<RoomOutputOptions>;\n  }): Promise<void> {\n    if (this.started) {\n      return;\n    }\n\n    this.agent = agent;\n    this._updateAgentState('initializing');\n\n    const tasks: Promise<void>[] = [];\n    // Check for existing input/output configuration and warn if needed\n    if (this.input.audio && inputOptions?.audioEnabled !== false) {\n      this.logger.warn('RoomIO audio input is enabled but input.audio is already set, ignoring..');\n    }\n\n    if (this.output.audio && outputOptions?.audioEnabled !== false) {\n      this.logger.warn(\n        'RoomIO audio output is enabled but output.audio is already set, ignoring..',\n      );\n    }\n\n    if (this.output.transcription && outputOptions?.transcriptionEnabled !== false) {\n      this.logger.warn(\n        'RoomIO transcription output is enabled but output.transcription is already set, ignoring..',\n      );\n    }\n\n    this.roomIO = new RoomIO({\n      agentSession: this,\n      room,\n      inputOptions,\n      outputOptions,\n    });\n    this.roomIO.start();\n\n    const ctx = getJobContext();\n    if (ctx && ctx.room === room && !room.isConnected) {\n      this.logger.debug('Auto-connecting to room via job context');\n      tasks.push(ctx.connect());\n    }\n    // TODO(AJS-265): add shutdown callback to job context\n    tasks.push(this.updateActivity(this.agent));\n\n    await Promise.allSettled(tasks);\n\n    // Log used IO configuration\n    this.logger.debug(\n      `using audio io: ${this.input.audio ? '`' + this.input.audio.constructor.name + '`' : '(none)'} -> \\`AgentSession\\` -> ${this.output.audio ? '`' + this.output.audio.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.logger.debug(\n      `using transcript io: \\`AgentSession\\` -> ${this.output.transcription ? '`' + this.output.transcription.constructor.name + '`' : '(none)'}`,\n    );\n\n    this.started = true;\n    this._updateAgentState('listening');\n  }\n\n  updateAgent(agent: Agent): void {\n    this.agent = agent;\n\n    if (this.started) {\n      this.updateActivity(agent);\n    }\n  }\n\n  commitUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    this.activity.commitUserTurn();\n  }\n\n  clearUserTurn() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    this.activity.clearUserTurn();\n  }\n\n  say(\n    text: string | ReadableStream<string>,\n    options?: {\n      audio?: ReadableStream<AudioFrame>;\n      allowInterruptions?: boolean;\n      addToChatCtx?: boolean;\n    },\n  ): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.activity.say(text, options);\n  }\n\n  interrupt() {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n    return this.activity.interrupt();\n  }\n\n  generateReply(options?: {\n    userInput?: string;\n    instructions?: string;\n    toolChoice?: ToolChoice;\n    allowInterruptions?: boolean;\n  }): SpeechHandle {\n    if (!this.activity) {\n      throw new Error('AgentSession is not running');\n    }\n\n    const userMessage = options?.userInput\n      ? new ChatMessage({\n          role: 'user',\n          content: options.userInput,\n        })\n      : undefined;\n\n    if (this.activity.draining) {\n      if (!this.nextActivity) {\n        throw new Error('AgentSession is closing, cannot use generateReply()');\n      }\n      return this.nextActivity.generateReply({ userMessage, ...options });\n    }\n\n    return this.activity.generateReply({ userMessage, ...options });\n  }\n\n  private async updateActivity(agent: Agent): Promise<void> {\n    // TODO(AJS-129): add lock to agent activity core lifecycle\n    this.nextActivity = new AgentActivity(agent, this);\n\n    if (this.activity) {\n      await this.activity.drain();\n      await this.activity.close();\n    }\n\n    this.activity = this.nextActivity;\n    this.nextActivity = undefined;\n\n    await this.activity.start();\n\n    if (this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  get chatCtx(): ChatContext {\n    return this._chatCtx.copy();\n  }\n\n  get agentState(): AgentState {\n    return this._agentState;\n  }\n\n  get currentAgent(): Agent {\n    if (!this.agent) {\n      throw new Error('AgentSession is not running');\n    }\n\n    return this.agent;\n  }\n\n  async close(): Promise<void> {\n    await this.closeImpl(CloseReason.USER_INITIATED);\n  }\n\n  /** @internal */\n  _closeSoon({\n    reason,\n    drain = false,\n    error = null,\n  }: {\n    reason: CloseReason;\n    drain?: boolean;\n    error?: RealtimeModelError | STTError | TTSError | LLMError | null;\n  }): void {\n    if (this.closingTask) {\n      return;\n    }\n    this.closeImpl(reason, error, drain);\n  }\n\n  /** @internal */\n  _onError(error: RealtimeModelError | STTError | TTSError | LLMError): void {\n    if (this.closingTask || error.recoverable) {\n      return;\n    }\n\n    this.logger.error(error, 'AgentSession is closing due to unrecoverable error');\n\n    this.closingTask = (async () => {\n      await this.closeImpl(CloseReason.ERROR, error);\n    })().then(() => {\n      this.closingTask = null;\n    });\n  }\n\n  /** @internal */\n  _conversationItemAdded(item: ChatMessage): void {\n    this._chatCtx.insert(item);\n    this.emit(AgentSessionEventTypes.ConversationItemAdded, createConversationItemAddedEvent(item));\n  }\n\n  /** @internal */\n  _updateAgentState(state: AgentState) {\n    if (this._agentState === state) {\n      return;\n    }\n\n    const oldState = this._agentState;\n    this._agentState = state;\n    this.emit(\n      AgentSessionEventTypes.AgentStateChanged,\n      createAgentStateChangedEvent(oldState, state),\n    );\n  }\n\n  /** @internal */\n  _updateUserState(state: UserState) {\n    if (this.userState === state) {\n      return;\n    }\n\n    const oldState = this.userState;\n    this.userState = state;\n    this.emit(\n      AgentSessionEventTypes.UserStateChanged,\n      createUserStateChangedEvent(oldState, state),\n    );\n  }\n\n  // -- User changed input/output streams/sinks --\n  private onAudioInputChanged(): void {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity && this._input.audio) {\n      this.activity.attachAudioInput(this._input.audio.stream);\n    }\n  }\n\n  private onAudioOutputChanged(): void {}\n\n  private onTextOutputChanged(): void {}\n\n  private async closeImpl(\n    reason: CloseReason,\n    error: RealtimeModelError | LLMError | TTSError | STTError | null = null,\n    drain: boolean = false,\n  ): Promise<void> {\n    if (!this.started) {\n      return;\n    }\n\n    if (this.activity) {\n      if (!drain) {\n        try {\n          this.activity.interrupt();\n        } catch (error) {\n          // uninterruptible speech [copied from python]\n          // TODO(shubhra): force interrupt or wait for it to finish?\n          // it might be an audio played from the error callback\n        }\n      }\n      await this.activity.drain();\n      // wait any uninterruptible speech to finish\n      await this.activity.currentSpeech?.waitForPlayout();\n      this.activity.detachAudioInput();\n    }\n\n    // detach the inputs and outputs\n    this.input.audio = null;\n    this.output.audio = null;\n    this.output.transcription = null;\n\n    await this.roomIO?.close();\n    this.roomIO = undefined;\n\n    await this.activity?.close();\n    this.activity = undefined;\n\n    this.started = false;\n\n    this.emit(AgentSessionEventTypes.Close, createCloseEvent(reason, error));\n\n    this.userState = 'listening';\n    this._agentState = 'initializing';\n\n    this.logger.info({ reason, error }, 'AgentSession closed');\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B;AAAA,EACE,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,OAIF;AACP,SAAS,qBAAqB;AAC9B,SAAS,aAAa,mBAAmB;AAGzC,SAAS,WAAW;AAMpB,SAAS,qBAAqB;AAE9B;AAAA,EACE;AAAA,EAIA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,YAAY,mBAAmB;AACxC,SAAS,cAA6D;AActE,MAAM,sBAAoC;AAAA,EACxC,oBAAoB;AAAA,EACpB,+BAA+B;AAAA,EAC/B,yBAAyB;AAAA,EACzB,sBAAsB;AAAA,EACtB,qBAAqB;AAAA,EACrB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AA0BO,MAAM,qBAEF,aAA+D;AAAA,EACxE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAES;AAAA,EAED;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV,YAAuB;AAAA,EAEvB;AAAA,EACA,SAAS,IAAI;AAAA,EAEb;AAAA,EACA;AAAA,EACA,cAA0B;AAAA,EAE1B;AAAA,EACA;AAAA,EAEA,cAAoC;AAAA,EAE5C,YAAY,MAAqC;AAC/C,UAAM;AAEN,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IACjB,IAAI;AAEJ,SAAK,MAAM;AAEX,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,MAAM,aAAa,gBAAgB,GAAG;AAAA,IAC7C,OAAO;AACL,WAAK,MAAM;AAAA,IACb;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,MAAM,aAAa,gBAAgB,GAAG;AAAA,IAC7C,OAAO;AACL,WAAK,MAAM;AAAA,IACb;AAEA,QAAI,OAAO,QAAQ,UAAU;AAC3B,WAAK,MAAM,aAAa,gBAAgB,GAAG;AAAA,IAC7C,OAAO;AACL,WAAK,MAAM;AAAA,IACb;AAEA,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAGjB,SAAK,SAAS,IAAI,WAAW,KAAK,mBAAmB;AACrD,SAAK,UAAU,IAAI,YAAY,KAAK,sBAAsB,KAAK,mBAAmB;AAGlF,SAAK,WAAW,YAAY,MAAM;AAClC,SAAK,UAAU,EAAE,GAAG,qBAAqB,GAAG,aAAa;AAAA,EAC3D;AAAA,EAEA,IAAI,QAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,WAAqB;AACvB,QAAI,KAAK,cAAc,QAAW;AAChC,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS,OAAiB;AAC5B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,MAAM;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAKkB;AAChB,QAAI,KAAK,SAAS;AAChB;AAAA,IACF;AAEA,SAAK,QAAQ;AACb,SAAK,kBAAkB,cAAc;AAErC,UAAM,QAAyB,CAAC;AAEhC,QAAI,KAAK,MAAM,UAAS,6CAAc,kBAAiB,OAAO;AAC5D,WAAK,OAAO,KAAK,0EAA0E;AAAA,IAC7F;AAEA,QAAI,KAAK,OAAO,UAAS,+CAAe,kBAAiB,OAAO;AAC9D,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,QAAI,KAAK,OAAO,kBAAiB,+CAAe,0BAAyB,OAAO;AAC9E,WAAK,OAAO;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,cAAc;AAAA,MACd;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,SAAK,OAAO,MAAM;AAElB,UAAM,MAAM,cAAc;AAC1B,QAAI,OAAO,IAAI,SAAS,QAAQ,CAAC,KAAK,aAAa;AACjD,WAAK,OAAO,MAAM,yCAAyC;AAC3D,YAAM,KAAK,IAAI,QAAQ,CAAC;AAAA,IAC1B;AAEA,UAAM,KAAK,KAAK,eAAe,KAAK,KAAK,CAAC;AAE1C,UAAM,QAAQ,WAAW,KAAK;AAG9B,SAAK,OAAO;AAAA,MACV,mBAAmB,KAAK,MAAM,QAAQ,MAAM,KAAK,MAAM,MAAM,YAAY,OAAO,MAAM,QAAQ,2BAA2B,KAAK,OAAO,QAAQ,MAAM,KAAK,OAAO,MAAM,YAAY,OAAO,MAAM,QAAQ;AAAA,IACxM;AAEA,SAAK,OAAO;AAAA,MACV,4CAA4C,KAAK,OAAO,gBAAgB,MAAM,KAAK,OAAO,cAAc,YAAY,OAAO,MAAM,QAAQ;AAAA,IAC3I;AAEA,SAAK,UAAU;AACf,SAAK,kBAAkB,WAAW;AAAA,EACpC;AAAA,EAEA,YAAY,OAAoB;AAC9B,SAAK,QAAQ;AAEb,QAAI,KAAK,SAAS;AAChB,WAAK,eAAe,KAAK;AAAA,IAC3B;AAAA,EACF;AAAA,EAEA,iBAAiB;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,SAAK,SAAS,eAAe;AAAA,EAC/B;AAAA,EAEA,gBAAgB;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,SAAK,SAAS,cAAc;AAAA,EAC9B;AAAA,EAEA,IACE,MACA,SAKc;AACd,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK,SAAS,IAAI,MAAM,OAAO;AAAA,EACxC;AAAA,EAEA,YAAY;AACV,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AACA,WAAO,KAAK,SAAS,UAAU;AAAA,EACjC;AAAA,EAEA,cAAc,SAKG;AACf,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,UAAM,eAAc,mCAAS,aACzB,IAAI,YAAY;AAAA,MACd,MAAM;AAAA,MACN,SAAS,QAAQ;AAAA,IACnB,CAAC,IACD;AAEJ,QAAI,KAAK,SAAS,UAAU;AAC1B,UAAI,CAAC,KAAK,cAAc;AACtB,cAAM,IAAI,MAAM,qDAAqD;AAAA,MACvE;AACA,aAAO,KAAK,aAAa,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,IACpE;AAEA,WAAO,KAAK,SAAS,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,eAAe,OAA6B;AAExD,SAAK,eAAe,IAAI,cAAc,OAAO,IAAI;AAEjD,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,SAAS,MAAM;AAC1B,YAAM,KAAK,SAAS,MAAM;AAAA,IAC5B;AAEA,SAAK,WAAW,KAAK;AACrB,SAAK,eAAe;AAEpB,UAAM,KAAK,SAAS,MAAM;AAE1B,QAAI,KAAK,OAAO,OAAO;AACrB,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEA,IAAI,UAAuB;AACzB,WAAO,KAAK,SAAS,KAAK;AAAA,EAC5B;AAAA,EAEA,IAAI,aAAyB;AAC3B,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,eAAsB;AACxB,QAAI,CAAC,KAAK,OAAO;AACf,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,QAAuB;AAC3B,UAAM,KAAK,UAAU,YAAY,cAAc;AAAA,EACjD;AAAA;AAAA,EAGA,WAAW;AAAA,IACT;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV,GAIS;AACP,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AACA,SAAK,UAAU,QAAQ,OAAO,KAAK;AAAA,EACrC;AAAA;AAAA,EAGA,SAAS,OAAkE;AACzE,QAAI,KAAK,eAAe,MAAM,aAAa;AACzC;AAAA,IACF;AAEA,SAAK,OAAO,MAAM,OAAO,oDAAoD;AAE7E,SAAK,eAAe,YAAY;AAC9B,YAAM,KAAK,UAAU,YAAY,OAAO,KAAK;AAAA,IAC/C,GAAG,EAAE,KAAK,MAAM;AACd,WAAK,cAAc;AAAA,IACrB,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,uBAAuB,MAAyB;AAC9C,SAAK,SAAS,OAAO,IAAI;AACzB,SAAK,KAAK,uBAAuB,uBAAuB,iCAAiC,IAAI,CAAC;AAAA,EAChG;AAAA;AAAA,EAGA,kBAAkB,OAAmB;AACnC,QAAI,KAAK,gBAAgB,OAAO;AAC9B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,cAAc;AACnB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,6BAA6B,UAAU,KAAK;AAAA,IAC9C;AAAA,EACF;AAAA;AAAA,EAGA,iBAAiB,OAAkB;AACjC,QAAI,KAAK,cAAc,OAAO;AAC5B;AAAA,IACF;AAEA,UAAM,WAAW,KAAK;AACtB,SAAK,YAAY;AACjB,SAAK;AAAA,MACH,uBAAuB;AAAA,MACvB,4BAA4B,UAAU,KAAK;AAAA,IAC7C;AAAA,EACF;AAAA;AAAA,EAGQ,sBAA4B;AAClC,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,YAAY,KAAK,OAAO,OAAO;AACtC,WAAK,SAAS,iBAAiB,KAAK,OAAO,MAAM,MAAM;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,uBAA6B;AAAA,EAAC;AAAA,EAE9B,sBAA4B;AAAA,EAAC;AAAA,EAErC,MAAc,UACZ,QACA,QAAoE,MACpE,QAAiB,OACF;AAvcnB;AAwcI,QAAI,CAAC,KAAK,SAAS;AACjB;AAAA,IACF;AAEA,QAAI,KAAK,UAAU;AACjB,UAAI,CAAC,OAAO;AACV,YAAI;AACF,eAAK,SAAS,UAAU;AAAA,QAC1B,SAASA,QAAO;AAAA,QAIhB;AAAA,MACF;AACA,YAAM,KAAK,SAAS,MAAM;AAE1B,cAAM,UAAK,SAAS,kBAAd,mBAA6B;AACnC,WAAK,SAAS,iBAAiB;AAAA,IACjC;AAGA,SAAK,MAAM,QAAQ;AACnB,SAAK,OAAO,QAAQ;AACpB,SAAK,OAAO,gBAAgB;AAE5B,YAAM,UAAK,WAAL,mBAAa;AACnB,SAAK,SAAS;AAEd,YAAM,UAAK,aAAL,mBAAe;AACrB,SAAK,WAAW;AAEhB,SAAK,UAAU;AAEf,SAAK,KAAK,uBAAuB,OAAO,iBAAiB,QAAQ,KAAK,CAAC;AAEvE,SAAK,YAAY;AACjB,SAAK,cAAc;AAEnB,SAAK,OAAO,KAAK,EAAE,QAAQ,MAAM,GAAG,qBAAqB;AAAA,EAC3D;AACF;","names":["error"]}

package/dist/voice/events.cjs

@@ -67,12 +67,14 @@ const createUserInputTranscribedEvent = ({
   transcript,
   isFinal,
   speakerId = null,
+  language = null,
   createdAt = Date.now()
 }) => ({
   type: "user_input_transcribed",
   transcript,
   isFinal,
   speakerId,
+  language,
   createdAt
 });
 const createMetricsCollectedEvent = ({

package/dist/voice/events.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/events.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n  RealtimeModelError,\n} from '../llm/index.js';\nimport type { LLM, RealtimeModel } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport type { AgentMetrics } from '../metrics/base.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS } from '../tts/index.js';\nimport type { TTSError } from '../tts/tts.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport enum AgentSessionEventTypes {\n  UserInputTranscribed = 'user_input_transcribed',\n  AgentStateChanged = 'agent_state_changed',\n  UserStateChanged = 'user_state_changed',\n  ConversationItemAdded = 'conversation_item_added',\n  FunctionToolsExecuted = 'function_tools_executed',\n  MetricsCollected = 'metrics_collected',\n  SpeechCreated = 'speech_created',\n  Error = 'error',\n  Close = 'close',\n}\n\nexport type UserState = 'speaking' | 'listening' | 'away';\nexport type AgentState = 'initializing' | 'idle' | 'listening' | 'thinking' | 'speaking';\n\nexport enum CloseReason {\n  ERROR = 'error',\n  JOB_SHUTDOWN = 'job_shutdown',\n  PARTICIPANT_DISCONNECTED = 'participant_disconnected',\n  USER_INITIATED = 'user_initiated',\n}\n\nexport type SpeechSource = 'say' | 'generate_reply' | 'tool_response';\n\nexport type UserStateChangedEvent = {\n  type: 'user_state_changed';\n  oldState: UserState;\n  newState: UserState;\n  createdAt: number;\n};\n\nexport const createUserStateChangedEvent = (\n  oldState: UserState,\n  newState: UserState,\n  createdAt: number = Date.now(),\n): UserStateChangedEvent => ({\n  type: 'user_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type AgentStateChangedEvent = {\n  type: 'agent_state_changed';\n  oldState: AgentState;\n  newState: AgentState;\n  createdAt: number;\n};\n\nexport const createAgentStateChangedEvent = (\n  oldState: AgentState,\n  newState: AgentState,\n  createdAt: number = Date.now(),\n): AgentStateChangedEvent => ({\n  type: 'agent_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type UserInputTranscribedEvent = {\n  type: 'user_input_transcribed';\n  transcript: string;\n  isFinal: boolean;\n  speakerId: string | null;\n  createdAt: number;\n};\n\nexport const createUserInputTranscribedEvent = ({\n  transcript,\n  isFinal,\n  speakerId = null,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  isFinal: boolean;\n  speakerId?: string | null;\n  createdAt?: number;\n}): UserInputTranscribedEvent => ({\n  type: 'user_input_transcribed',\n  transcript,\n  isFinal,\n  speakerId,\n  createdAt,\n});\n\nexport type MetricsCollectedEvent = {\n  type: 'metrics_collected';\n  metrics: AgentMetrics;\n  createdAt: number;\n};\n\nexport const createMetricsCollectedEvent = ({\n  metrics,\n  createdAt = Date.now(),\n}: {\n  metrics: AgentMetrics;\n  createdAt?: number;\n}): MetricsCollectedEvent => ({\n  type: 'metrics_collected',\n  metrics,\n  createdAt,\n});\n\nexport type ConversationItemAddedEvent = {\n  type: 'conversation_item_added';\n  item: ChatMessage;\n  createdAt: number;\n};\n\nexport const createConversationItemAddedEvent = (\n  item: ChatMessage,\n  createdAt: number = Date.now(),\n): ConversationItemAddedEvent => ({\n  type: 'conversation_item_added',\n  item,\n  createdAt,\n});\n\nexport type FunctionToolsExecutedEvent = {\n  type: 'function_tools_executed';\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt: number;\n};\n\nexport const createFunctionToolsExecutedEvent = ({\n  functionCalls,\n  functionCallOutputs,\n  createdAt = Date.now(),\n}: {\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt?: number;\n}): FunctionToolsExecutedEvent => {\n  return {\n    type: 'function_tools_executed',\n    functionCalls,\n    functionCallOutputs,\n    createdAt,\n  };\n};\n\nexport const zipFunctionCallsAndOutputs = (\n  event: FunctionToolsExecutedEvent,\n): Array<[FunctionCall, FunctionCallOutput]> => {\n  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]!]);\n};\n\nexport type SpeechCreatedEvent = {\n  type: 'speech_created';\n  /**\n   * True if the speech was created using public methods like `say` or `generate_reply`\n   */\n  userInitiated: boolean;\n  /**\n   * Source indicating how the speech handle was created\n   */\n  source: SpeechSource;\n  /**\n   * The speech handle that was created\n   */\n  // TODO(shubhra): we need to make sure this doesn't get serialized\n  speechHandle: SpeechHandle;\n  /**\n   * The timestamp when the speech handle was created\n   */\n  createdAt: number;\n};\n\nexport const createSpeechCreatedEvent = ({\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt = Date.now(),\n}: {\n  userInitiated: boolean;\n  source: SpeechSource;\n  speechHandle: SpeechHandle;\n  createdAt?: number;\n}): SpeechCreatedEvent => ({\n  type: 'speech_created',\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt,\n});\n\nexport type ErrorEvent = {\n  type: 'error';\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown;\n  source: LLM | STT | TTS | RealtimeModel | unknown;\n  createdAt: number;\n};\n\nexport const createErrorEvent = (\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown,\n  source: LLM | STT | TTS | RealtimeModel | unknown,\n  createdAt: number = Date.now(),\n): ErrorEvent => ({\n  type: 'error',\n  error,\n  source,\n  createdAt,\n});\n\nexport type CloseEvent = {\n  type: 'close';\n  error: RealtimeModelError | STTError | TTSError | LLMError | null;\n  reason: CloseReason;\n  createdAt: number;\n};\n\nexport const createCloseEvent = (\n  reason: CloseReason,\n  error: RealtimeModelError | STTError | TTSError | LLMError | null = null,\n  createdAt: number = Date.now(),\n): CloseEvent => ({\n  type: 'close',\n  error,\n  reason,\n  createdAt,\n});\n\nexport type AgentEvent =\n  | UserInputTranscribedEvent\n  | UserStateChangedEvent\n  | AgentStateChangedEvent\n  | MetricsCollectedEvent\n  | ConversationItemAddedEvent\n  | FunctionToolsExecutedEvent\n  | SpeechCreatedEvent\n  | ErrorEvent\n  | CloseEvent;\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkBO,IAAK,yBAAL,kBAAKA,4BAAL;AACL,EAAAA,wBAAA,0BAAuB;AACvB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,WAAQ;AACR,EAAAA,wBAAA,WAAQ;AATE,SAAAA;AAAA,GAAA;AAeL,IAAK,cAAL,kBAAKC,iBAAL;AACL,EAAAA,aAAA,WAAQ;AACR,EAAAA,aAAA,kBAAe;AACf,EAAAA,aAAA,8BAA2B;AAC3B,EAAAA,aAAA,oBAAiB;AAJP,SAAAA;AAAA,GAAA;AAgBL,MAAM,8BAA8B,CACzC,UACA,UACA,YAAoB,KAAK,IAAI,OACF;AAAA,EAC3B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,+BAA+B,CAC1C,UACA,UACA,YAAoB,KAAK,IAAI,OACD;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAUO,MAAM,kCAAkC,CAAC;AAAA,EAC9C;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,YAAY,KAAK,IAAI;AACvB,OAKkC;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAQO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAG8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,mCAAmC,CAC9C,MACA,YAAoB,KAAK,IAAI,OACG;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AACF;AASO,MAAM,mCAAmC,CAAC;AAAA,EAC/C;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,MAIkC;AAChC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,MAAM,6BAA6B,CACxC,UAC8C;AAC9C,SAAO,MAAM,cAAc,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,MAAM,oBAAoB,KAAK,CAAE,CAAC;AAC3F;AAuBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAK2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,OACA,QACA,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,QACA,QAAoE,MACpE,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;","names":["AgentSessionEventTypes","CloseReason"]}
+{"version":3,"sources":["../../src/voice/events.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n  RealtimeModelError,\n} from '../llm/index.js';\nimport type { LLM, RealtimeModel } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport type { AgentMetrics } from '../metrics/base.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS } from '../tts/index.js';\nimport type { TTSError } from '../tts/tts.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport enum AgentSessionEventTypes {\n  UserInputTranscribed = 'user_input_transcribed',\n  AgentStateChanged = 'agent_state_changed',\n  UserStateChanged = 'user_state_changed',\n  ConversationItemAdded = 'conversation_item_added',\n  FunctionToolsExecuted = 'function_tools_executed',\n  MetricsCollected = 'metrics_collected',\n  SpeechCreated = 'speech_created',\n  Error = 'error',\n  Close = 'close',\n}\n\nexport type UserState = 'speaking' | 'listening' | 'away';\nexport type AgentState = 'initializing' | 'idle' | 'listening' | 'thinking' | 'speaking';\n\nexport enum CloseReason {\n  ERROR = 'error',\n  JOB_SHUTDOWN = 'job_shutdown',\n  PARTICIPANT_DISCONNECTED = 'participant_disconnected',\n  USER_INITIATED = 'user_initiated',\n}\n\nexport type SpeechSource = 'say' | 'generate_reply' | 'tool_response';\n\nexport type UserStateChangedEvent = {\n  type: 'user_state_changed';\n  oldState: UserState;\n  newState: UserState;\n  createdAt: number;\n};\n\nexport const createUserStateChangedEvent = (\n  oldState: UserState,\n  newState: UserState,\n  createdAt: number = Date.now(),\n): UserStateChangedEvent => ({\n  type: 'user_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type AgentStateChangedEvent = {\n  type: 'agent_state_changed';\n  oldState: AgentState;\n  newState: AgentState;\n  createdAt: number;\n};\n\nexport const createAgentStateChangedEvent = (\n  oldState: AgentState,\n  newState: AgentState,\n  createdAt: number = Date.now(),\n): AgentStateChangedEvent => ({\n  type: 'agent_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type UserInputTranscribedEvent = {\n  type: 'user_input_transcribed';\n  transcript: string;\n  isFinal: boolean;\n  // TODO(AJS-106): add multi participant support\n  /** Not supported yet. Always null by default. */\n  speakerId: string | null;\n  createdAt: number;\n  language: string | null;\n};\n\nexport const createUserInputTranscribedEvent = ({\n  transcript,\n  isFinal,\n  speakerId = null,\n  language = null,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  isFinal: boolean;\n  speakerId?: string | null;\n  language?: string | null;\n  createdAt?: number;\n}): UserInputTranscribedEvent => ({\n  type: 'user_input_transcribed',\n  transcript,\n  isFinal,\n  speakerId,\n  language,\n  createdAt,\n});\n\nexport type MetricsCollectedEvent = {\n  type: 'metrics_collected';\n  metrics: AgentMetrics;\n  createdAt: number;\n};\n\nexport const createMetricsCollectedEvent = ({\n  metrics,\n  createdAt = Date.now(),\n}: {\n  metrics: AgentMetrics;\n  createdAt?: number;\n}): MetricsCollectedEvent => ({\n  type: 'metrics_collected',\n  metrics,\n  createdAt,\n});\n\nexport type ConversationItemAddedEvent = {\n  type: 'conversation_item_added';\n  item: ChatMessage;\n  createdAt: number;\n};\n\nexport const createConversationItemAddedEvent = (\n  item: ChatMessage,\n  createdAt: number = Date.now(),\n): ConversationItemAddedEvent => ({\n  type: 'conversation_item_added',\n  item,\n  createdAt,\n});\n\nexport type FunctionToolsExecutedEvent = {\n  type: 'function_tools_executed';\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt: number;\n};\n\nexport const createFunctionToolsExecutedEvent = ({\n  functionCalls,\n  functionCallOutputs,\n  createdAt = Date.now(),\n}: {\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt?: number;\n}): FunctionToolsExecutedEvent => {\n  return {\n    type: 'function_tools_executed',\n    functionCalls,\n    functionCallOutputs,\n    createdAt,\n  };\n};\n\nexport const zipFunctionCallsAndOutputs = (\n  event: FunctionToolsExecutedEvent,\n): Array<[FunctionCall, FunctionCallOutput]> => {\n  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]!]);\n};\n\nexport type SpeechCreatedEvent = {\n  type: 'speech_created';\n  /**\n   * True if the speech was created using public methods like `say` or `generate_reply`\n   */\n  userInitiated: boolean;\n  /**\n   * Source indicating how the speech handle was created\n   */\n  source: SpeechSource;\n  /**\n   * The speech handle that was created\n   */\n  // TODO(shubhra): we need to make sure this doesn't get serialized\n  speechHandle: SpeechHandle;\n  /**\n   * The timestamp when the speech handle was created\n   */\n  createdAt: number;\n};\n\nexport const createSpeechCreatedEvent = ({\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt = Date.now(),\n}: {\n  userInitiated: boolean;\n  source: SpeechSource;\n  speechHandle: SpeechHandle;\n  createdAt?: number;\n}): SpeechCreatedEvent => ({\n  type: 'speech_created',\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt,\n});\n\nexport type ErrorEvent = {\n  type: 'error';\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown;\n  source: LLM | STT | TTS | RealtimeModel | unknown;\n  createdAt: number;\n};\n\nexport const createErrorEvent = (\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown,\n  source: LLM | STT | TTS | RealtimeModel | unknown,\n  createdAt: number = Date.now(),\n): ErrorEvent => ({\n  type: 'error',\n  error,\n  source,\n  createdAt,\n});\n\nexport type CloseEvent = {\n  type: 'close';\n  error: RealtimeModelError | STTError | TTSError | LLMError | null;\n  reason: CloseReason;\n  createdAt: number;\n};\n\nexport const createCloseEvent = (\n  reason: CloseReason,\n  error: RealtimeModelError | STTError | TTSError | LLMError | null = null,\n  createdAt: number = Date.now(),\n): CloseEvent => ({\n  type: 'close',\n  error,\n  reason,\n  createdAt,\n});\n\nexport type AgentEvent =\n  | UserInputTranscribedEvent\n  | UserStateChangedEvent\n  | AgentStateChangedEvent\n  | MetricsCollectedEvent\n  | ConversationItemAddedEvent\n  | FunctionToolsExecutedEvent\n  | SpeechCreatedEvent\n  | ErrorEvent\n  | CloseEvent;\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkBO,IAAK,yBAAL,kBAAKA,4BAAL;AACL,EAAAA,wBAAA,0BAAuB;AACvB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,WAAQ;AACR,EAAAA,wBAAA,WAAQ;AATE,SAAAA;AAAA,GAAA;AAeL,IAAK,cAAL,kBAAKC,iBAAL;AACL,EAAAA,aAAA,WAAQ;AACR,EAAAA,aAAA,kBAAe;AACf,EAAAA,aAAA,8BAA2B;AAC3B,EAAAA,aAAA,oBAAiB;AAJP,SAAAA;AAAA,GAAA;AAgBL,MAAM,8BAA8B,CACzC,UACA,UACA,YAAoB,KAAK,IAAI,OACF;AAAA,EAC3B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,+BAA+B,CAC1C,UACA,UACA,YAAoB,KAAK,IAAI,OACD;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAaO,MAAM,kCAAkC,CAAC;AAAA,EAC9C;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,YAAY,KAAK,IAAI;AACvB,OAMkC;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAQO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAG8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,mCAAmC,CAC9C,MACA,YAAoB,KAAK,IAAI,OACG;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AACF;AASO,MAAM,mCAAmC,CAAC;AAAA,EAC/C;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,MAIkC;AAChC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,MAAM,6BAA6B,CACxC,UAC8C;AAC9C,SAAO,MAAM,cAAc,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,MAAM,oBAAoB,KAAK,CAAE,CAAC;AAC3F;AAuBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAK2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,OACA,QACA,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,QACA,QAAoE,MACpE,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;","names":["AgentSessionEventTypes","CloseReason"]}

package/dist/voice/events.d.cts

@@ -45,13 +45,16 @@ export type UserInputTranscribedEvent = {
     type: 'user_input_transcribed';
     transcript: string;
     isFinal: boolean;
+    /** Not supported yet. Always null by default. */
     speakerId: string | null;
     createdAt: number;
+    language: string | null;
 };
-export declare const createUserInputTranscribedEvent: ({ transcript, isFinal, speakerId, createdAt, }: {
+export declare const createUserInputTranscribedEvent: ({ transcript, isFinal, speakerId, language, createdAt, }: {
     transcript: string;
     isFinal: boolean;
     speakerId?: string | null;
+    language?: string | null;
     createdAt?: number;
 }) => UserInputTranscribedEvent;
 export type MetricsCollectedEvent = {

package/dist/voice/events.d.ts

@@ -45,13 +45,16 @@ export type UserInputTranscribedEvent = {
     type: 'user_input_transcribed';
     transcript: string;
     isFinal: boolean;
+    /** Not supported yet. Always null by default. */
     speakerId: string | null;
     createdAt: number;
+    language: string | null;
 };
-export declare const createUserInputTranscribedEvent: ({ transcript, isFinal, speakerId, createdAt, }: {
+export declare const createUserInputTranscribedEvent: ({ transcript, isFinal, speakerId, language, createdAt, }: {
     transcript: string;
     isFinal: boolean;
     speakerId?: string | null;
+    language?: string | null;
     createdAt?: number;
 }) => UserInputTranscribedEvent;
 export type MetricsCollectedEvent = {

package/dist/voice/events.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/voice/events.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,oBAAY,sBAAsB;IAChC,oBAAoB,2BAA2B;IAC/C,iBAAiB,wBAAwB;IACzC,gBAAgB,uBAAuB;IACvC,qBAAqB,4BAA4B;IACjD,qBAAqB,4BAA4B;IACjD,gBAAgB,sBAAsB;IACtC,aAAa,mBAAmB;IAChC,KAAK,UAAU;IACf,KAAK,UAAU;CAChB;AAED,MAAM,MAAM,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,MAAM,CAAC;AAC1D,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,UAAU,CAAC;AAEzF,oBAAY,WAAW;IACrB,KAAK,UAAU;IACf,YAAY,iBAAiB;IAC7B,wBAAwB,6BAA6B;IACrD,cAAc,mBAAmB;CAClC;AAED,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,gBAAgB,GAAG,eAAe,CAAC;AAEtE,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,oBAAoB,CAAC;IAC3B,QAAQ,EAAE,SAAS,CAAC;IACpB,QAAQ,EAAE,SAAS,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAC5B,SAAS,YACT,SAAS,cACR,MAAM,KAChB,qBAKD,CAAC;AAEH,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,qBAAqB,CAAC;IAC5B,QAAQ,EAAE,UAAU,CAAC;IACrB,QAAQ,EAAE,UAAU,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,4BAA4B,aAC7B,UAAU,YACV,UAAU,cACT,MAAM,KAChB,sBAKD,CAAC;AAEH,MAAM,MAAM,yBAAyB,GAAG;IACtC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,+BAA+B,mDAKzC;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,yBAMF,CAAC;AAEH,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,mBAAmB,CAAC;IAC1B,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,2BAA2B,4BAGrC;IACD,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,qBAIF,CAAC;AAEH,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,yBAAyB,CAAC;IAChC,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gCAAgC,SACrC,WAAW,cACN,MAAM,KAChB,0BAID,CAAC;AAEH,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,yBAAyB,CAAC;IAChC,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;IAC1C,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gCAAgC,uDAI1C;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,0BAOH,CAAC;AAEF,eAAO,MAAM,0BAA0B,UAC9B,0BAA0B,KAChC,MAAM,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAE1C,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,gBAAgB,CAAC;IACvB;;OAEG;IACH,aAAa,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;OAEG;IAEH,YAAY,EAAE,YAAY,CAAC;IAC3B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,wBAAwB,wDAKlC;IACD,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,kBAMF,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;IACrE,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,aAAa,GAAG,OAAO,CAAC;IAClD,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gBAAgB,UACpB,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,UAC5D,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,aAAa,GAAG,OAAO,cACtC,MAAM,KAChB,UAKD,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;IAClE,MAAM,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gBAAgB,WACnB,WAAW,UACZ,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,cACtD,MAAM,KAChB,UAKD,CAAC;AAEH,MAAM,MAAM,UAAU,GAClB,yBAAyB,GACzB,qBAAqB,GACrB,sBAAsB,GACtB,qBAAqB,GACrB,0BAA0B,GAC1B,0BAA0B,GAC1B,kBAAkB,GAClB,UAAU,GACV,UAAU,CAAC"}
+{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/voice/events.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,oBAAY,sBAAsB;IAChC,oBAAoB,2BAA2B;IAC/C,iBAAiB,wBAAwB;IACzC,gBAAgB,uBAAuB;IACvC,qBAAqB,4BAA4B;IACjD,qBAAqB,4BAA4B;IACjD,gBAAgB,sBAAsB;IACtC,aAAa,mBAAmB;IAChC,KAAK,UAAU;IACf,KAAK,UAAU;CAChB;AAED,MAAM,MAAM,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,MAAM,CAAC;AAC1D,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,UAAU,CAAC;AAEzF,oBAAY,WAAW;IACrB,KAAK,UAAU;IACf,YAAY,iBAAiB;IAC7B,wBAAwB,6BAA6B;IACrD,cAAc,mBAAmB;CAClC;AAED,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,gBAAgB,GAAG,eAAe,CAAC;AAEtE,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,oBAAoB,CAAC;IAC3B,QAAQ,EAAE,SAAS,CAAC;IACpB,QAAQ,EAAE,SAAS,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,2BAA2B,aAC5B,SAAS,YACT,SAAS,cACR,MAAM,KAChB,qBAKD,CAAC;AAEH,MAAM,MAAM,sBAAsB,GAAG;IACnC,IAAI,EAAE,qBAAqB,CAAC;IAC5B,QAAQ,EAAE,UAAU,CAAC;IACrB,QAAQ,EAAE,UAAU,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,4BAA4B,aAC7B,UAAU,YACV,UAAU,cACT,MAAM,KAChB,sBAKD,CAAC;AAEH,MAAM,MAAM,yBAAyB,GAAG;IACtC,IAAI,EAAE,wBAAwB,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IAEjB,iDAAiD;IACjD,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEF,eAAO,MAAM,+BAA+B,6DAMzC;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,yBAOF,CAAC;AAEH,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,mBAAmB,CAAC;IAC1B,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,2BAA2B,4BAGrC;IACD,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,qBAIF,CAAC;AAEH,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,yBAAyB,CAAC;IAChC,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gCAAgC,SACrC,WAAW,cACN,MAAM,KAChB,0BAID,CAAC;AAEH,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,yBAAyB,CAAC;IAChC,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;IAC1C,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gCAAgC,uDAI1C;IACD,aAAa,EAAE,YAAY,EAAE,CAAC;IAC9B,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,0BAOH,CAAC;AAEF,eAAO,MAAM,0BAA0B,UAC9B,0BAA0B,KAChC,MAAM,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAE1C,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,gBAAgB,CAAC;IACvB;;OAEG;IACH,aAAa,EAAE,OAAO,CAAC;IACvB;;OAEG;IACH,MAAM,EAAE,YAAY,CAAC;IACrB;;OAEG;IAEH,YAAY,EAAE,YAAY,CAAC;IAC3B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,wBAAwB,wDAKlC;IACD,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,YAAY,EAAE,YAAY,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,kBAMF,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;IACrE,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,aAAa,GAAG,OAAO,CAAC;IAClD,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gBAAgB,UACpB,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,UAC5D,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,aAAa,GAAG,OAAO,cACtC,MAAM,KAChB,UAKD,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;IAClE,MAAM,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,eAAO,MAAM,gBAAgB,WACnB,WAAW,UACZ,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,IAAI,cACtD,MAAM,KAChB,UAKD,CAAC;AAEH,MAAM,MAAM,UAAU,GAClB,yBAAyB,GACzB,qBAAqB,GACrB,sBAAsB,GACtB,qBAAqB,GACrB,0BAA0B,GAC1B,0BAA0B,GAC1B,kBAAkB,GAClB,UAAU,GACV,UAAU,CAAC"}

package/dist/voice/events.js

@@ -33,12 +33,14 @@ const createUserInputTranscribedEvent = ({
   transcript,
   isFinal,
   speakerId = null,
+  language = null,
   createdAt = Date.now()
 }) => ({
   type: "user_input_transcribed",
   transcript,
   isFinal,
   speakerId,
+  language,
   createdAt
 });
 const createMetricsCollectedEvent = ({

package/dist/voice/events.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/events.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n  RealtimeModelError,\n} from '../llm/index.js';\nimport type { LLM, RealtimeModel } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport type { AgentMetrics } from '../metrics/base.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS } from '../tts/index.js';\nimport type { TTSError } from '../tts/tts.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport enum AgentSessionEventTypes {\n  UserInputTranscribed = 'user_input_transcribed',\n  AgentStateChanged = 'agent_state_changed',\n  UserStateChanged = 'user_state_changed',\n  ConversationItemAdded = 'conversation_item_added',\n  FunctionToolsExecuted = 'function_tools_executed',\n  MetricsCollected = 'metrics_collected',\n  SpeechCreated = 'speech_created',\n  Error = 'error',\n  Close = 'close',\n}\n\nexport type UserState = 'speaking' | 'listening' | 'away';\nexport type AgentState = 'initializing' | 'idle' | 'listening' | 'thinking' | 'speaking';\n\nexport enum CloseReason {\n  ERROR = 'error',\n  JOB_SHUTDOWN = 'job_shutdown',\n  PARTICIPANT_DISCONNECTED = 'participant_disconnected',\n  USER_INITIATED = 'user_initiated',\n}\n\nexport type SpeechSource = 'say' | 'generate_reply' | 'tool_response';\n\nexport type UserStateChangedEvent = {\n  type: 'user_state_changed';\n  oldState: UserState;\n  newState: UserState;\n  createdAt: number;\n};\n\nexport const createUserStateChangedEvent = (\n  oldState: UserState,\n  newState: UserState,\n  createdAt: number = Date.now(),\n): UserStateChangedEvent => ({\n  type: 'user_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type AgentStateChangedEvent = {\n  type: 'agent_state_changed';\n  oldState: AgentState;\n  newState: AgentState;\n  createdAt: number;\n};\n\nexport const createAgentStateChangedEvent = (\n  oldState: AgentState,\n  newState: AgentState,\n  createdAt: number = Date.now(),\n): AgentStateChangedEvent => ({\n  type: 'agent_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type UserInputTranscribedEvent = {\n  type: 'user_input_transcribed';\n  transcript: string;\n  isFinal: boolean;\n  speakerId: string | null;\n  createdAt: number;\n};\n\nexport const createUserInputTranscribedEvent = ({\n  transcript,\n  isFinal,\n  speakerId = null,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  isFinal: boolean;\n  speakerId?: string | null;\n  createdAt?: number;\n}): UserInputTranscribedEvent => ({\n  type: 'user_input_transcribed',\n  transcript,\n  isFinal,\n  speakerId,\n  createdAt,\n});\n\nexport type MetricsCollectedEvent = {\n  type: 'metrics_collected';\n  metrics: AgentMetrics;\n  createdAt: number;\n};\n\nexport const createMetricsCollectedEvent = ({\n  metrics,\n  createdAt = Date.now(),\n}: {\n  metrics: AgentMetrics;\n  createdAt?: number;\n}): MetricsCollectedEvent => ({\n  type: 'metrics_collected',\n  metrics,\n  createdAt,\n});\n\nexport type ConversationItemAddedEvent = {\n  type: 'conversation_item_added';\n  item: ChatMessage;\n  createdAt: number;\n};\n\nexport const createConversationItemAddedEvent = (\n  item: ChatMessage,\n  createdAt: number = Date.now(),\n): ConversationItemAddedEvent => ({\n  type: 'conversation_item_added',\n  item,\n  createdAt,\n});\n\nexport type FunctionToolsExecutedEvent = {\n  type: 'function_tools_executed';\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt: number;\n};\n\nexport const createFunctionToolsExecutedEvent = ({\n  functionCalls,\n  functionCallOutputs,\n  createdAt = Date.now(),\n}: {\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt?: number;\n}): FunctionToolsExecutedEvent => {\n  return {\n    type: 'function_tools_executed',\n    functionCalls,\n    functionCallOutputs,\n    createdAt,\n  };\n};\n\nexport const zipFunctionCallsAndOutputs = (\n  event: FunctionToolsExecutedEvent,\n): Array<[FunctionCall, FunctionCallOutput]> => {\n  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]!]);\n};\n\nexport type SpeechCreatedEvent = {\n  type: 'speech_created';\n  /**\n   * True if the speech was created using public methods like `say` or `generate_reply`\n   */\n  userInitiated: boolean;\n  /**\n   * Source indicating how the speech handle was created\n   */\n  source: SpeechSource;\n  /**\n   * The speech handle that was created\n   */\n  // TODO(shubhra): we need to make sure this doesn't get serialized\n  speechHandle: SpeechHandle;\n  /**\n   * The timestamp when the speech handle was created\n   */\n  createdAt: number;\n};\n\nexport const createSpeechCreatedEvent = ({\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt = Date.now(),\n}: {\n  userInitiated: boolean;\n  source: SpeechSource;\n  speechHandle: SpeechHandle;\n  createdAt?: number;\n}): SpeechCreatedEvent => ({\n  type: 'speech_created',\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt,\n});\n\nexport type ErrorEvent = {\n  type: 'error';\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown;\n  source: LLM | STT | TTS | RealtimeModel | unknown;\n  createdAt: number;\n};\n\nexport const createErrorEvent = (\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown,\n  source: LLM | STT | TTS | RealtimeModel | unknown,\n  createdAt: number = Date.now(),\n): ErrorEvent => ({\n  type: 'error',\n  error,\n  source,\n  createdAt,\n});\n\nexport type CloseEvent = {\n  type: 'close';\n  error: RealtimeModelError | STTError | TTSError | LLMError | null;\n  reason: CloseReason;\n  createdAt: number;\n};\n\nexport const createCloseEvent = (\n  reason: CloseReason,\n  error: RealtimeModelError | STTError | TTSError | LLMError | null = null,\n  createdAt: number = Date.now(),\n): CloseEvent => ({\n  type: 'close',\n  error,\n  reason,\n  createdAt,\n});\n\nexport type AgentEvent =\n  | UserInputTranscribedEvent\n  | UserStateChangedEvent\n  | AgentStateChangedEvent\n  | MetricsCollectedEvent\n  | ConversationItemAddedEvent\n  | FunctionToolsExecutedEvent\n  | SpeechCreatedEvent\n  | ErrorEvent\n  | CloseEvent;\n"],"mappings":"AAkBO,IAAK,yBAAL,kBAAKA,4BAAL;AACL,EAAAA,wBAAA,0BAAuB;AACvB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,WAAQ;AACR,EAAAA,wBAAA,WAAQ;AATE,SAAAA;AAAA,GAAA;AAeL,IAAK,cAAL,kBAAKC,iBAAL;AACL,EAAAA,aAAA,WAAQ;AACR,EAAAA,aAAA,kBAAe;AACf,EAAAA,aAAA,8BAA2B;AAC3B,EAAAA,aAAA,oBAAiB;AAJP,SAAAA;AAAA,GAAA;AAgBL,MAAM,8BAA8B,CACzC,UACA,UACA,YAAoB,KAAK,IAAI,OACF;AAAA,EAC3B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,+BAA+B,CAC1C,UACA,UACA,YAAoB,KAAK,IAAI,OACD;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAUO,MAAM,kCAAkC,CAAC;AAAA,EAC9C;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,YAAY,KAAK,IAAI;AACvB,OAKkC;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAQO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAG8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,mCAAmC,CAC9C,MACA,YAAoB,KAAK,IAAI,OACG;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AACF;AASO,MAAM,mCAAmC,CAAC;AAAA,EAC/C;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,MAIkC;AAChC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,MAAM,6BAA6B,CACxC,UAC8C;AAC9C,SAAO,MAAM,cAAc,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,MAAM,oBAAoB,KAAK,CAAE,CAAC;AAC3F;AAuBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAK2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,OACA,QACA,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,QACA,QAAoE,MACpE,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;","names":["AgentSessionEventTypes","CloseReason"]}
+{"version":3,"sources":["../../src/voice/events.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n  RealtimeModelError,\n} from '../llm/index.js';\nimport type { LLM, RealtimeModel } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport type { AgentMetrics } from '../metrics/base.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS } from '../tts/index.js';\nimport type { TTSError } from '../tts/tts.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport enum AgentSessionEventTypes {\n  UserInputTranscribed = 'user_input_transcribed',\n  AgentStateChanged = 'agent_state_changed',\n  UserStateChanged = 'user_state_changed',\n  ConversationItemAdded = 'conversation_item_added',\n  FunctionToolsExecuted = 'function_tools_executed',\n  MetricsCollected = 'metrics_collected',\n  SpeechCreated = 'speech_created',\n  Error = 'error',\n  Close = 'close',\n}\n\nexport type UserState = 'speaking' | 'listening' | 'away';\nexport type AgentState = 'initializing' | 'idle' | 'listening' | 'thinking' | 'speaking';\n\nexport enum CloseReason {\n  ERROR = 'error',\n  JOB_SHUTDOWN = 'job_shutdown',\n  PARTICIPANT_DISCONNECTED = 'participant_disconnected',\n  USER_INITIATED = 'user_initiated',\n}\n\nexport type SpeechSource = 'say' | 'generate_reply' | 'tool_response';\n\nexport type UserStateChangedEvent = {\n  type: 'user_state_changed';\n  oldState: UserState;\n  newState: UserState;\n  createdAt: number;\n};\n\nexport const createUserStateChangedEvent = (\n  oldState: UserState,\n  newState: UserState,\n  createdAt: number = Date.now(),\n): UserStateChangedEvent => ({\n  type: 'user_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type AgentStateChangedEvent = {\n  type: 'agent_state_changed';\n  oldState: AgentState;\n  newState: AgentState;\n  createdAt: number;\n};\n\nexport const createAgentStateChangedEvent = (\n  oldState: AgentState,\n  newState: AgentState,\n  createdAt: number = Date.now(),\n): AgentStateChangedEvent => ({\n  type: 'agent_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type UserInputTranscribedEvent = {\n  type: 'user_input_transcribed';\n  transcript: string;\n  isFinal: boolean;\n  // TODO(AJS-106): add multi participant support\n  /** Not supported yet. Always null by default. */\n  speakerId: string | null;\n  createdAt: number;\n  language: string | null;\n};\n\nexport const createUserInputTranscribedEvent = ({\n  transcript,\n  isFinal,\n  speakerId = null,\n  language = null,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  isFinal: boolean;\n  speakerId?: string | null;\n  language?: string | null;\n  createdAt?: number;\n}): UserInputTranscribedEvent => ({\n  type: 'user_input_transcribed',\n  transcript,\n  isFinal,\n  speakerId,\n  language,\n  createdAt,\n});\n\nexport type MetricsCollectedEvent = {\n  type: 'metrics_collected';\n  metrics: AgentMetrics;\n  createdAt: number;\n};\n\nexport const createMetricsCollectedEvent = ({\n  metrics,\n  createdAt = Date.now(),\n}: {\n  metrics: AgentMetrics;\n  createdAt?: number;\n}): MetricsCollectedEvent => ({\n  type: 'metrics_collected',\n  metrics,\n  createdAt,\n});\n\nexport type ConversationItemAddedEvent = {\n  type: 'conversation_item_added';\n  item: ChatMessage;\n  createdAt: number;\n};\n\nexport const createConversationItemAddedEvent = (\n  item: ChatMessage,\n  createdAt: number = Date.now(),\n): ConversationItemAddedEvent => ({\n  type: 'conversation_item_added',\n  item,\n  createdAt,\n});\n\nexport type FunctionToolsExecutedEvent = {\n  type: 'function_tools_executed';\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt: number;\n};\n\nexport const createFunctionToolsExecutedEvent = ({\n  functionCalls,\n  functionCallOutputs,\n  createdAt = Date.now(),\n}: {\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt?: number;\n}): FunctionToolsExecutedEvent => {\n  return {\n    type: 'function_tools_executed',\n    functionCalls,\n    functionCallOutputs,\n    createdAt,\n  };\n};\n\nexport const zipFunctionCallsAndOutputs = (\n  event: FunctionToolsExecutedEvent,\n): Array<[FunctionCall, FunctionCallOutput]> => {\n  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]!]);\n};\n\nexport type SpeechCreatedEvent = {\n  type: 'speech_created';\n  /**\n   * True if the speech was created using public methods like `say` or `generate_reply`\n   */\n  userInitiated: boolean;\n  /**\n   * Source indicating how the speech handle was created\n   */\n  source: SpeechSource;\n  /**\n   * The speech handle that was created\n   */\n  // TODO(shubhra): we need to make sure this doesn't get serialized\n  speechHandle: SpeechHandle;\n  /**\n   * The timestamp when the speech handle was created\n   */\n  createdAt: number;\n};\n\nexport const createSpeechCreatedEvent = ({\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt = Date.now(),\n}: {\n  userInitiated: boolean;\n  source: SpeechSource;\n  speechHandle: SpeechHandle;\n  createdAt?: number;\n}): SpeechCreatedEvent => ({\n  type: 'speech_created',\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt,\n});\n\nexport type ErrorEvent = {\n  type: 'error';\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown;\n  source: LLM | STT | TTS | RealtimeModel | unknown;\n  createdAt: number;\n};\n\nexport const createErrorEvent = (\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown,\n  source: LLM | STT | TTS | RealtimeModel | unknown,\n  createdAt: number = Date.now(),\n): ErrorEvent => ({\n  type: 'error',\n  error,\n  source,\n  createdAt,\n});\n\nexport type CloseEvent = {\n  type: 'close';\n  error: RealtimeModelError | STTError | TTSError | LLMError | null;\n  reason: CloseReason;\n  createdAt: number;\n};\n\nexport const createCloseEvent = (\n  reason: CloseReason,\n  error: RealtimeModelError | STTError | TTSError | LLMError | null = null,\n  createdAt: number = Date.now(),\n): CloseEvent => ({\n  type: 'close',\n  error,\n  reason,\n  createdAt,\n});\n\nexport type AgentEvent =\n  | UserInputTranscribedEvent\n  | UserStateChangedEvent\n  | AgentStateChangedEvent\n  | MetricsCollectedEvent\n  | ConversationItemAddedEvent\n  | FunctionToolsExecutedEvent\n  | SpeechCreatedEvent\n  | ErrorEvent\n  | CloseEvent;\n"],"mappings":"AAkBO,IAAK,yBAAL,kBAAKA,4BAAL;AACL,EAAAA,wBAAA,0BAAuB;AACvB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,WAAQ;AACR,EAAAA,wBAAA,WAAQ;AATE,SAAAA;AAAA,GAAA;AAeL,IAAK,cAAL,kBAAKC,iBAAL;AACL,EAAAA,aAAA,WAAQ;AACR,EAAAA,aAAA,kBAAe;AACf,EAAAA,aAAA,8BAA2B;AAC3B,EAAAA,aAAA,oBAAiB;AAJP,SAAAA;AAAA,GAAA;AAgBL,MAAM,8BAA8B,CACzC,UACA,UACA,YAAoB,KAAK,IAAI,OACF;AAAA,EAC3B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,+BAA+B,CAC1C,UACA,UACA,YAAoB,KAAK,IAAI,OACD;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAaO,MAAM,kCAAkC,CAAC;AAAA,EAC9C;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,YAAY,KAAK,IAAI;AACvB,OAMkC;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAQO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAG8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,mCAAmC,CAC9C,MACA,YAAoB,KAAK,IAAI,OACG;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AACF;AASO,MAAM,mCAAmC,CAAC;AAAA,EAC/C;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,MAIkC;AAChC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,MAAM,6BAA6B,CACxC,UAC8C;AAC9C,SAAO,MAAM,cAAc,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,MAAM,oBAAoB,KAAK,CAAE,CAAC;AAC3F;AAuBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAK2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,OACA,QACA,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,QACA,QAAoE,MACpE,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;","names":["AgentSessionEventTypes","CloseReason"]}

package/dist/voice/generation.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/generation.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AudioResampler } from '@livekit/rtc-node';\nimport type { ReadableStream, ReadableStreamDefaultReader } from 'stream/web';\nimport { ZodObject } from 'zod';\nimport {\n  type ChatContext,\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n} from '../llm/chat_context.js';\nimport type { ChatChunk } from '../llm/llm.js';\nimport {\n  type ToolChoice,\n  type ToolContext,\n  isAgentHandoff,\n  isFunctionTool,\n  isToolError,\n} from '../llm/tool_context.js';\nimport { log } from '../log.js';\nimport { IdentityTransform } from '../stream/identity_transform.js';\nimport { Future, Task, shortuuid, toError } from '../utils.js';\nimport { type Agent, type ModelSettings, asyncLocalStorage, isStopResponse } from './agent.js';\nimport type { AgentSession } from './agent_session.js';\nimport type { AudioOutput, LLMNode, TTSNode, TextOutput } from './io.js';\nimport { RunContext } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\n/* @internal */\nexport class _LLMGenerationData {\n  generatedText: string = '';\n  generatedToolCalls: FunctionCall[];\n  id: string;\n\n  constructor(\n    public readonly textStream: ReadableStream<string>,\n    public readonly toolCallStream: ReadableStream<FunctionCall>,\n  ) {\n    this.id = shortuuid('item_');\n    this.generatedToolCalls = [];\n  }\n}\n\n// TODO(brian): remove this class in favor of ToolOutput\nexport class _ToolOutput {\n  output: _JsOutput[];\n  firstToolFut: Future;\n\n  constructor() {\n    this.output = [];\n    this.firstToolFut = new Future();\n  }\n}\n\n// TODO(brian): remove this class in favor of ToolExecutionOutput\nexport class _SanitizedOutput {\n  toolCall: FunctionCall;\n  toolCallOutput?: FunctionCallOutput;\n  replyRequired: boolean;\n  agentTask?: Agent;\n\n  constructor(\n    toolCall: FunctionCall,\n    toolCallOutput: FunctionCallOutput | undefined,\n    replyRequired: boolean,\n    agentTask: Agent | undefined,\n  ) {\n    this.toolCall = toolCall;\n    this.toolCallOutput = toolCallOutput;\n    this.replyRequired = replyRequired;\n    this.agentTask = agentTask;\n  }\n\n  static create(params: {\n    toolCall: FunctionCall;\n    toolCallOutput?: FunctionCallOutput;\n    replyRequired?: boolean;\n    agentTask?: Agent;\n  }) {\n    const { toolCall, toolCallOutput, replyRequired = true, agentTask } = params;\n    return new _SanitizedOutput(toolCall, toolCallOutput, replyRequired, agentTask);\n  }\n}\n\nfunction isValidToolOutput(toolOutput: unknown): boolean {\n  const validTypes = ['string', 'number', 'boolean'];\n\n  if (validTypes.includes(typeof toolOutput)) {\n    return true;\n  }\n\n  if (toolOutput === undefined || toolOutput === null) {\n    return true;\n  }\n\n  if (Array.isArray(toolOutput)) {\n    return toolOutput.every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Set) {\n    return Array.from(toolOutput).every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Map) {\n    return Array.from(toolOutput.values()).every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Object) {\n    return Object.entries(toolOutput).every(\n      ([key, value]) => validTypes.includes(typeof key) && isValidToolOutput(value),\n    );\n  }\n\n  return false;\n}\n\nexport class ToolExecutionOutput {\n  constructor(\n    public readonly toolCall: FunctionCall,\n    public readonly toolCallOutput: FunctionCallOutput | undefined,\n    public readonly agentTask: Agent | undefined,\n    public readonly rawOutput: unknown,\n    public readonly rawException: Error | undefined,\n    public readonly replyRequired: boolean,\n  ) {}\n\n  static create(params: {\n    toolCall: FunctionCall;\n    toolCallOutput?: FunctionCallOutput;\n    agentTask?: Agent;\n    rawOutput: unknown;\n    rawException?: Error;\n    replyRequired?: boolean;\n  }) {\n    const {\n      toolCall,\n      toolCallOutput,\n      agentTask,\n      rawOutput,\n      rawException,\n      replyRequired = true,\n    } = params;\n    return new ToolExecutionOutput(\n      toolCall,\n      toolCallOutput,\n      agentTask,\n      rawOutput,\n      rawException,\n      replyRequired,\n    );\n  }\n}\n\nexport interface ToolOutput {\n  output: ToolExecutionOutput[];\n  firstToolStartedFuture: Future<void>;\n}\n\n// TODO(brian): remove this class in favor of ToolExecutionOutput\nexport class _JsOutput {\n  toolCall: FunctionCall;\n  output: unknown;\n  exception?: Error;\n\n  #logger = log();\n\n  constructor(toolCall: FunctionCall, output: unknown, exception: Error | undefined) {\n    this.toolCall = toolCall;\n    this.output = output;\n    this.exception = exception;\n  }\n\n  static create(params: { toolCall: FunctionCall; output?: unknown; exception?: Error }) {\n    const { toolCall, output = undefined, exception = undefined } = params;\n    return new _JsOutput(toolCall, output, exception);\n  }\n\n  sanitize(): _SanitizedOutput {\n    if (isToolError(this.exception)) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: FunctionCallOutput.create({\n          name: this.toolCall.name,\n          callId: this.toolCall.callId,\n          output: this.exception.message,\n          isError: true,\n        }),\n      });\n    }\n\n    if (isStopResponse(this.exception)) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n      });\n    }\n\n    if (this.exception !== undefined) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: FunctionCallOutput.create({\n          name: this.toolCall.name,\n          callId: this.toolCall.callId,\n          output: 'An internal error occurred while executing the tool.', // Don't send the actual error message, as it may contain sensitive information\n          isError: true,\n        }),\n      });\n    }\n\n    let agentTask: Agent | undefined = undefined;\n    let toolOutput: unknown = this.output;\n    if (isAgentHandoff(this.output)) {\n      agentTask = this.output.agent;\n      toolOutput = this.output.returns;\n    }\n\n    if (!isValidToolOutput(toolOutput)) {\n      this.#logger.error(\n        {\n          callId: this.toolCall.callId,\n          function: this.toolCall.name,\n        },\n        `AI function ${this.toolCall.name} returned an invalid output`,\n      );\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: undefined,\n      });\n    }\n\n    return _SanitizedOutput.create({\n      toolCall: FunctionCall.create({ ...this.toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: this.toolCall.name,\n        callId: this.toolCall.callId,\n        output: toolOutput !== undefined ? JSON.stringify(toolOutput) : '', // take the string representation of the output\n        isError: false,\n      }),\n      replyRequired: toolOutput !== undefined, // require a reply if the tool returned an output\n      agentTask,\n    });\n  }\n}\n\nexport function createToolOutput(params: {\n  toolCall: FunctionCall;\n  output?: unknown;\n  exception?: Error;\n}): ToolExecutionOutput {\n  const { toolCall, output, exception } = params;\n  const logger = log();\n\n  // support returning Exception instead of raising them (for devex purposes inside evals)\n  let finalOutput = output;\n  let finalException = exception;\n  if (output instanceof Error) {\n    finalException = output;\n    finalOutput = undefined;\n  }\n\n  if (isToolError(finalException)) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: toolCall.name,\n        callId: toolCall.callId,\n        output: finalException.message,\n        isError: true,\n      }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  if (isStopResponse(finalException)) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  if (finalException !== undefined) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: toolCall.name,\n        callId: toolCall.callId,\n        output: 'An internal error occurred', // Don't send the actual error message, as it may contain sensitive information\n        isError: true,\n      }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  let agentTask: Agent | undefined = undefined;\n  let toolOutput: unknown = finalOutput;\n  if (isAgentHandoff(finalOutput)) {\n    agentTask = finalOutput.agent;\n    toolOutput = finalOutput.returns;\n  }\n\n  if (!isValidToolOutput(toolOutput)) {\n    logger.error(\n      {\n        callId: toolCall.callId,\n        output: finalOutput,\n      },\n      `AI function ${toolCall.name} returned an invalid output`,\n    );\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  return ToolExecutionOutput.create({\n    toolCall: FunctionCall.create({ ...toolCall }),\n    toolCallOutput: FunctionCallOutput.create({\n      name: toolCall.name,\n      callId: toolCall.callId,\n      output: toolOutput !== undefined ? JSON.stringify(toolOutput) : '', // take the string representation of the output\n      isError: false,\n    }),\n    replyRequired: toolOutput !== undefined, // require a reply if the tool returned an output\n    agentTask,\n    rawOutput: finalOutput,\n    rawException: finalException,\n  });\n}\n\nconst INSTRUCTIONS_MESSAGE_ID = 'lk.agent_task.instructions';\n\n/**\n * Update the instruction message in the chat context or insert a new one if missing.\n *\n * This function looks for an existing instruction message in the chat context using the identifier\n * 'INSTRUCTIONS_MESSAGE_ID'.\n *\n * @param options - The options for updating the instructions.\n * @param options.chatCtx - The chat context to update.\n * @param options.instructions - The instructions to add.\n * @param options.addIfMissing - Whether to add the instructions if they are missing.\n */\nexport function updateInstructions(options: {\n  chatCtx: ChatContext;\n  instructions: string;\n  addIfMissing: boolean;\n}) {\n  const { chatCtx, instructions, addIfMissing } = options;\n\n  const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID);\n  if (idx !== undefined) {\n    if (chatCtx.items[idx]!.type === 'message') {\n      // create a new instance to avoid mutating the original\n      chatCtx.items[idx] = ChatMessage.create({\n        id: INSTRUCTIONS_MESSAGE_ID,\n        role: 'system',\n        content: [instructions],\n        createdAt: chatCtx.items[idx]!.createdAt,\n      });\n    } else {\n      throw new Error('expected the instructions inside the chatCtx to be of type \"message\"');\n    }\n  } else if (addIfMissing) {\n    // insert the instructions at the beginning of the chat context\n    chatCtx.items.unshift(\n      ChatMessage.create({\n        id: INSTRUCTIONS_MESSAGE_ID,\n        role: 'system',\n        content: [instructions],\n      }),\n    );\n  }\n}\n\nexport function performLLMInference(\n  node: LLMNode,\n  chatCtx: ChatContext,\n  toolCtx: ToolContext,\n  modelSettings: ModelSettings,\n  controller: AbortController,\n): [Task<void>, _LLMGenerationData] {\n  const textStream = new IdentityTransform<string>();\n  const toolCallStream = new IdentityTransform<FunctionCall>();\n\n  const textWriter = textStream.writable.getWriter();\n  const toolCallWriter = toolCallStream.writable.getWriter();\n  const data = new _LLMGenerationData(textStream.readable, toolCallStream.readable);\n\n  const inferenceTask = async (signal: AbortSignal) => {\n    let llmStreamReader: ReadableStreamDefaultReader<string | ChatChunk> | null = null;\n    let llmStream: ReadableStream<string | ChatChunk> | null = null;\n\n    try {\n      llmStream = await node(chatCtx, toolCtx, modelSettings);\n      if (llmStream === null) {\n        await textWriter.close();\n        return;\n      }\n\n      // TODO(brian): add support for dynamic tools\n\n      llmStreamReader = llmStream.getReader();\n      while (true) {\n        if (signal.aborted) {\n          break;\n        }\n        const { done, value: chunk } = await llmStreamReader.read();\n        if (done) {\n          break;\n        }\n\n        if (typeof chunk === 'string') {\n          data.generatedText += chunk;\n          await textWriter.write(chunk);\n          // TODO(shubhra): better way to check??\n        } else {\n          if (chunk.delta === undefined) {\n            continue;\n          }\n\n          if (chunk.delta.toolCalls) {\n            for (const tool of chunk.delta.toolCalls) {\n              if (tool.type !== 'function_call') continue;\n\n              const toolCall = FunctionCall.create({\n                callId: `${data.id}/fnc_${data.generatedToolCalls.length}`,\n                name: tool.name,\n                args: tool.args,\n              });\n\n              data.generatedToolCalls.push(toolCall);\n              await toolCallWriter.write(toolCall);\n            }\n          }\n\n          if (chunk.delta.content) {\n            data.generatedText += chunk.delta.content;\n            await textWriter.write(chunk.delta.content);\n          }\n        }\n\n        // No need to check if chunk is of type other than ChatChunk or string like in\n        // Python since chunk is defined in the type ChatChunk | string in TypeScript\n      }\n    } catch (error) {\n      if (error instanceof DOMException && error.name === 'AbortError') {\n        // Abort signal was triggered, handle gracefully\n        return;\n      }\n      throw error;\n    } finally {\n      llmStreamReader?.releaseLock();\n      await llmStream?.cancel();\n      await textWriter.close();\n      await toolCallWriter.close();\n    }\n  };\n\n  return [\n    Task.from((controller) => inferenceTask(controller.signal), controller, 'performLLMInference'),\n    data,\n  ];\n}\n\nexport function performTTSInference(\n  node: TTSNode,\n  text: ReadableStream<string>,\n  modelSettings: ModelSettings,\n  controller: AbortController,\n): [Task<void>, ReadableStream<AudioFrame>] {\n  const audioStream = new IdentityTransform<AudioFrame>();\n  const outputWriter = audioStream.writable.getWriter();\n  const audioOutputStream = audioStream.readable;\n\n  const inferenceTask = async (signal: AbortSignal) => {\n    let ttsStreamReader: ReadableStreamDefaultReader<AudioFrame> | null = null;\n    let ttsStream: ReadableStream<AudioFrame> | null = null;\n\n    try {\n      ttsStream = await node(text, modelSettings);\n      if (ttsStream === null) {\n        await outputWriter.close();\n        return;\n      }\n\n      ttsStreamReader = ttsStream.getReader();\n      while (true) {\n        if (signal.aborted) {\n          break;\n        }\n        const { done, value: chunk } = await ttsStreamReader.read();\n        if (done) {\n          break;\n        }\n        await outputWriter.write(chunk);\n      }\n    } catch (error) {\n      if (error instanceof DOMException && error.name === 'AbortError') {\n        // Abort signal was triggered, handle gracefully\n        return;\n      }\n      throw error;\n    } finally {\n      ttsStreamReader?.releaseLock();\n      await ttsStream?.cancel();\n      await outputWriter.close();\n    }\n  };\n\n  return [\n    Task.from((controller) => inferenceTask(controller.signal), controller, 'performTTSInference'),\n    audioOutputStream,\n  ];\n}\n\nexport interface _TextOut {\n  text: string;\n  firstTextFut: Future;\n}\n\nasync function forwardText(\n  source: ReadableStream<string>,\n  out: _TextOut,\n  signal: AbortSignal,\n  textOutput: TextOutput | null,\n): Promise<void> {\n  const reader = source.getReader();\n  try {\n    while (true) {\n      if (signal.aborted) {\n        break;\n      }\n      const { done, value: delta } = await reader.read();\n      if (done) break;\n      out.text += delta;\n      if (textOutput !== null) {\n        await textOutput.captureText(delta);\n      }\n      if (!out.firstTextFut.done) {\n        out.firstTextFut.resolve();\n      }\n    }\n  } finally {\n    if (textOutput !== null) {\n      textOutput.flush();\n    }\n    reader?.releaseLock();\n  }\n}\n\nexport function performTextForwarding(\n  source: ReadableStream<string>,\n  controller: AbortController,\n  textOutput: TextOutput | null,\n): [Task<void>, _TextOut] {\n  const out = {\n    text: '',\n    firstTextFut: new Future(),\n  };\n  return [\n    Task.from(\n      (controller) => forwardText(source, out, controller.signal, textOutput),\n      controller,\n      'performTextForwarding',\n    ),\n    out,\n  ];\n}\n\nexport interface _AudioOut {\n  audio: Array<AudioFrame>;\n  firstFrameFut: Future;\n}\n\nasync function forwardAudio(\n  ttsStream: ReadableStream<AudioFrame>,\n  audioOuput: AudioOutput,\n  out: _AudioOut,\n  signal?: AbortSignal,\n): Promise<void> {\n  const reader = ttsStream.getReader();\n  let resampler: AudioResampler | null = null;\n\n  try {\n    while (true) {\n      if (signal?.aborted) {\n        break;\n      }\n\n      const { done, value: frame } = await reader.read();\n      if (done) break;\n\n      out.audio.push(frame);\n\n      if (\n        !out.firstFrameFut.done &&\n        audioOuput.sampleRate &&\n        audioOuput.sampleRate !== frame.sampleRate &&\n        !resampler\n      ) {\n        resampler = new AudioResampler(frame.sampleRate, audioOuput.sampleRate, 1);\n      }\n\n      if (resampler) {\n        for (const f of resampler.push(frame)) {\n          await audioOuput.captureFrame(f);\n        }\n      } else {\n        await audioOuput.captureFrame(frame);\n      }\n\n      // set the first frame future if not already set\n      // (after completing the first frame)\n      if (!out.firstFrameFut.done) {\n        out.firstFrameFut.resolve();\n      }\n    }\n  } finally {\n    reader?.releaseLock();\n    if (resampler) {\n      for (const f of resampler.flush()) {\n        await audioOuput.captureFrame(f);\n      }\n    }\n    audioOuput.flush();\n  }\n}\n\nexport function performAudioForwarding(\n  ttsStream: ReadableStream<AudioFrame>,\n  audioOutput: AudioOutput,\n  controller: AbortController,\n): [Task<void>, _AudioOut] {\n  const out = {\n    audio: [],\n    firstFrameFut: new Future(),\n  };\n  return [\n    Task.from(\n      (controller) => forwardAudio(ttsStream, audioOutput, out, controller.signal),\n      controller,\n      'performAudioForwarding',\n    ),\n    out,\n  ];\n}\n\nexport function performToolExecutions({\n  session,\n  speechHandle,\n  toolCtx,\n  toolChoice,\n  toolCallStream,\n  onToolExecutionStarted = () => {},\n  onToolExecutionCompleted = () => {},\n  controller,\n}: {\n  session: AgentSession;\n  speechHandle: SpeechHandle;\n  toolCtx: ToolContext;\n  toolChoice?: ToolChoice;\n  toolCallStream: ReadableStream<FunctionCall>;\n  onToolExecutionStarted?: (toolCall: FunctionCall) => void;\n  onToolExecutionCompleted?: (toolExecutionOutput: ToolExecutionOutput) => void;\n  controller: AbortController;\n}): [Task<void>, ToolOutput] {\n  const logger = log();\n  const toolOutput: ToolOutput = {\n    output: [],\n    firstToolStartedFuture: new Future(),\n  };\n\n  const toolCompleted = (out: ToolExecutionOutput) => {\n    onToolExecutionCompleted(out);\n    toolOutput.output.push(out);\n  };\n\n  const executeToolsTask = async (controller: AbortController) => {\n    const signal = controller.signal;\n    const reader = toolCallStream.getReader();\n\n    const tasks: Promise<any>[] = [];\n    while (!signal.aborted) {\n      const { done, value: toolCall } = await reader.read();\n      if (signal.aborted) break;\n      if (done) break;\n\n      if (toolChoice === 'none') {\n        logger.error(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          \"received a tool call with toolChoice set to 'none', ignoring\",\n        );\n        continue;\n      }\n\n      // TODO(brian): assert other toolChoice values\n\n      const tool = toolCtx[toolCall.name];\n      if (!tool) {\n        logger.warn(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          `unknown AI function ${toolCall.name}`,\n        );\n        continue;\n      }\n\n      if (!isFunctionTool(tool)) {\n        logger.error(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          `unknown tool type: ${typeof tool}`,\n        );\n        continue;\n      }\n\n      let parsedArgs: object | undefined;\n\n      // Ensure valid arguments\n      try {\n        const jsonArgs = JSON.parse(toolCall.args);\n\n        if (tool.parameters instanceof ZodObject) {\n          parsedArgs = tool.parameters.parse(jsonArgs);\n        } else {\n          parsedArgs = jsonArgs;\n        }\n      } catch (rawError) {\n        const error = toError(rawError);\n        logger.error(\n          {\n            function: toolCall.name,\n            arguments: toolCall.args,\n            speech_id: speechHandle.id,\n            error: error.message,\n          },\n          `tried to call AI function ${toolCall.name} with invalid arguments`,\n        );\n        toolCompleted(\n          createToolOutput({\n            toolCall,\n            exception: error,\n          }),\n        );\n        continue;\n      }\n\n      if (!toolOutput.firstToolStartedFuture.done) {\n        toolOutput.firstToolStartedFuture.resolve();\n      }\n\n      onToolExecutionStarted(toolCall);\n\n      logger.info(\n        {\n          function: toolCall.name,\n          arguments: parsedArgs,\n          speech_id: speechHandle.id,\n        },\n        'Executing LLM tool call',\n      );\n\n      const toolExecution = asyncLocalStorage.run({ functionCall: toolCall }, async () => {\n        return await tool.execute(parsedArgs, {\n          ctx: new RunContext(session, speechHandle, toolCall),\n          toolCallId: toolCall.callId,\n          abortSignal: signal,\n        });\n      });\n\n      const tracableToolExecution = async (toolExecTask: Promise<unknown>) => {\n        // TODO(brian): add tracing\n\n        // await for task to complete, if task is aborted, set exception\n        let toolOutput: ToolExecutionOutput | undefined;\n        try {\n          const { result, isAborted } = await waitUntilAborted(toolExecTask, signal);\n          toolOutput = createToolOutput({\n            toolCall,\n            exception: isAborted ? new Error('tool call was aborted') : undefined,\n            output: isAborted ? undefined : result,\n          });\n        } catch (rawError) {\n          logger.error(\n            {\n              function: toolCall.name,\n              speech_id: speechHandle.id,\n              error: toError(rawError).message,\n            },\n            'exception occurred while executing tool',\n          );\n          toolOutput = createToolOutput({\n            toolCall,\n            exception: toError(rawError),\n          });\n        } finally {\n          if (!toolOutput) throw new Error('toolOutput is undefined');\n          toolCompleted(toolOutput);\n        }\n      };\n\n      // wait, not cancelling all tool calling tasks\n      tasks.push(tracableToolExecution(toolExecution));\n    }\n\n    await Promise.allSettled(tasks);\n    if (toolOutput.output.length > 0) {\n      logger.debug(\n        {\n          speech_id: speechHandle.id,\n        },\n        'tools execution completed',\n      );\n    }\n  };\n\n  return [Task.from(executeToolsTask, controller, 'performToolExecutions'), toolOutput];\n}\n\ntype Aborted<T> =\n  | {\n      result: T;\n      isAborted: false;\n    }\n  | {\n      result: undefined;\n      isAborted: true;\n    };\n\nasync function waitUntilAborted<T>(promise: Promise<T>, signal: AbortSignal): Promise<Aborted<T>> {\n  const abortFut = new Future<Aborted<T>>();\n\n  const resolveAbort = () => {\n    if (!abortFut.done) {\n      abortFut.resolve({ result: undefined, isAborted: true });\n    }\n  };\n\n  signal.addEventListener('abort', resolveAbort);\n\n  promise\n    .then((r) => {\n      if (!abortFut.done) {\n        abortFut.resolve({ result: r, isAborted: false });\n      }\n    })\n    .catch((e) => {\n      if (!abortFut.done) {\n        abortFut.reject(e);\n      }\n    })\n    .finally(() => {\n      signal.removeEventListener('abort', resolveAbort);\n    });\n\n  return await abortFut.await;\n}\n\nexport function removeInstructions(chatCtx: ChatContext) {\n  // loop in case there are items with the same id (shouldn't happen!)\n  while (true) {\n    const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID);\n    if (idx !== undefined) {\n      chatCtx.items.splice(idx, 1);\n    } else {\n      break;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,sBAA+B;AAE/B,iBAA0B;AAC1B,0BAKO;AAEP,0BAMO;AACP,iBAAoB;AACpB,gCAAkC;AAClC,mBAAiD;AACjD,mBAAkF;AAGlF,yBAA2B;AAIpB,MAAM,mBAAmB;AAAA,EAK9B,YACkB,YACA,gBAChB;AAFgB;AACA;AAEhB,SAAK,SAAK,wBAAU,OAAO;AAC3B,SAAK,qBAAqB,CAAC;AAAA,EAC7B;AAAA,EAVA,gBAAwB;AAAA,EACxB;AAAA,EACA;AASF;AAGO,MAAM,YAAY;AAAA,EACvB;AAAA,EACA;AAAA,EAEA,cAAc;AACZ,SAAK,SAAS,CAAC;AACf,SAAK,eAAe,IAAI,oBAAO;AAAA,EACjC;AACF;AAGO,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,UACA,gBACA,eACA,WACA;AACA,SAAK,WAAW;AAChB,SAAK,iBAAiB;AACtB,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,OAAO,OAAO,QAKX;AACD,UAAM,EAAE,UAAU,gBAAgB,gBAAgB,MAAM,UAAU,IAAI;AACtE,WAAO,IAAI,iBAAiB,UAAU,gBAAgB,eAAe,SAAS;AAAA,EAChF;AACF;AAEA,SAAS,kBAAkB,YAA8B;AACvD,QAAM,aAAa,CAAC,UAAU,UAAU,SAAS;AAEjD,MAAI,WAAW,SAAS,OAAO,UAAU,GAAG;AAC1C,WAAO;AAAA,EACT;AAEA,MAAI,eAAe,UAAa,eAAe,MAAM;AACnD,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,WAAO,WAAW,MAAM,iBAAiB;AAAA,EAC3C;AAEA,MAAI,sBAAsB,KAAK;AAC7B,WAAO,MAAM,KAAK,UAAU,EAAE,MAAM,iBAAiB;AAAA,EACvD;AAEA,MAAI,sBAAsB,KAAK;AAC7B,WAAO,MAAM,KAAK,WAAW,OAAO,CAAC,EAAE,MAAM,iBAAiB;AAAA,EAChE;AAEA,MAAI,sBAAsB,QAAQ;AAChC,WAAO,OAAO,QAAQ,UAAU,EAAE;AAAA,MAChC,CAAC,CAAC,KAAK,KAAK,MAAM,WAAW,SAAS,OAAO,GAAG,KAAK,kBAAkB,KAAK;AAAA,IAC9E;AAAA,EACF;AAEA,SAAO;AACT;AAEO,MAAM,oBAAoB;AAAA,EAC/B,YACkB,UACA,gBACA,WACA,WACA,cACA,eAChB;AANgB;AACA;AACA;AACA;AACA;AACA;AAAA,EACf;AAAA,EAEH,OAAO,OAAO,QAOX;AACD,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,IAClB,IAAI;AACJ,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAQO,MAAM,UAAU;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,cAAU,gBAAI;AAAA,EAEd,YAAY,UAAwB,QAAiB,WAA8B;AACjF,SAAK,WAAW;AAChB,SAAK,SAAS;AACd,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,OAAO,OAAO,QAAyE;AACrF,UAAM,EAAE,UAAU,SAAS,QAAW,YAAY,OAAU,IAAI;AAChE,WAAO,IAAI,UAAU,UAAU,QAAQ,SAAS;AAAA,EAClD;AAAA,EAEA,WAA6B;AAC3B,YAAI,iCAAY,KAAK,SAAS,GAAG;AAC/B,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB,uCAAmB,OAAO;AAAA,UACxC,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,QAAQ,KAAK,UAAU;AAAA,UACvB,SAAS;AAAA,QACX,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAEA,YAAI,6BAAe,KAAK,SAAS,GAAG;AAClC,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,MACpD,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,cAAc,QAAW;AAChC,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB,uCAAmB,OAAO;AAAA,UACxC,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,QAAQ;AAAA;AAAA,UACR,SAAS;AAAA,QACX,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAEA,QAAI,YAA+B;AACnC,QAAI,aAAsB,KAAK;AAC/B,YAAI,oCAAe,KAAK,MAAM,GAAG;AAC/B,kBAAY,KAAK,OAAO;AACxB,mBAAa,KAAK,OAAO;AAAA,IAC3B;AAEA,QAAI,CAAC,kBAAkB,UAAU,GAAG;AAClC,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK,SAAS;AAAA,UACtB,UAAU,KAAK,SAAS;AAAA,QAC1B;AAAA,QACA,eAAe,KAAK,SAAS,IAAI;AAAA,MACnC;AACA,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB;AAAA,MAClB,CAAC;AAAA,IACH;AAEA,WAAO,iBAAiB,OAAO;AAAA,MAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,MAClD,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,KAAK,SAAS;AAAA,QACpB,QAAQ,KAAK,SAAS;AAAA,QACtB,QAAQ,eAAe,SAAY,KAAK,UAAU,UAAU,IAAI;AAAA;AAAA,QAChE,SAAS;AAAA,MACX,CAAC;AAAA,MACD,eAAe,eAAe;AAAA;AAAA,MAC9B;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEO,SAAS,iBAAiB,QAIT;AACtB,QAAM,EAAE,UAAU,QAAQ,UAAU,IAAI;AACxC,QAAM,aAAS,gBAAI;AAGnB,MAAI,cAAc;AAClB,MAAI,iBAAiB;AACrB,MAAI,kBAAkB,OAAO;AAC3B,qBAAiB;AACjB,kBAAc;AAAA,EAChB;AAEA,UAAI,iCAAY,cAAc,GAAG;AAC/B,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,SAAS;AAAA,QACf,QAAQ,SAAS;AAAA,QACjB,QAAQ,eAAe;AAAA,QACvB,SAAS;AAAA,MACX,CAAC;AAAA,MACD,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,UAAI,6BAAe,cAAc,GAAG;AAClC,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,MAAI,mBAAmB,QAAW;AAChC,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,SAAS;AAAA,QACf,QAAQ,SAAS;AAAA,QACjB,QAAQ;AAAA;AAAA,QACR,SAAS;AAAA,MACX,CAAC;AAAA,MACD,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,MAAI,YAA+B;AACnC,MAAI,aAAsB;AAC1B,UAAI,oCAAe,WAAW,GAAG;AAC/B,gBAAY,YAAY;AACxB,iBAAa,YAAY;AAAA,EAC3B;AAEA,MAAI,CAAC,kBAAkB,UAAU,GAAG;AAClC,WAAO;AAAA,MACL;AAAA,QACE,QAAQ,SAAS;AAAA,QACjB,QAAQ;AAAA,MACV;AAAA,MACA,eAAe,SAAS,IAAI;AAAA,IAC9B;AACA,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,SAAO,oBAAoB,OAAO;AAAA,IAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,IAC7C,gBAAgB,uCAAmB,OAAO;AAAA,MACxC,MAAM,SAAS;AAAA,MACf,QAAQ,SAAS;AAAA,MACjB,QAAQ,eAAe,SAAY,KAAK,UAAU,UAAU,IAAI;AAAA;AAAA,MAChE,SAAS;AAAA,IACX,CAAC;AAAA,IACD,eAAe,eAAe;AAAA;AAAA,IAC9B;AAAA,IACA,WAAW;AAAA,IACX,cAAc;AAAA,EAChB,CAAC;AACH;AAEA,MAAM,0BAA0B;AAazB,SAAS,mBAAmB,SAIhC;AACD,QAAM,EAAE,SAAS,cAAc,aAAa,IAAI;AAEhD,QAAM,MAAM,QAAQ,UAAU,uBAAuB;AACrD,MAAI,QAAQ,QAAW;AACrB,QAAI,QAAQ,MAAM,GAAG,EAAG,SAAS,WAAW;AAE1C,cAAQ,MAAM,GAAG,IAAI,gCAAY,OAAO;AAAA,QACtC,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,WAAW,QAAQ,MAAM,GAAG,EAAG;AAAA,MACjC,CAAC;AAAA,IACH,OAAO;AACL,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAAA,EACF,WAAW,cAAc;AAEvB,YAAQ,MAAM;AAAA,MACZ,gCAAY,OAAO;AAAA,QACjB,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,MACxB,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEO,SAAS,oBACd,MACA,SACA,SACA,eACA,YACkC;AAClC,QAAM,aAAa,IAAI,4CAA0B;AACjD,QAAM,iBAAiB,IAAI,4CAAgC;AAE3D,QAAM,aAAa,WAAW,SAAS,UAAU;AACjD,QAAM,iBAAiB,eAAe,SAAS,UAAU;AACzD,QAAM,OAAO,IAAI,mBAAmB,WAAW,UAAU,eAAe,QAAQ;AAEhF,QAAM,gBAAgB,OAAO,WAAwB;AACnD,QAAI,kBAA0E;AAC9E,QAAI,YAAuD;AAE3D,QAAI;AACF,kBAAY,MAAM,KAAK,SAAS,SAAS,aAAa;AACtD,UAAI,cAAc,MAAM;AACtB,cAAM,WAAW,MAAM;AACvB;AAAA,MACF;AAIA,wBAAkB,UAAU,UAAU;AACtC,aAAO,MAAM;AACX,YAAI,OAAO,SAAS;AAClB;AAAA,QACF;AACA,cAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,gBAAgB,KAAK;AAC1D,YAAI,MAAM;AACR;AAAA,QACF;AAEA,YAAI,OAAO,UAAU,UAAU;AAC7B,eAAK,iBAAiB;AACtB,gBAAM,WAAW,MAAM,KAAK;AAAA,QAE9B,OAAO;AACL,cAAI,MAAM,UAAU,QAAW;AAC7B;AAAA,UACF;AAEA,cAAI,MAAM,MAAM,WAAW;AACzB,uBAAW,QAAQ,MAAM,MAAM,WAAW;AACxC,kBAAI,KAAK,SAAS,gBAAiB;AAEnC,oBAAM,WAAW,iCAAa,OAAO;AAAA,gBACnC,QAAQ,GAAG,KAAK,EAAE,QAAQ,KAAK,mBAAmB,MAAM;AAAA,gBACxD,MAAM,KAAK;AAAA,gBACX,MAAM,KAAK;AAAA,cACb,CAAC;AAED,mBAAK,mBAAmB,KAAK,QAAQ;AACrC,oBAAM,eAAe,MAAM,QAAQ;AAAA,YACrC;AAAA,UACF;AAEA,cAAI,MAAM,MAAM,SAAS;AACvB,iBAAK,iBAAiB,MAAM,MAAM;AAClC,kBAAM,WAAW,MAAM,MAAM,MAAM,OAAO;AAAA,UAC5C;AAAA,QACF;AAAA,MAIF;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,gBAAgB,MAAM,SAAS,cAAc;AAEhE;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AACA,yDAAiB;AACjB,aAAM,uCAAW;AACjB,YAAM,WAAW,MAAM;AACvB,YAAM,eAAe,MAAM;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL,kBAAK,KAAK,CAACA,gBAAe,cAAcA,YAAW,MAAM,GAAG,YAAY,qBAAqB;AAAA,IAC7F;AAAA,EACF;AACF;AAEO,SAAS,oBACd,MACA,MACA,eACA,YAC0C;AAC1C,QAAM,cAAc,IAAI,4CAA8B;AACtD,QAAM,eAAe,YAAY,SAAS,UAAU;AACpD,QAAM,oBAAoB,YAAY;AAEtC,QAAM,gBAAgB,OAAO,WAAwB;AACnD,QAAI,kBAAkE;AACtE,QAAI,YAA+C;AAEnD,QAAI;AACF,kBAAY,MAAM,KAAK,MAAM,aAAa;AAC1C,UAAI,cAAc,MAAM;AACtB,cAAM,aAAa,MAAM;AACzB;AAAA,MACF;AAEA,wBAAkB,UAAU,UAAU;AACtC,aAAO,MAAM;AACX,YAAI,OAAO,SAAS;AAClB;AAAA,QACF;AACA,cAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,gBAAgB,KAAK;AAC1D,YAAI,MAAM;AACR;AAAA,QACF;AACA,cAAM,aAAa,MAAM,KAAK;AAAA,MAChC;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,gBAAgB,MAAM,SAAS,cAAc;AAEhE;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AACA,yDAAiB;AACjB,aAAM,uCAAW;AACjB,YAAM,aAAa,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL,kBAAK,KAAK,CAACA,gBAAe,cAAcA,YAAW,MAAM,GAAG,YAAY,qBAAqB;AAAA,IAC7F;AAAA,EACF;AACF;AAOA,eAAe,YACb,QACA,KACA,QACA,YACe;AACf,QAAM,SAAS,OAAO,UAAU;AAChC,MAAI;AACF,WAAO,MAAM;AACX,UAAI,OAAO,SAAS;AAClB;AAAA,MACF;AACA,YAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,OAAO,KAAK;AACjD,UAAI,KAAM;AACV,UAAI,QAAQ;AACZ,UAAI,eAAe,MAAM;AACvB,cAAM,WAAW,YAAY,KAAK;AAAA,MACpC;AACA,UAAI,CAAC,IAAI,aAAa,MAAM;AAC1B,YAAI,aAAa,QAAQ;AAAA,MAC3B;AAAA,IACF;AAAA,EACF,UAAE;AACA,QAAI,eAAe,MAAM;AACvB,iBAAW,MAAM;AAAA,IACnB;AACA,qCAAQ;AAAA,EACV;AACF;AAEO,SAAS,sBACd,QACA,YACA,YACwB;AACxB,QAAM,MAAM;AAAA,IACV,MAAM;AAAA,IACN,cAAc,IAAI,oBAAO;AAAA,EAC3B;AACA,SAAO;AAAA,IACL,kBAAK;AAAA,MACH,CAACA,gBAAe,YAAY,QAAQ,KAAKA,YAAW,QAAQ,UAAU;AAAA,MACtE;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAOA,eAAe,aACb,WACA,YACA,KACA,QACe;AACf,QAAM,SAAS,UAAU,UAAU;AACnC,MAAI,YAAmC;AAEvC,MAAI;AACF,WAAO,MAAM;AACX,UAAI,iCAAQ,SAAS;AACnB;AAAA,MACF;AAEA,YAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,OAAO,KAAK;AACjD,UAAI,KAAM;AAEV,UAAI,MAAM,KAAK,KAAK;AAEpB,UACE,CAAC,IAAI,cAAc,QACnB,WAAW,cACX,WAAW,eAAe,MAAM,cAChC,CAAC,WACD;AACA,oBAAY,IAAI,+BAAe,MAAM,YAAY,WAAW,YAAY,CAAC;AAAA,MAC3E;AAEA,UAAI,WAAW;AACb,mBAAW,KAAK,UAAU,KAAK,KAAK,GAAG;AACrC,gBAAM,WAAW,aAAa,CAAC;AAAA,QACjC;AAAA,MACF,OAAO;AACL,cAAM,WAAW,aAAa,KAAK;AAAA,MACrC;AAIA,UAAI,CAAC,IAAI,cAAc,MAAM;AAC3B,YAAI,cAAc,QAAQ;AAAA,MAC5B;AAAA,IACF;AAAA,EACF,UAAE;AACA,qCAAQ;AACR,QAAI,WAAW;AACb,iBAAW,KAAK,UAAU,MAAM,GAAG;AACjC,cAAM,WAAW,aAAa,CAAC;AAAA,MACjC;AAAA,IACF;AACA,eAAW,MAAM;AAAA,EACnB;AACF;AAEO,SAAS,uBACd,WACA,aACA,YACyB;AACzB,QAAM,MAAM;AAAA,IACV,OAAO,CAAC;AAAA,IACR,eAAe,IAAI,oBAAO;AAAA,EAC5B;AACA,SAAO;AAAA,IACL,kBAAK;AAAA,MACH,CAACA,gBAAe,aAAa,WAAW,aAAa,KAAKA,YAAW,MAAM;AAAA,MAC3E;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,sBAAsB;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,yBAAyB,MAAM;AAAA,EAAC;AAAA,EAChC,2BAA2B,MAAM;AAAA,EAAC;AAAA,EAClC;AACF,GAS6B;AAC3B,QAAM,aAAS,gBAAI;AACnB,QAAM,aAAyB;AAAA,IAC7B,QAAQ,CAAC;AAAA,IACT,wBAAwB,IAAI,oBAAO;AAAA,EACrC;AAEA,QAAM,gBAAgB,CAAC,QAA6B;AAClD,6BAAyB,GAAG;AAC5B,eAAW,OAAO,KAAK,GAAG;AAAA,EAC5B;AAEA,QAAM,mBAAmB,OAAOA,gBAAgC;AAC9D,UAAM,SAASA,YAAW;AAC1B,UAAM,SAAS,eAAe,UAAU;AAExC,UAAM,QAAwB,CAAC;AAC/B,WAAO,CAAC,OAAO,SAAS;AACtB,YAAM,EAAE,MAAM,OAAO,SAAS,IAAI,MAAM,OAAO,KAAK;AACpD,UAAI,OAAO,QAAS;AACpB,UAAI,KAAM;AAEV,UAAI,eAAe,QAAQ;AACzB,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA;AAAA,QACF;AACA;AAAA,MACF;AAIA,YAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,UAAI,CAAC,MAAM;AACT,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA,uBAAuB,SAAS,IAAI;AAAA,QACtC;AACA;AAAA,MACF;AAEA,UAAI,KAAC,oCAAe,IAAI,GAAG;AACzB,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA,sBAAsB,OAAO,IAAI;AAAA,QACnC;AACA;AAAA,MACF;AAEA,UAAI;AAGJ,UAAI;AACF,cAAM,WAAW,KAAK,MAAM,SAAS,IAAI;AAEzC,YAAI,KAAK,sBAAsB,sBAAW;AACxC,uBAAa,KAAK,WAAW,MAAM,QAAQ;AAAA,QAC7C,OAAO;AACL,uBAAa;AAAA,QACf;AAAA,MACF,SAAS,UAAU;AACjB,cAAM,YAAQ,sBAAQ,QAAQ;AAC9B,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,SAAS;AAAA,YACpB,WAAW,aAAa;AAAA,YACxB,OAAO,MAAM;AAAA,UACf;AAAA,UACA,6BAA6B,SAAS,IAAI;AAAA,QAC5C;AACA;AAAA,UACE,iBAAiB;AAAA,YACf;AAAA,YACA,WAAW;AAAA,UACb,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,UAAI,CAAC,WAAW,uBAAuB,MAAM;AAC3C,mBAAW,uBAAuB,QAAQ;AAAA,MAC5C;AAEA,6BAAuB,QAAQ;AAE/B,aAAO;AAAA,QACL;AAAA,UACE,UAAU,SAAS;AAAA,UACnB,WAAW;AAAA,UACX,WAAW,aAAa;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AAEA,YAAM,gBAAgB,+BAAkB,IAAI,EAAE,cAAc,SAAS,GAAG,YAAY;AAClF,eAAO,MAAM,KAAK,QAAQ,YAAY;AAAA,UACpC,KAAK,IAAI,8BAAW,SAAS,cAAc,QAAQ;AAAA,UACnD,YAAY,SAAS;AAAA,UACrB,aAAa;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AAED,YAAM,wBAAwB,OAAO,iBAAmC;AAItE,YAAIC;AACJ,YAAI;AACF,gBAAM,EAAE,QAAQ,UAAU,IAAI,MAAM,iBAAiB,cAAc,MAAM;AACzE,UAAAA,cAAa,iBAAiB;AAAA,YAC5B;AAAA,YACA,WAAW,YAAY,IAAI,MAAM,uBAAuB,IAAI;AAAA,YAC5D,QAAQ,YAAY,SAAY;AAAA,UAClC,CAAC;AAAA,QACH,SAAS,UAAU;AACjB,iBAAO;AAAA,YACL;AAAA,cACE,UAAU,SAAS;AAAA,cACnB,WAAW,aAAa;AAAA,cACxB,WAAO,sBAAQ,QAAQ,EAAE;AAAA,YAC3B;AAAA,YACA;AAAA,UACF;AACA,UAAAA,cAAa,iBAAiB;AAAA,YAC5B;AAAA,YACA,eAAW,sBAAQ,QAAQ;AAAA,UAC7B,CAAC;AAAA,QACH,UAAE;AACA,cAAI,CAACA,YAAY,OAAM,IAAI,MAAM,yBAAyB;AAC1D,wBAAcA,WAAU;AAAA,QAC1B;AAAA,MACF;AAGA,YAAM,KAAK,sBAAsB,aAAa,CAAC;AAAA,IACjD;AAEA,UAAM,QAAQ,WAAW,KAAK;AAC9B,QAAI,WAAW,OAAO,SAAS,GAAG;AAChC,aAAO;AAAA,QACL;AAAA,UACE,WAAW,aAAa;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO,CAAC,kBAAK,KAAK,kBAAkB,YAAY,uBAAuB,GAAG,UAAU;AACtF;AAYA,eAAe,iBAAoB,SAAqB,QAA0C;AAChG,QAAM,WAAW,IAAI,oBAAmB;AAExC,QAAM,eAAe,MAAM;AACzB,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,QAAQ,EAAE,QAAQ,QAAW,WAAW,KAAK,CAAC;AAAA,IACzD;AAAA,EACF;AAEA,SAAO,iBAAiB,SAAS,YAAY;AAE7C,UACG,KAAK,CAAC,MAAM;AACX,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,QAAQ,EAAE,QAAQ,GAAG,WAAW,MAAM,CAAC;AAAA,IAClD;AAAA,EACF,CAAC,EACA,MAAM,CAAC,MAAM;AACZ,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,OAAO,CAAC;AAAA,IACnB;AAAA,EACF,CAAC,EACA,QAAQ,MAAM;AACb,WAAO,oBAAoB,SAAS,YAAY;AAAA,EAClD,CAAC;AAEH,SAAO,MAAM,SAAS;AACxB;AAEO,SAAS,mBAAmB,SAAsB;AAEvD,SAAO,MAAM;AACX,UAAM,MAAM,QAAQ,UAAU,uBAAuB;AACrD,QAAI,QAAQ,QAAW;AACrB,cAAQ,MAAM,OAAO,KAAK,CAAC;AAAA,IAC7B,OAAO;AACL;AAAA,IACF;AAAA,EACF;AACF;","names":["controller","toolOutput"]}
+{"version":3,"sources":["../../src/voice/generation.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { AudioResampler } from '@livekit/rtc-node';\nimport type { ReadableStream, ReadableStreamDefaultReader } from 'stream/web';\nimport { ZodObject } from 'zod';\nimport {\n  type ChatContext,\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n} from '../llm/chat_context.js';\nimport type { ChatChunk } from '../llm/llm.js';\nimport {\n  type ToolChoice,\n  type ToolContext,\n  isAgentHandoff,\n  isFunctionTool,\n  isToolError,\n} from '../llm/tool_context.js';\nimport { log } from '../log.js';\nimport { IdentityTransform } from '../stream/identity_transform.js';\nimport { Future, Task, shortuuid, toError } from '../utils.js';\nimport { type Agent, type ModelSettings, asyncLocalStorage, isStopResponse } from './agent.js';\nimport type { AgentSession } from './agent_session.js';\nimport type { AudioOutput, LLMNode, TTSNode, TextOutput } from './io.js';\nimport { RunContext } from './run_context.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\n/** @internal */\nexport class _LLMGenerationData {\n  generatedText: string = '';\n  generatedToolCalls: FunctionCall[];\n  id: string;\n\n  constructor(\n    public readonly textStream: ReadableStream<string>,\n    public readonly toolCallStream: ReadableStream<FunctionCall>,\n  ) {\n    this.id = shortuuid('item_');\n    this.generatedToolCalls = [];\n  }\n}\n\n// TODO(brian): remove this class in favor of ToolOutput\nexport class _ToolOutput {\n  output: _JsOutput[];\n  firstToolFut: Future;\n\n  constructor() {\n    this.output = [];\n    this.firstToolFut = new Future();\n  }\n}\n\n// TODO(brian): remove this class in favor of ToolExecutionOutput\nexport class _SanitizedOutput {\n  toolCall: FunctionCall;\n  toolCallOutput?: FunctionCallOutput;\n  replyRequired: boolean;\n  agentTask?: Agent;\n\n  constructor(\n    toolCall: FunctionCall,\n    toolCallOutput: FunctionCallOutput | undefined,\n    replyRequired: boolean,\n    agentTask: Agent | undefined,\n  ) {\n    this.toolCall = toolCall;\n    this.toolCallOutput = toolCallOutput;\n    this.replyRequired = replyRequired;\n    this.agentTask = agentTask;\n  }\n\n  static create(params: {\n    toolCall: FunctionCall;\n    toolCallOutput?: FunctionCallOutput;\n    replyRequired?: boolean;\n    agentTask?: Agent;\n  }) {\n    const { toolCall, toolCallOutput, replyRequired = true, agentTask } = params;\n    return new _SanitizedOutput(toolCall, toolCallOutput, replyRequired, agentTask);\n  }\n}\n\nfunction isValidToolOutput(toolOutput: unknown): boolean {\n  const validTypes = ['string', 'number', 'boolean'];\n\n  if (validTypes.includes(typeof toolOutput)) {\n    return true;\n  }\n\n  if (toolOutput === undefined || toolOutput === null) {\n    return true;\n  }\n\n  if (Array.isArray(toolOutput)) {\n    return toolOutput.every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Set) {\n    return Array.from(toolOutput).every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Map) {\n    return Array.from(toolOutput.values()).every(isValidToolOutput);\n  }\n\n  if (toolOutput instanceof Object) {\n    return Object.entries(toolOutput).every(\n      ([key, value]) => validTypes.includes(typeof key) && isValidToolOutput(value),\n    );\n  }\n\n  return false;\n}\n\nexport class ToolExecutionOutput {\n  constructor(\n    public readonly toolCall: FunctionCall,\n    public readonly toolCallOutput: FunctionCallOutput | undefined,\n    public readonly agentTask: Agent | undefined,\n    public readonly rawOutput: unknown,\n    public readonly rawException: Error | undefined,\n    public readonly replyRequired: boolean,\n  ) {}\n\n  static create(params: {\n    toolCall: FunctionCall;\n    toolCallOutput?: FunctionCallOutput;\n    agentTask?: Agent;\n    rawOutput: unknown;\n    rawException?: Error;\n    replyRequired?: boolean;\n  }) {\n    const {\n      toolCall,\n      toolCallOutput,\n      agentTask,\n      rawOutput,\n      rawException,\n      replyRequired = true,\n    } = params;\n    return new ToolExecutionOutput(\n      toolCall,\n      toolCallOutput,\n      agentTask,\n      rawOutput,\n      rawException,\n      replyRequired,\n    );\n  }\n}\n\nexport interface ToolOutput {\n  output: ToolExecutionOutput[];\n  firstToolStartedFuture: Future<void>;\n}\n\n// TODO(brian): remove this class in favor of ToolExecutionOutput\nexport class _JsOutput {\n  toolCall: FunctionCall;\n  output: unknown;\n  exception?: Error;\n\n  #logger = log();\n\n  constructor(toolCall: FunctionCall, output: unknown, exception: Error | undefined) {\n    this.toolCall = toolCall;\n    this.output = output;\n    this.exception = exception;\n  }\n\n  static create(params: { toolCall: FunctionCall; output?: unknown; exception?: Error }) {\n    const { toolCall, output = undefined, exception = undefined } = params;\n    return new _JsOutput(toolCall, output, exception);\n  }\n\n  sanitize(): _SanitizedOutput {\n    if (isToolError(this.exception)) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: FunctionCallOutput.create({\n          name: this.toolCall.name,\n          callId: this.toolCall.callId,\n          output: this.exception.message,\n          isError: true,\n        }),\n      });\n    }\n\n    if (isStopResponse(this.exception)) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n      });\n    }\n\n    if (this.exception !== undefined) {\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: FunctionCallOutput.create({\n          name: this.toolCall.name,\n          callId: this.toolCall.callId,\n          output: 'An internal error occurred while executing the tool.', // Don't send the actual error message, as it may contain sensitive information\n          isError: true,\n        }),\n      });\n    }\n\n    let agentTask: Agent | undefined = undefined;\n    let toolOutput: unknown = this.output;\n    if (isAgentHandoff(this.output)) {\n      agentTask = this.output.agent;\n      toolOutput = this.output.returns;\n    }\n\n    if (!isValidToolOutput(toolOutput)) {\n      this.#logger.error(\n        {\n          callId: this.toolCall.callId,\n          function: this.toolCall.name,\n        },\n        `AI function ${this.toolCall.name} returned an invalid output`,\n      );\n      return _SanitizedOutput.create({\n        toolCall: FunctionCall.create({ ...this.toolCall }),\n        toolCallOutput: undefined,\n      });\n    }\n\n    return _SanitizedOutput.create({\n      toolCall: FunctionCall.create({ ...this.toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: this.toolCall.name,\n        callId: this.toolCall.callId,\n        output: toolOutput !== undefined ? JSON.stringify(toolOutput) : '', // take the string representation of the output\n        isError: false,\n      }),\n      replyRequired: toolOutput !== undefined, // require a reply if the tool returned an output\n      agentTask,\n    });\n  }\n}\n\nexport function createToolOutput(params: {\n  toolCall: FunctionCall;\n  output?: unknown;\n  exception?: Error;\n}): ToolExecutionOutput {\n  const { toolCall, output, exception } = params;\n  const logger = log();\n\n  // support returning Exception instead of raising them (for devex purposes inside evals)\n  let finalOutput = output;\n  let finalException = exception;\n  if (output instanceof Error) {\n    finalException = output;\n    finalOutput = undefined;\n  }\n\n  if (isToolError(finalException)) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: toolCall.name,\n        callId: toolCall.callId,\n        output: finalException.message,\n        isError: true,\n      }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  if (isStopResponse(finalException)) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  if (finalException !== undefined) {\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      toolCallOutput: FunctionCallOutput.create({\n        name: toolCall.name,\n        callId: toolCall.callId,\n        output: 'An internal error occurred', // Don't send the actual error message, as it may contain sensitive information\n        isError: true,\n      }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  let agentTask: Agent | undefined = undefined;\n  let toolOutput: unknown = finalOutput;\n  if (isAgentHandoff(finalOutput)) {\n    agentTask = finalOutput.agent;\n    toolOutput = finalOutput.returns;\n  }\n\n  if (!isValidToolOutput(toolOutput)) {\n    logger.error(\n      {\n        callId: toolCall.callId,\n        output: finalOutput,\n      },\n      `AI function ${toolCall.name} returned an invalid output`,\n    );\n    return ToolExecutionOutput.create({\n      toolCall: FunctionCall.create({ ...toolCall }),\n      rawOutput: finalOutput,\n      rawException: finalException,\n    });\n  }\n\n  return ToolExecutionOutput.create({\n    toolCall: FunctionCall.create({ ...toolCall }),\n    toolCallOutput: FunctionCallOutput.create({\n      name: toolCall.name,\n      callId: toolCall.callId,\n      output: toolOutput !== undefined ? JSON.stringify(toolOutput) : '', // take the string representation of the output\n      isError: false,\n    }),\n    replyRequired: toolOutput !== undefined, // require a reply if the tool returned an output\n    agentTask,\n    rawOutput: finalOutput,\n    rawException: finalException,\n  });\n}\n\nconst INSTRUCTIONS_MESSAGE_ID = 'lk.agent_task.instructions';\n\n/**\n * Update the instruction message in the chat context or insert a new one if missing.\n *\n * This function looks for an existing instruction message in the chat context using the identifier\n * 'INSTRUCTIONS_MESSAGE_ID'.\n *\n * @param options - The options for updating the instructions.\n * @param options.chatCtx - The chat context to update.\n * @param options.instructions - The instructions to add.\n * @param options.addIfMissing - Whether to add the instructions if they are missing.\n */\nexport function updateInstructions(options: {\n  chatCtx: ChatContext;\n  instructions: string;\n  addIfMissing: boolean;\n}) {\n  const { chatCtx, instructions, addIfMissing } = options;\n\n  const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID);\n  if (idx !== undefined) {\n    if (chatCtx.items[idx]!.type === 'message') {\n      // create a new instance to avoid mutating the original\n      chatCtx.items[idx] = ChatMessage.create({\n        id: INSTRUCTIONS_MESSAGE_ID,\n        role: 'system',\n        content: [instructions],\n        createdAt: chatCtx.items[idx]!.createdAt,\n      });\n    } else {\n      throw new Error('expected the instructions inside the chatCtx to be of type \"message\"');\n    }\n  } else if (addIfMissing) {\n    // insert the instructions at the beginning of the chat context\n    chatCtx.items.unshift(\n      ChatMessage.create({\n        id: INSTRUCTIONS_MESSAGE_ID,\n        role: 'system',\n        content: [instructions],\n      }),\n    );\n  }\n}\n\nexport function performLLMInference(\n  node: LLMNode,\n  chatCtx: ChatContext,\n  toolCtx: ToolContext,\n  modelSettings: ModelSettings,\n  controller: AbortController,\n): [Task<void>, _LLMGenerationData] {\n  const textStream = new IdentityTransform<string>();\n  const toolCallStream = new IdentityTransform<FunctionCall>();\n\n  const textWriter = textStream.writable.getWriter();\n  const toolCallWriter = toolCallStream.writable.getWriter();\n  const data = new _LLMGenerationData(textStream.readable, toolCallStream.readable);\n\n  const inferenceTask = async (signal: AbortSignal) => {\n    let llmStreamReader: ReadableStreamDefaultReader<string | ChatChunk> | null = null;\n    let llmStream: ReadableStream<string | ChatChunk> | null = null;\n\n    try {\n      llmStream = await node(chatCtx, toolCtx, modelSettings);\n      if (llmStream === null) {\n        await textWriter.close();\n        return;\n      }\n\n      // TODO(brian): add support for dynamic tools\n\n      llmStreamReader = llmStream.getReader();\n      while (true) {\n        if (signal.aborted) {\n          break;\n        }\n        const { done, value: chunk } = await llmStreamReader.read();\n        if (done) {\n          break;\n        }\n\n        if (typeof chunk === 'string') {\n          data.generatedText += chunk;\n          await textWriter.write(chunk);\n          // TODO(shubhra): better way to check??\n        } else {\n          if (chunk.delta === undefined) {\n            continue;\n          }\n\n          if (chunk.delta.toolCalls) {\n            for (const tool of chunk.delta.toolCalls) {\n              if (tool.type !== 'function_call') continue;\n\n              const toolCall = FunctionCall.create({\n                callId: `${data.id}/fnc_${data.generatedToolCalls.length}`,\n                name: tool.name,\n                args: tool.args,\n              });\n\n              data.generatedToolCalls.push(toolCall);\n              await toolCallWriter.write(toolCall);\n            }\n          }\n\n          if (chunk.delta.content) {\n            data.generatedText += chunk.delta.content;\n            await textWriter.write(chunk.delta.content);\n          }\n        }\n\n        // No need to check if chunk is of type other than ChatChunk or string like in\n        // Python since chunk is defined in the type ChatChunk | string in TypeScript\n      }\n    } catch (error) {\n      if (error instanceof DOMException && error.name === 'AbortError') {\n        // Abort signal was triggered, handle gracefully\n        return;\n      }\n      throw error;\n    } finally {\n      llmStreamReader?.releaseLock();\n      await llmStream?.cancel();\n      await textWriter.close();\n      await toolCallWriter.close();\n    }\n  };\n\n  return [\n    Task.from((controller) => inferenceTask(controller.signal), controller, 'performLLMInference'),\n    data,\n  ];\n}\n\nexport function performTTSInference(\n  node: TTSNode,\n  text: ReadableStream<string>,\n  modelSettings: ModelSettings,\n  controller: AbortController,\n): [Task<void>, ReadableStream<AudioFrame>] {\n  const audioStream = new IdentityTransform<AudioFrame>();\n  const outputWriter = audioStream.writable.getWriter();\n  const audioOutputStream = audioStream.readable;\n\n  const inferenceTask = async (signal: AbortSignal) => {\n    let ttsStreamReader: ReadableStreamDefaultReader<AudioFrame> | null = null;\n    let ttsStream: ReadableStream<AudioFrame> | null = null;\n\n    try {\n      ttsStream = await node(text, modelSettings);\n      if (ttsStream === null) {\n        await outputWriter.close();\n        return;\n      }\n\n      ttsStreamReader = ttsStream.getReader();\n      while (true) {\n        if (signal.aborted) {\n          break;\n        }\n        const { done, value: chunk } = await ttsStreamReader.read();\n        if (done) {\n          break;\n        }\n        await outputWriter.write(chunk);\n      }\n    } catch (error) {\n      if (error instanceof DOMException && error.name === 'AbortError') {\n        // Abort signal was triggered, handle gracefully\n        return;\n      }\n      throw error;\n    } finally {\n      ttsStreamReader?.releaseLock();\n      await ttsStream?.cancel();\n      await outputWriter.close();\n    }\n  };\n\n  return [\n    Task.from((controller) => inferenceTask(controller.signal), controller, 'performTTSInference'),\n    audioOutputStream,\n  ];\n}\n\nexport interface _TextOut {\n  text: string;\n  firstTextFut: Future;\n}\n\nasync function forwardText(\n  source: ReadableStream<string>,\n  out: _TextOut,\n  signal: AbortSignal,\n  textOutput: TextOutput | null,\n): Promise<void> {\n  const reader = source.getReader();\n  try {\n    while (true) {\n      if (signal.aborted) {\n        break;\n      }\n      const { done, value: delta } = await reader.read();\n      if (done) break;\n      out.text += delta;\n      if (textOutput !== null) {\n        await textOutput.captureText(delta);\n      }\n      if (!out.firstTextFut.done) {\n        out.firstTextFut.resolve();\n      }\n    }\n  } finally {\n    if (textOutput !== null) {\n      textOutput.flush();\n    }\n    reader?.releaseLock();\n  }\n}\n\nexport function performTextForwarding(\n  source: ReadableStream<string>,\n  controller: AbortController,\n  textOutput: TextOutput | null,\n): [Task<void>, _TextOut] {\n  const out = {\n    text: '',\n    firstTextFut: new Future(),\n  };\n  return [\n    Task.from(\n      (controller) => forwardText(source, out, controller.signal, textOutput),\n      controller,\n      'performTextForwarding',\n    ),\n    out,\n  ];\n}\n\nexport interface _AudioOut {\n  audio: Array<AudioFrame>;\n  firstFrameFut: Future;\n}\n\nasync function forwardAudio(\n  ttsStream: ReadableStream<AudioFrame>,\n  audioOuput: AudioOutput,\n  out: _AudioOut,\n  signal?: AbortSignal,\n): Promise<void> {\n  const reader = ttsStream.getReader();\n  let resampler: AudioResampler | null = null;\n\n  try {\n    while (true) {\n      if (signal?.aborted) {\n        break;\n      }\n\n      const { done, value: frame } = await reader.read();\n      if (done) break;\n\n      out.audio.push(frame);\n\n      if (\n        !out.firstFrameFut.done &&\n        audioOuput.sampleRate &&\n        audioOuput.sampleRate !== frame.sampleRate &&\n        !resampler\n      ) {\n        resampler = new AudioResampler(frame.sampleRate, audioOuput.sampleRate, 1);\n      }\n\n      if (resampler) {\n        for (const f of resampler.push(frame)) {\n          await audioOuput.captureFrame(f);\n        }\n      } else {\n        await audioOuput.captureFrame(frame);\n      }\n\n      // set the first frame future if not already set\n      // (after completing the first frame)\n      if (!out.firstFrameFut.done) {\n        out.firstFrameFut.resolve();\n      }\n    }\n  } finally {\n    reader?.releaseLock();\n    if (resampler) {\n      for (const f of resampler.flush()) {\n        await audioOuput.captureFrame(f);\n      }\n    }\n    audioOuput.flush();\n  }\n}\n\nexport function performAudioForwarding(\n  ttsStream: ReadableStream<AudioFrame>,\n  audioOutput: AudioOutput,\n  controller: AbortController,\n): [Task<void>, _AudioOut] {\n  const out = {\n    audio: [],\n    firstFrameFut: new Future(),\n  };\n  return [\n    Task.from(\n      (controller) => forwardAudio(ttsStream, audioOutput, out, controller.signal),\n      controller,\n      'performAudioForwarding',\n    ),\n    out,\n  ];\n}\n\nexport function performToolExecutions({\n  session,\n  speechHandle,\n  toolCtx,\n  toolChoice,\n  toolCallStream,\n  onToolExecutionStarted = () => {},\n  onToolExecutionCompleted = () => {},\n  controller,\n}: {\n  session: AgentSession;\n  speechHandle: SpeechHandle;\n  toolCtx: ToolContext;\n  toolChoice?: ToolChoice;\n  toolCallStream: ReadableStream<FunctionCall>;\n  onToolExecutionStarted?: (toolCall: FunctionCall) => void;\n  onToolExecutionCompleted?: (toolExecutionOutput: ToolExecutionOutput) => void;\n  controller: AbortController;\n}): [Task<void>, ToolOutput] {\n  const logger = log();\n  const toolOutput: ToolOutput = {\n    output: [],\n    firstToolStartedFuture: new Future(),\n  };\n\n  const toolCompleted = (out: ToolExecutionOutput) => {\n    onToolExecutionCompleted(out);\n    toolOutput.output.push(out);\n  };\n\n  const executeToolsTask = async (controller: AbortController) => {\n    const signal = controller.signal;\n    const reader = toolCallStream.getReader();\n\n    const tasks: Promise<any>[] = [];\n    while (!signal.aborted) {\n      const { done, value: toolCall } = await reader.read();\n      if (signal.aborted) break;\n      if (done) break;\n\n      if (toolChoice === 'none') {\n        logger.error(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          \"received a tool call with toolChoice set to 'none', ignoring\",\n        );\n        continue;\n      }\n\n      // TODO(brian): assert other toolChoice values\n\n      const tool = toolCtx[toolCall.name];\n      if (!tool) {\n        logger.warn(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          `unknown AI function ${toolCall.name}`,\n        );\n        continue;\n      }\n\n      if (!isFunctionTool(tool)) {\n        logger.error(\n          {\n            function: toolCall.name,\n            speech_id: speechHandle.id,\n          },\n          `unknown tool type: ${typeof tool}`,\n        );\n        continue;\n      }\n\n      let parsedArgs: object | undefined;\n\n      // Ensure valid arguments\n      try {\n        const jsonArgs = JSON.parse(toolCall.args);\n\n        if (tool.parameters instanceof ZodObject) {\n          parsedArgs = tool.parameters.parse(jsonArgs);\n        } else {\n          parsedArgs = jsonArgs;\n        }\n      } catch (rawError) {\n        const error = toError(rawError);\n        logger.error(\n          {\n            function: toolCall.name,\n            arguments: toolCall.args,\n            speech_id: speechHandle.id,\n            error: error.message,\n          },\n          `tried to call AI function ${toolCall.name} with invalid arguments`,\n        );\n        toolCompleted(\n          createToolOutput({\n            toolCall,\n            exception: error,\n          }),\n        );\n        continue;\n      }\n\n      if (!toolOutput.firstToolStartedFuture.done) {\n        toolOutput.firstToolStartedFuture.resolve();\n      }\n\n      onToolExecutionStarted(toolCall);\n\n      logger.info(\n        {\n          function: toolCall.name,\n          arguments: parsedArgs,\n          speech_id: speechHandle.id,\n        },\n        'Executing LLM tool call',\n      );\n\n      const toolExecution = asyncLocalStorage.run({ functionCall: toolCall }, async () => {\n        return await tool.execute(parsedArgs, {\n          ctx: new RunContext(session, speechHandle, toolCall),\n          toolCallId: toolCall.callId,\n          abortSignal: signal,\n        });\n      });\n\n      const tracableToolExecution = async (toolExecTask: Promise<unknown>) => {\n        // TODO(brian): add tracing\n\n        // await for task to complete, if task is aborted, set exception\n        let toolOutput: ToolExecutionOutput | undefined;\n        try {\n          const { result, isAborted } = await waitUntilAborted(toolExecTask, signal);\n          toolOutput = createToolOutput({\n            toolCall,\n            exception: isAborted ? new Error('tool call was aborted') : undefined,\n            output: isAborted ? undefined : result,\n          });\n        } catch (rawError) {\n          logger.error(\n            {\n              function: toolCall.name,\n              speech_id: speechHandle.id,\n              error: toError(rawError).message,\n            },\n            'exception occurred while executing tool',\n          );\n          toolOutput = createToolOutput({\n            toolCall,\n            exception: toError(rawError),\n          });\n        } finally {\n          if (!toolOutput) throw new Error('toolOutput is undefined');\n          toolCompleted(toolOutput);\n        }\n      };\n\n      // wait, not cancelling all tool calling tasks\n      tasks.push(tracableToolExecution(toolExecution));\n    }\n\n    await Promise.allSettled(tasks);\n    if (toolOutput.output.length > 0) {\n      logger.debug(\n        {\n          speech_id: speechHandle.id,\n        },\n        'tools execution completed',\n      );\n    }\n  };\n\n  return [Task.from(executeToolsTask, controller, 'performToolExecutions'), toolOutput];\n}\n\ntype Aborted<T> =\n  | {\n      result: T;\n      isAborted: false;\n    }\n  | {\n      result: undefined;\n      isAborted: true;\n    };\n\nasync function waitUntilAborted<T>(promise: Promise<T>, signal: AbortSignal): Promise<Aborted<T>> {\n  const abortFut = new Future<Aborted<T>>();\n\n  const resolveAbort = () => {\n    if (!abortFut.done) {\n      abortFut.resolve({ result: undefined, isAborted: true });\n    }\n  };\n\n  signal.addEventListener('abort', resolveAbort);\n\n  promise\n    .then((r) => {\n      if (!abortFut.done) {\n        abortFut.resolve({ result: r, isAborted: false });\n      }\n    })\n    .catch((e) => {\n      if (!abortFut.done) {\n        abortFut.reject(e);\n      }\n    })\n    .finally(() => {\n      signal.removeEventListener('abort', resolveAbort);\n    });\n\n  return await abortFut.await;\n}\n\nexport function removeInstructions(chatCtx: ChatContext) {\n  // loop in case there are items with the same id (shouldn't happen!)\n  while (true) {\n    const idx = chatCtx.indexById(INSTRUCTIONS_MESSAGE_ID);\n    if (idx !== undefined) {\n      chatCtx.items.splice(idx, 1);\n    } else {\n      break;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,sBAA+B;AAE/B,iBAA0B;AAC1B,0BAKO;AAEP,0BAMO;AACP,iBAAoB;AACpB,gCAAkC;AAClC,mBAAiD;AACjD,mBAAkF;AAGlF,yBAA2B;AAIpB,MAAM,mBAAmB;AAAA,EAK9B,YACkB,YACA,gBAChB;AAFgB;AACA;AAEhB,SAAK,SAAK,wBAAU,OAAO;AAC3B,SAAK,qBAAqB,CAAC;AAAA,EAC7B;AAAA,EAVA,gBAAwB;AAAA,EACxB;AAAA,EACA;AASF;AAGO,MAAM,YAAY;AAAA,EACvB;AAAA,EACA;AAAA,EAEA,cAAc;AACZ,SAAK,SAAS,CAAC;AACf,SAAK,eAAe,IAAI,oBAAO;AAAA,EACjC;AACF;AAGO,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YACE,UACA,gBACA,eACA,WACA;AACA,SAAK,WAAW;AAChB,SAAK,iBAAiB;AACtB,SAAK,gBAAgB;AACrB,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,OAAO,OAAO,QAKX;AACD,UAAM,EAAE,UAAU,gBAAgB,gBAAgB,MAAM,UAAU,IAAI;AACtE,WAAO,IAAI,iBAAiB,UAAU,gBAAgB,eAAe,SAAS;AAAA,EAChF;AACF;AAEA,SAAS,kBAAkB,YAA8B;AACvD,QAAM,aAAa,CAAC,UAAU,UAAU,SAAS;AAEjD,MAAI,WAAW,SAAS,OAAO,UAAU,GAAG;AAC1C,WAAO;AAAA,EACT;AAEA,MAAI,eAAe,UAAa,eAAe,MAAM;AACnD,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,WAAO,WAAW,MAAM,iBAAiB;AAAA,EAC3C;AAEA,MAAI,sBAAsB,KAAK;AAC7B,WAAO,MAAM,KAAK,UAAU,EAAE,MAAM,iBAAiB;AAAA,EACvD;AAEA,MAAI,sBAAsB,KAAK;AAC7B,WAAO,MAAM,KAAK,WAAW,OAAO,CAAC,EAAE,MAAM,iBAAiB;AAAA,EAChE;AAEA,MAAI,sBAAsB,QAAQ;AAChC,WAAO,OAAO,QAAQ,UAAU,EAAE;AAAA,MAChC,CAAC,CAAC,KAAK,KAAK,MAAM,WAAW,SAAS,OAAO,GAAG,KAAK,kBAAkB,KAAK;AAAA,IAC9E;AAAA,EACF;AAEA,SAAO;AACT;AAEO,MAAM,oBAAoB;AAAA,EAC/B,YACkB,UACA,gBACA,WACA,WACA,cACA,eAChB;AANgB;AACA;AACA;AACA;AACA;AACA;AAAA,EACf;AAAA,EAEH,OAAO,OAAO,QAOX;AACD,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,IAClB,IAAI;AACJ,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAQO,MAAM,UAAU;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EAEA,cAAU,gBAAI;AAAA,EAEd,YAAY,UAAwB,QAAiB,WAA8B;AACjF,SAAK,WAAW;AAChB,SAAK,SAAS;AACd,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,OAAO,OAAO,QAAyE;AACrF,UAAM,EAAE,UAAU,SAAS,QAAW,YAAY,OAAU,IAAI;AAChE,WAAO,IAAI,UAAU,UAAU,QAAQ,SAAS;AAAA,EAClD;AAAA,EAEA,WAA6B;AAC3B,YAAI,iCAAY,KAAK,SAAS,GAAG;AAC/B,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB,uCAAmB,OAAO;AAAA,UACxC,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,QAAQ,KAAK,UAAU;AAAA,UACvB,SAAS;AAAA,QACX,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAEA,YAAI,6BAAe,KAAK,SAAS,GAAG;AAClC,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,MACpD,CAAC;AAAA,IACH;AAEA,QAAI,KAAK,cAAc,QAAW;AAChC,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB,uCAAmB,OAAO;AAAA,UACxC,MAAM,KAAK,SAAS;AAAA,UACpB,QAAQ,KAAK,SAAS;AAAA,UACtB,QAAQ;AAAA;AAAA,UACR,SAAS;AAAA,QACX,CAAC;AAAA,MACH,CAAC;AAAA,IACH;AAEA,QAAI,YAA+B;AACnC,QAAI,aAAsB,KAAK;AAC/B,YAAI,oCAAe,KAAK,MAAM,GAAG;AAC/B,kBAAY,KAAK,OAAO;AACxB,mBAAa,KAAK,OAAO;AAAA,IAC3B;AAEA,QAAI,CAAC,kBAAkB,UAAU,GAAG;AAClC,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,QAAQ,KAAK,SAAS;AAAA,UACtB,UAAU,KAAK,SAAS;AAAA,QAC1B;AAAA,QACA,eAAe,KAAK,SAAS,IAAI;AAAA,MACnC;AACA,aAAO,iBAAiB,OAAO;AAAA,QAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,QAClD,gBAAgB;AAAA,MAClB,CAAC;AAAA,IACH;AAEA,WAAO,iBAAiB,OAAO;AAAA,MAC7B,UAAU,iCAAa,OAAO,EAAE,GAAG,KAAK,SAAS,CAAC;AAAA,MAClD,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,KAAK,SAAS;AAAA,QACpB,QAAQ,KAAK,SAAS;AAAA,QACtB,QAAQ,eAAe,SAAY,KAAK,UAAU,UAAU,IAAI;AAAA;AAAA,QAChE,SAAS;AAAA,MACX,CAAC;AAAA,MACD,eAAe,eAAe;AAAA;AAAA,MAC9B;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEO,SAAS,iBAAiB,QAIT;AACtB,QAAM,EAAE,UAAU,QAAQ,UAAU,IAAI;AACxC,QAAM,aAAS,gBAAI;AAGnB,MAAI,cAAc;AAClB,MAAI,iBAAiB;AACrB,MAAI,kBAAkB,OAAO;AAC3B,qBAAiB;AACjB,kBAAc;AAAA,EAChB;AAEA,UAAI,iCAAY,cAAc,GAAG;AAC/B,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,SAAS;AAAA,QACf,QAAQ,SAAS;AAAA,QACjB,QAAQ,eAAe;AAAA,QACvB,SAAS;AAAA,MACX,CAAC;AAAA,MACD,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,UAAI,6BAAe,cAAc,GAAG;AAClC,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,MAAI,mBAAmB,QAAW;AAChC,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,gBAAgB,uCAAmB,OAAO;AAAA,QACxC,MAAM,SAAS;AAAA,QACf,QAAQ,SAAS;AAAA,QACjB,QAAQ;AAAA;AAAA,QACR,SAAS;AAAA,MACX,CAAC;AAAA,MACD,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,MAAI,YAA+B;AACnC,MAAI,aAAsB;AAC1B,UAAI,oCAAe,WAAW,GAAG;AAC/B,gBAAY,YAAY;AACxB,iBAAa,YAAY;AAAA,EAC3B;AAEA,MAAI,CAAC,kBAAkB,UAAU,GAAG;AAClC,WAAO;AAAA,MACL;AAAA,QACE,QAAQ,SAAS;AAAA,QACjB,QAAQ;AAAA,MACV;AAAA,MACA,eAAe,SAAS,IAAI;AAAA,IAC9B;AACA,WAAO,oBAAoB,OAAO;AAAA,MAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,MAC7C,WAAW;AAAA,MACX,cAAc;AAAA,IAChB,CAAC;AAAA,EACH;AAEA,SAAO,oBAAoB,OAAO;AAAA,IAChC,UAAU,iCAAa,OAAO,EAAE,GAAG,SAAS,CAAC;AAAA,IAC7C,gBAAgB,uCAAmB,OAAO;AAAA,MACxC,MAAM,SAAS;AAAA,MACf,QAAQ,SAAS;AAAA,MACjB,QAAQ,eAAe,SAAY,KAAK,UAAU,UAAU,IAAI;AAAA;AAAA,MAChE,SAAS;AAAA,IACX,CAAC;AAAA,IACD,eAAe,eAAe;AAAA;AAAA,IAC9B;AAAA,IACA,WAAW;AAAA,IACX,cAAc;AAAA,EAChB,CAAC;AACH;AAEA,MAAM,0BAA0B;AAazB,SAAS,mBAAmB,SAIhC;AACD,QAAM,EAAE,SAAS,cAAc,aAAa,IAAI;AAEhD,QAAM,MAAM,QAAQ,UAAU,uBAAuB;AACrD,MAAI,QAAQ,QAAW;AACrB,QAAI,QAAQ,MAAM,GAAG,EAAG,SAAS,WAAW;AAE1C,cAAQ,MAAM,GAAG,IAAI,gCAAY,OAAO;AAAA,QACtC,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,WAAW,QAAQ,MAAM,GAAG,EAAG;AAAA,MACjC,CAAC;AAAA,IACH,OAAO;AACL,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAAA,EACF,WAAW,cAAc;AAEvB,YAAQ,MAAM;AAAA,MACZ,gCAAY,OAAO;AAAA,QACjB,IAAI;AAAA,QACJ,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,MACxB,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEO,SAAS,oBACd,MACA,SACA,SACA,eACA,YACkC;AAClC,QAAM,aAAa,IAAI,4CAA0B;AACjD,QAAM,iBAAiB,IAAI,4CAAgC;AAE3D,QAAM,aAAa,WAAW,SAAS,UAAU;AACjD,QAAM,iBAAiB,eAAe,SAAS,UAAU;AACzD,QAAM,OAAO,IAAI,mBAAmB,WAAW,UAAU,eAAe,QAAQ;AAEhF,QAAM,gBAAgB,OAAO,WAAwB;AACnD,QAAI,kBAA0E;AAC9E,QAAI,YAAuD;AAE3D,QAAI;AACF,kBAAY,MAAM,KAAK,SAAS,SAAS,aAAa;AACtD,UAAI,cAAc,MAAM;AACtB,cAAM,WAAW,MAAM;AACvB;AAAA,MACF;AAIA,wBAAkB,UAAU,UAAU;AACtC,aAAO,MAAM;AACX,YAAI,OAAO,SAAS;AAClB;AAAA,QACF;AACA,cAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,gBAAgB,KAAK;AAC1D,YAAI,MAAM;AACR;AAAA,QACF;AAEA,YAAI,OAAO,UAAU,UAAU;AAC7B,eAAK,iBAAiB;AACtB,gBAAM,WAAW,MAAM,KAAK;AAAA,QAE9B,OAAO;AACL,cAAI,MAAM,UAAU,QAAW;AAC7B;AAAA,UACF;AAEA,cAAI,MAAM,MAAM,WAAW;AACzB,uBAAW,QAAQ,MAAM,MAAM,WAAW;AACxC,kBAAI,KAAK,SAAS,gBAAiB;AAEnC,oBAAM,WAAW,iCAAa,OAAO;AAAA,gBACnC,QAAQ,GAAG,KAAK,EAAE,QAAQ,KAAK,mBAAmB,MAAM;AAAA,gBACxD,MAAM,KAAK;AAAA,gBACX,MAAM,KAAK;AAAA,cACb,CAAC;AAED,mBAAK,mBAAmB,KAAK,QAAQ;AACrC,oBAAM,eAAe,MAAM,QAAQ;AAAA,YACrC;AAAA,UACF;AAEA,cAAI,MAAM,MAAM,SAAS;AACvB,iBAAK,iBAAiB,MAAM,MAAM;AAClC,kBAAM,WAAW,MAAM,MAAM,MAAM,OAAO;AAAA,UAC5C;AAAA,QACF;AAAA,MAIF;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,gBAAgB,MAAM,SAAS,cAAc;AAEhE;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AACA,yDAAiB;AACjB,aAAM,uCAAW;AACjB,YAAM,WAAW,MAAM;AACvB,YAAM,eAAe,MAAM;AAAA,IAC7B;AAAA,EACF;AAEA,SAAO;AAAA,IACL,kBAAK,KAAK,CAACA,gBAAe,cAAcA,YAAW,MAAM,GAAG,YAAY,qBAAqB;AAAA,IAC7F;AAAA,EACF;AACF;AAEO,SAAS,oBACd,MACA,MACA,eACA,YAC0C;AAC1C,QAAM,cAAc,IAAI,4CAA8B;AACtD,QAAM,eAAe,YAAY,SAAS,UAAU;AACpD,QAAM,oBAAoB,YAAY;AAEtC,QAAM,gBAAgB,OAAO,WAAwB;AACnD,QAAI,kBAAkE;AACtE,QAAI,YAA+C;AAEnD,QAAI;AACF,kBAAY,MAAM,KAAK,MAAM,aAAa;AAC1C,UAAI,cAAc,MAAM;AACtB,cAAM,aAAa,MAAM;AACzB;AAAA,MACF;AAEA,wBAAkB,UAAU,UAAU;AACtC,aAAO,MAAM;AACX,YAAI,OAAO,SAAS;AAClB;AAAA,QACF;AACA,cAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,gBAAgB,KAAK;AAC1D,YAAI,MAAM;AACR;AAAA,QACF;AACA,cAAM,aAAa,MAAM,KAAK;AAAA,MAChC;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,gBAAgB,MAAM,SAAS,cAAc;AAEhE;AAAA,MACF;AACA,YAAM;AAAA,IACR,UAAE;AACA,yDAAiB;AACjB,aAAM,uCAAW;AACjB,YAAM,aAAa,MAAM;AAAA,IAC3B;AAAA,EACF;AAEA,SAAO;AAAA,IACL,kBAAK,KAAK,CAACA,gBAAe,cAAcA,YAAW,MAAM,GAAG,YAAY,qBAAqB;AAAA,IAC7F;AAAA,EACF;AACF;AAOA,eAAe,YACb,QACA,KACA,QACA,YACe;AACf,QAAM,SAAS,OAAO,UAAU;AAChC,MAAI;AACF,WAAO,MAAM;AACX,UAAI,OAAO,SAAS;AAClB;AAAA,MACF;AACA,YAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,OAAO,KAAK;AACjD,UAAI,KAAM;AACV,UAAI,QAAQ;AACZ,UAAI,eAAe,MAAM;AACvB,cAAM,WAAW,YAAY,KAAK;AAAA,MACpC;AACA,UAAI,CAAC,IAAI,aAAa,MAAM;AAC1B,YAAI,aAAa,QAAQ;AAAA,MAC3B;AAAA,IACF;AAAA,EACF,UAAE;AACA,QAAI,eAAe,MAAM;AACvB,iBAAW,MAAM;AAAA,IACnB;AACA,qCAAQ;AAAA,EACV;AACF;AAEO,SAAS,sBACd,QACA,YACA,YACwB;AACxB,QAAM,MAAM;AAAA,IACV,MAAM;AAAA,IACN,cAAc,IAAI,oBAAO;AAAA,EAC3B;AACA,SAAO;AAAA,IACL,kBAAK;AAAA,MACH,CAACA,gBAAe,YAAY,QAAQ,KAAKA,YAAW,QAAQ,UAAU;AAAA,MACtE;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAOA,eAAe,aACb,WACA,YACA,KACA,QACe;AACf,QAAM,SAAS,UAAU,UAAU;AACnC,MAAI,YAAmC;AAEvC,MAAI;AACF,WAAO,MAAM;AACX,UAAI,iCAAQ,SAAS;AACnB;AAAA,MACF;AAEA,YAAM,EAAE,MAAM,OAAO,MAAM,IAAI,MAAM,OAAO,KAAK;AACjD,UAAI,KAAM;AAEV,UAAI,MAAM,KAAK,KAAK;AAEpB,UACE,CAAC,IAAI,cAAc,QACnB,WAAW,cACX,WAAW,eAAe,MAAM,cAChC,CAAC,WACD;AACA,oBAAY,IAAI,+BAAe,MAAM,YAAY,WAAW,YAAY,CAAC;AAAA,MAC3E;AAEA,UAAI,WAAW;AACb,mBAAW,KAAK,UAAU,KAAK,KAAK,GAAG;AACrC,gBAAM,WAAW,aAAa,CAAC;AAAA,QACjC;AAAA,MACF,OAAO;AACL,cAAM,WAAW,aAAa,KAAK;AAAA,MACrC;AAIA,UAAI,CAAC,IAAI,cAAc,MAAM;AAC3B,YAAI,cAAc,QAAQ;AAAA,MAC5B;AAAA,IACF;AAAA,EACF,UAAE;AACA,qCAAQ;AACR,QAAI,WAAW;AACb,iBAAW,KAAK,UAAU,MAAM,GAAG;AACjC,cAAM,WAAW,aAAa,CAAC;AAAA,MACjC;AAAA,IACF;AACA,eAAW,MAAM;AAAA,EACnB;AACF;AAEO,SAAS,uBACd,WACA,aACA,YACyB;AACzB,QAAM,MAAM;AAAA,IACV,OAAO,CAAC;AAAA,IACR,eAAe,IAAI,oBAAO;AAAA,EAC5B;AACA,SAAO;AAAA,IACL,kBAAK;AAAA,MACH,CAACA,gBAAe,aAAa,WAAW,aAAa,KAAKA,YAAW,MAAM;AAAA,MAC3E;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;AAEO,SAAS,sBAAsB;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,yBAAyB,MAAM;AAAA,EAAC;AAAA,EAChC,2BAA2B,MAAM;AAAA,EAAC;AAAA,EAClC;AACF,GAS6B;AAC3B,QAAM,aAAS,gBAAI;AACnB,QAAM,aAAyB;AAAA,IAC7B,QAAQ,CAAC;AAAA,IACT,wBAAwB,IAAI,oBAAO;AAAA,EACrC;AAEA,QAAM,gBAAgB,CAAC,QAA6B;AAClD,6BAAyB,GAAG;AAC5B,eAAW,OAAO,KAAK,GAAG;AAAA,EAC5B;AAEA,QAAM,mBAAmB,OAAOA,gBAAgC;AAC9D,UAAM,SAASA,YAAW;AAC1B,UAAM,SAAS,eAAe,UAAU;AAExC,UAAM,QAAwB,CAAC;AAC/B,WAAO,CAAC,OAAO,SAAS;AACtB,YAAM,EAAE,MAAM,OAAO,SAAS,IAAI,MAAM,OAAO,KAAK;AACpD,UAAI,OAAO,QAAS;AACpB,UAAI,KAAM;AAEV,UAAI,eAAe,QAAQ;AACzB,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA;AAAA,QACF;AACA;AAAA,MACF;AAIA,YAAM,OAAO,QAAQ,SAAS,IAAI;AAClC,UAAI,CAAC,MAAM;AACT,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA,uBAAuB,SAAS,IAAI;AAAA,QACtC;AACA;AAAA,MACF;AAEA,UAAI,KAAC,oCAAe,IAAI,GAAG;AACzB,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,aAAa;AAAA,UAC1B;AAAA,UACA,sBAAsB,OAAO,IAAI;AAAA,QACnC;AACA;AAAA,MACF;AAEA,UAAI;AAGJ,UAAI;AACF,cAAM,WAAW,KAAK,MAAM,SAAS,IAAI;AAEzC,YAAI,KAAK,sBAAsB,sBAAW;AACxC,uBAAa,KAAK,WAAW,MAAM,QAAQ;AAAA,QAC7C,OAAO;AACL,uBAAa;AAAA,QACf;AAAA,MACF,SAAS,UAAU;AACjB,cAAM,YAAQ,sBAAQ,QAAQ;AAC9B,eAAO;AAAA,UACL;AAAA,YACE,UAAU,SAAS;AAAA,YACnB,WAAW,SAAS;AAAA,YACpB,WAAW,aAAa;AAAA,YACxB,OAAO,MAAM;AAAA,UACf;AAAA,UACA,6BAA6B,SAAS,IAAI;AAAA,QAC5C;AACA;AAAA,UACE,iBAAiB;AAAA,YACf;AAAA,YACA,WAAW;AAAA,UACb,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,UAAI,CAAC,WAAW,uBAAuB,MAAM;AAC3C,mBAAW,uBAAuB,QAAQ;AAAA,MAC5C;AAEA,6BAAuB,QAAQ;AAE/B,aAAO;AAAA,QACL;AAAA,UACE,UAAU,SAAS;AAAA,UACnB,WAAW;AAAA,UACX,WAAW,aAAa;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AAEA,YAAM,gBAAgB,+BAAkB,IAAI,EAAE,cAAc,SAAS,GAAG,YAAY;AAClF,eAAO,MAAM,KAAK,QAAQ,YAAY;AAAA,UACpC,KAAK,IAAI,8BAAW,SAAS,cAAc,QAAQ;AAAA,UACnD,YAAY,SAAS;AAAA,UACrB,aAAa;AAAA,QACf,CAAC;AAAA,MACH,CAAC;AAED,YAAM,wBAAwB,OAAO,iBAAmC;AAItE,YAAIC;AACJ,YAAI;AACF,gBAAM,EAAE,QAAQ,UAAU,IAAI,MAAM,iBAAiB,cAAc,MAAM;AACzE,UAAAA,cAAa,iBAAiB;AAAA,YAC5B;AAAA,YACA,WAAW,YAAY,IAAI,MAAM,uBAAuB,IAAI;AAAA,YAC5D,QAAQ,YAAY,SAAY;AAAA,UAClC,CAAC;AAAA,QACH,SAAS,UAAU;AACjB,iBAAO;AAAA,YACL;AAAA,cACE,UAAU,SAAS;AAAA,cACnB,WAAW,aAAa;AAAA,cACxB,WAAO,sBAAQ,QAAQ,EAAE;AAAA,YAC3B;AAAA,YACA;AAAA,UACF;AACA,UAAAA,cAAa,iBAAiB;AAAA,YAC5B;AAAA,YACA,eAAW,sBAAQ,QAAQ;AAAA,UAC7B,CAAC;AAAA,QACH,UAAE;AACA,cAAI,CAACA,YAAY,OAAM,IAAI,MAAM,yBAAyB;AAC1D,wBAAcA,WAAU;AAAA,QAC1B;AAAA,MACF;AAGA,YAAM,KAAK,sBAAsB,aAAa,CAAC;AAAA,IACjD;AAEA,UAAM,QAAQ,WAAW,KAAK;AAC9B,QAAI,WAAW,OAAO,SAAS,GAAG;AAChC,aAAO;AAAA,QACL;AAAA,UACE,WAAW,aAAa;AAAA,QAC1B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO,CAAC,kBAAK,KAAK,kBAAkB,YAAY,uBAAuB,GAAG,UAAU;AACtF;AAYA,eAAe,iBAAoB,SAAqB,QAA0C;AAChG,QAAM,WAAW,IAAI,oBAAmB;AAExC,QAAM,eAAe,MAAM;AACzB,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,QAAQ,EAAE,QAAQ,QAAW,WAAW,KAAK,CAAC;AAAA,IACzD;AAAA,EACF;AAEA,SAAO,iBAAiB,SAAS,YAAY;AAE7C,UACG,KAAK,CAAC,MAAM;AACX,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,QAAQ,EAAE,QAAQ,GAAG,WAAW,MAAM,CAAC;AAAA,IAClD;AAAA,EACF,CAAC,EACA,MAAM,CAAC,MAAM;AACZ,QAAI,CAAC,SAAS,MAAM;AAClB,eAAS,OAAO,CAAC;AAAA,IACnB;AAAA,EACF,CAAC,EACA,QAAQ,MAAM;AACb,WAAO,oBAAoB,SAAS,YAAY;AAAA,EAClD,CAAC;AAEH,SAAO,MAAM,SAAS;AACxB;AAEO,SAAS,mBAAmB,SAAsB;AAEvD,SAAO,MAAM;AACX,UAAM,MAAM,QAAQ,UAAU,uBAAuB;AACrD,QAAI,QAAQ,QAAW;AACrB,cAAQ,MAAM,OAAO,KAAK,CAAC;AAAA,IAC7B,OAAO;AACL;AAAA,IACF;AAAA,EACF;AACF;","names":["controller","toolOutput"]}

package/dist/voice/generation.d.cts

@@ -8,6 +8,7 @@ import { type Agent, type ModelSettings } from './agent.js';
 import type { AgentSession } from './agent_session.js';
 import type { AudioOutput, LLMNode, TTSNode, TextOutput } from './io.js';
 import type { SpeechHandle } from './speech_handle.js';
+/** @internal */
 export declare class _LLMGenerationData {
     readonly textStream: ReadableStream<string>;
     readonly toolCallStream: ReadableStream<FunctionCall>;

package/dist/voice/generation.d.ts

@@ -8,6 +8,7 @@ import { type Agent, type ModelSettings } from './agent.js';
 import type { AgentSession } from './agent_session.js';
 import type { AudioOutput, LLMNode, TTSNode, TextOutput } from './io.js';
 import type { SpeechHandle } from './speech_handle.js';
+/** @internal */
 export declare class _LLMGenerationData {
     readonly textStream: ReadableStream<string>;
     readonly toolCallStream: ReadableStream<FunctionCall>;