@livekit/agents 1.1.0 → 1.2.0

package/dist/voice/events.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/voice/events.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type {\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n  RealtimeModelError,\n} from '../llm/index.js';\nimport type { LLM, RealtimeModel } from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport type { AgentMetrics } from '../metrics/base.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS } from '../tts/index.js';\nimport type { TTSError } from '../tts/tts.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport enum AgentSessionEventTypes {\n  UserInputTranscribed = 'user_input_transcribed',\n  AgentStateChanged = 'agent_state_changed',\n  UserStateChanged = 'user_state_changed',\n  ConversationItemAdded = 'conversation_item_added',\n  FunctionToolsExecuted = 'function_tools_executed',\n  MetricsCollected = 'metrics_collected',\n  SpeechCreated = 'speech_created',\n  Error = 'error',\n  Close = 'close',\n}\n\nexport type UserState = 'speaking' | 'listening' | 'away';\nexport type AgentState = 'initializing' | 'idle' | 'listening' | 'thinking' | 'speaking';\n\nexport enum CloseReason {\n  ERROR = 'error',\n  JOB_SHUTDOWN = 'job_shutdown',\n  PARTICIPANT_DISCONNECTED = 'participant_disconnected',\n  USER_INITIATED = 'user_initiated',\n}\n\nexport type SpeechSource = 'say' | 'generate_reply' | 'tool_response';\n\nexport type UserStateChangedEvent = {\n  type: 'user_state_changed';\n  oldState: UserState;\n  newState: UserState;\n  createdAt: number;\n};\n\nexport const createUserStateChangedEvent = (\n  oldState: UserState,\n  newState: UserState,\n  createdAt: number = Date.now(),\n): UserStateChangedEvent => ({\n  type: 'user_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type AgentStateChangedEvent = {\n  type: 'agent_state_changed';\n  oldState: AgentState;\n  newState: AgentState;\n  createdAt: number;\n};\n\nexport const createAgentStateChangedEvent = (\n  oldState: AgentState,\n  newState: AgentState,\n  createdAt: number = Date.now(),\n): AgentStateChangedEvent => ({\n  type: 'agent_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type UserInputTranscribedEvent = {\n  type: 'user_input_transcribed';\n  transcript: string;\n  isFinal: boolean;\n  speakerId: string | null;\n  createdAt: number;\n};\n\nexport const createUserInputTranscribedEvent = ({\n  transcript,\n  isFinal,\n  speakerId = null,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  isFinal: boolean;\n  speakerId?: string | null;\n  createdAt?: number;\n}): UserInputTranscribedEvent => ({\n  type: 'user_input_transcribed',\n  transcript,\n  isFinal,\n  speakerId,\n  createdAt,\n});\n\nexport type MetricsCollectedEvent = {\n  type: 'metrics_collected';\n  metrics: AgentMetrics;\n  createdAt: number;\n};\n\nexport const createMetricsCollectedEvent = ({\n  metrics,\n  createdAt = Date.now(),\n}: {\n  metrics: AgentMetrics;\n  createdAt?: number;\n}): MetricsCollectedEvent => ({\n  type: 'metrics_collected',\n  metrics,\n  createdAt,\n});\n\nexport type ConversationItemAddedEvent = {\n  type: 'conversation_item_added';\n  item: ChatMessage;\n  createdAt: number;\n};\n\nexport const createConversationItemAddedEvent = (\n  item: ChatMessage,\n  createdAt: number = Date.now(),\n): ConversationItemAddedEvent => ({\n  type: 'conversation_item_added',\n  item,\n  createdAt,\n});\n\nexport type FunctionToolsExecutedEvent = {\n  type: 'function_tools_executed';\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt: number;\n};\n\nexport const createFunctionToolsExecutedEvent = ({\n  functionCalls,\n  functionCallOutputs,\n  createdAt = Date.now(),\n}: {\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt?: number;\n}): FunctionToolsExecutedEvent => {\n  return {\n    type: 'function_tools_executed',\n    functionCalls,\n    functionCallOutputs,\n    createdAt,\n  };\n};\n\nexport const zipFunctionCallsAndOutputs = (\n  event: FunctionToolsExecutedEvent,\n): Array<[FunctionCall, FunctionCallOutput]> => {\n  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]!]);\n};\n\nexport type SpeechCreatedEvent = {\n  type: 'speech_created';\n  /**\n   * True if the speech was created using public methods like `say` or `generate_reply`\n   */\n  userInitiated: boolean;\n  /**\n   * Source indicating how the speech handle was created\n   */\n  source: SpeechSource;\n  /**\n   * The speech handle that was created\n   */\n  // TODO(shubhra): we need to make sure this doesn't get serialized\n  speechHandle: SpeechHandle;\n  /**\n   * The timestamp when the speech handle was created\n   */\n  createdAt: number;\n};\n\nexport const createSpeechCreatedEvent = ({\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt = Date.now(),\n}: {\n  userInitiated: boolean;\n  source: SpeechSource;\n  speechHandle: SpeechHandle;\n  createdAt?: number;\n}): SpeechCreatedEvent => ({\n  type: 'speech_created',\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt,\n});\n\nexport type ErrorEvent = {\n  type: 'error';\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown;\n  source: LLM | STT | TTS | RealtimeModel | unknown;\n  createdAt: number;\n};\n\nexport const createErrorEvent = (\n  error: RealtimeModelError | STTError | TTSError | LLMError | unknown,\n  source: LLM | STT | TTS | RealtimeModel | unknown,\n  createdAt: number = Date.now(),\n): ErrorEvent => ({\n  type: 'error',\n  error,\n  source,\n  createdAt,\n});\n\nexport type CloseEvent = {\n  type: 'close';\n  error: RealtimeModelError | STTError | TTSError | LLMError | null;\n  reason: CloseReason;\n  createdAt: number;\n};\n\nexport const createCloseEvent = (\n  reason: CloseReason,\n  error: RealtimeModelError | STTError | TTSError | LLMError | null = null,\n  createdAt: number = Date.now(),\n): CloseEvent => ({\n  type: 'close',\n  error,\n  reason,\n  createdAt,\n});\n\nexport type AgentEvent =\n  | UserInputTranscribedEvent\n  | UserStateChangedEvent\n  | AgentStateChangedEvent\n  | MetricsCollectedEvent\n  | ConversationItemAddedEvent\n  | FunctionToolsExecutedEvent\n  | SpeechCreatedEvent\n  | ErrorEvent\n  | CloseEvent;\n"],"mappings":"AAkBO,IAAK,yBAAL,kBAAKA,4BAAL;AACL,EAAAA,wBAAA,0BAAuB;AACvB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,WAAQ;AACR,EAAAA,wBAAA,WAAQ;AATE,SAAAA;AAAA,GAAA;AAeL,IAAK,cAAL,kBAAKC,iBAAL;AACL,EAAAA,aAAA,WAAQ;AACR,EAAAA,aAAA,kBAAe;AACf,EAAAA,aAAA,8BAA2B;AAC3B,EAAAA,aAAA,oBAAiB;AAJP,SAAAA;AAAA,GAAA;AAgBL,MAAM,8BAA8B,CACzC,UACA,UACA,YAAoB,KAAK,IAAI,OACF;AAAA,EAC3B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,+BAA+B,CAC1C,UACA,UACA,YAAoB,KAAK,IAAI,OACD;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAUO,MAAM,kCAAkC,CAAC;AAAA,EAC9C;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,YAAY,KAAK,IAAI;AACvB,OAKkC;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAQO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAG8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,mCAAmC,CAC9C,MACA,YAAoB,KAAK,IAAI,OACG;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AACF;AASO,MAAM,mCAAmC,CAAC;AAAA,EAC/C;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,MAIkC;AAChC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,MAAM,6BAA6B,CACxC,UAC8C;AAC9C,SAAO,MAAM,cAAc,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,MAAM,oBAAoB,KAAK,CAAE,CAAC;AAC3F;AAuBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAK2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,OACA,QACA,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,QACA,QAAoE,MACpE,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;","names":["AgentSessionEventTypes","CloseReason"]}
+{"version":3,"sources":["../../src/voice/events.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { InterruptionDetectionError } from '../inference/interruption/errors.js';\nimport type { OverlappingSpeechEvent } from '../inference/interruption/types.js';\nimport type { LanguageCode } from '../language.js';\nimport type {\n  ChatMessage,\n  FunctionCall,\n  FunctionCallOutput,\n  LLM,\n  RealtimeModel,\n  RealtimeModelError,\n} from '../llm/index.js';\nimport type { LLMError } from '../llm/llm.js';\nimport type { AgentMetrics } from '../metrics/base.js';\nimport type { STT } from '../stt/index.js';\nimport type { STTError } from '../stt/stt.js';\nimport type { TTS } from '../tts/index.js';\nimport type { TTSError } from '../tts/tts.js';\nimport type { SpeechHandle } from './speech_handle.js';\n\nexport enum AgentSessionEventTypes {\n  UserInputTranscribed = 'user_input_transcribed',\n  AgentStateChanged = 'agent_state_changed',\n  UserStateChanged = 'user_state_changed',\n  ConversationItemAdded = 'conversation_item_added',\n  FunctionToolsExecuted = 'function_tools_executed',\n  MetricsCollected = 'metrics_collected',\n  SpeechCreated = 'speech_created',\n  OverlappingSpeech = 'overlapping_speech',\n  Error = 'error',\n  Close = 'close',\n}\n\nexport type UserState = 'speaking' | 'listening' | 'away';\nexport type AgentState = 'initializing' | 'idle' | 'listening' | 'thinking' | 'speaking';\n\nexport enum CloseReason {\n  ERROR = 'error',\n  JOB_SHUTDOWN = 'job_shutdown',\n  PARTICIPANT_DISCONNECTED = 'participant_disconnected',\n  USER_INITIATED = 'user_initiated',\n}\n\nexport type ShutdownReason = CloseReason | string;\n\nexport type SpeechSource = 'say' | 'generate_reply' | 'tool_response';\n\nexport type UserStateChangedEvent = {\n  type: 'user_state_changed';\n  oldState: UserState;\n  newState: UserState;\n  createdAt: number;\n};\n\nexport const createUserStateChangedEvent = (\n  oldState: UserState,\n  newState: UserState,\n  createdAt: number = Date.now(),\n): UserStateChangedEvent => ({\n  type: 'user_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type AgentStateChangedEvent = {\n  type: 'agent_state_changed';\n  oldState: AgentState;\n  newState: AgentState;\n  createdAt: number;\n};\n\nexport const createAgentStateChangedEvent = (\n  oldState: AgentState,\n  newState: AgentState,\n  createdAt: number = Date.now(),\n): AgentStateChangedEvent => ({\n  type: 'agent_state_changed',\n  oldState,\n  newState,\n  createdAt,\n});\n\nexport type UserInputTranscribedEvent = {\n  type: 'user_input_transcribed';\n  transcript: string;\n  isFinal: boolean;\n  // TODO(AJS-106): add multi participant support\n  /** Not supported yet. Always null by default. */\n  speakerId: string | null;\n  createdAt: number;\n  language: LanguageCode | null;\n};\n\nexport const createUserInputTranscribedEvent = ({\n  transcript,\n  isFinal,\n  speakerId = null,\n  language = null,\n  createdAt = Date.now(),\n}: {\n  transcript: string;\n  isFinal: boolean;\n  speakerId?: string | null;\n  language?: LanguageCode | null;\n  createdAt?: number;\n}): UserInputTranscribedEvent => ({\n  type: 'user_input_transcribed',\n  transcript,\n  isFinal,\n  speakerId,\n  language,\n  createdAt,\n});\n\nexport type MetricsCollectedEvent = {\n  type: 'metrics_collected';\n  metrics: AgentMetrics;\n  createdAt: number;\n};\n\nexport const createMetricsCollectedEvent = ({\n  metrics,\n  createdAt = Date.now(),\n}: {\n  metrics: AgentMetrics;\n  createdAt?: number;\n}): MetricsCollectedEvent => ({\n  type: 'metrics_collected',\n  metrics,\n  createdAt,\n});\n\nexport type ConversationItemAddedEvent = {\n  type: 'conversation_item_added';\n  item: ChatMessage;\n  createdAt: number;\n};\n\nexport const createConversationItemAddedEvent = (\n  item: ChatMessage,\n  createdAt: number = Date.now(),\n): ConversationItemAddedEvent => ({\n  type: 'conversation_item_added',\n  item,\n  createdAt,\n});\n\nexport type FunctionToolsExecutedEvent = {\n  type: 'function_tools_executed';\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt: number;\n};\n\nexport const createFunctionToolsExecutedEvent = ({\n  functionCalls,\n  functionCallOutputs,\n  createdAt = Date.now(),\n}: {\n  functionCalls: FunctionCall[];\n  functionCallOutputs: FunctionCallOutput[];\n  createdAt?: number;\n}): FunctionToolsExecutedEvent => {\n  return {\n    type: 'function_tools_executed',\n    functionCalls,\n    functionCallOutputs,\n    createdAt,\n  };\n};\n\nexport const zipFunctionCallsAndOutputs = (\n  event: FunctionToolsExecutedEvent,\n): Array<[FunctionCall, FunctionCallOutput]> => {\n  return event.functionCalls.map((call, index) => [call, event.functionCallOutputs[index]!]);\n};\n\nexport type SpeechCreatedEvent = {\n  type: 'speech_created';\n  /**\n   * True if the speech was created using public methods like `say` or `generate_reply`\n   */\n  userInitiated: boolean;\n  /**\n   * Source indicating how the speech handle was created\n   */\n  source: SpeechSource;\n  /**\n   * The speech handle that was created\n   */\n  // TODO(shubhra): we need to make sure this doesn't get serialized\n  speechHandle: SpeechHandle;\n  /**\n   * The timestamp when the speech handle was created\n   */\n  createdAt: number;\n};\n\nexport const createSpeechCreatedEvent = ({\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt = Date.now(),\n}: {\n  userInitiated: boolean;\n  source: SpeechSource;\n  speechHandle: SpeechHandle;\n  createdAt?: number;\n}): SpeechCreatedEvent => ({\n  type: 'speech_created',\n  userInitiated,\n  source,\n  speechHandle,\n  createdAt,\n});\n\nexport type ErrorEvent = {\n  type: 'error';\n  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | unknown;\n  source: LLM | STT | TTS | RealtimeModel | unknown;\n  createdAt: number;\n};\n\nexport const createErrorEvent = (\n  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | unknown,\n  source: LLM | STT | TTS | RealtimeModel | unknown,\n  createdAt: number = Date.now(),\n): ErrorEvent => ({\n  type: 'error',\n  error,\n  source,\n  createdAt,\n});\n\nexport type CloseEvent = {\n  type: 'close';\n  error: RealtimeModelError | STTError | TTSError | LLMError | InterruptionDetectionError | null;\n  reason: ShutdownReason;\n  createdAt: number;\n};\n\nexport const createCloseEvent = (\n  reason: ShutdownReason,\n  error:\n    | RealtimeModelError\n    | STTError\n    | TTSError\n    | LLMError\n    | InterruptionDetectionError\n    | null = null,\n  createdAt: number = Date.now(),\n): CloseEvent => ({\n  type: 'close',\n  error,\n  reason,\n  createdAt,\n});\n\nexport type AgentEvent =\n  | UserInputTranscribedEvent\n  | UserStateChangedEvent\n  | AgentStateChangedEvent\n  | MetricsCollectedEvent\n  | ConversationItemAddedEvent\n  | FunctionToolsExecutedEvent\n  | SpeechCreatedEvent\n  | OverlappingSpeechEvent\n  | ErrorEvent\n  | CloseEvent;\n"],"mappings":"AAsBO,IAAK,yBAAL,kBAAKA,4BAAL;AACL,EAAAA,wBAAA,0BAAuB;AACvB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,2BAAwB;AACxB,EAAAA,wBAAA,sBAAmB;AACnB,EAAAA,wBAAA,mBAAgB;AAChB,EAAAA,wBAAA,uBAAoB;AACpB,EAAAA,wBAAA,WAAQ;AACR,EAAAA,wBAAA,WAAQ;AAVE,SAAAA;AAAA,GAAA;AAgBL,IAAK,cAAL,kBAAKC,iBAAL;AACL,EAAAA,aAAA,WAAQ;AACR,EAAAA,aAAA,kBAAe;AACf,EAAAA,aAAA,8BAA2B;AAC3B,EAAAA,aAAA,oBAAiB;AAJP,SAAAA;AAAA,GAAA;AAkBL,MAAM,8BAA8B,CACzC,UACA,UACA,YAAoB,KAAK,IAAI,OACF;AAAA,EAC3B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,+BAA+B,CAC1C,UACA,UACA,YAAoB,KAAK,IAAI,OACD;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AAaO,MAAM,kCAAkC,CAAC;AAAA,EAC9C;AAAA,EACA;AAAA,EACA,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,YAAY,KAAK,IAAI;AACvB,OAMkC;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAQO,MAAM,8BAA8B,CAAC;AAAA,EAC1C;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAG8B;AAAA,EAC5B,MAAM;AAAA,EACN;AAAA,EACA;AACF;AAQO,MAAM,mCAAmC,CAC9C,MACA,YAAoB,KAAK,IAAI,OACG;AAAA,EAChC,MAAM;AAAA,EACN;AAAA,EACA;AACF;AASO,MAAM,mCAAmC,CAAC;AAAA,EAC/C;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,MAIkC;AAChC,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEO,MAAM,6BAA6B,CACxC,UAC8C;AAC9C,SAAO,MAAM,cAAc,IAAI,CAAC,MAAM,UAAU,CAAC,MAAM,MAAM,oBAAoB,KAAK,CAAE,CAAC;AAC3F;AAuBO,MAAM,2BAA2B,CAAC;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,YAAY,KAAK,IAAI;AACvB,OAK2B;AAAA,EACzB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,OACA,QACA,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;AASO,MAAM,mBAAmB,CAC9B,QACA,QAMW,MACX,YAAoB,KAAK,IAAI,OACb;AAAA,EAChB,MAAM;AAAA,EACN;AAAA,EACA;AAAA,EACA;AACF;","names":["AgentSessionEventTypes","CloseReason"]}

package/dist/voice/generation.cjs

@@ -34,13 +34,17 @@ __export(generation_exports, {
 });
 module.exports = __toCommonJS(generation_exports);
 var import_rtc_node = require("@livekit/rtc-node");
-var import_zod = require("zod");
+var import_api = require("@opentelemetry/api");
 var import_chat_context = require("../llm/chat_context.cjs");
 var import_tool_context = require("../llm/tool_context.cjs");
+var import_zod_utils = require("../llm/zod-utils.cjs");
 var import_log = require("../log.cjs");
 var import_identity_transform = require("../stream/identity_transform.cjs");
+var import_telemetry = require("../telemetry/index.cjs");
+var import_types = require("../types.cjs");
 var import_utils = require("../utils.cjs");
 var import_agent = require("./agent.cjs");
+var import_io = require("./io.cjs");
 var import_run_context = require("./run_context.cjs");
 class _LLMGenerationData {
   constructor(textStream, toolCallStream) {
@@ -52,6 +56,7 @@ class _LLMGenerationData {
   generatedText = "";
   generatedToolCalls;
   id;
+  ttft;
 }
 class _ToolOutput {
   output;
@@ -310,29 +315,45 @@ function updateInstructions(options) {
     );
   }
 }
-function performLLMInference(node, chatCtx, toolCtx, modelSettings, controller) {
+function performLLMInference(node, chatCtx, toolCtx, modelSettings, controller, model, provider) {
   const textStream = new import_identity_transform.IdentityTransform();
   const toolCallStream = new import_identity_transform.IdentityTransform();
   const textWriter = textStream.writable.getWriter();
   const toolCallWriter = toolCallStream.writable.getWriter();
   const data = new _LLMGenerationData(textStream.readable, toolCallStream.readable);
-  const inferenceTask = async (signal) => {
+  const _performLLMInferenceImpl = async (signal, span) => {
+    span.setAttribute(
+      import_telemetry.traceTypes.ATTR_CHAT_CTX,
+      JSON.stringify(chatCtx.toJSON({ excludeTimestamp: false }))
+    );
+    span.setAttribute(import_telemetry.traceTypes.ATTR_FUNCTION_TOOLS, JSON.stringify(Object.keys(toolCtx)));
+    if (model) {
+      span.setAttribute(import_telemetry.traceTypes.ATTR_GEN_AI_REQUEST_MODEL, model);
+    }
+    if (provider) {
+      span.setAttribute(import_telemetry.traceTypes.ATTR_GEN_AI_PROVIDER_NAME, provider);
+    }
     let llmStreamReader = null;
     let llmStream = null;
+    const startTime = performance.now() / 1e3;
+    let firstTokenReceived = false;
     try {
       llmStream = await node(chatCtx, toolCtx, modelSettings);
       if (llmStream === null) {
         await textWriter.close();
         return;
       }
+      const abortPromise = (0, import_utils.waitForAbort)(signal);
       llmStreamReader = llmStream.getReader();
       while (true) {
-        if (signal.aborted) {
-          break;
-        }
-        const { done, value: chunk } = await llmStreamReader.read();
-        if (done) {
-          break;
+        if (signal.aborted) break;
+        const result = await Promise.race([llmStreamReader.read(), abortPromise]);
+        if (result === void 0) break;
+        const { done, value: chunk } = result;
+        if (done) break;
+        if (!firstTokenReceived) {
+          firstTokenReceived = true;
+          data.ttft = performance.now() / 1e3 - startTime;
         }
         if (typeof chunk === "string") {
           data.generatedText += chunk;
@@ -347,7 +368,10 @@ function performLLMInference(node, chatCtx, toolCtx, modelSettings, controller)
               const toolCall = import_chat_context.FunctionCall.create({
                 callId: `${data.id}/fnc_${data.generatedToolCalls.length}`,
                 name: tool.name,
-                args: tool.args
+                args: tool.args,
+                // Preserve thought signature for Gemini 3+ thinking mode
+                thoughtSignature: tool.thoughtSignature,
+                extra: tool.extra || {}
               });
               data.generatedToolCalls.push(toolCall);
               await toolCallWriter.write(toolCall);
@@ -359,6 +383,10 @@ function performLLMInference(node, chatCtx, toolCtx, modelSettings, controller)
           }
         }
       }
+      span.setAttribute(import_telemetry.traceTypes.ATTR_RESPONSE_TEXT, data.generatedText);
+      if (data.ttft !== void 0) {
+        span.setAttribute(import_telemetry.traceTypes.ATTR_RESPONSE_TTFT, data.ttft);
+      }
     } catch (error) {
       if (error instanceof DOMException && error.name === "AbortError") {
         return;
@@ -371,34 +399,104 @@ function performLLMInference(node, chatCtx, toolCtx, modelSettings, controller)
       await toolCallWriter.close();
     }
   };
+  const currentContext = import_api.context.active();
+  const inferenceTask = async (signal) => import_telemetry.tracer.startActiveSpan(async (span) => _performLLMInferenceImpl(signal, span), {
+    name: "llm_node",
+    context: currentContext
+  });
   return [
     import_utils.Task.from((controller2) => inferenceTask(controller2.signal), controller, "performLLMInference"),
     data
   ];
 }
-function performTTSInference(node, text, modelSettings, controller) {
+function performTTSInference(node, text, modelSettings, controller, model, provider) {
   const audioStream = new import_identity_transform.IdentityTransform();
   const outputWriter = audioStream.writable.getWriter();
   const audioOutputStream = audioStream.readable;
-  const inferenceTask = async (signal) => {
+  const timedTextsFut = new import_utils.Future();
+  const timedTextsStream = new import_identity_transform.IdentityTransform();
+  const timedTextsWriter = timedTextsStream.writable.getWriter();
+  const textOnlyStream = new import_identity_transform.IdentityTransform();
+  const textOnlyWriter = textOnlyStream.writable.getWriter();
+  (async () => {
+    const reader = text.getReader();
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
+          break;
+        }
+        const textValue = typeof value === "string" ? value : value.text;
+        await textOnlyWriter.write(textValue);
+      }
+      await textOnlyWriter.close();
+    } catch (e) {
+      await textOnlyWriter.abort(e);
+    } finally {
+      reader.releaseLock();
+    }
+  })();
+  let ttfb;
+  const genData = {
+    audioStream: audioOutputStream,
+    timedTextsFut,
+    ttfb: void 0
+  };
+  const _performTTSInferenceImpl = async (signal, span) => {
+    if (model) {
+      span.setAttribute(import_telemetry.traceTypes.ATTR_GEN_AI_REQUEST_MODEL, model);
+    }
+    if (provider) {
+      span.setAttribute(import_telemetry.traceTypes.ATTR_GEN_AI_PROVIDER_NAME, provider);
+    }
     let ttsStreamReader = null;
     let ttsStream = null;
+    let pushedDuration = 0;
+    const startTime = performance.now() / 1e3;
+    let firstByteReceived = false;
     try {
-      ttsStream = await node(text, modelSettings);
+      ttsStream = await node(textOnlyStream.readable, modelSettings);
       if (ttsStream === null) {
+        timedTextsFut.resolve(null);
         await outputWriter.close();
+        await timedTextsWriter.close();
         return;
       }
+      if (!timedTextsFut.done) {
+        timedTextsFut.resolve(timedTextsStream.readable);
+      }
       ttsStreamReader = ttsStream.getReader();
+      const initialPushedDuration = pushedDuration;
       while (true) {
         if (signal.aborted) {
           break;
         }
-        const { done, value: chunk } = await ttsStreamReader.read();
+        const { done, value: frame } = await ttsStreamReader.read();
         if (done) {
           break;
         }
-        await outputWriter.write(chunk);
+        if (!firstByteReceived) {
+          firstByteReceived = true;
+          ttfb = performance.now() / 1e3 - startTime;
+          genData.ttfb = ttfb;
+          span.setAttribute(import_telemetry.traceTypes.ATTR_RESPONSE_TTFB, ttfb);
+        }
+        await outputWriter.write(frame);
+        const timedTranscripts = frame.userdata[import_types.USERDATA_TIMED_TRANSCRIPT];
+        if (timedTranscripts && timedTranscripts.length > 0) {
+          for (const timedText of timedTranscripts) {
+            const adjustedTimedText = (0, import_io.createTimedString)({
+              text: timedText.text,
+              startTime: timedText.startTime !== void 0 ? timedText.startTime + initialPushedDuration : void 0,
+              endTime: timedText.endTime !== void 0 ? timedText.endTime + initialPushedDuration : void 0,
+              confidence: timedText.confidence,
+              startTimeOffset: timedText.startTimeOffset
+            });
+            await timedTextsWriter.write(adjustedTimedText);
+          }
+        }
+        const frameDuration = frame.samplesPerChannel / frame.sampleRate;
+        pushedDuration += frameDuration;
       }
     } catch (error) {
       if (error instanceof DOMException && error.name === "AbortError") {
@@ -406,14 +504,23 @@ function performTTSInference(node, text, modelSettings, controller) {
       }
       throw error;
     } finally {
+      if (!timedTextsFut.done) {
+        timedTextsFut.resolve(null);
+      }
       ttsStreamReader == null ? void 0 : ttsStreamReader.releaseLock();
       await (ttsStream == null ? void 0 : ttsStream.cancel());
       await outputWriter.close();
+      await timedTextsWriter.close();
     }
   };
+  const currentContext = import_api.context.active();
+  const inferenceTask = async (signal) => import_telemetry.tracer.startActiveSpan(async (span) => _performTTSInferenceImpl(signal, span), {
+    name: "tts_node",
+    context: currentContext
+  });
   return [
     import_utils.Task.from((controller2) => inferenceTask(controller2.signal), controller, "performTTSInference"),
-    audioOutputStream
+    genData
   ];
 }
 async function forwardText(source, out, signal, textOutput) {
@@ -425,7 +532,9 @@ async function forwardText(source, out, signal, textOutput) {
       }
       const { done, value: delta } = await reader.read();
       if (done) break;
-      out.text += delta;
+      const deltaIsTimedString = (0, import_io.isTimedString)(delta);
+      const textDelta = deltaIsTimedString ? delta.text : delta;
+      out.text += textDelta;
       if (textOutput !== null) {
         await textOutput.captureText(delta);
       }
@@ -454,10 +563,17 @@ function performTextForwarding(source, controller, textOutput) {
     out
   ];
 }
-async function forwardAudio(ttsStream, audioOuput, out, signal) {
+async function forwardAudio(ttsStream, audioOutput, out, signal) {
   const reader = ttsStream.getReader();
   let resampler = null;
+  const onPlaybackStarted = (ev) => {
+    if (!out.firstFrameFut.done) {
+      out.firstFrameFut.resolve(ev.createdAt);
+    }
+  };
   try {
+    audioOutput.on(import_io.AudioOutput.EVENT_PLAYBACK_STARTED, onPlaybackStarted);
+    audioOutput.resume();
     while (true) {
       if (signal == null ? void 0 : signal.aborted) {
         break;
@@ -465,28 +581,29 @@ async function forwardAudio(ttsStream, audioOuput, out, signal) {
       const { done, value: frame } = await reader.read();
       if (done) break;
       out.audio.push(frame);
-      if (!out.firstFrameFut.done && audioOuput.sampleRate && audioOuput.sampleRate !== frame.sampleRate && !resampler) {
-        resampler = new import_rtc_node.AudioResampler(frame.sampleRate, audioOuput.sampleRate, 1);
+      if (!out.firstFrameFut.done && audioOutput.sampleRate && audioOutput.sampleRate !== frame.sampleRate && !resampler) {
+        resampler = new import_rtc_node.AudioResampler(frame.sampleRate, audioOutput.sampleRate, 1);
       }
       if (resampler) {
         for (const f of resampler.push(frame)) {
-          await audioOuput.captureFrame(f);
+          await audioOutput.captureFrame(f);
         }
       } else {
-        await audioOuput.captureFrame(frame);
-      }
-      if (!out.firstFrameFut.done) {
-        out.firstFrameFut.resolve();
+        await audioOutput.captureFrame(frame);
       }
     }
-  } finally {
-    reader == null ? void 0 : reader.releaseLock();
     if (resampler) {
       for (const f of resampler.flush()) {
-        await audioOuput.captureFrame(f);
+        await audioOutput.captureFrame(f);
       }
     }
-    audioOuput.flush();
+  } finally {
+    audioOutput.off(import_io.AudioOutput.EVENT_PLAYBACK_STARTED, onPlaybackStarted);
+    if (!out.firstFrameFut.done) {
+      out.firstFrameFut.reject(new Error("audio forwarding cancelled before playback started"));
+    }
+    reader == null ? void 0 : reader.releaseLock();
+    audioOutput.flush();
   }
 }
 function performAudioForwarding(ttsStream, audioOutput, controller) {
@@ -566,8 +683,13 @@ function performToolExecutions({
       let parsedArgs;
       try {
         const jsonArgs = JSON.parse(toolCall.args);
-        if (tool.parameters instanceof import_zod.ZodObject) {
-          parsedArgs = tool.parameters.parse(jsonArgs);
+        if ((0, import_zod_utils.isZodSchema)(tool.parameters)) {
+          const result = await (0, import_zod_utils.parseZodSchema)(tool.parameters, jsonArgs);
+          if (result.success) {
+            parsedArgs = result.data;
+          } else {
+            throw result.error;
+          }
         } else {
           parsedArgs = jsonArgs;
         }
@@ -602,14 +724,9 @@ function performToolExecutions({
         },
         "Executing LLM tool call"
       );
-      const toolExecution = import_agent.asyncLocalStorage.run({ functionCall: toolCall }, async () => {
-        return await tool.execute(parsedArgs, {
-          ctx: new import_run_context.RunContext(session, speechHandle, toolCall),
-          toolCallId: toolCall.callId,
-          abortSignal: signal
-        });
-      });
-      const tracableToolExecution = async (toolExecTask) => {
+      const _tracableToolExecutionImpl = async (toolExecTask, span) => {
+        span.setAttribute(import_telemetry.traceTypes.ATTR_FUNCTION_TOOL_NAME, toolCall.name);
+        span.setAttribute(import_telemetry.traceTypes.ATTR_FUNCTION_TOOL_ARGS, toolCall.args);
         let toolOutput2;
         try {
           const { result, isAborted } = await waitUntilAborted(toolExecTask, signal);
@@ -618,6 +735,16 @@ function performToolExecutions({
             exception: isAborted ? new Error("tool call was aborted") : void 0,
             output: isAborted ? void 0 : result
           });
+          if (toolOutput2.toolCallOutput) {
+            span.setAttribute(
+              import_telemetry.traceTypes.ATTR_FUNCTION_TOOL_OUTPUT,
+              toolOutput2.toolCallOutput.output
+            );
+            span.setAttribute(
+              import_telemetry.traceTypes.ATTR_FUNCTION_TOOL_IS_ERROR,
+              toolOutput2.toolCallOutput.isError
+            );
+          }
         } catch (rawError) {
           logger.error(
             {
@@ -631,14 +758,51 @@ function performToolExecutions({
             toolCall,
             exception: (0, import_utils.toError)(rawError)
           });
+          if (toolOutput2.toolCallOutput) {
+            span.setAttribute(
+              import_telemetry.traceTypes.ATTR_FUNCTION_TOOL_OUTPUT,
+              toolOutput2.toolCallOutput.output
+            );
+            span.setAttribute(import_telemetry.traceTypes.ATTR_FUNCTION_TOOL_IS_ERROR, true);
+          }
         } finally {
           if (!toolOutput2) throw new Error("toolOutput is undefined");
           toolCompleted(toolOutput2);
         }
       };
-      tasks.push(tracableToolExecution(toolExecution));
+      const tracableToolExecution = (toolExecTask) => import_telemetry.tracer.startActiveSpan(async (span) => _tracableToolExecutionImpl(toolExecTask, span), {
+        name: "function_tool"
+      });
+      const toolTask = import_utils.Task.from(
+        async () => {
+          const currentTask = import_utils.Task.current();
+          if (currentTask) {
+            (0, import_agent._setActivityTaskInfo)(currentTask, {
+              speechHandle,
+              functionCall: toolCall,
+              inlineTask: true
+            });
+          }
+          const toolExecution = import_agent.functionCallStorage.run({ functionCall: toolCall }, async () => {
+            return await tool.execute(parsedArgs, {
+              ctx: new import_run_context.RunContext(session, speechHandle, toolCall),
+              toolCallId: toolCall.callId,
+              abortSignal: signal
+            });
+          });
+          await tracableToolExecution(toolExecution);
+        },
+        controller2,
+        `performToolExecution:${toolCall.name}`
+      );
+      (0, import_agent._setActivityTaskInfo)(toolTask, {
+        speechHandle,
+        functionCall: toolCall,
+        inlineTask: true
+      });
+      tasks.push(toolTask);
     }
-    await Promise.allSettled(tasks);
+    await Promise.allSettled(tasks.map((task) => task.result));
     if (toolOutput.output.length > 0) {
       logger.debug(
         {