@livekit/agents-plugin-openai 1.0.40 → 1.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/realtime/api_proto.cjs.map +1 -1
- package/dist/realtime/api_proto.d.cts +1 -0
- package/dist/realtime/api_proto.d.ts +1 -0
- package/dist/realtime/api_proto.d.ts.map +1 -1
- package/dist/realtime/api_proto.js.map +1 -1
- package/dist/realtime/realtime_model.cjs +5 -2
- package/dist/realtime/realtime_model.cjs.map +1 -1
- package/dist/realtime/realtime_model.d.ts.map +1 -1
- package/dist/realtime/realtime_model.js +6 -2
- package/dist/realtime/realtime_model.js.map +1 -1
- package/dist/realtime/realtime_model_beta.cjs +5 -2
- package/dist/realtime/realtime_model_beta.cjs.map +1 -1
- package/dist/realtime/realtime_model_beta.d.ts.map +1 -1
- package/dist/realtime/realtime_model_beta.js +6 -2
- package/dist/realtime/realtime_model_beta.js.map +1 -1
- package/package.json +5 -5
- package/src/realtime/api_proto.ts +2 -0
- package/src/realtime/realtime_model.ts +11 -6
- package/src/realtime/realtime_model_beta.ts +11 -6
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/realtime/realtime_model.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { metrics } from '@livekit/agents';\nimport {\n type APIConnectOptions,\n APIConnectionError,\n APIError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Future,\n Queue,\n Task,\n cancelAndWait,\n delay,\n isAPIError,\n llm,\n log,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport type { AudioResampler } from '@livekit/rtc-node';\nimport { AudioFrame, combineAudioFrames } from '@livekit/rtc-node';\nimport { type MessageEvent, WebSocket } from 'ws';\nimport * as api_proto from './api_proto.js';\n\n// if LK_OPENAI_DEBUG convert it to a number, otherwise set it to 0\nconst lkOaiDebug = process.env.LK_OPENAI_DEBUG ? Number(process.env.LK_OPENAI_DEBUG) : 0;\n\nconst SAMPLE_RATE = 24000;\nconst NUM_CHANNELS = 1;\nconst BASE_URL = 'https://api.openai.com/v1';\n\nconst MOCK_AUDIO_ID_PREFIX = 'lk_mock_audio_item_';\n\ntype Modality = 'text' | 'audio';\n\ninterface RealtimeOptions {\n model: api_proto.Model;\n voice: api_proto.Voice;\n toolChoice?: llm.ToolChoice;\n inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n inputAudioNoiseReduction?: api_proto.NoiseReduction | null;\n turnDetection?: api_proto.TurnDetectionType | null;\n maxResponseOutputTokens?: number | 'inf';\n speed?: number;\n tracing?: api_proto.TracingConfig | null;\n apiKey?: string;\n baseURL: string;\n isAzure: boolean;\n azureDeployment?: string;\n entraToken?: string;\n apiVersion?: string;\n maxSessionDuration: number;\n // reset the connection after this many seconds if provided\n connOptions: APIConnectOptions;\n modalities: Modality[];\n}\n\ninterface MessageGeneration {\n messageId: string;\n textChannel: stream.StreamChannel<string>;\n audioChannel: stream.StreamChannel<AudioFrame>;\n audioTranscript: string;\n modalities: Future<('text' | 'audio')[]>;\n}\n\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel<llm.MessageGeneration>;\n functionChannel: stream.StreamChannel<llm.FunctionCall>;\n messages: Map<string, MessageGeneration>;\n\n /** @internal */\n _doneFut: Future;\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n}\n\nclass CreateResponseHandle {\n instructions?: string;\n doneFut: Future<llm.GenerationCreatedEvent>;\n // TODO(shubhra): add timeout\n constructor({ instructions }: { instructions?: string }) {\n this.instructions = instructions;\n this.doneFut = new Future();\n }\n}\n\nconst DEFAULT_FIRST_RETRY_INTERVAL_MS = 100;\nconst DEFAULT_TURN_DETECTION: api_proto.TurnDetectionType = {\n type: 'semantic_vad',\n eagerness: 'medium',\n create_response: true,\n interrupt_response: true,\n};\nconst DEFAULT_INPUT_AUDIO_TRANSCRIPTION: api_proto.InputAudioTranscription = {\n model: 'gpt-4o-mini-transcribe',\n};\nconst DEFAULT_TOOL_CHOICE: llm.ToolChoice = 'auto';\nconst DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS: number | 'inf' = 'inf';\n\nconst AZURE_DEFAULT_INPUT_AUDIO_TRANSCRIPTION: api_proto.InputAudioTranscription = {\n model: 'whisper-1',\n};\n\nconst AZURE_DEFAULT_TURN_DETECTION: api_proto.TurnDetectionType = {\n type: 'server_vad',\n threshold: 0.5,\n prefix_padding_ms: 300,\n silence_duration_ms: 200,\n create_response: true,\n};\n\nconst DEFAULT_MAX_SESSION_DURATION = 20 * 60 * 1000; // 20 minutes\n\nconst DEFAULT_REALTIME_MODEL_OPTIONS = {\n model: 'gpt-realtime',\n voice: 'marin',\n inputAudioTranscription: DEFAULT_INPUT_AUDIO_TRANSCRIPTION,\n inputAudioNoiseReduction: undefined as api_proto.NoiseReduction | undefined,\n turnDetection: DEFAULT_TURN_DETECTION,\n toolChoice: DEFAULT_TOOL_CHOICE,\n maxResponseOutputTokens: DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS,\n maxSessionDuration: DEFAULT_MAX_SESSION_DURATION,\n connOptions: DEFAULT_API_CONNECT_OPTIONS,\n modalities: ['text', 'audio'] as Modality[],\n tracing: undefined as api_proto.TracingConfig | undefined,\n};\nexport class RealtimeModel extends llm.RealtimeModel {\n sampleRate = api_proto.SAMPLE_RATE;\n numChannels = api_proto.NUM_CHANNELS;\n inFrameSize = api_proto.IN_FRAME_SIZE;\n outFrameSize = api_proto.OUT_FRAME_SIZE;\n\n /* @internal */\n _options: RealtimeOptions;\n\n get model(): string {\n return this._options.model;\n }\n\n constructor(\n options: {\n model?: string;\n voice?: string;\n /** @deprecated Unused in GA API (v1). Temperature is no longer supported. */\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n baseURL?: string;\n modalities?: Modality[];\n inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n inputAudioNoiseReduction?: api_proto.NoiseReduction | null;\n turnDetection?: api_proto.TurnDetectionType | null;\n speed?: number;\n tracing?: api_proto.TracingConfig | null;\n azureDeployment?: string;\n apiKey?: string;\n entraToken?: string;\n apiVersion?: string;\n maxSessionDuration?: number;\n connOptions?: APIConnectOptions;\n } = {},\n ) {\n const modalities = (options.modalities ||\n DEFAULT_REALTIME_MODEL_OPTIONS.modalities) as Modality[];\n\n super({\n messageTruncation: true,\n turnDetection: options.turnDetection !== null,\n userTranscription: options.inputAudioTranscription !== null,\n autoToolReplyGeneration: false,\n audioOutput: modalities.includes('audio'),\n });\n\n const isAzure = !!(options.apiVersion || options.entraToken || options.azureDeployment);\n\n if (options.apiKey === '' && !isAzure) {\n throw new Error(\n 'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environment variable',\n );\n }\n\n const apiKey = options.apiKey || process.env.OPENAI_API_KEY;\n\n if (!apiKey && !isAzure) {\n throw new Error(\n 'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environment variable',\n );\n }\n\n if (!options.baseURL && isAzure) {\n const azureEndpoint = process.env.AZURE_OPENAI_ENDPOINT;\n if (!azureEndpoint) {\n throw new Error(\n 'Missing Azure endpoint. Please pass base_url or set AZURE_OPENAI_ENDPOINT environment variable.',\n );\n }\n options.baseURL = `${azureEndpoint.replace(/\\/$/, '')}/openai`;\n }\n\n const { modalities: _, ...optionsWithoutModalities } = options;\n this._options = {\n ...DEFAULT_REALTIME_MODEL_OPTIONS,\n ...optionsWithoutModalities,\n baseURL: options.baseURL || BASE_URL,\n apiKey,\n isAzure,\n model: options.model || DEFAULT_REALTIME_MODEL_OPTIONS.model,\n modalities,\n };\n }\n\n /**\n * Create a RealtimeModel instance configured for Azure OpenAI Service.\n *\n * @param azureDeployment - The name of your Azure OpenAI deployment.\n * @param azureEndpoint - The endpoint URL for your Azure OpenAI resource. If undefined, will attempt to read from the environment variable AZURE_OPENAI_ENDPOINT.\n * @param apiVersion - API version to use with Azure OpenAI Service. If undefined, will attempt to read from the environment variable OPENAI_API_VERSION.\n * @param apiKey - Azure OpenAI API key. If undefined, will attempt to read from the environment variable AZURE_OPENAI_API_KEY.\n * @param entraToken - Azure Entra authentication token. Required if not using API key authentication.\n * @param baseURL - Base URL for the API endpoint. If undefined, constructed from the azure_endpoint.\n * @param voice - Voice setting for audio outputs. Defaults to \"alloy\".\n * @param inputAudioTranscription - Options for transcribing input audio. Defaults to @see DEFAULT_INPUT_AUDIO_TRANSCRIPTION.\n * @param inputAudioNoiseReduction - Options for noise reduction. Defaults to undefined.\n * @param turnDetection - Options for server-based voice activity detection (VAD). Defaults to @see DEFAULT_SERVER_VAD_OPTIONS.\n * @param speed - Speed of the audio output. Defaults to 1.0.\n * @param tracing - Tracing configuration. Defaults to undefined.\n *\n * @returns A RealtimeModel instance configured for Azure OpenAI Service.\n *\n * @throws Error if required Azure parameters are missing or invalid.\n */\n static withAzure({\n azureDeployment,\n azureEndpoint,\n apiVersion,\n apiKey,\n entraToken,\n baseURL,\n voice = 'alloy',\n temperature, // eslint-disable-line @typescript-eslint/no-unused-vars\n inputAudioTranscription = AZURE_DEFAULT_INPUT_AUDIO_TRANSCRIPTION,\n inputAudioNoiseReduction,\n turnDetection = AZURE_DEFAULT_TURN_DETECTION,\n speed,\n tracing,\n }: {\n azureDeployment: string;\n azureEndpoint?: string;\n apiVersion?: string;\n apiKey?: string;\n entraToken?: string;\n baseURL?: string;\n voice?: string;\n /** @deprecated Unused in GA API (v1). Temperature is no longer supported. */\n temperature?: number;\n inputAudioTranscription?: api_proto.InputAudioTranscription;\n inputAudioNoiseReduction?: api_proto.NoiseReduction;\n turnDetection?: api_proto.TurnDetectionType;\n speed?: number;\n tracing?: api_proto.TracingConfig;\n }) {\n apiKey = apiKey || process.env.AZURE_OPENAI_API_KEY;\n if (!apiKey && !entraToken) {\n throw new Error(\n 'Missing credentials. Please pass one of `apiKey`, `entraToken`, or the `AZURE_OPENAI_API_KEY` environment variable.',\n );\n }\n\n apiVersion = apiVersion || process.env.OPENAI_API_VERSION;\n if (!apiVersion) {\n throw new Error(\n 'Must provide either the `apiVersion` argument or the `OPENAI_API_VERSION` environment variable',\n );\n }\n\n if (!baseURL) {\n azureEndpoint = azureEndpoint || process.env.AZURE_OPENAI_ENDPOINT;\n if (!azureEndpoint) {\n throw new Error(\n 'Missing Azure endpoint. Please pass the `azure_endpoint` parameter or set the `AZURE_OPENAI_ENDPOINT` environment variable.',\n );\n }\n baseURL = `${azureEndpoint.replace(/\\/$/, '')}/openai`;\n }\n\n return new RealtimeModel({\n voice,\n inputAudioTranscription,\n inputAudioNoiseReduction,\n turnDetection,\n speed,\n tracing,\n apiKey,\n azureDeployment,\n apiVersion,\n entraToken,\n baseURL,\n });\n }\n\n session() {\n return new RealtimeSession(this);\n }\n\n async close() {\n return;\n }\n}\n\nfunction processBaseURL({\n baseURL,\n model,\n isAzure = false,\n azureDeployment,\n apiVersion,\n}: {\n baseURL: string;\n model: string;\n isAzure: boolean;\n azureDeployment?: string;\n apiVersion?: string;\n}): string {\n const url = new URL([baseURL, 'realtime'].join('/'));\n\n if (url.protocol === 'https:') {\n url.protocol = 'wss:';\n }\n\n // ensure \"/realtime\" is added if the path is empty OR \"/v1\"\n if (!url.pathname || ['', '/v1', '/openai'].includes(url.pathname.replace(/\\/$/, ''))) {\n url.pathname = url.pathname.replace(/\\/$/, '') + '/realtime';\n } else {\n url.pathname = url.pathname.replace(/\\/$/, '');\n }\n\n const queryParams: Record<string, string> = {};\n if (isAzure) {\n if (apiVersion) {\n queryParams['api-version'] = apiVersion;\n }\n if (azureDeployment) {\n queryParams['deployment'] = azureDeployment;\n }\n } else {\n queryParams['model'] = model;\n }\n\n for (const [key, value] of Object.entries(queryParams)) {\n url.searchParams.set(key, value);\n }\n\n return url.toString();\n}\n\n/**\n * A session for the OpenAI Realtime API.\n *\n * This class is used to interact with the OpenAI Realtime API.\n * It is responsible for sending events to the OpenAI Realtime API and receiving events from it.\n *\n * It exposes two more events:\n * - openai_server_event_received: expose the raw server events from the OpenAI Realtime API\n * - openai_client_event_queued: expose the raw client events sent to the OpenAI Realtime API\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private remoteChatCtx: llm.RemoteChatContext = new llm.RemoteChatContext();\n private messageChannel = new Queue<api_proto.ClientEvent>();\n private inputResampler?: AudioResampler;\n private instructions?: string;\n private oaiRealtimeModel: RealtimeModel;\n private currentGeneration?: ResponseGeneration;\n private responseCreatedFutures: { [id: string]: CreateResponseHandle } = {};\n\n private textModeRecoveryRetries: number = 0;\n\n private itemCreateFutures: { [id: string]: Future } = {};\n private itemDeleteFutures: { [id: string]: Future } = {};\n\n // Track items that have real server-side audio (created in current session, not restored)\n // Items restored after reconnection are text-only and cannot be truncated\n private audioCapableItemIds: Set<string> = new Set();\n\n private updateChatCtxLock = new Mutex();\n private updateFuncCtxLock = new Mutex();\n\n // 100ms chunks\n private bstream = new AudioByteStream(SAMPLE_RATE, NUM_CHANNELS, SAMPLE_RATE / 10);\n\n private pushedDurationMs: number = 0;\n\n #logger = log();\n #task: Task<void>;\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.oaiRealtimeModel = realtimeModel;\n\n this.#task = Task.from(({ signal }) => this.#mainTask(signal));\n\n this.sendEvent(this.createSessionUpdateEvent());\n }\n\n sendEvent(command: api_proto.ClientEvent): void {\n this.messageChannel.put(command);\n }\n\n private createSessionUpdateEvent(): api_proto.SessionUpdateEvent {\n const audioFormat: api_proto.AudioFormat = { type: 'audio/pcm', rate: SAMPLE_RATE };\n\n const modality: Modality = this.oaiRealtimeModel._options.modalities.includes('audio')\n ? 'audio'\n : 'text';\n\n return {\n type: 'session.update',\n session: {\n type: 'realtime',\n model: this.oaiRealtimeModel._options.model,\n output_modalities: [modality],\n audio: {\n input: {\n format: audioFormat,\n noise_reduction: this.oaiRealtimeModel._options.inputAudioNoiseReduction,\n transcription: this.oaiRealtimeModel._options.inputAudioTranscription,\n turn_detection: this.oaiRealtimeModel._options.turnDetection,\n },\n output: {\n format: audioFormat,\n speed: this.oaiRealtimeModel._options.speed,\n voice: this.oaiRealtimeModel._options.voice,\n },\n },\n max_output_tokens:\n this.oaiRealtimeModel._options.maxResponseOutputTokens === Infinity\n ? 'inf'\n : this.oaiRealtimeModel._options.maxResponseOutputTokens,\n tool_choice: toOaiToolChoice(this.oaiRealtimeModel._options.toolChoice),\n tracing: this.oaiRealtimeModel._options.tracing,\n instructions: this.instructions,\n },\n };\n }\n\n get chatCtx() {\n return this.remoteChatCtx.toChatCtx();\n }\n\n get tools() {\n return { ...this._tools } as llm.ToolContext;\n }\n\n async updateChatCtx(_chatCtx: llm.ChatContext): Promise<void> {\n const unlock = await this.updateChatCtxLock.lock();\n const events = this.createChatCtxUpdateEvents(_chatCtx);\n const futures: Future<void>[] = [];\n\n for (const event of events) {\n const future = new Future<void>();\n futures.push(future);\n\n if (event.type === 'conversation.item.create') {\n this.itemCreateFutures[event.item.id] = future;\n } else if (event.type == 'conversation.item.delete') {\n this.itemDeleteFutures[event.item_id] = future;\n }\n\n this.sendEvent(event);\n }\n\n if (futures.length === 0) {\n unlock();\n return;\n }\n\n try {\n // wait for futures to resolve or timeout\n await Promise.race([\n Promise.all(futures),\n delay(5000).then(() => {\n throw new Error('Chat ctx update events timed out');\n }),\n ]);\n } catch (e) {\n this.#logger.error((e as Error).message);\n throw e;\n } finally {\n unlock();\n }\n }\n\n private createChatCtxUpdateEvents(\n chatCtx: llm.ChatContext,\n addMockAudio: boolean = false,\n ): (api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[] {\n const newChatCtx = chatCtx.copy();\n if (addMockAudio) {\n newChatCtx.items.push(createMockAudioItem());\n } else {\n // clean up existing mock audio items\n newChatCtx.items = newChatCtx.items.filter(\n (item) => !item.id.startsWith(MOCK_AUDIO_ID_PREFIX),\n );\n }\n\n const events: (\n | api_proto.ConversationItemCreateEvent\n | api_proto.ConversationItemDeleteEvent\n )[] = [];\n\n const diffOps = llm.computeChatCtxDiff(this.chatCtx, newChatCtx);\n for (const op of diffOps.toRemove) {\n events.push({\n type: 'conversation.item.delete',\n item_id: op,\n event_id: shortuuid('chat_ctx_delete_'),\n } as api_proto.ConversationItemDeleteEvent);\n }\n\n for (const [previousId, id] of diffOps.toCreate) {\n const chatItem = newChatCtx.getById(id);\n if (!chatItem) {\n throw new Error(`Chat item ${id} not found`);\n }\n events.push({\n type: 'conversation.item.create',\n item: livekitItemToOpenAIItem(chatItem),\n previous_item_id: previousId ?? undefined,\n event_id: shortuuid('chat_ctx_create_'),\n } as api_proto.ConversationItemCreateEvent);\n }\n return events;\n }\n\n async updateTools(_tools: llm.ToolContext): Promise<void> {\n const unlock = await this.updateFuncCtxLock.lock();\n const ev = this.createToolsUpdateEvent(_tools);\n this.sendEvent(ev);\n\n if (!ev.session.tools) {\n throw new Error('Tools are missing in the session update event');\n }\n\n // TODO(brian): these logics below are noops I think, leaving it here to keep\n // parity with the python but we should remove them later\n const retainedToolNames = new Set(ev.session.tools.map((tool) => tool.name));\n const retainedTools = Object.fromEntries(\n Object.entries(_tools).filter(\n ([name, tool]) => llm.isFunctionTool(tool) && retainedToolNames.has(name),\n ),\n );\n\n this._tools = retainedTools as llm.ToolContext;\n\n unlock();\n }\n\n private createToolsUpdateEvent(_tools: llm.ToolContext): api_proto.SessionUpdateEvent {\n const oaiTools: api_proto.Tool[] = [];\n\n for (const [name, tool] of Object.entries(_tools)) {\n if (!llm.isFunctionTool(tool)) {\n this.#logger.error({ name, tool }, \"OpenAI Realtime API doesn't support this tool type\");\n continue;\n }\n\n const { parameters: toolParameters, description } = tool;\n try {\n const parameters = llm.toJsonSchema(\n toolParameters,\n ) as unknown as api_proto.Tool['parameters'];\n\n oaiTools.push({\n name,\n description,\n parameters: parameters,\n type: 'function',\n });\n } catch (e) {\n this.#logger.error({ name, tool }, \"OpenAI Realtime API doesn't support this tool type\");\n continue;\n }\n }\n\n return {\n type: 'session.update',\n session: {\n type: 'realtime',\n model: this.oaiRealtimeModel._options.model,\n tools: oaiTools,\n },\n event_id: shortuuid('tools_update_'),\n };\n }\n\n async updateInstructions(_instructions: string): Promise<void> {\n const eventId = shortuuid('instructions_update_');\n this.sendEvent({\n type: 'session.update',\n session: {\n type: 'realtime',\n instructions: _instructions,\n },\n event_id: eventId,\n } as api_proto.SessionUpdateEvent);\n this.instructions = _instructions;\n }\n\n updateOptions({ toolChoice }: { toolChoice?: llm.ToolChoice }): void {\n const options: api_proto.SessionUpdateEvent['session'] = {\n type: 'realtime',\n };\n\n this.oaiRealtimeModel._options.toolChoice = toolChoice;\n options.tool_choice = toOaiToolChoice(toolChoice);\n\n // TODO(brian): add other options here\n\n this.sendEvent({\n type: 'session.update',\n session: options,\n event_id: shortuuid('options_update_'),\n });\n }\n\n pushAudio(frame: AudioFrame): void {\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n this.sendEvent({\n type: 'input_audio_buffer.append',\n audio: Buffer.from(nf.data.buffer).toString('base64'),\n } as api_proto.InputAudioBufferAppendEvent);\n // TODO(AJS-102): use frame.durationMs once available in rtc-node\n this.pushedDurationMs += (nf.samplesPerChannel / nf.sampleRate) * 1000;\n }\n }\n }\n\n async commitAudio(): Promise<void> {\n if (this.pushedDurationMs > 100) {\n // OpenAI requires at least 100ms of audio\n this.sendEvent({\n type: 'input_audio_buffer.commit',\n } as api_proto.InputAudioBufferCommitEvent);\n this.pushedDurationMs = 0;\n }\n }\n\n async clearAudio(): Promise<void> {\n this.sendEvent({\n type: 'input_audio_buffer.clear',\n } as api_proto.InputAudioBufferClearEvent);\n this.pushedDurationMs = 0;\n }\n\n async generateReply(instructions?: string): Promise<llm.GenerationCreatedEvent> {\n const handle = this.createResponse({ instructions, userInitiated: true });\n this.textModeRecoveryRetries = 0;\n return handle.doneFut.await;\n }\n\n async interrupt(): Promise<void> {\n this.sendEvent({\n type: 'response.cancel',\n } as api_proto.ResponseCancelEvent);\n }\n\n async truncate(_options: {\n messageId: string;\n audioEndMs: number;\n modalities?: Modality[];\n audioTranscript?: string;\n }): Promise<void> {\n // Check if modalities include audio AND the item has real server-side audio\n // Items restored after reconnection are text-only and cannot be truncated\n const hasAudioModality = !_options.modalities || _options.modalities.includes('audio');\n const hasServerSideAudio = this.audioCapableItemIds.has(_options.messageId);\n\n if (hasAudioModality && hasServerSideAudio) {\n this.sendEvent({\n type: 'conversation.item.truncate',\n content_index: 0,\n item_id: _options.messageId,\n audio_end_ms: _options.audioEndMs,\n } as api_proto.ConversationItemTruncateEvent);\n } else if (_options.audioTranscript !== undefined) {\n // sync it to the remote chat context\n const chatCtx = this.chatCtx.copy();\n const idx = chatCtx.indexById(_options.messageId);\n if (idx !== undefined) {\n const item = chatCtx.items[idx];\n if (item && item.type === 'message') {\n const newItem = llm.ChatMessage.create({\n ...item,\n content: [_options.audioTranscript],\n });\n chatCtx.items[idx] = newItem;\n const events = this.createChatCtxUpdateEvents(chatCtx);\n for (const ev of events) {\n this.sendEvent(ev);\n }\n }\n }\n }\n }\n\n private loggableEvent(\n event: api_proto.ClientEvent | api_proto.ServerEvent,\n ): Record<string, unknown> {\n const untypedEvent: Record<string, unknown> = {};\n for (const [key, value] of Object.entries(event)) {\n if (value !== undefined) {\n untypedEvent[key] = value;\n }\n }\n\n if (untypedEvent.audio && typeof untypedEvent.audio === 'string') {\n return { ...untypedEvent, audio: '...' };\n }\n if (\n untypedEvent.delta &&\n typeof untypedEvent.delta === 'string' &&\n event.type === 'response.audio.delta'\n ) {\n return { ...untypedEvent, delta: '...' };\n }\n return untypedEvent;\n }\n\n private async createWsConn(): Promise<WebSocket> {\n const headers: Record<string, string> = {\n 'User-Agent': 'LiveKit-Agents-JS',\n };\n\n if (this.oaiRealtimeModel._options.isAzure) {\n // Microsoft API has two ways of authentication\n // 1. Entra token set as `Bearer` token\n // 2. API key set as `api_key` header (also accepts query string)\n if (this.oaiRealtimeModel._options.entraToken) {\n headers.Authorization = `Bearer ${this.oaiRealtimeModel._options.entraToken}`;\n } else if (this.oaiRealtimeModel._options.apiKey) {\n headers['api-key'] = this.oaiRealtimeModel._options.apiKey;\n } else {\n throw new Error('Microsoft API key or entraToken is required');\n }\n } else {\n if (!this.oaiRealtimeModel._options.apiKey) {\n throw new Error(\n 'OpenAI API key is required but not set. Check OPENAI_API_KEY environment variable.',\n );\n }\n headers.Authorization = `Bearer ${this.oaiRealtimeModel._options.apiKey}`;\n }\n\n const url = processBaseURL({\n baseURL: this.oaiRealtimeModel._options.baseURL,\n model: this.oaiRealtimeModel._options.model,\n isAzure: this.oaiRealtimeModel._options.isAzure,\n apiVersion: this.oaiRealtimeModel._options.apiVersion,\n azureDeployment: this.oaiRealtimeModel._options.azureDeployment,\n });\n\n if (lkOaiDebug) {\n this.#logger.debug(`Connecting to OpenAI Realtime API at ${url}`);\n }\n\n return new Promise((resolve, reject) => {\n const ws = new WebSocket(url, { headers });\n let waiting = true;\n\n const timeout = setTimeout(() => {\n ws.close();\n reject(new Error('WebSocket connection timeout'));\n }, this.oaiRealtimeModel._options.connOptions.timeoutMs);\n\n ws.once('open', () => {\n if (!waiting) return;\n waiting = false;\n clearTimeout(timeout);\n resolve(ws);\n });\n\n ws.once('close', () => {\n if (!waiting) return;\n waiting = false;\n clearTimeout(timeout);\n reject(new Error('OpenAI Realtime API connection closed'));\n });\n });\n }\n\n async #mainTask(signal: AbortSignal): Promise<void> {\n let reconnecting = false;\n let numRetries = 0;\n let wsConn: WebSocket | null = null;\n const maxRetries = this.oaiRealtimeModel._options.connOptions.maxRetry;\n\n const reconnect = async () => {\n this.#logger.debug(\n {\n maxSessionDuration: this.oaiRealtimeModel._options.maxSessionDuration,\n },\n 'Reconnecting to OpenAI Realtime API',\n );\n\n // Clean up pending futures from old connection to prevent memory leaks\n for (const fut of Object.values(this.itemCreateFutures)) {\n if (!fut.done) fut.reject(new Error('Session reconnected'));\n }\n this.itemCreateFutures = {};\n\n for (const fut of Object.values(this.itemDeleteFutures)) {\n if (!fut.done) fut.reject(new Error('Session reconnected'));\n }\n this.itemDeleteFutures = {};\n\n // Clear audio-capable item tracking - restored items are text-only on the server\n this.audioCapableItemIds.clear();\n\n const events: api_proto.ClientEvent[] = [];\n\n // options and instructions\n events.push(this.createSessionUpdateEvent());\n\n // tools\n if (Object.keys(this._tools).length > 0) {\n events.push(this.createToolsUpdateEvent(this._tools));\n }\n\n // chat context\n const chatCtx = this.chatCtx.copy({\n excludeFunctionCall: true,\n excludeInstructions: true,\n excludeEmptyMessage: true,\n });\n\n const oldChatCtx = this.remoteChatCtx;\n this.remoteChatCtx = new llm.RemoteChatContext();\n events.push(...this.createChatCtxUpdateEvents(chatCtx));\n\n try {\n for (const ev of events) {\n this.emit('openai_client_event_queued', ev);\n wsConn!.send(JSON.stringify(ev));\n }\n } catch (error) {\n this.remoteChatCtx = oldChatCtx;\n throw new APIConnectionError({\n message: 'Failed to send message to OpenAI Realtime API during session re-connection',\n });\n }\n\n this.#logger.debug('Reconnected to OpenAI Realtime API');\n\n this.emit('session_reconnected', {} as llm.RealtimeSessionReconnectedEvent);\n };\n\n reconnecting = false;\n while (!this.#closed && !signal.aborted) {\n this.#logger.debug('Creating WebSocket connection to OpenAI Realtime API');\n wsConn = await this.createWsConn();\n if (signal.aborted) break;\n\n try {\n if (reconnecting) {\n await reconnect();\n if (signal.aborted) break;\n numRetries = 0;\n }\n\n await this.runWs(wsConn);\n if (signal.aborted) break;\n } catch (error) {\n if (!isAPIError(error)) {\n this.emitError({ error: error as Error, recoverable: false });\n throw error;\n }\n\n if (maxRetries === 0 || !error.retryable) {\n this.emitError({ error: error as Error, recoverable: false });\n throw error;\n }\n\n if (numRetries === maxRetries) {\n this.emitError({ error: error as Error, recoverable: false });\n throw new APIConnectionError({\n message: `OpenAI Realtime API connection failed after ${numRetries} attempts`,\n options: {\n body: error,\n retryable: false,\n },\n });\n }\n\n this.emitError({ error: error as Error, recoverable: true });\n const retryInterval =\n numRetries === 0\n ? DEFAULT_FIRST_RETRY_INTERVAL_MS\n : this.oaiRealtimeModel._options.connOptions.retryIntervalMs;\n this.#logger.warn(\n {\n attempt: numRetries,\n maxRetries,\n error,\n },\n `OpenAI Realtime API connection failed, retrying in ${retryInterval / 1000}s`,\n );\n\n await delay(retryInterval);\n numRetries++;\n }\n\n reconnecting = true;\n }\n }\n\n private async runWs(wsConn: WebSocket): Promise<void> {\n const forwardEvents = async (signal: AbortSignal): Promise<void> => {\n const abortFuture = new Future<void>();\n signal.addEventListener('abort', () => abortFuture.resolve());\n\n while (!this.#closed && wsConn.readyState === WebSocket.OPEN && !signal.aborted) {\n try {\n const event = await Promise.race([this.messageChannel.get(), abortFuture.await]);\n if (signal.aborted || abortFuture.done || event === undefined) {\n break;\n }\n\n if (lkOaiDebug) {\n this.#logger.debug(this.loggableEvent(event), `(client) -> ${event.type}`);\n }\n\n this.emit('openai_client_event_queued', event);\n wsConn.send(JSON.stringify(event));\n } catch (error) {\n break;\n }\n }\n\n wsConn.close();\n };\n\n const wsCloseFuture = new Future<void | Error>();\n\n wsConn.onerror = (error) => {\n wsCloseFuture.resolve(new APIConnectionError({ message: error.message }));\n };\n wsConn.onclose = () => {\n wsCloseFuture.resolve();\n };\n\n wsConn.onmessage = (message: MessageEvent) => {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const event: any = JSON.parse(message.data as string);\n\n this.emit('openai_server_event_received', event);\n if (lkOaiDebug) {\n this.#logger.debug(this.loggableEvent(event), `(server) <- ${event.type}`);\n }\n\n switch (event.type) {\n case 'input_audio_buffer.speech_started':\n this.handleInputAudioBufferSpeechStarted(event);\n break;\n case 'input_audio_buffer.speech_stopped':\n this.handleInputAudioBufferSpeechStopped(event);\n break;\n case 'response.created':\n this.handleResponseCreated(event);\n break;\n case 'response.output_item.added':\n this.handleResponseOutputItemAdded(event);\n break;\n case 'conversation.item.added':\n case 'conversation.item.created': // Beta: kept for backward compatibility\n this.handleConversationItemCreated(event);\n break;\n case 'conversation.item.deleted':\n this.handleConversationItemDeleted(event);\n break;\n case 'conversation.item.input_audio_transcription.completed':\n this.handleConversationItemInputAudioTranscriptionCompleted(event);\n break;\n case 'conversation.item.input_audio_transcription.failed':\n this.handleConversationItemInputAudioTranscriptionFailed(event);\n break;\n case 'response.content_part.added':\n this.handleResponseContentPartAdded(event);\n break;\n case 'response.content_part.done':\n this.handleResponseContentPartDone(event);\n break;\n case 'response.output_text.delta':\n case 'response.text.delta': // Beta: kept for backward compatibility\n this.handleResponseTextDelta(event);\n break;\n case 'response.output_text.done':\n case 'response.text.done': // Beta: kept for backward compatibility\n this.handleResponseTextDone(event);\n break;\n case 'response.output_audio_transcript.delta':\n case 'response.audio_transcript.delta': // Beta: kept for backward compatibility\n this.handleResponseAudioTranscriptDelta(event);\n break;\n case 'response.output_audio.delta':\n case 'response.audio.delta': // Beta: kept for backward compatibility\n this.handleResponseAudioDelta(event);\n break;\n case 'response.output_audio_transcript.done':\n case 'response.audio_transcript.done': // Beta: kept for backward compatibility\n this.handleResponseAudioTranscriptDone(event);\n break;\n case 'response.output_audio.done':\n case 'response.audio.done': // Beta: kept for backward compatibility\n this.handleResponseAudioDone(event);\n break;\n case 'response.output_item.done':\n this.handleResponseOutputItemDone(event);\n break;\n case 'response.done':\n this.handleResponseDone(event);\n break;\n case 'error':\n this.handleError(event);\n break;\n default:\n if (lkOaiDebug) {\n this.#logger.debug(`unhandled event: ${event.type}`);\n }\n break;\n }\n };\n\n const sendTask = Task.from(({ signal }) => forwardEvents(signal));\n\n const wsTask = Task.from(({ signal }) => {\n const abortPromise = new Promise<void>((resolve) => {\n signal.addEventListener('abort', () => {\n resolve();\n });\n });\n\n return Promise.race([wsCloseFuture.await, abortPromise]);\n });\n\n const waitReconnectTask = Task.from(async ({ signal }) => {\n await delay(this.oaiRealtimeModel._options.maxSessionDuration, { signal });\n return new APIConnectionError({\n message: 'OpenAI Realtime API connection timeout',\n });\n });\n\n try {\n const result = await Promise.race([wsTask.result, sendTask.result, waitReconnectTask.result]);\n\n if (waitReconnectTask.done && this.currentGeneration) {\n await this.currentGeneration._doneFut.await;\n }\n\n if (result instanceof Error) {\n throw result;\n }\n } finally {\n await cancelAndWait([wsTask, sendTask, waitReconnectTask], 2000);\n wsConn.close();\n }\n }\n\n async close() {\n super.close();\n this.#closed = true;\n await this.#task;\n\n // Clean up pending futures to prevent memory leaks\n for (const handle of Object.values(this.responseCreatedFutures)) {\n if (!handle.doneFut.done) {\n handle.doneFut.reject(new Error('Session closed'));\n }\n }\n this.responseCreatedFutures = {};\n\n for (const fut of Object.values(this.itemCreateFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed'));\n }\n }\n this.itemCreateFutures = {};\n\n for (const fut of Object.values(this.itemDeleteFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed'));\n }\n }\n this.itemDeleteFutures = {};\n\n // Clean up current generation if exists\n if (this.currentGeneration) {\n for (const gen of this.currentGeneration.messages.values()) {\n gen.textChannel.close();\n gen.audioChannel.close();\n if (!gen.modalities.done) {\n gen.modalities.resolve(this.oaiRealtimeModel._options.modalities);\n }\n }\n this.currentGeneration.messages.clear();\n this.currentGeneration.messageChannel.close();\n this.currentGeneration.functionChannel.close();\n if (!this.currentGeneration._doneFut.done) {\n this.currentGeneration._doneFut.resolve();\n }\n this.currentGeneration = undefined;\n }\n\n // Clear the message queue\n this.messageChannel.items.length = 0;\n }\n\n private handleInputAudioBufferSpeechStarted(\n _event: api_proto.InputAudioBufferSpeechStartedEvent,\n ): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputAudioBufferSpeechStopped(\n _event: api_proto.InputAudioBufferSpeechStoppedEvent,\n ): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: this.oaiRealtimeModel._options.inputAudioTranscription !== null,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleResponseCreated(event: api_proto.ResponseCreatedEvent): void {\n if (!event.response.id) {\n throw new Error('response.id is missing');\n }\n\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel<llm.MessageGeneration>(),\n functionChannel: stream.createStreamChannel<llm.FunctionCall>(),\n messages: new Map(),\n _doneFut: new Future(),\n _createdTimestamp: Date.now(),\n };\n\n // Build generation event and resolve client future (if any) before emitting,\n // matching Python behavior.\n const generationEv = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n responseId: event.response.id,\n } as llm.GenerationCreatedEvent;\n\n const clientEventId = event.response.metadata?.client_event_id;\n if (clientEventId) {\n const handle = this.responseCreatedFutures[clientEventId];\n if (handle) {\n delete this.responseCreatedFutures[clientEventId];\n generationEv.userInitiated = true;\n if (!handle.doneFut.done) {\n handle.doneFut.resolve(generationEv);\n }\n }\n }\n\n this.emit('generation_created', generationEv);\n }\n\n private handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n if (!event.item.type) {\n throw new Error('item.type is not set');\n }\n\n const itemType = event.item.type;\n\n if (itemType !== 'message') {\n // non-message items (e.g. function calls) don't need additional handling here\n // the generation event was already emitted in handleResponseCreated\n this.textModeRecoveryRetries = 0;\n return;\n }\n\n const itemId = event.item.id;\n if (!itemId) {\n throw new Error('item.id is not set');\n }\n\n const modalitiesFut = new Future<Modality[]>();\n const itemGeneration: MessageGeneration = {\n messageId: itemId,\n textChannel: stream.createStreamChannel<string>(),\n audioChannel: stream.createStreamChannel<AudioFrame>(),\n audioTranscript: '',\n modalities: modalitiesFut,\n };\n\n // If audioOutput is not supported, close audio channel immediately\n if (!this.oaiRealtimeModel.capabilities.audioOutput) {\n itemGeneration.audioChannel.close();\n modalitiesFut.resolve(['text']);\n }\n\n this.currentGeneration.messageChannel.write({\n messageId: itemId,\n textStream: itemGeneration.textChannel.stream(),\n audioStream: itemGeneration.audioChannel.stream(),\n modalities: modalitiesFut.await,\n });\n\n this.currentGeneration.messages.set(itemId, itemGeneration);\n }\n\n private handleConversationItemCreated(event: api_proto.ConversationItemCreatedEvent): void {\n if (!event.item.id) {\n throw new Error('item.id is not set');\n }\n\n try {\n this.remoteChatCtx.insert(event.previous_item_id, openAIItemToLivekitItem(event.item));\n } catch (error) {\n this.#logger.error({ error, itemId: event.item.id }, 'failed to insert conversation item');\n }\n\n const fut = this.itemCreateFutures[event.item.id];\n if (fut) {\n fut.resolve();\n delete this.itemCreateFutures[event.item.id];\n }\n }\n\n private handleConversationItemDeleted(event: api_proto.ConversationItemDeletedEvent): void {\n if (!event.item_id) {\n throw new Error('item_id is not set');\n }\n\n // Clean up audio-capable tracking for deleted items\n this.audioCapableItemIds.delete(event.item_id);\n\n try {\n this.remoteChatCtx.delete(event.item_id);\n } catch (error) {\n this.#logger.error({ error, itemId: event.item_id }, 'failed to delete conversation item');\n }\n\n const fut = this.itemDeleteFutures[event.item_id];\n if (fut) {\n fut.resolve();\n delete this.itemDeleteFutures[event.item_id];\n }\n }\n\n private handleConversationItemInputAudioTranscriptionCompleted(\n event: api_proto.ConversationItemInputAudioTranscriptionCompletedEvent,\n ): void {\n const remoteItem = this.remoteChatCtx.get(event.item_id);\n if (!remoteItem) {\n return;\n }\n\n const item = remoteItem.item;\n if (item instanceof llm.ChatMessage) {\n item.content.push(event.transcript);\n } else {\n throw new Error('item is not a chat message');\n }\n\n this.emit('input_audio_transcription_completed', {\n itemId: event.item_id,\n transcript: event.transcript,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n }\n\n private handleConversationItemInputAudioTranscriptionFailed(\n event: api_proto.ConversationItemInputAudioTranscriptionFailedEvent,\n ): void {\n this.#logger.error(\n { error: event.error },\n 'OpenAI Realtime API failed to transcribe input audio',\n );\n }\n\n private handleResponseContentPartAdded(event: api_proto.ResponseContentPartAddedEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n const itemId = event.item_id;\n const itemType = event.part.type;\n\n const itemGeneration = this.currentGeneration.messages.get(itemId);\n if (!itemGeneration) {\n this.#logger.warn(`itemGeneration not found for itemId=${itemId}`);\n return;\n }\n\n const isTextType = itemType === 'text' || itemType === 'output_text';\n if (isTextType && this.oaiRealtimeModel.capabilities.audioOutput) {\n this.#logger.warn('Text response received from OpenAI Realtime API in audio modality.');\n }\n\n if (!itemGeneration.modalities.done) {\n const modalityResult: Modality[] = isTextType ? ['text'] : ['audio', 'text'];\n itemGeneration.modalities.resolve(modalityResult);\n\n // Track items with real server-side audio for truncation eligibility\n if (!isTextType) {\n this.audioCapableItemIds.add(itemId);\n }\n }\n\n if (this.currentGeneration._firstTokenTimestamp === undefined) {\n this.currentGeneration._firstTokenTimestamp = Date.now();\n }\n }\n\n private handleResponseContentPartDone(event: api_proto.ResponseContentPartDoneEvent): void {\n if (!event.part) {\n return;\n }\n if (event.part.type !== 'text') {\n return;\n }\n\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n // TODO(shubhra): handle text mode recovery\n }\n\n private handleResponseTextDelta(event: api_proto.ResponseTextDeltaEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n const itemGeneration = this.currentGeneration.messages.get(event.item_id);\n if (!itemGeneration) {\n throw new Error('itemGeneration is not set');\n }\n\n if (\n !this.oaiRealtimeModel.capabilities.audioOutput &&\n !this.currentGeneration._firstTokenTimestamp\n ) {\n this.currentGeneration._firstTokenTimestamp = Date.now();\n }\n\n itemGeneration.textChannel.write(event.delta);\n itemGeneration.audioTranscript += event.delta;\n }\n\n private handleResponseTextDone(_event: api_proto.ResponseTextDoneEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n }\n\n private handleResponseAudioTranscriptDelta(\n event: api_proto.ResponseAudioTranscriptDeltaEvent,\n ): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n const itemId = event.item_id;\n const delta = event.delta;\n\n // TODO (shubhra): add timed string support\n\n const itemGeneration = this.currentGeneration.messages.get(itemId);\n if (!itemGeneration) {\n throw new Error('itemGeneration is not set');\n } else {\n itemGeneration.textChannel.write(delta);\n itemGeneration.audioTranscript += delta;\n }\n }\n\n private handleResponseAudioDelta(event: api_proto.ResponseAudioDeltaEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n const itemGeneration = this.currentGeneration.messages.get(event.item_id);\n if (!itemGeneration) {\n throw new Error('itemGeneration is not set');\n }\n\n if (this.currentGeneration._firstTokenTimestamp === undefined) {\n this.currentGeneration._firstTokenTimestamp = Date.now();\n }\n\n if (!itemGeneration.modalities.done) {\n itemGeneration.modalities.resolve(['audio', 'text']);\n }\n\n const binaryString = atob(event.delta);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n itemGeneration.audioChannel.write(\n new AudioFrame(\n new Int16Array(bytes.buffer),\n api_proto.SAMPLE_RATE,\n api_proto.NUM_CHANNELS,\n bytes.length / 2,\n ),\n );\n }\n\n private handleResponseAudioTranscriptDone(\n _event: api_proto.ResponseAudioTranscriptDoneEvent,\n ): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n }\n\n private handleResponseAudioDone(_event: api_proto.ResponseAudioDoneEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n }\n\n private handleResponseOutputItemDone(event: api_proto.ResponseOutputItemDoneEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n const itemId = event.item.id;\n const itemType = event.item.type;\n\n if (itemType === 'function_call') {\n const item = event.item;\n if (!item.call_id || !item.name || !item.arguments) {\n throw new Error('item is not a function call');\n }\n this.currentGeneration.functionChannel.write(\n llm.FunctionCall.create({\n callId: item.call_id,\n name: item.name,\n args: item.arguments,\n }),\n );\n } else if (itemType === 'message') {\n const itemGeneration = this.currentGeneration.messages.get(itemId);\n if (!itemGeneration) {\n return;\n }\n // text response doesn't have itemGeneration\n itemGeneration.textChannel.close();\n itemGeneration.audioChannel.close();\n if (!itemGeneration.modalities.done) {\n // In case message modalities is not set, this shouldn't happen\n itemGeneration.modalities.resolve(this.oaiRealtimeModel._options.modalities);\n }\n }\n }\n\n private handleResponseDone(_event: api_proto.ResponseDoneEvent): void {\n if (!this.currentGeneration) {\n // OpenAI has a race condition where we could receive response.done without any\n // previous response.created (This happens generally during interruption)\n return;\n }\n\n const createdTimestamp = this.currentGeneration._createdTimestamp;\n const firstTokenTimestamp = this.currentGeneration._firstTokenTimestamp;\n\n this.#logger.debug(\n {\n messageCount: this.currentGeneration.messages.size,\n },\n 'Closing generation channels in handleResponseDone',\n );\n\n for (const generation of this.currentGeneration.messages.values()) {\n generation.textChannel.close();\n generation.audioChannel.close();\n if (!generation.modalities.done) {\n generation.modalities.resolve(this.oaiRealtimeModel._options.modalities);\n }\n }\n\n this.currentGeneration.functionChannel.close();\n this.currentGeneration.messageChannel.close();\n\n for (const itemId of this.currentGeneration.messages.keys()) {\n const remoteItem = this.remoteChatCtx.get(itemId);\n if (remoteItem && remoteItem.item instanceof llm.ChatMessage) {\n remoteItem.item.content.push(this.currentGeneration.messages.get(itemId)!.audioTranscript);\n }\n }\n\n this.currentGeneration._doneFut.resolve();\n this.currentGeneration = undefined;\n\n // Calculate and emit metrics\n const usage = _event.response.usage;\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = Date.now() - createdTimestamp;\n\n const realtimeMetrics: metrics.RealtimeModelMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: _event.response.id || '',\n ttftMs,\n durationMs,\n cancelled: _event.response.status === 'cancelled',\n label: 'openai_realtime',\n inputTokens: usage?.input_tokens ?? 0,\n outputTokens: usage?.output_tokens ?? 0,\n totalTokens: usage?.total_tokens ?? 0,\n tokensPerSecond: durationMs > 0 ? (usage?.output_tokens ?? 0) / (durationMs / 1000) : 0,\n inputTokenDetails: {\n audioTokens: usage?.input_token_details?.audio_tokens ?? 0,\n textTokens: usage?.input_token_details?.text_tokens ?? 0,\n imageTokens: 0, // Not supported yet\n cachedTokens: usage?.input_token_details?.cached_tokens ?? 0,\n cachedTokensDetails: usage?.input_token_details?.cached_tokens_details\n ? {\n audioTokens: usage?.input_token_details?.cached_tokens_details?.audio_tokens ?? 0,\n textTokens: usage?.input_token_details?.cached_tokens_details?.text_tokens ?? 0,\n imageTokens: usage?.input_token_details?.cached_tokens_details?.image_tokens ?? 0,\n }\n : undefined,\n },\n outputTokenDetails: {\n textTokens: usage?.output_token_details?.text_tokens ?? 0,\n audioTokens: usage?.output_token_details?.audio_tokens ?? 0,\n imageTokens: 0,\n },\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n // TODO(brian): handle response done but not complete\n }\n\n private handleError(event: api_proto.ErrorEvent): void {\n if (event.error.message.startsWith('Cancellation failed')) {\n return;\n }\n\n this.#logger.error({ error: event.error }, 'OpenAI Realtime API returned an error');\n this.emitError({\n error: new APIError(event.error.message, {\n body: event.error,\n retryable: true,\n }),\n recoverable: true,\n });\n\n // TODO(brian): set error for response future if it exists\n }\n\n private emitError({ error, recoverable }: { error: Error; recoverable: boolean }): void {\n // IMPORTANT: only emit error if there are listeners; otherwise emit will throw an error\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label\n label: '',\n error,\n recoverable,\n } as llm.RealtimeModelError);\n }\n\n private *resampleAudio(frame: AudioFrame): Generator<AudioFrame> {\n yield frame;\n }\n\n private createResponse({\n userInitiated,\n instructions,\n oldHandle,\n }: {\n userInitiated: boolean;\n instructions?: string;\n oldHandle?: CreateResponseHandle;\n }): CreateResponseHandle {\n const handle = oldHandle || new CreateResponseHandle({ instructions });\n if (oldHandle && instructions) {\n handle.instructions = instructions;\n }\n\n const eventId = shortuuid('response_create_');\n if (userInitiated) {\n this.responseCreatedFutures[eventId] = handle;\n }\n\n const response: api_proto.ResponseCreateEvent['response'] = {};\n if (instructions) response.instructions = instructions;\n if (userInitiated) response.metadata = { client_event_id: eventId };\n\n this.sendEvent({\n type: 'response.create',\n event_id: eventId,\n response: Object.keys(response).length > 0 ? response : undefined,\n });\n\n return handle;\n }\n}\n\n/** @internal Exported for testing purposes */\nexport function livekitItemToOpenAIItem(item: llm.ChatItem): api_proto.ItemResource {\n switch (item.type) {\n case 'function_call':\n return {\n id: item.id,\n type: 'function_call',\n call_id: item.callId,\n name: item.name,\n arguments: item.args,\n } as api_proto.FunctionCallItem;\n case 'function_call_output':\n return {\n id: item.id,\n type: 'function_call_output',\n call_id: item.callId,\n output: item.output,\n } as api_proto.FunctionCallOutputItem;\n case 'message':\n const role = item.role === 'developer' ? 'system' : item.role;\n const contentList: api_proto.Content[] = [];\n for (const c of item.content) {\n if (typeof c === 'string') {\n contentList.push({\n type: role === 'assistant' ? 'output_text' : 'input_text',\n text: c,\n } as api_proto.InputTextContent | api_proto.OutputTextContent);\n } else if (c.type === 'image_content') {\n // not supported for now\n continue;\n } else if (c.type === 'audio_content') {\n if (role === 'user') {\n const encodedAudio = Buffer.from(combineAudioFrames(c.frame).data).toString('base64');\n contentList.push({\n type: 'input_audio',\n audio: encodedAudio,\n } as api_proto.InputAudioContent);\n }\n }\n }\n return {\n id: item.id,\n type: 'message',\n role,\n content: contentList,\n } as api_proto.UserItem | api_proto.AssistantItem | api_proto.SystemItem;\n default:\n throw new Error(`Unsupported item type: ${(item as any).type}`);\n }\n}\n\nfunction openAIItemToLivekitItem(item: api_proto.ItemResource): llm.ChatItem {\n if (!item.id) {\n throw new Error('item.id is not set');\n }\n\n switch (item.type) {\n case 'function_call':\n return llm.FunctionCall.create({\n id: item.id,\n callId: item.call_id,\n name: item.name,\n args: item.arguments,\n });\n case 'function_call_output':\n return llm.FunctionCallOutput.create({\n id: item.id,\n callId: item.call_id,\n output: item.output,\n isError: false,\n });\n case 'message':\n const content: llm.ChatContent[] = [];\n // item.content can be a single object or an array; normalize to array\n const contents = Array.isArray(item.content) ? item.content : [item.content];\n for (const c of contents) {\n if (c.type === 'text' || c.type === 'input_text') {\n content.push(c.text);\n }\n }\n return llm.ChatMessage.create({\n id: item.id,\n role: item.role,\n content,\n });\n }\n}\n\nfunction createMockAudioItem(durationSeconds: number = 2): llm.ChatMessage {\n const audioData = Buffer.alloc(durationSeconds * SAMPLE_RATE);\n return llm.ChatMessage.create({\n id: shortuuid(MOCK_AUDIO_ID_PREFIX),\n role: 'user',\n content: [\n {\n type: 'audio_content',\n frame: [\n new AudioFrame(\n new Int16Array(audioData.buffer),\n SAMPLE_RATE,\n NUM_CHANNELS,\n audioData.length / 2,\n ),\n ],\n } as llm.AudioContent,\n ],\n });\n}\n\nfunction toOaiToolChoice(toolChoice?: llm.ToolChoice): api_proto.ToolChoice {\n if (typeof toolChoice === 'string') {\n return toolChoice;\n }\n\n if (toolChoice?.type === 'function') {\n return toolChoice.function.name;\n }\n\n return 'auto';\n}\n"],"mappings":"AAIA;AAAA,EAEE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,aAAa;AAEtB,SAAS,YAAY,0BAA0B;AAC/C,SAA4B,iBAAiB;AAC7C,YAAY,eAAe;AAG3B,MAAM,aAAa,QAAQ,IAAI,kBAAkB,OAAO,QAAQ,IAAI,eAAe,IAAI;AAEvF,MAAM,cAAc;AACpB,MAAM,eAAe;AACrB,MAAM,WAAW;AAEjB,MAAM,uBAAuB;AA+C7B,MAAM,qBAAqB;AAAA,EACzB;AAAA,EACA;AAAA;AAAA,EAEA,YAAY,EAAE,aAAa,GAA8B;AACvD,SAAK,eAAe;AACpB,SAAK,UAAU,IAAI,OAAO;AAAA,EAC5B;AACF;AAEA,MAAM,kCAAkC;AACxC,MAAM,yBAAsD;AAAA,EAC1D,MAAM;AAAA,EACN,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,oBAAoB;AACtB;AACA,MAAM,oCAAuE;AAAA,EAC3E,OAAO;AACT;AACA,MAAM,sBAAsC;AAC5C,MAAM,qCAAqD;AAE3D,MAAM,0CAA6E;AAAA,EACjF,OAAO;AACT;AAEA,MAAM,+BAA4D;AAAA,EAChE,MAAM;AAAA,EACN,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,iBAAiB;AACnB;AAEA,MAAM,+BAA+B,KAAK,KAAK;AAE/C,MAAM,iCAAiC;AAAA,EACrC,OAAO;AAAA,EACP,OAAO;AAAA,EACP,yBAAyB;AAAA,EACzB,0BAA0B;AAAA,EAC1B,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,yBAAyB;AAAA,EACzB,oBAAoB;AAAA,EACpB,aAAa;AAAA,EACb,YAAY,CAAC,QAAQ,OAAO;AAAA,EAC5B,SAAS;AACX;AACO,MAAM,sBAAsB,IAAI,cAAc;AAAA,EACnD,aAAa,UAAU;AAAA,EACvB,cAAc,UAAU;AAAA,EACxB,cAAc,UAAU;AAAA,EACxB,eAAe,UAAU;AAAA;AAAA,EAGzB;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,YACE,UAmBI,CAAC,GACL;AACA,UAAM,aAAc,QAAQ,cAC1B,+BAA+B;AAEjC,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe,QAAQ,kBAAkB;AAAA,MACzC,mBAAmB,QAAQ,4BAA4B;AAAA,MACvD,yBAAyB;AAAA,MACzB,aAAa,WAAW,SAAS,OAAO;AAAA,IAC1C,CAAC;AAED,UAAM,UAAU,CAAC,EAAE,QAAQ,cAAc,QAAQ,cAAc,QAAQ;AAEvE,QAAI,QAAQ,WAAW,MAAM,CAAC,SAAS;AACrC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAE7C,QAAI,CAAC,UAAU,CAAC,SAAS;AACvB,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,QAAQ,WAAW,SAAS;AAC/B,YAAM,gBAAgB,QAAQ,IAAI;AAClC,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,cAAQ,UAAU,GAAG,cAAc,QAAQ,OAAO,EAAE,CAAC;AAAA,IACvD;AAEA,UAAM,EAAE,YAAY,GAAG,GAAG,yBAAyB,IAAI;AACvD,SAAK,WAAW;AAAA,MACd,GAAG;AAAA,MACH,GAAG;AAAA,MACH,SAAS,QAAQ,WAAW;AAAA,MAC5B;AAAA,MACA;AAAA,MACA,OAAO,QAAQ,SAAS,+BAA+B;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAsBA,OAAO,UAAU;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,IACR;AAAA;AAAA,IACA,0BAA0B;AAAA,IAC1B;AAAA,IACA,gBAAgB;AAAA,IAChB;AAAA,IACA;AAAA,EACF,GAeG;AACD,aAAS,UAAU,QAAQ,IAAI;AAC/B,QAAI,CAAC,UAAU,CAAC,YAAY;AAC1B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,iBAAa,cAAc,QAAQ,IAAI;AACvC,QAAI,CAAC,YAAY;AACf,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,SAAS;AACZ,sBAAgB,iBAAiB,QAAQ,IAAI;AAC7C,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,gBAAU,GAAG,cAAc,QAAQ,OAAO,EAAE,CAAC;AAAA,IAC/C;AAEA,WAAO,IAAI,cAAc;AAAA,MACvB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA,EAEA,MAAM,QAAQ;AACZ;AAAA,EACF;AACF;AAEA,SAAS,eAAe;AAAA,EACtB;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV;AAAA,EACA;AACF,GAMW;AACT,QAAM,MAAM,IAAI,IAAI,CAAC,SAAS,UAAU,EAAE,KAAK,GAAG,CAAC;AAEnD,MAAI,IAAI,aAAa,UAAU;AAC7B,QAAI,WAAW;AAAA,EACjB;AAGA,MAAI,CAAC,IAAI,YAAY,CAAC,IAAI,OAAO,SAAS,EAAE,SAAS,IAAI,SAAS,QAAQ,OAAO,EAAE,CAAC,GAAG;AACrF,QAAI,WAAW,IAAI,SAAS,QAAQ,OAAO,EAAE,IAAI;AAAA,EACnD,OAAO;AACL,QAAI,WAAW,IAAI,SAAS,QAAQ,OAAO,EAAE;AAAA,EAC/C;AAEA,QAAM,cAAsC,CAAC;AAC7C,MAAI,SAAS;AACX,QAAI,YAAY;AACd,kBAAY,aAAa,IAAI;AAAA,IAC/B;AACA,QAAI,iBAAiB;AACnB,kBAAY,YAAY,IAAI;AAAA,IAC9B;AAAA,EACF,OAAO;AACL,gBAAY,OAAO,IAAI;AAAA,EACzB;AAEA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,WAAW,GAAG;AACtD,QAAI,aAAa,IAAI,KAAK,KAAK;AAAA,EACjC;AAEA,SAAO,IAAI,SAAS;AACtB;AAYO,MAAM,wBAAwB,IAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,gBAAuC,IAAI,IAAI,kBAAkB;AAAA,EACjE,iBAAiB,IAAI,MAA6B;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,yBAAiE,CAAC;AAAA,EAElE,0BAAkC;AAAA,EAElC,oBAA8C,CAAC;AAAA,EAC/C,oBAA8C,CAAC;AAAA;AAAA;AAAA,EAI/C,sBAAmC,oBAAI,IAAI;AAAA,EAE3C,oBAAoB,IAAI,MAAM;AAAA,EAC9B,oBAAoB,IAAI,MAAM;AAAA;AAAA,EAG9B,UAAU,IAAI,gBAAgB,aAAa,cAAc,cAAc,EAAE;AAAA,EAEzE,mBAA2B;AAAA,EAEnC,UAAU,IAAI;AAAA,EACd;AAAA,EACA,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,mBAAmB;AAExB,SAAK,QAAQ,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,UAAU,MAAM,CAAC;AAE7D,SAAK,UAAU,KAAK,yBAAyB,CAAC;AAAA,EAChD;AAAA,EAEA,UAAU,SAAsC;AAC9C,SAAK,eAAe,IAAI,OAAO;AAAA,EACjC;AAAA,EAEQ,2BAAyD;AAC/D,UAAM,cAAqC,EAAE,MAAM,aAAa,MAAM,YAAY;AAElF,UAAM,WAAqB,KAAK,iBAAiB,SAAS,WAAW,SAAS,OAAO,IACjF,UACA;AAEJ,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,MAAM;AAAA,QACN,OAAO,KAAK,iBAAiB,SAAS;AAAA,QACtC,mBAAmB,CAAC,QAAQ;AAAA,QAC5B,OAAO;AAAA,UACL,OAAO;AAAA,YACL,QAAQ;AAAA,YACR,iBAAiB,KAAK,iBAAiB,SAAS;AAAA,YAChD,eAAe,KAAK,iBAAiB,SAAS;AAAA,YAC9C,gBAAgB,KAAK,iBAAiB,SAAS;AAAA,UACjD;AAAA,UACA,QAAQ;AAAA,YACN,QAAQ;AAAA,YACR,OAAO,KAAK,iBAAiB,SAAS;AAAA,YACtC,OAAO,KAAK,iBAAiB,SAAS;AAAA,UACxC;AAAA,QACF;AAAA,QACA,mBACE,KAAK,iBAAiB,SAAS,4BAA4B,WACvD,QACA,KAAK,iBAAiB,SAAS;AAAA,QACrC,aAAa,gBAAgB,KAAK,iBAAiB,SAAS,UAAU;AAAA,QACtE,SAAS,KAAK,iBAAiB,SAAS;AAAA,QACxC,cAAc,KAAK;AAAA,MACrB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,UAAU;AACZ,WAAO,KAAK,cAAc,UAAU;AAAA,EACtC;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,MAAM,cAAc,UAA0C;AAC5D,UAAM,SAAS,MAAM,KAAK,kBAAkB,KAAK;AACjD,UAAM,SAAS,KAAK,0BAA0B,QAAQ;AACtD,UAAM,UAA0B,CAAC;AAEjC,eAAW,SAAS,QAAQ;AAC1B,YAAM,SAAS,IAAI,OAAa;AAChC,cAAQ,KAAK,MAAM;AAEnB,UAAI,MAAM,SAAS,4BAA4B;AAC7C,aAAK,kBAAkB,MAAM,KAAK,EAAE,IAAI;AAAA,MAC1C,WAAW,MAAM,QAAQ,4BAA4B;AACnD,aAAK,kBAAkB,MAAM,OAAO,IAAI;AAAA,MAC1C;AAEA,WAAK,UAAU,KAAK;AAAA,IACtB;AAEA,QAAI,QAAQ,WAAW,GAAG;AACxB,aAAO;AACP;AAAA,IACF;AAEA,QAAI;AAEF,YAAM,QAAQ,KAAK;AAAA,QACjB,QAAQ,IAAI,OAAO;AAAA,QACnB,MAAM,GAAI,EAAE,KAAK,MAAM;AACrB,gBAAM,IAAI,MAAM,kCAAkC;AAAA,QACpD,CAAC;AAAA,MACH,CAAC;AAAA,IACH,SAAS,GAAG;AACV,WAAK,QAAQ,MAAO,EAAY,OAAO;AACvC,YAAM;AAAA,IACR,UAAE;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEQ,0BACN,SACA,eAAwB,OAC2D;AACnF,UAAM,aAAa,QAAQ,KAAK;AAChC,QAAI,cAAc;AAChB,iBAAW,MAAM,KAAK,oBAAoB,CAAC;AAAA,IAC7C,OAAO;AAEL,iBAAW,QAAQ,WAAW,MAAM;AAAA,QAClC,CAAC,SAAS,CAAC,KAAK,GAAG,WAAW,oBAAoB;AAAA,MACpD;AAAA,IACF;AAEA,UAAM,SAGA,CAAC;AAEP,UAAM,UAAU,IAAI,mBAAmB,KAAK,SAAS,UAAU;AAC/D,eAAW,MAAM,QAAQ,UAAU;AACjC,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,SAAS;AAAA,QACT,UAAU,UAAU,kBAAkB;AAAA,MACxC,CAA0C;AAAA,IAC5C;AAEA,eAAW,CAAC,YAAY,EAAE,KAAK,QAAQ,UAAU;AAC/C,YAAM,WAAW,WAAW,QAAQ,EAAE;AACtC,UAAI,CAAC,UAAU;AACb,cAAM,IAAI,MAAM,aAAa,EAAE,YAAY;AAAA,MAC7C;AACA,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,MAAM,wBAAwB,QAAQ;AAAA,QACtC,kBAAkB,cAAc;AAAA,QAChC,UAAU,UAAU,kBAAkB;AAAA,MACxC,CAA0C;AAAA,IAC5C;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,YAAY,QAAwC;AACxD,UAAM,SAAS,MAAM,KAAK,kBAAkB,KAAK;AACjD,UAAM,KAAK,KAAK,uBAAuB,MAAM;AAC7C,SAAK,UAAU,EAAE;AAEjB,QAAI,CAAC,GAAG,QAAQ,OAAO;AACrB,YAAM,IAAI,MAAM,+CAA+C;AAAA,IACjE;AAIA,UAAM,oBAAoB,IAAI,IAAI,GAAG,QAAQ,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC;AAC3E,UAAM,gBAAgB,OAAO;AAAA,MAC3B,OAAO,QAAQ,MAAM,EAAE;AAAA,QACrB,CAAC,CAAC,MAAM,IAAI,MAAM,IAAI,eAAe,IAAI,KAAK,kBAAkB,IAAI,IAAI;AAAA,MAC1E;AAAA,IACF;AAEA,SAAK,SAAS;AAEd,WAAO;AAAA,EACT;AAAA,EAEQ,uBAAuB,QAAuD;AACpF,UAAM,WAA6B,CAAC;AAEpC,eAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,MAAM,GAAG;AACjD,UAAI,CAAC,IAAI,eAAe,IAAI,GAAG;AAC7B,aAAK,QAAQ,MAAM,EAAE,MAAM,KAAK,GAAG,oDAAoD;AACvF;AAAA,MACF;AAEA,YAAM,EAAE,YAAY,gBAAgB,YAAY,IAAI;AACpD,UAAI;AACF,cAAM,aAAa,IAAI;AAAA,UACrB;AAAA,QACF;AAEA,iBAAS,KAAK;AAAA,UACZ;AAAA,UACA;AAAA,UACA;AAAA,UACA,MAAM;AAAA,QACR,CAAC;AAAA,MACH,SAAS,GAAG;AACV,aAAK,QAAQ,MAAM,EAAE,MAAM,KAAK,GAAG,oDAAoD;AACvF;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,MAAM;AAAA,QACN,OAAO,KAAK,iBAAiB,SAAS;AAAA,QACtC,OAAO;AAAA,MACT;AAAA,MACA,UAAU,UAAU,eAAe;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,eAAsC;AAC7D,UAAM,UAAU,UAAU,sBAAsB;AAChD,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,QACP,MAAM;AAAA,QACN,cAAc;AAAA,MAChB;AAAA,MACA,UAAU;AAAA,IACZ,CAAiC;AACjC,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,cAAc,EAAE,WAAW,GAA0C;AACnE,UAAM,UAAmD;AAAA,MACvD,MAAM;AAAA,IACR;AAEA,SAAK,iBAAiB,SAAS,aAAa;AAC5C,YAAQ,cAAc,gBAAgB,UAAU;AAIhD,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,MACT,UAAU,UAAU,iBAAiB;AAAA,IACvC,CAAC;AAAA,EACH;AAAA,EAEA,UAAU,OAAyB;AACjC,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,aAAK,UAAU;AAAA,UACb,MAAM;AAAA,UACN,OAAO,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,QACtD,CAA0C;AAE1C,aAAK,oBAAqB,GAAG,oBAAoB,GAAG,aAAc;AAAA,MACpE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAA6B;AACjC,QAAI,KAAK,mBAAmB,KAAK;AAE/B,WAAK,UAAU;AAAA,QACb,MAAM;AAAA,MACR,CAA0C;AAC1C,WAAK,mBAAmB;AAAA,IAC1B;AAAA,EACF;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,IACR,CAAyC;AACzC,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,UAAM,SAAS,KAAK,eAAe,EAAE,cAAc,eAAe,KAAK,CAAC;AACxE,SAAK,0BAA0B;AAC/B,WAAO,OAAO,QAAQ;AAAA,EACxB;AAAA,EAEA,MAAM,YAA2B;AAC/B,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,IACR,CAAkC;AAAA,EACpC;AAAA,EAEA,MAAM,SAAS,UAKG;AAGhB,UAAM,mBAAmB,CAAC,SAAS,cAAc,SAAS,WAAW,SAAS,OAAO;AACrF,UAAM,qBAAqB,KAAK,oBAAoB,IAAI,SAAS,SAAS;AAE1E,QAAI,oBAAoB,oBAAoB;AAC1C,WAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,eAAe;AAAA,QACf,SAAS,SAAS;AAAA,QAClB,cAAc,SAAS;AAAA,MACzB,CAA4C;AAAA,IAC9C,WAAW,SAAS,oBAAoB,QAAW;AAEjD,YAAM,UAAU,KAAK,QAAQ,KAAK;AAClC,YAAM,MAAM,QAAQ,UAAU,SAAS,SAAS;AAChD,UAAI,QAAQ,QAAW;AACrB,cAAM,OAAO,QAAQ,MAAM,GAAG;AAC9B,YAAI,QAAQ,KAAK,SAAS,WAAW;AACnC,gBAAM,UAAU,IAAI,YAAY,OAAO;AAAA,YACrC,GAAG;AAAA,YACH,SAAS,CAAC,SAAS,eAAe;AAAA,UACpC,CAAC;AACD,kBAAQ,MAAM,GAAG,IAAI;AACrB,gBAAM,SAAS,KAAK,0BAA0B,OAAO;AACrD,qBAAW,MAAM,QAAQ;AACvB,iBAAK,UAAU,EAAE;AAAA,UACnB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,cACN,OACyB;AACzB,UAAM,eAAwC,CAAC;AAC/C,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAI,UAAU,QAAW;AACvB,qBAAa,GAAG,IAAI;AAAA,MACtB;AAAA,IACF;AAEA,QAAI,aAAa,SAAS,OAAO,aAAa,UAAU,UAAU;AAChE,aAAO,EAAE,GAAG,cAAc,OAAO,MAAM;AAAA,IACzC;AACA,QACE,aAAa,SACb,OAAO,aAAa,UAAU,YAC9B,MAAM,SAAS,wBACf;AACA,aAAO,EAAE,GAAG,cAAc,OAAO,MAAM;AAAA,IACzC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,eAAmC;AAC/C,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,IAChB;AAEA,QAAI,KAAK,iBAAiB,SAAS,SAAS;AAI1C,UAAI,KAAK,iBAAiB,SAAS,YAAY;AAC7C,gBAAQ,gBAAgB,UAAU,KAAK,iBAAiB,SAAS,UAAU;AAAA,MAC7E,WAAW,KAAK,iBAAiB,SAAS,QAAQ;AAChD,gBAAQ,SAAS,IAAI,KAAK,iBAAiB,SAAS;AAAA,MACtD,OAAO;AACL,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAAA,IACF,OAAO;AACL,UAAI,CAAC,KAAK,iBAAiB,SAAS,QAAQ;AAC1C,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,cAAQ,gBAAgB,UAAU,KAAK,iBAAiB,SAAS,MAAM;AAAA,IACzE;AAEA,UAAM,MAAM,eAAe;AAAA,MACzB,SAAS,KAAK,iBAAiB,SAAS;AAAA,MACxC,OAAO,KAAK,iBAAiB,SAAS;AAAA,MACtC,SAAS,KAAK,iBAAiB,SAAS;AAAA,MACxC,YAAY,KAAK,iBAAiB,SAAS;AAAA,MAC3C,iBAAiB,KAAK,iBAAiB,SAAS;AAAA,IAClD,CAAC;AAED,QAAI,YAAY;AACd,WAAK,QAAQ,MAAM,wCAAwC,GAAG,EAAE;AAAA,IAClE;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,KAAK,IAAI,UAAU,KAAK,EAAE,QAAQ,CAAC;AACzC,UAAI,UAAU;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,WAAG,MAAM;AACT,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,iBAAiB,SAAS,YAAY,SAAS;AAEvD,SAAG,KAAK,QAAQ,MAAM;AACpB,YAAI,CAAC,QAAS;AACd,kBAAU;AACV,qBAAa,OAAO;AACpB,gBAAQ,EAAE;AAAA,MACZ,CAAC;AAED,SAAG,KAAK,SAAS,MAAM;AACrB,YAAI,CAAC,QAAS;AACd,kBAAU;AACV,qBAAa,OAAO;AACpB,eAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,MAC3D,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,QAAoC;AAClD,QAAI,eAAe;AACnB,QAAI,aAAa;AACjB,QAAI,SAA2B;AAC/B,UAAM,aAAa,KAAK,iBAAiB,SAAS,YAAY;AAE9D,UAAM,YAAY,YAAY;AAC5B,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,oBAAoB,KAAK,iBAAiB,SAAS;AAAA,QACrD;AAAA,QACA;AAAA,MACF;AAGA,iBAAW,OAAO,OAAO,OAAO,KAAK,iBAAiB,GAAG;AACvD,YAAI,CAAC,IAAI,KAAM,KAAI,OAAO,IAAI,MAAM,qBAAqB,CAAC;AAAA,MAC5D;AACA,WAAK,oBAAoB,CAAC;AAE1B,iBAAW,OAAO,OAAO,OAAO,KAAK,iBAAiB,GAAG;AACvD,YAAI,CAAC,IAAI,KAAM,KAAI,OAAO,IAAI,MAAM,qBAAqB,CAAC;AAAA,MAC5D;AACA,WAAK,oBAAoB,CAAC;AAG1B,WAAK,oBAAoB,MAAM;AAE/B,YAAM,SAAkC,CAAC;AAGzC,aAAO,KAAK,KAAK,yBAAyB,CAAC;AAG3C,UAAI,OAAO,KAAK,KAAK,MAAM,EAAE,SAAS,GAAG;AACvC,eAAO,KAAK,KAAK,uBAAuB,KAAK,MAAM,CAAC;AAAA,MACtD;AAGA,YAAM,UAAU,KAAK,QAAQ,KAAK;AAAA,QAChC,qBAAqB;AAAA,QACrB,qBAAqB;AAAA,QACrB,qBAAqB;AAAA,MACvB,CAAC;AAED,YAAM,aAAa,KAAK;AACxB,WAAK,gBAAgB,IAAI,IAAI,kBAAkB;AAC/C,aAAO,KAAK,GAAG,KAAK,0BAA0B,OAAO,CAAC;AAEtD,UAAI;AACF,mBAAW,MAAM,QAAQ;AACvB,eAAK,KAAK,8BAA8B,EAAE;AAC1C,iBAAQ,KAAK,KAAK,UAAU,EAAE,CAAC;AAAA,QACjC;AAAA,MACF,SAAS,OAAO;AACd,aAAK,gBAAgB;AACrB,cAAM,IAAI,mBAAmB;AAAA,UAC3B,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAEA,WAAK,QAAQ,MAAM,oCAAoC;AAEvD,WAAK,KAAK,uBAAuB,CAAC,CAAwC;AAAA,IAC5E;AAEA,mBAAe;AACf,WAAO,CAAC,KAAK,WAAW,CAAC,OAAO,SAAS;AACvC,WAAK,QAAQ,MAAM,sDAAsD;AACzE,eAAS,MAAM,KAAK,aAAa;AACjC,UAAI,OAAO,QAAS;AAEpB,UAAI;AACF,YAAI,cAAc;AAChB,gBAAM,UAAU;AAChB,cAAI,OAAO,QAAS;AACpB,uBAAa;AAAA,QACf;AAEA,cAAM,KAAK,MAAM,MAAM;AACvB,YAAI,OAAO,QAAS;AAAA,MACtB,SAAS,OAAO;AACd,YAAI,CAAC,WAAW,KAAK,GAAG;AACtB,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAEA,YAAI,eAAe,KAAK,CAAC,MAAM,WAAW;AACxC,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAEA,YAAI,eAAe,YAAY;AAC7B,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS,+CAA+C,UAAU;AAAA,YAClE,SAAS;AAAA,cACP,MAAM;AAAA,cACN,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAEA,aAAK,UAAU,EAAE,OAAuB,aAAa,KAAK,CAAC;AAC3D,cAAM,gBACJ,eAAe,IACX,kCACA,KAAK,iBAAiB,SAAS,YAAY;AACjD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS;AAAA,YACT;AAAA,YACA;AAAA,UACF;AAAA,UACA,sDAAsD,gBAAgB,GAAI;AAAA,QAC5E;AAEA,cAAM,MAAM,aAAa;AACzB;AAAA,MACF;AAEA,qBAAe;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,MAAc,MAAM,QAAkC;AACpD,UAAM,gBAAgB,OAAO,WAAuC;AAClE,YAAM,cAAc,IAAI,OAAa;AACrC,aAAO,iBAAiB,SAAS,MAAM,YAAY,QAAQ,CAAC;AAE5D,aAAO,CAAC,KAAK,WAAW,OAAO,eAAe,UAAU,QAAQ,CAAC,OAAO,SAAS;AAC/E,YAAI;AACF,gBAAM,QAAQ,MAAM,QAAQ,KAAK,CAAC,KAAK,eAAe,IAAI,GAAG,YAAY,KAAK,CAAC;AAC/E,cAAI,OAAO,WAAW,YAAY,QAAQ,UAAU,QAAW;AAC7D;AAAA,UACF;AAEA,cAAI,YAAY;AACd,iBAAK,QAAQ,MAAM,KAAK,cAAc,KAAK,GAAG,eAAe,MAAM,IAAI,EAAE;AAAA,UAC3E;AAEA,eAAK,KAAK,8BAA8B,KAAK;AAC7C,iBAAO,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,QACnC,SAAS,OAAO;AACd;AAAA,QACF;AAAA,MACF;AAEA,aAAO,MAAM;AAAA,IACf;AAEA,UAAM,gBAAgB,IAAI,OAAqB;AAE/C,WAAO,UAAU,CAAC,UAAU;AAC1B,oBAAc,QAAQ,IAAI,mBAAmB,EAAE,SAAS,MAAM,QAAQ,CAAC,CAAC;AAAA,IAC1E;AACA,WAAO,UAAU,MAAM;AACrB,oBAAc,QAAQ;AAAA,IACxB;AAEA,WAAO,YAAY,CAAC,YAA0B;AAE5C,YAAM,QAAa,KAAK,MAAM,QAAQ,IAAc;AAEpD,WAAK,KAAK,gCAAgC,KAAK;AAC/C,UAAI,YAAY;AACd,aAAK,QAAQ,MAAM,KAAK,cAAc,KAAK,GAAG,eAAe,MAAM,IAAI,EAAE;AAAA,MAC3E;AAEA,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH,eAAK,oCAAoC,KAAK;AAC9C;AAAA,QACF,KAAK;AACH,eAAK,oCAAoC,KAAK;AAC9C;AAAA,QACF,KAAK;AACH,eAAK,sBAAsB,KAAK;AAChC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,uDAAuD,KAAK;AACjE;AAAA,QACF,KAAK;AACH,eAAK,oDAAoD,KAAK;AAC9D;AAAA,QACF,KAAK;AACH,eAAK,+BAA+B,KAAK;AACzC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,wBAAwB,KAAK;AAClC;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,uBAAuB,KAAK;AACjC;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,mCAAmC,KAAK;AAC7C;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,yBAAyB,KAAK;AACnC;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,kCAAkC,KAAK;AAC5C;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,wBAAwB,KAAK;AAClC;AAAA,QACF,KAAK;AACH,eAAK,6BAA6B,KAAK;AACvC;AAAA,QACF,KAAK;AACH,eAAK,mBAAmB,KAAK;AAC7B;AAAA,QACF,KAAK;AACH,eAAK,YAAY,KAAK;AACtB;AAAA,QACF;AACE,cAAI,YAAY;AACd,iBAAK,QAAQ,MAAM,oBAAoB,MAAM,IAAI,EAAE;AAAA,UACrD;AACA;AAAA,MACJ;AAAA,IACF;AAEA,UAAM,WAAW,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,cAAc,MAAM,CAAC;AAEhE,UAAM,SAAS,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AACvC,YAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,eAAO,iBAAiB,SAAS,MAAM;AACrC,kBAAQ;AAAA,QACV,CAAC;AAAA,MACH,CAAC;AAED,aAAO,QAAQ,KAAK,CAAC,cAAc,OAAO,YAAY,CAAC;AAAA,IACzD,CAAC;AAED,UAAM,oBAAoB,KAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AACxD,YAAM,MAAM,KAAK,iBAAiB,SAAS,oBAAoB,EAAE,OAAO,CAAC;AACzE,aAAO,IAAI,mBAAmB;AAAA,QAC5B,SAAS;AAAA,MACX,CAAC;AAAA,IACH,CAAC;AAED,QAAI;AACF,YAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,OAAO,QAAQ,SAAS,QAAQ,kBAAkB,MAAM,CAAC;AAE5F,UAAI,kBAAkB,QAAQ,KAAK,mBAAmB;AACpD,cAAM,KAAK,kBAAkB,SAAS;AAAA,MACxC;AAEA,UAAI,kBAAkB,OAAO;AAC3B,cAAM;AAAA,MACR;AAAA,IACF,UAAE;AACA,YAAM,cAAc,CAAC,QAAQ,UAAU,iBAAiB,GAAG,GAAI;AAC/D,aAAO,MAAM;AAAA,IACf;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,MAAM;AACZ,SAAK,UAAU;AACf,UAAM,KAAK;AAGX,eAAW,UAAU,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC/D,UAAI,CAAC,OAAO,QAAQ,MAAM;AACxB,eAAO,QAAQ,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,MACnD;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,eAAW,OAAO,OAAO,OAAO,KAAK,iBAAiB,GAAG;AACvD,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,MACxC;AAAA,IACF;AACA,SAAK,oBAAoB,CAAC;AAE1B,eAAW,OAAO,OAAO,OAAO,KAAK,iBAAiB,GAAG;AACvD,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,MACxC;AAAA,IACF;AACA,SAAK,oBAAoB,CAAC;AAG1B,QAAI,KAAK,mBAAmB;AAC1B,iBAAW,OAAO,KAAK,kBAAkB,SAAS,OAAO,GAAG;AAC1D,YAAI,YAAY,MAAM;AACtB,YAAI,aAAa,MAAM;AACvB,YAAI,CAAC,IAAI,WAAW,MAAM;AACxB,cAAI,WAAW,QAAQ,KAAK,iBAAiB,SAAS,UAAU;AAAA,QAClE;AAAA,MACF;AACA,WAAK,kBAAkB,SAAS,MAAM;AACtC,WAAK,kBAAkB,eAAe,MAAM;AAC5C,WAAK,kBAAkB,gBAAgB,MAAM;AAC7C,UAAI,CAAC,KAAK,kBAAkB,SAAS,MAAM;AACzC,aAAK,kBAAkB,SAAS,QAAQ;AAAA,MAC1C;AACA,WAAK,oBAAoB;AAAA,IAC3B;AAGA,SAAK,eAAe,MAAM,SAAS;AAAA,EACrC;AAAA,EAEQ,oCACN,QACM;AACN,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,oCACN,QACM;AACN,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B,KAAK,iBAAiB,SAAS,4BAA4B;AAAA,IACvF,CAAgC;AAAA,EAClC;AAAA,EAEQ,sBAAsB,OAA6C;AAjnC7E;AAknCI,QAAI,CAAC,MAAM,SAAS,IAAI;AACtB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AAEA,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,OAAO,oBAA2C;AAAA,MAClE,iBAAiB,OAAO,oBAAsC;AAAA,MAC9D,UAAU,oBAAI,IAAI;AAAA,MAClB,UAAU,IAAI,OAAO;AAAA,MACrB,mBAAmB,KAAK,IAAI;AAAA,IAC9B;AAIA,UAAM,eAAe;AAAA,MACnB,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,MACf,YAAY,MAAM,SAAS;AAAA,IAC7B;AAEA,UAAM,iBAAgB,WAAM,SAAS,aAAf,mBAAyB;AAC/C,QAAI,eAAe;AACjB,YAAM,SAAS,KAAK,uBAAuB,aAAa;AACxD,UAAI,QAAQ;AACV,eAAO,KAAK,uBAAuB,aAAa;AAChD,qBAAa,gBAAgB;AAC7B,YAAI,CAAC,OAAO,QAAQ,MAAM;AACxB,iBAAO,QAAQ,QAAQ,YAAY;AAAA,QACrC;AAAA,MACF;AAAA,IACF;AAEA,SAAK,KAAK,sBAAsB,YAAY;AAAA,EAC9C;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,QAAI,CAAC,MAAM,KAAK,MAAM;AACpB,YAAM,IAAI,MAAM,sBAAsB;AAAA,IACxC;AAEA,UAAM,WAAW,MAAM,KAAK;AAE5B,QAAI,aAAa,WAAW;AAG1B,WAAK,0BAA0B;AAC/B;AAAA,IACF;AAEA,UAAM,SAAS,MAAM,KAAK;AAC1B,QAAI,CAAC,QAAQ;AACX,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,gBAAgB,IAAI,OAAmB;AAC7C,UAAM,iBAAoC;AAAA,MACxC,WAAW;AAAA,MACX,aAAa,OAAO,oBAA4B;AAAA,MAChD,cAAc,OAAO,oBAAgC;AAAA,MACrD,iBAAiB;AAAA,MACjB,YAAY;AAAA,IACd;AAGA,QAAI,CAAC,KAAK,iBAAiB,aAAa,aAAa;AACnD,qBAAe,aAAa,MAAM;AAClC,oBAAc,QAAQ,CAAC,MAAM,CAAC;AAAA,IAChC;AAEA,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,eAAe,YAAY,OAAO;AAAA,MAC9C,aAAa,eAAe,aAAa,OAAO;AAAA,MAChD,YAAY,cAAc;AAAA,IAC5B,CAAC;AAED,SAAK,kBAAkB,SAAS,IAAI,QAAQ,cAAc;AAAA,EAC5D;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,KAAK,IAAI;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,QAAI;AACF,WAAK,cAAc,OAAO,MAAM,kBAAkB,wBAAwB,MAAM,IAAI,CAAC;AAAA,IACvF,SAAS,OAAO;AACd,WAAK,QAAQ,MAAM,EAAE,OAAO,QAAQ,MAAM,KAAK,GAAG,GAAG,oCAAoC;AAAA,IAC3F;AAEA,UAAM,MAAM,KAAK,kBAAkB,MAAM,KAAK,EAAE;AAChD,QAAI,KAAK;AACP,UAAI,QAAQ;AACZ,aAAO,KAAK,kBAAkB,MAAM,KAAK,EAAE;AAAA,IAC7C;AAAA,EACF;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAGA,SAAK,oBAAoB,OAAO,MAAM,OAAO;AAE7C,QAAI;AACF,WAAK,cAAc,OAAO,MAAM,OAAO;AAAA,IACzC,SAAS,OAAO;AACd,WAAK,QAAQ,MAAM,EAAE,OAAO,QAAQ,MAAM,QAAQ,GAAG,oCAAoC;AAAA,IAC3F;AAEA,UAAM,MAAM,KAAK,kBAAkB,MAAM,OAAO;AAChD,QAAI,KAAK;AACP,UAAI,QAAQ;AACZ,aAAO,KAAK,kBAAkB,MAAM,OAAO;AAAA,IAC7C;AAAA,EACF;AAAA,EAEQ,uDACN,OACM;AACN,UAAM,aAAa,KAAK,cAAc,IAAI,MAAM,OAAO;AACvD,QAAI,CAAC,YAAY;AACf;AAAA,IACF;AAEA,UAAM,OAAO,WAAW;AACxB,QAAI,gBAAgB,IAAI,aAAa;AACnC,WAAK,QAAQ,KAAK,MAAM,UAAU;AAAA,IACpC,OAAO;AACL,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,SAAK,KAAK,uCAAuC;AAAA,MAC/C,QAAQ,MAAM;AAAA,MACd,YAAY,MAAM;AAAA,MAClB,SAAS;AAAA,IACX,CAAoC;AAAA,EACtC;AAAA,EAEQ,oDACN,OACM;AACN,SAAK,QAAQ;AAAA,MACX,EAAE,OAAO,MAAM,MAAM;AAAA,MACrB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,+BAA+B,OAAsD;AAC3F,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM;AACrB,UAAM,WAAW,MAAM,KAAK;AAE5B,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,QAAI,CAAC,gBAAgB;AACnB,WAAK,QAAQ,KAAK,uCAAuC,MAAM,EAAE;AACjE;AAAA,IACF;AAEA,UAAM,aAAa,aAAa,UAAU,aAAa;AACvD,QAAI,cAAc,KAAK,iBAAiB,aAAa,aAAa;AAChE,WAAK,QAAQ,KAAK,oEAAoE;AAAA,IACxF;AAEA,QAAI,CAAC,eAAe,WAAW,MAAM;AACnC,YAAM,iBAA6B,aAAa,CAAC,MAAM,IAAI,CAAC,SAAS,MAAM;AAC3E,qBAAe,WAAW,QAAQ,cAAc;AAGhD,UAAI,CAAC,YAAY;AACf,aAAK,oBAAoB,IAAI,MAAM;AAAA,MACrC;AAAA,IACF;AAEA,QAAI,KAAK,kBAAkB,yBAAyB,QAAW;AAC7D,WAAK,kBAAkB,uBAAuB,KAAK,IAAI;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,MAAM;AACf;AAAA,IACF;AACA,QAAI,MAAM,KAAK,SAAS,QAAQ;AAC9B;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EAGF;AAAA,EAEQ,wBAAwB,OAA+C;AAC7E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM,OAAO;AACxE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AAEA,QACE,CAAC,KAAK,iBAAiB,aAAa,eACpC,CAAC,KAAK,kBAAkB,sBACxB;AACA,WAAK,kBAAkB,uBAAuB,KAAK,IAAI;AAAA,IACzD;AAEA,mBAAe,YAAY,MAAM,MAAM,KAAK;AAC5C,mBAAe,mBAAmB,MAAM;AAAA,EAC1C;AAAA,EAEQ,uBAAuB,QAA+C;AAC5E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,mCACN,OACM;AACN,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM;AACrB,UAAM,QAAQ,MAAM;AAIpB,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C,OAAO;AACL,qBAAe,YAAY,MAAM,KAAK;AACtC,qBAAe,mBAAmB;AAAA,IACpC;AAAA,EACF;AAAA,EAEQ,yBAAyB,OAAgD;AAC/E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM,OAAO;AACxE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AAEA,QAAI,KAAK,kBAAkB,yBAAyB,QAAW;AAC7D,WAAK,kBAAkB,uBAAuB,KAAK,IAAI;AAAA,IACzD;AAEA,QAAI,CAAC,eAAe,WAAW,MAAM;AACnC,qBAAe,WAAW,QAAQ,CAAC,SAAS,MAAM,CAAC;AAAA,IACrD;AAEA,UAAM,eAAe,KAAK,MAAM,KAAK;AACrC,UAAM,MAAM,aAAa;AACzB,UAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,aAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,YAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,IACtC;AAEA,mBAAe,aAAa;AAAA,MAC1B,IAAI;AAAA,QACF,IAAI,WAAW,MAAM,MAAM;AAAA,QAC3B,UAAU;AAAA,QACV,UAAU;AAAA,QACV,MAAM,SAAS;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,kCACN,QACM;AACN,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,wBAAwB,QAAgD;AAC9E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,6BAA6B,OAAoD;AACvF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM,KAAK;AAC1B,UAAM,WAAW,MAAM,KAAK;AAE5B,QAAI,aAAa,iBAAiB;AAChC,YAAM,OAAO,MAAM;AACnB,UAAI,CAAC,KAAK,WAAW,CAAC,KAAK,QAAQ,CAAC,KAAK,WAAW;AAClD,cAAM,IAAI,MAAM,6BAA6B;AAAA,MAC/C;AACA,WAAK,kBAAkB,gBAAgB;AAAA,QACrC,IAAI,aAAa,OAAO;AAAA,UACtB,QAAQ,KAAK;AAAA,UACb,MAAM,KAAK;AAAA,UACX,MAAM,KAAK;AAAA,QACb,CAAC;AAAA,MACH;AAAA,IACF,WAAW,aAAa,WAAW;AACjC,YAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,UAAI,CAAC,gBAAgB;AACnB;AAAA,MACF;AAEA,qBAAe,YAAY,MAAM;AACjC,qBAAe,aAAa,MAAM;AAClC,UAAI,CAAC,eAAe,WAAW,MAAM;AAEnC,uBAAe,WAAW,QAAQ,KAAK,iBAAiB,SAAS,UAAU;AAAA,MAC7E;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,mBAAmB,QAA2C;AAj8CxE;AAk8CI,QAAI,CAAC,KAAK,mBAAmB;AAG3B;AAAA,IACF;AAEA,UAAM,mBAAmB,KAAK,kBAAkB;AAChD,UAAM,sBAAsB,KAAK,kBAAkB;AAEnD,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,cAAc,KAAK,kBAAkB,SAAS;AAAA,MAChD;AAAA,MACA;AAAA,IACF;AAEA,eAAW,cAAc,KAAK,kBAAkB,SAAS,OAAO,GAAG;AACjE,iBAAW,YAAY,MAAM;AAC7B,iBAAW,aAAa,MAAM;AAC9B,UAAI,CAAC,WAAW,WAAW,MAAM;AAC/B,mBAAW,WAAW,QAAQ,KAAK,iBAAiB,SAAS,UAAU;AAAA,MACzE;AAAA,IACF;AAEA,SAAK,kBAAkB,gBAAgB,MAAM;AAC7C,SAAK,kBAAkB,eAAe,MAAM;AAE5C,eAAW,UAAU,KAAK,kBAAkB,SAAS,KAAK,GAAG;AAC3D,YAAM,aAAa,KAAK,cAAc,IAAI,MAAM;AAChD,UAAI,cAAc,WAAW,gBAAgB,IAAI,aAAa;AAC5D,mBAAW,KAAK,QAAQ,KAAK,KAAK,kBAAkB,SAAS,IAAI,MAAM,EAAG,eAAe;AAAA,MAC3F;AAAA,IACF;AAEA,SAAK,kBAAkB,SAAS,QAAQ;AACxC,SAAK,oBAAoB;AAGzB,UAAM,QAAQ,OAAO,SAAS;AAC9B,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,KAAK,IAAI,IAAI;AAEhC,UAAM,kBAAgD;AAAA,MACpD,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,OAAO,SAAS,MAAM;AAAA,MACjC;AAAA,MACA;AAAA,MACA,WAAW,OAAO,SAAS,WAAW;AAAA,MACtC,OAAO;AAAA,MACP,cAAa,+BAAO,iBAAgB;AAAA,MACpC,eAAc,+BAAO,kBAAiB;AAAA,MACtC,cAAa,+BAAO,iBAAgB;AAAA,MACpC,iBAAiB,aAAa,MAAK,+BAAO,kBAAiB,MAAM,aAAa,OAAQ;AAAA,MACtF,mBAAmB;AAAA,QACjB,eAAa,oCAAO,wBAAP,mBAA4B,iBAAgB;AAAA,QACzD,cAAY,oCAAO,wBAAP,mBAA4B,gBAAe;AAAA,QACvD,aAAa;AAAA;AAAA,QACb,gBAAc,oCAAO,wBAAP,mBAA4B,kBAAiB;AAAA,QAC3D,uBAAqB,oCAAO,wBAAP,mBAA4B,yBAC7C;AAAA,UACE,eAAa,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,iBAAgB;AAAA,UAChF,cAAY,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,gBAAe;AAAA,UAC9E,eAAa,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,iBAAgB;AAAA,QAClF,IACA;AAAA,MACN;AAAA,MACA,oBAAoB;AAAA,QAClB,cAAY,oCAAO,yBAAP,mBAA6B,gBAAe;AAAA,QACxD,eAAa,oCAAO,yBAAP,mBAA6B,iBAAgB;AAAA,QAC1D,aAAa;AAAA,MACf;AAAA,IACF;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAEhD;AAAA,EAEQ,YAAY,OAAmC;AACrD,QAAI,MAAM,MAAM,QAAQ,WAAW,qBAAqB,GAAG;AACzD;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,OAAO,MAAM,MAAM,GAAG,uCAAuC;AAClF,SAAK,UAAU;AAAA,MACb,OAAO,IAAI,SAAS,MAAM,MAAM,SAAS;AAAA,QACvC,MAAM,MAAM;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AAAA,MACD,aAAa;AAAA,IACf,CAAC;AAAA,EAGH;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAAiD;AAEtF,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAA2B;AAAA,EAC7B;AAAA,EAEA,CAAS,cAAc,OAA0C;AAC/D,UAAM;AAAA,EACR;AAAA,EAEQ,eAAe;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIyB;AACvB,UAAM,SAAS,aAAa,IAAI,qBAAqB,EAAE,aAAa,CAAC;AACrE,QAAI,aAAa,cAAc;AAC7B,aAAO,eAAe;AAAA,IACxB;AAEA,UAAM,UAAU,UAAU,kBAAkB;AAC5C,QAAI,eAAe;AACjB,WAAK,uBAAuB,OAAO,IAAI;AAAA,IACzC;AAEA,UAAM,WAAsD,CAAC;AAC7D,QAAI,aAAc,UAAS,eAAe;AAC1C,QAAI,cAAe,UAAS,WAAW,EAAE,iBAAiB,QAAQ;AAElE,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,UAAU;AAAA,MACV,UAAU,OAAO,KAAK,QAAQ,EAAE,SAAS,IAAI,WAAW;AAAA,IAC1D,CAAC;AAED,WAAO;AAAA,EACT;AACF;AAGO,SAAS,wBAAwB,MAA4C;AAClF,UAAQ,KAAK,MAAM;AAAA,IACjB,KAAK;AACH,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN,SAAS,KAAK;AAAA,QACd,MAAM,KAAK;AAAA,QACX,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,KAAK;AACH,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN,SAAS,KAAK;AAAA,QACd,QAAQ,KAAK;AAAA,MACf;AAAA,IACF,KAAK;AACH,YAAM,OAAO,KAAK,SAAS,cAAc,WAAW,KAAK;AACzD,YAAM,cAAmC,CAAC;AAC1C,iBAAW,KAAK,KAAK,SAAS;AAC5B,YAAI,OAAO,MAAM,UAAU;AACzB,sBAAY,KAAK;AAAA,YACf,MAAM,SAAS,cAAc,gBAAgB;AAAA,YAC7C,MAAM;AAAA,UACR,CAA6D;AAAA,QAC/D,WAAW,EAAE,SAAS,iBAAiB;AAErC;AAAA,QACF,WAAW,EAAE,SAAS,iBAAiB;AACrC,cAAI,SAAS,QAAQ;AACnB,kBAAM,eAAe,OAAO,KAAK,mBAAmB,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,QAAQ;AACpF,wBAAY,KAAK;AAAA,cACf,MAAM;AAAA,cACN,OAAO;AAAA,YACT,CAAgC;AAAA,UAClC;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN;AAAA,QACA,SAAS;AAAA,MACX;AAAA,IACF;AACE,YAAM,IAAI,MAAM,0BAA2B,KAAa,IAAI,EAAE;AAAA,EAClE;AACF;AAEA,SAAS,wBAAwB,MAA4C;AAC3E,MAAI,CAAC,KAAK,IAAI;AACZ,UAAM,IAAI,MAAM,oBAAoB;AAAA,EACtC;AAEA,UAAQ,KAAK,MAAM;AAAA,IACjB,KAAK;AACH,aAAO,IAAI,aAAa,OAAO;AAAA,QAC7B,IAAI,KAAK;AAAA,QACT,QAAQ,KAAK;AAAA,QACb,MAAM,KAAK;AAAA,QACX,MAAM,KAAK;AAAA,MACb,CAAC;AAAA,IACH,KAAK;AACH,aAAO,IAAI,mBAAmB,OAAO;AAAA,QACnC,IAAI,KAAK;AAAA,QACT,QAAQ,KAAK;AAAA,QACb,QAAQ,KAAK;AAAA,QACb,SAAS;AAAA,MACX,CAAC;AAAA,IACH,KAAK;AACH,YAAM,UAA6B,CAAC;AAEpC,YAAM,WAAW,MAAM,QAAQ,KAAK,OAAO,IAAI,KAAK,UAAU,CAAC,KAAK,OAAO;AAC3E,iBAAW,KAAK,UAAU;AACxB,YAAI,EAAE,SAAS,UAAU,EAAE,SAAS,cAAc;AAChD,kBAAQ,KAAK,EAAE,IAAI;AAAA,QACrB;AAAA,MACF;AACA,aAAO,IAAI,YAAY,OAAO;AAAA,QAC5B,IAAI,KAAK;AAAA,QACT,MAAM,KAAK;AAAA,QACX;AAAA,MACF,CAAC;AAAA,EACL;AACF;AAEA,SAAS,oBAAoB,kBAA0B,GAAoB;AACzE,QAAM,YAAY,OAAO,MAAM,kBAAkB,WAAW;AAC5D,SAAO,IAAI,YAAY,OAAO;AAAA,IAC5B,IAAI,UAAU,oBAAoB;AAAA,IAClC,MAAM;AAAA,IACN,SAAS;AAAA,MACP;AAAA,QACE,MAAM;AAAA,QACN,OAAO;AAAA,UACL,IAAI;AAAA,YACF,IAAI,WAAW,UAAU,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,YACA,UAAU,SAAS;AAAA,UACrB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,SAAS,gBAAgB,YAAmD;AAC1E,MAAI,OAAO,eAAe,UAAU;AAClC,WAAO;AAAA,EACT;AAEA,OAAI,yCAAY,UAAS,YAAY;AACnC,WAAO,WAAW,SAAS;AAAA,EAC7B;AAEA,SAAO;AACT;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/realtime/realtime_model.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n type APIConnectOptions,\n APIConnectionError,\n APIError,\n AudioByteStream,\n DEFAULT_API_CONNECT_OPTIONS,\n Future,\n Queue,\n Task,\n type TimedString,\n cancelAndWait,\n createTimedString,\n delay,\n isAPIError,\n llm,\n log,\n type metrics,\n shortuuid,\n stream,\n} from '@livekit/agents';\nimport { Mutex } from '@livekit/mutex';\nimport type { AudioResampler } from '@livekit/rtc-node';\nimport { AudioFrame, combineAudioFrames } from '@livekit/rtc-node';\nimport { type MessageEvent, WebSocket } from 'ws';\nimport * as api_proto from './api_proto.js';\n\n// if LK_OPENAI_DEBUG convert it to a number, otherwise set it to 0\nconst lkOaiDebug = process.env.LK_OPENAI_DEBUG ? Number(process.env.LK_OPENAI_DEBUG) : 0;\n\nconst SAMPLE_RATE = 24000;\nconst NUM_CHANNELS = 1;\nconst BASE_URL = 'https://api.openai.com/v1';\n\nconst MOCK_AUDIO_ID_PREFIX = 'lk_mock_audio_item_';\n\ntype Modality = 'text' | 'audio';\n\ninterface RealtimeOptions {\n model: api_proto.Model;\n voice: api_proto.Voice;\n toolChoice?: llm.ToolChoice;\n inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n inputAudioNoiseReduction?: api_proto.NoiseReduction | null;\n turnDetection?: api_proto.TurnDetectionType | null;\n maxResponseOutputTokens?: number | 'inf';\n speed?: number;\n tracing?: api_proto.TracingConfig | null;\n apiKey?: string;\n baseURL: string;\n isAzure: boolean;\n azureDeployment?: string;\n entraToken?: string;\n apiVersion?: string;\n maxSessionDuration: number;\n // reset the connection after this many seconds if provided\n connOptions: APIConnectOptions;\n modalities: Modality[];\n}\n\ninterface MessageGeneration {\n messageId: string;\n textChannel: stream.StreamChannel<string | TimedString>;\n audioChannel: stream.StreamChannel<AudioFrame>;\n audioTranscript: string;\n modalities: Future<('text' | 'audio')[]>;\n}\n\ninterface ResponseGeneration {\n messageChannel: stream.StreamChannel<llm.MessageGeneration>;\n functionChannel: stream.StreamChannel<llm.FunctionCall>;\n messages: Map<string, MessageGeneration>;\n\n /** @internal */\n _doneFut: Future;\n /** @internal */\n _createdTimestamp: number;\n /** @internal */\n _firstTokenTimestamp?: number;\n}\n\nclass CreateResponseHandle {\n instructions?: string;\n doneFut: Future<llm.GenerationCreatedEvent>;\n // TODO(shubhra): add timeout\n constructor({ instructions }: { instructions?: string }) {\n this.instructions = instructions;\n this.doneFut = new Future();\n }\n}\n\nconst DEFAULT_FIRST_RETRY_INTERVAL_MS = 100;\nconst DEFAULT_TURN_DETECTION: api_proto.TurnDetectionType = {\n type: 'semantic_vad',\n eagerness: 'medium',\n create_response: true,\n interrupt_response: true,\n};\nconst DEFAULT_INPUT_AUDIO_TRANSCRIPTION: api_proto.InputAudioTranscription = {\n model: 'gpt-4o-mini-transcribe',\n};\nconst DEFAULT_TOOL_CHOICE: llm.ToolChoice = 'auto';\nconst DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS: number | 'inf' = 'inf';\n\nconst AZURE_DEFAULT_INPUT_AUDIO_TRANSCRIPTION: api_proto.InputAudioTranscription = {\n model: 'whisper-1',\n};\n\nconst AZURE_DEFAULT_TURN_DETECTION: api_proto.TurnDetectionType = {\n type: 'server_vad',\n threshold: 0.5,\n prefix_padding_ms: 300,\n silence_duration_ms: 200,\n create_response: true,\n};\n\nconst DEFAULT_MAX_SESSION_DURATION = 20 * 60 * 1000; // 20 minutes\n\nconst DEFAULT_REALTIME_MODEL_OPTIONS = {\n model: 'gpt-realtime',\n voice: 'marin',\n inputAudioTranscription: DEFAULT_INPUT_AUDIO_TRANSCRIPTION,\n inputAudioNoiseReduction: undefined as api_proto.NoiseReduction | undefined,\n turnDetection: DEFAULT_TURN_DETECTION,\n toolChoice: DEFAULT_TOOL_CHOICE,\n maxResponseOutputTokens: DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS,\n maxSessionDuration: DEFAULT_MAX_SESSION_DURATION,\n connOptions: DEFAULT_API_CONNECT_OPTIONS,\n modalities: ['text', 'audio'] as Modality[],\n tracing: undefined as api_proto.TracingConfig | undefined,\n};\nexport class RealtimeModel extends llm.RealtimeModel {\n sampleRate = api_proto.SAMPLE_RATE;\n numChannels = api_proto.NUM_CHANNELS;\n inFrameSize = api_proto.IN_FRAME_SIZE;\n outFrameSize = api_proto.OUT_FRAME_SIZE;\n\n /* @internal */\n _options: RealtimeOptions;\n\n get model(): string {\n return this._options.model;\n }\n\n constructor(\n options: {\n model?: string;\n voice?: string;\n /** @deprecated Unused in GA API (v1). Temperature is no longer supported. */\n temperature?: number;\n toolChoice?: llm.ToolChoice;\n baseURL?: string;\n modalities?: Modality[];\n inputAudioTranscription?: api_proto.InputAudioTranscription | null;\n inputAudioNoiseReduction?: api_proto.NoiseReduction | null;\n turnDetection?: api_proto.TurnDetectionType | null;\n speed?: number;\n tracing?: api_proto.TracingConfig | null;\n azureDeployment?: string;\n apiKey?: string;\n entraToken?: string;\n apiVersion?: string;\n maxSessionDuration?: number;\n connOptions?: APIConnectOptions;\n } = {},\n ) {\n const modalities = (options.modalities ||\n DEFAULT_REALTIME_MODEL_OPTIONS.modalities) as Modality[];\n\n super({\n messageTruncation: true,\n turnDetection: options.turnDetection !== null,\n userTranscription: options.inputAudioTranscription !== null,\n autoToolReplyGeneration: false,\n audioOutput: modalities.includes('audio'),\n });\n\n const isAzure = !!(options.apiVersion || options.entraToken || options.azureDeployment);\n\n if (options.apiKey === '' && !isAzure) {\n throw new Error(\n 'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environment variable',\n );\n }\n\n const apiKey = options.apiKey || process.env.OPENAI_API_KEY;\n\n if (!apiKey && !isAzure) {\n throw new Error(\n 'OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environment variable',\n );\n }\n\n if (!options.baseURL && isAzure) {\n const azureEndpoint = process.env.AZURE_OPENAI_ENDPOINT;\n if (!azureEndpoint) {\n throw new Error(\n 'Missing Azure endpoint. Please pass base_url or set AZURE_OPENAI_ENDPOINT environment variable.',\n );\n }\n options.baseURL = `${azureEndpoint.replace(/\\/$/, '')}/openai`;\n }\n\n const { modalities: _, ...optionsWithoutModalities } = options;\n this._options = {\n ...DEFAULT_REALTIME_MODEL_OPTIONS,\n ...optionsWithoutModalities,\n baseURL: options.baseURL || BASE_URL,\n apiKey,\n isAzure,\n model: options.model || DEFAULT_REALTIME_MODEL_OPTIONS.model,\n modalities,\n };\n }\n\n /**\n * Create a RealtimeModel instance configured for Azure OpenAI Service.\n *\n * @param azureDeployment - The name of your Azure OpenAI deployment.\n * @param azureEndpoint - The endpoint URL for your Azure OpenAI resource. If undefined, will attempt to read from the environment variable AZURE_OPENAI_ENDPOINT.\n * @param apiVersion - API version to use with Azure OpenAI Service. If undefined, will attempt to read from the environment variable OPENAI_API_VERSION.\n * @param apiKey - Azure OpenAI API key. If undefined, will attempt to read from the environment variable AZURE_OPENAI_API_KEY.\n * @param entraToken - Azure Entra authentication token. Required if not using API key authentication.\n * @param baseURL - Base URL for the API endpoint. If undefined, constructed from the azure_endpoint.\n * @param voice - Voice setting for audio outputs. Defaults to \"alloy\".\n * @param inputAudioTranscription - Options for transcribing input audio. Defaults to @see DEFAULT_INPUT_AUDIO_TRANSCRIPTION.\n * @param inputAudioNoiseReduction - Options for noise reduction. Defaults to undefined.\n * @param turnDetection - Options for server-based voice activity detection (VAD). Defaults to @see DEFAULT_SERVER_VAD_OPTIONS.\n * @param speed - Speed of the audio output. Defaults to 1.0.\n * @param tracing - Tracing configuration. Defaults to undefined.\n *\n * @returns A RealtimeModel instance configured for Azure OpenAI Service.\n *\n * @throws Error if required Azure parameters are missing or invalid.\n */\n static withAzure({\n azureDeployment,\n azureEndpoint,\n apiVersion,\n apiKey,\n entraToken,\n baseURL,\n voice = 'alloy',\n temperature, // eslint-disable-line @typescript-eslint/no-unused-vars\n inputAudioTranscription = AZURE_DEFAULT_INPUT_AUDIO_TRANSCRIPTION,\n inputAudioNoiseReduction,\n turnDetection = AZURE_DEFAULT_TURN_DETECTION,\n speed,\n tracing,\n }: {\n azureDeployment: string;\n azureEndpoint?: string;\n apiVersion?: string;\n apiKey?: string;\n entraToken?: string;\n baseURL?: string;\n voice?: string;\n /** @deprecated Unused in GA API (v1). Temperature is no longer supported. */\n temperature?: number;\n inputAudioTranscription?: api_proto.InputAudioTranscription;\n inputAudioNoiseReduction?: api_proto.NoiseReduction;\n turnDetection?: api_proto.TurnDetectionType;\n speed?: number;\n tracing?: api_proto.TracingConfig;\n }) {\n apiKey = apiKey || process.env.AZURE_OPENAI_API_KEY;\n if (!apiKey && !entraToken) {\n throw new Error(\n 'Missing credentials. Please pass one of `apiKey`, `entraToken`, or the `AZURE_OPENAI_API_KEY` environment variable.',\n );\n }\n\n apiVersion = apiVersion || process.env.OPENAI_API_VERSION;\n if (!apiVersion) {\n throw new Error(\n 'Must provide either the `apiVersion` argument or the `OPENAI_API_VERSION` environment variable',\n );\n }\n\n if (!baseURL) {\n azureEndpoint = azureEndpoint || process.env.AZURE_OPENAI_ENDPOINT;\n if (!azureEndpoint) {\n throw new Error(\n 'Missing Azure endpoint. Please pass the `azure_endpoint` parameter or set the `AZURE_OPENAI_ENDPOINT` environment variable.',\n );\n }\n baseURL = `${azureEndpoint.replace(/\\/$/, '')}/openai`;\n }\n\n return new RealtimeModel({\n voice,\n inputAudioTranscription,\n inputAudioNoiseReduction,\n turnDetection,\n speed,\n tracing,\n apiKey,\n azureDeployment,\n apiVersion,\n entraToken,\n baseURL,\n });\n }\n\n session() {\n return new RealtimeSession(this);\n }\n\n async close() {\n return;\n }\n}\n\nfunction processBaseURL({\n baseURL,\n model,\n isAzure = false,\n azureDeployment,\n apiVersion,\n}: {\n baseURL: string;\n model: string;\n isAzure: boolean;\n azureDeployment?: string;\n apiVersion?: string;\n}): string {\n const url = new URL([baseURL, 'realtime'].join('/'));\n\n if (url.protocol === 'https:') {\n url.protocol = 'wss:';\n }\n\n // ensure \"/realtime\" is added if the path is empty OR \"/v1\"\n if (!url.pathname || ['', '/v1', '/openai'].includes(url.pathname.replace(/\\/$/, ''))) {\n url.pathname = url.pathname.replace(/\\/$/, '') + '/realtime';\n } else {\n url.pathname = url.pathname.replace(/\\/$/, '');\n }\n\n const queryParams: Record<string, string> = {};\n if (isAzure) {\n if (apiVersion) {\n queryParams['api-version'] = apiVersion;\n }\n if (azureDeployment) {\n queryParams['deployment'] = azureDeployment;\n }\n } else {\n queryParams['model'] = model;\n }\n\n for (const [key, value] of Object.entries(queryParams)) {\n url.searchParams.set(key, value);\n }\n\n return url.toString();\n}\n\n/**\n * A session for the OpenAI Realtime API.\n *\n * This class is used to interact with the OpenAI Realtime API.\n * It is responsible for sending events to the OpenAI Realtime API and receiving events from it.\n *\n * It exposes two more events:\n * - openai_server_event_received: expose the raw server events from the OpenAI Realtime API\n * - openai_client_event_queued: expose the raw client events sent to the OpenAI Realtime API\n */\nexport class RealtimeSession extends llm.RealtimeSession {\n private _tools: llm.ToolContext = {};\n private remoteChatCtx: llm.RemoteChatContext = new llm.RemoteChatContext();\n private messageChannel = new Queue<api_proto.ClientEvent>();\n private inputResampler?: AudioResampler;\n private instructions?: string;\n private oaiRealtimeModel: RealtimeModel;\n private currentGeneration?: ResponseGeneration;\n private responseCreatedFutures: { [id: string]: CreateResponseHandle } = {};\n\n private textModeRecoveryRetries: number = 0;\n\n private itemCreateFutures: { [id: string]: Future } = {};\n private itemDeleteFutures: { [id: string]: Future } = {};\n\n // Track items that have real server-side audio (created in current session, not restored)\n // Items restored after reconnection are text-only and cannot be truncated\n private audioCapableItemIds: Set<string> = new Set();\n\n private updateChatCtxLock = new Mutex();\n private updateFuncCtxLock = new Mutex();\n\n // 100ms chunks\n private bstream = new AudioByteStream(SAMPLE_RATE, NUM_CHANNELS, SAMPLE_RATE / 10);\n\n private pushedDurationMs: number = 0;\n\n #logger = log();\n #task: Task<void>;\n #closed = false;\n\n constructor(realtimeModel: RealtimeModel) {\n super(realtimeModel);\n\n this.oaiRealtimeModel = realtimeModel;\n\n this.#task = Task.from(({ signal }) => this.#mainTask(signal));\n\n this.sendEvent(this.createSessionUpdateEvent());\n }\n\n sendEvent(command: api_proto.ClientEvent): void {\n this.messageChannel.put(command);\n }\n\n private createSessionUpdateEvent(): api_proto.SessionUpdateEvent {\n const audioFormat: api_proto.AudioFormat = { type: 'audio/pcm', rate: SAMPLE_RATE };\n\n const modality: Modality = this.oaiRealtimeModel._options.modalities.includes('audio')\n ? 'audio'\n : 'text';\n\n return {\n type: 'session.update',\n session: {\n type: 'realtime',\n model: this.oaiRealtimeModel._options.model,\n output_modalities: [modality],\n audio: {\n input: {\n format: audioFormat,\n noise_reduction: this.oaiRealtimeModel._options.inputAudioNoiseReduction,\n transcription: this.oaiRealtimeModel._options.inputAudioTranscription,\n turn_detection: this.oaiRealtimeModel._options.turnDetection,\n },\n output: {\n format: audioFormat,\n speed: this.oaiRealtimeModel._options.speed,\n voice: this.oaiRealtimeModel._options.voice,\n },\n },\n max_output_tokens:\n this.oaiRealtimeModel._options.maxResponseOutputTokens === Infinity\n ? 'inf'\n : this.oaiRealtimeModel._options.maxResponseOutputTokens,\n tool_choice: toOaiToolChoice(this.oaiRealtimeModel._options.toolChoice),\n tracing: this.oaiRealtimeModel._options.tracing,\n instructions: this.instructions,\n },\n };\n }\n\n get chatCtx() {\n return this.remoteChatCtx.toChatCtx();\n }\n\n get tools() {\n return { ...this._tools } as llm.ToolContext;\n }\n\n async updateChatCtx(_chatCtx: llm.ChatContext): Promise<void> {\n const unlock = await this.updateChatCtxLock.lock();\n const events = this.createChatCtxUpdateEvents(_chatCtx);\n const futures: Future<void>[] = [];\n\n for (const event of events) {\n const future = new Future<void>();\n futures.push(future);\n\n if (event.type === 'conversation.item.create') {\n this.itemCreateFutures[event.item.id] = future;\n } else if (event.type == 'conversation.item.delete') {\n this.itemDeleteFutures[event.item_id] = future;\n }\n\n this.sendEvent(event);\n }\n\n if (futures.length === 0) {\n unlock();\n return;\n }\n\n try {\n // wait for futures to resolve or timeout\n await Promise.race([\n Promise.all(futures),\n delay(5000).then(() => {\n throw new Error('Chat ctx update events timed out');\n }),\n ]);\n } catch (e) {\n this.#logger.error((e as Error).message);\n throw e;\n } finally {\n unlock();\n }\n }\n\n private createChatCtxUpdateEvents(\n chatCtx: llm.ChatContext,\n addMockAudio: boolean = false,\n ): (api_proto.ConversationItemCreateEvent | api_proto.ConversationItemDeleteEvent)[] {\n const newChatCtx = chatCtx.copy();\n if (addMockAudio) {\n newChatCtx.items.push(createMockAudioItem());\n } else {\n // clean up existing mock audio items\n newChatCtx.items = newChatCtx.items.filter(\n (item) => !item.id.startsWith(MOCK_AUDIO_ID_PREFIX),\n );\n }\n\n const events: (\n | api_proto.ConversationItemCreateEvent\n | api_proto.ConversationItemDeleteEvent\n )[] = [];\n\n const diffOps = llm.computeChatCtxDiff(this.chatCtx, newChatCtx);\n for (const op of diffOps.toRemove) {\n events.push({\n type: 'conversation.item.delete',\n item_id: op,\n event_id: shortuuid('chat_ctx_delete_'),\n } as api_proto.ConversationItemDeleteEvent);\n }\n\n for (const [previousId, id] of diffOps.toCreate) {\n const chatItem = newChatCtx.getById(id);\n if (!chatItem) {\n throw new Error(`Chat item ${id} not found`);\n }\n events.push({\n type: 'conversation.item.create',\n item: livekitItemToOpenAIItem(chatItem),\n previous_item_id: previousId ?? undefined,\n event_id: shortuuid('chat_ctx_create_'),\n } as api_proto.ConversationItemCreateEvent);\n }\n return events;\n }\n\n async updateTools(_tools: llm.ToolContext): Promise<void> {\n const unlock = await this.updateFuncCtxLock.lock();\n const ev = this.createToolsUpdateEvent(_tools);\n this.sendEvent(ev);\n\n if (!ev.session.tools) {\n throw new Error('Tools are missing in the session update event');\n }\n\n // TODO(brian): these logics below are noops I think, leaving it here to keep\n // parity with the python but we should remove them later\n const retainedToolNames = new Set(ev.session.tools.map((tool) => tool.name));\n const retainedTools = Object.fromEntries(\n Object.entries(_tools).filter(\n ([name, tool]) => llm.isFunctionTool(tool) && retainedToolNames.has(name),\n ),\n );\n\n this._tools = retainedTools as llm.ToolContext;\n\n unlock();\n }\n\n private createToolsUpdateEvent(_tools: llm.ToolContext): api_proto.SessionUpdateEvent {\n const oaiTools: api_proto.Tool[] = [];\n\n for (const [name, tool] of Object.entries(_tools)) {\n if (!llm.isFunctionTool(tool)) {\n this.#logger.error({ name, tool }, \"OpenAI Realtime API doesn't support this tool type\");\n continue;\n }\n\n const { parameters: toolParameters, description } = tool;\n try {\n const parameters = llm.toJsonSchema(\n toolParameters,\n ) as unknown as api_proto.Tool['parameters'];\n\n oaiTools.push({\n name,\n description,\n parameters: parameters,\n type: 'function',\n });\n } catch (e) {\n this.#logger.error({ name, tool }, \"OpenAI Realtime API doesn't support this tool type\");\n continue;\n }\n }\n\n return {\n type: 'session.update',\n session: {\n type: 'realtime',\n model: this.oaiRealtimeModel._options.model,\n tools: oaiTools,\n },\n event_id: shortuuid('tools_update_'),\n };\n }\n\n async updateInstructions(_instructions: string): Promise<void> {\n const eventId = shortuuid('instructions_update_');\n this.sendEvent({\n type: 'session.update',\n session: {\n type: 'realtime',\n instructions: _instructions,\n },\n event_id: eventId,\n } as api_proto.SessionUpdateEvent);\n this.instructions = _instructions;\n }\n\n updateOptions({ toolChoice }: { toolChoice?: llm.ToolChoice }): void {\n const options: api_proto.SessionUpdateEvent['session'] = {\n type: 'realtime',\n };\n\n this.oaiRealtimeModel._options.toolChoice = toolChoice;\n options.tool_choice = toOaiToolChoice(toolChoice);\n\n // TODO(brian): add other options here\n\n this.sendEvent({\n type: 'session.update',\n session: options,\n event_id: shortuuid('options_update_'),\n });\n }\n\n pushAudio(frame: AudioFrame): void {\n for (const f of this.resampleAudio(frame)) {\n for (const nf of this.bstream.write(f.data.buffer as ArrayBuffer)) {\n this.sendEvent({\n type: 'input_audio_buffer.append',\n audio: Buffer.from(nf.data.buffer).toString('base64'),\n } as api_proto.InputAudioBufferAppendEvent);\n // TODO(AJS-102): use frame.durationMs once available in rtc-node\n this.pushedDurationMs += (nf.samplesPerChannel / nf.sampleRate) * 1000;\n }\n }\n }\n\n async commitAudio(): Promise<void> {\n if (this.pushedDurationMs > 100) {\n // OpenAI requires at least 100ms of audio\n this.sendEvent({\n type: 'input_audio_buffer.commit',\n } as api_proto.InputAudioBufferCommitEvent);\n this.pushedDurationMs = 0;\n }\n }\n\n async clearAudio(): Promise<void> {\n this.sendEvent({\n type: 'input_audio_buffer.clear',\n } as api_proto.InputAudioBufferClearEvent);\n this.pushedDurationMs = 0;\n }\n\n async generateReply(instructions?: string): Promise<llm.GenerationCreatedEvent> {\n const handle = this.createResponse({ instructions, userInitiated: true });\n this.textModeRecoveryRetries = 0;\n return handle.doneFut.await;\n }\n\n async interrupt(): Promise<void> {\n this.sendEvent({\n type: 'response.cancel',\n } as api_proto.ResponseCancelEvent);\n }\n\n async truncate(_options: {\n messageId: string;\n audioEndMs: number;\n modalities?: Modality[];\n audioTranscript?: string;\n }): Promise<void> {\n // Check if modalities include audio AND the item has real server-side audio\n // Items restored after reconnection are text-only and cannot be truncated\n const hasAudioModality = !_options.modalities || _options.modalities.includes('audio');\n const hasServerSideAudio = this.audioCapableItemIds.has(_options.messageId);\n\n if (hasAudioModality && hasServerSideAudio) {\n this.sendEvent({\n type: 'conversation.item.truncate',\n content_index: 0,\n item_id: _options.messageId,\n audio_end_ms: _options.audioEndMs,\n } as api_proto.ConversationItemTruncateEvent);\n } else if (_options.audioTranscript !== undefined) {\n // sync it to the remote chat context\n const chatCtx = this.chatCtx.copy();\n const idx = chatCtx.indexById(_options.messageId);\n if (idx !== undefined) {\n const item = chatCtx.items[idx];\n if (item && item.type === 'message') {\n const newItem = llm.ChatMessage.create({\n ...item,\n content: [_options.audioTranscript],\n });\n chatCtx.items[idx] = newItem;\n const events = this.createChatCtxUpdateEvents(chatCtx);\n for (const ev of events) {\n this.sendEvent(ev);\n }\n }\n }\n }\n }\n\n private loggableEvent(\n event: api_proto.ClientEvent | api_proto.ServerEvent,\n ): Record<string, unknown> {\n const untypedEvent: Record<string, unknown> = {};\n for (const [key, value] of Object.entries(event)) {\n if (value !== undefined) {\n untypedEvent[key] = value;\n }\n }\n\n if (untypedEvent.audio && typeof untypedEvent.audio === 'string') {\n return { ...untypedEvent, audio: '...' };\n }\n if (\n untypedEvent.delta &&\n typeof untypedEvent.delta === 'string' &&\n event.type === 'response.audio.delta'\n ) {\n return { ...untypedEvent, delta: '...' };\n }\n return untypedEvent;\n }\n\n private async createWsConn(): Promise<WebSocket> {\n const headers: Record<string, string> = {\n 'User-Agent': 'LiveKit-Agents-JS',\n };\n\n if (this.oaiRealtimeModel._options.isAzure) {\n // Microsoft API has two ways of authentication\n // 1. Entra token set as `Bearer` token\n // 2. API key set as `api_key` header (also accepts query string)\n if (this.oaiRealtimeModel._options.entraToken) {\n headers.Authorization = `Bearer ${this.oaiRealtimeModel._options.entraToken}`;\n } else if (this.oaiRealtimeModel._options.apiKey) {\n headers['api-key'] = this.oaiRealtimeModel._options.apiKey;\n } else {\n throw new Error('Microsoft API key or entraToken is required');\n }\n } else {\n if (!this.oaiRealtimeModel._options.apiKey) {\n throw new Error(\n 'OpenAI API key is required but not set. Check OPENAI_API_KEY environment variable.',\n );\n }\n headers.Authorization = `Bearer ${this.oaiRealtimeModel._options.apiKey}`;\n }\n\n const url = processBaseURL({\n baseURL: this.oaiRealtimeModel._options.baseURL,\n model: this.oaiRealtimeModel._options.model,\n isAzure: this.oaiRealtimeModel._options.isAzure,\n apiVersion: this.oaiRealtimeModel._options.apiVersion,\n azureDeployment: this.oaiRealtimeModel._options.azureDeployment,\n });\n\n if (lkOaiDebug) {\n this.#logger.debug(`Connecting to OpenAI Realtime API at ${url}`);\n }\n\n return new Promise((resolve, reject) => {\n const ws = new WebSocket(url, { headers });\n let waiting = true;\n\n const timeout = setTimeout(() => {\n ws.close();\n reject(new Error('WebSocket connection timeout'));\n }, this.oaiRealtimeModel._options.connOptions.timeoutMs);\n\n ws.once('open', () => {\n if (!waiting) return;\n waiting = false;\n clearTimeout(timeout);\n resolve(ws);\n });\n\n ws.once('close', () => {\n if (!waiting) return;\n waiting = false;\n clearTimeout(timeout);\n reject(new Error('OpenAI Realtime API connection closed'));\n });\n });\n }\n\n async #mainTask(signal: AbortSignal): Promise<void> {\n let reconnecting = false;\n let numRetries = 0;\n let wsConn: WebSocket | null = null;\n const maxRetries = this.oaiRealtimeModel._options.connOptions.maxRetry;\n\n const reconnect = async () => {\n this.#logger.debug(\n {\n maxSessionDuration: this.oaiRealtimeModel._options.maxSessionDuration,\n },\n 'Reconnecting to OpenAI Realtime API',\n );\n\n // Clean up pending futures from old connection to prevent memory leaks\n for (const fut of Object.values(this.itemCreateFutures)) {\n if (!fut.done) fut.reject(new Error('Session reconnected'));\n }\n this.itemCreateFutures = {};\n\n for (const fut of Object.values(this.itemDeleteFutures)) {\n if (!fut.done) fut.reject(new Error('Session reconnected'));\n }\n this.itemDeleteFutures = {};\n\n // Clear audio-capable item tracking - restored items are text-only on the server\n this.audioCapableItemIds.clear();\n\n const events: api_proto.ClientEvent[] = [];\n\n // options and instructions\n events.push(this.createSessionUpdateEvent());\n\n // tools\n if (Object.keys(this._tools).length > 0) {\n events.push(this.createToolsUpdateEvent(this._tools));\n }\n\n // chat context\n const chatCtx = this.chatCtx.copy({\n excludeFunctionCall: true,\n excludeInstructions: true,\n excludeEmptyMessage: true,\n });\n\n const oldChatCtx = this.remoteChatCtx;\n this.remoteChatCtx = new llm.RemoteChatContext();\n events.push(...this.createChatCtxUpdateEvents(chatCtx));\n\n try {\n for (const ev of events) {\n this.emit('openai_client_event_queued', ev);\n wsConn!.send(JSON.stringify(ev));\n }\n } catch (error) {\n this.remoteChatCtx = oldChatCtx;\n throw new APIConnectionError({\n message: 'Failed to send message to OpenAI Realtime API during session re-connection',\n });\n }\n\n this.#logger.debug('Reconnected to OpenAI Realtime API');\n\n this.emit('session_reconnected', {} as llm.RealtimeSessionReconnectedEvent);\n };\n\n reconnecting = false;\n while (!this.#closed && !signal.aborted) {\n this.#logger.debug('Creating WebSocket connection to OpenAI Realtime API');\n wsConn = await this.createWsConn();\n if (signal.aborted) break;\n\n try {\n if (reconnecting) {\n await reconnect();\n if (signal.aborted) break;\n numRetries = 0;\n }\n\n await this.runWs(wsConn);\n if (signal.aborted) break;\n } catch (error) {\n if (!isAPIError(error)) {\n this.emitError({ error: error as Error, recoverable: false });\n throw error;\n }\n\n if (maxRetries === 0 || !error.retryable) {\n this.emitError({ error: error as Error, recoverable: false });\n throw error;\n }\n\n if (numRetries === maxRetries) {\n this.emitError({ error: error as Error, recoverable: false });\n throw new APIConnectionError({\n message: `OpenAI Realtime API connection failed after ${numRetries} attempts`,\n options: {\n body: error,\n retryable: false,\n },\n });\n }\n\n this.emitError({ error: error as Error, recoverable: true });\n const retryInterval =\n numRetries === 0\n ? DEFAULT_FIRST_RETRY_INTERVAL_MS\n : this.oaiRealtimeModel._options.connOptions.retryIntervalMs;\n this.#logger.warn(\n {\n attempt: numRetries,\n maxRetries,\n error,\n },\n `OpenAI Realtime API connection failed, retrying in ${retryInterval / 1000}s`,\n );\n\n await delay(retryInterval);\n numRetries++;\n }\n\n reconnecting = true;\n }\n }\n\n private async runWs(wsConn: WebSocket): Promise<void> {\n const forwardEvents = async (signal: AbortSignal): Promise<void> => {\n const abortFuture = new Future<void>();\n signal.addEventListener('abort', () => abortFuture.resolve());\n\n while (!this.#closed && wsConn.readyState === WebSocket.OPEN && !signal.aborted) {\n try {\n const event = await Promise.race([this.messageChannel.get(), abortFuture.await]);\n if (signal.aborted || abortFuture.done || event === undefined) {\n break;\n }\n\n if (lkOaiDebug) {\n this.#logger.debug(this.loggableEvent(event), `(client) -> ${event.type}`);\n }\n\n this.emit('openai_client_event_queued', event);\n wsConn.send(JSON.stringify(event));\n } catch (error) {\n break;\n }\n }\n\n wsConn.close();\n };\n\n const wsCloseFuture = new Future<void | Error>();\n\n wsConn.onerror = (error) => {\n wsCloseFuture.resolve(new APIConnectionError({ message: error.message }));\n };\n wsConn.onclose = () => {\n wsCloseFuture.resolve();\n };\n\n wsConn.onmessage = (message: MessageEvent) => {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n const event: any = JSON.parse(message.data as string);\n\n this.emit('openai_server_event_received', event);\n if (lkOaiDebug) {\n this.#logger.debug(this.loggableEvent(event), `(server) <- ${event.type}`);\n }\n\n switch (event.type) {\n case 'input_audio_buffer.speech_started':\n this.handleInputAudioBufferSpeechStarted(event);\n break;\n case 'input_audio_buffer.speech_stopped':\n this.handleInputAudioBufferSpeechStopped(event);\n break;\n case 'response.created':\n this.handleResponseCreated(event);\n break;\n case 'response.output_item.added':\n this.handleResponseOutputItemAdded(event);\n break;\n case 'conversation.item.added':\n case 'conversation.item.created': // Beta: kept for backward compatibility\n this.handleConversationItemCreated(event);\n break;\n case 'conversation.item.deleted':\n this.handleConversationItemDeleted(event);\n break;\n case 'conversation.item.input_audio_transcription.completed':\n this.handleConversationItemInputAudioTranscriptionCompleted(event);\n break;\n case 'conversation.item.input_audio_transcription.failed':\n this.handleConversationItemInputAudioTranscriptionFailed(event);\n break;\n case 'response.content_part.added':\n this.handleResponseContentPartAdded(event);\n break;\n case 'response.content_part.done':\n this.handleResponseContentPartDone(event);\n break;\n case 'response.output_text.delta':\n case 'response.text.delta': // Beta: kept for backward compatibility\n this.handleResponseTextDelta(event);\n break;\n case 'response.output_text.done':\n case 'response.text.done': // Beta: kept for backward compatibility\n this.handleResponseTextDone(event);\n break;\n case 'response.output_audio_transcript.delta':\n case 'response.audio_transcript.delta': // Beta: kept for backward compatibility\n this.handleResponseAudioTranscriptDelta(event);\n break;\n case 'response.output_audio.delta':\n case 'response.audio.delta': // Beta: kept for backward compatibility\n this.handleResponseAudioDelta(event);\n break;\n case 'response.output_audio_transcript.done':\n case 'response.audio_transcript.done': // Beta: kept for backward compatibility\n this.handleResponseAudioTranscriptDone(event);\n break;\n case 'response.output_audio.done':\n case 'response.audio.done': // Beta: kept for backward compatibility\n this.handleResponseAudioDone(event);\n break;\n case 'response.output_item.done':\n this.handleResponseOutputItemDone(event);\n break;\n case 'response.done':\n this.handleResponseDone(event);\n break;\n case 'error':\n this.handleError(event);\n break;\n default:\n if (lkOaiDebug) {\n this.#logger.debug(`unhandled event: ${event.type}`);\n }\n break;\n }\n };\n\n const sendTask = Task.from(({ signal }) => forwardEvents(signal));\n\n const wsTask = Task.from(({ signal }) => {\n const abortPromise = new Promise<void>((resolve) => {\n signal.addEventListener('abort', () => {\n resolve();\n });\n });\n\n return Promise.race([wsCloseFuture.await, abortPromise]);\n });\n\n const waitReconnectTask = Task.from(async ({ signal }) => {\n await delay(this.oaiRealtimeModel._options.maxSessionDuration, { signal });\n return new APIConnectionError({\n message: 'OpenAI Realtime API connection timeout',\n });\n });\n\n try {\n const result = await Promise.race([wsTask.result, sendTask.result, waitReconnectTask.result]);\n\n if (waitReconnectTask.done && this.currentGeneration) {\n await this.currentGeneration._doneFut.await;\n }\n\n if (result instanceof Error) {\n throw result;\n }\n } finally {\n await cancelAndWait([wsTask, sendTask, waitReconnectTask], 2000);\n wsConn.close();\n }\n }\n\n async close() {\n super.close();\n this.#closed = true;\n await this.#task;\n\n // Clean up pending futures to prevent memory leaks\n for (const handle of Object.values(this.responseCreatedFutures)) {\n if (!handle.doneFut.done) {\n handle.doneFut.reject(new Error('Session closed'));\n }\n }\n this.responseCreatedFutures = {};\n\n for (const fut of Object.values(this.itemCreateFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed'));\n }\n }\n this.itemCreateFutures = {};\n\n for (const fut of Object.values(this.itemDeleteFutures)) {\n if (!fut.done) {\n fut.reject(new Error('Session closed'));\n }\n }\n this.itemDeleteFutures = {};\n\n // Clean up current generation if exists\n if (this.currentGeneration) {\n for (const gen of this.currentGeneration.messages.values()) {\n gen.textChannel.close();\n gen.audioChannel.close();\n if (!gen.modalities.done) {\n gen.modalities.resolve(this.oaiRealtimeModel._options.modalities);\n }\n }\n this.currentGeneration.messages.clear();\n this.currentGeneration.messageChannel.close();\n this.currentGeneration.functionChannel.close();\n if (!this.currentGeneration._doneFut.done) {\n this.currentGeneration._doneFut.resolve();\n }\n this.currentGeneration = undefined;\n }\n\n // Clear the message queue\n this.messageChannel.items.length = 0;\n }\n\n private handleInputAudioBufferSpeechStarted(\n _event: api_proto.InputAudioBufferSpeechStartedEvent,\n ): void {\n this.emit('input_speech_started', {} as llm.InputSpeechStartedEvent);\n }\n\n private handleInputAudioBufferSpeechStopped(\n _event: api_proto.InputAudioBufferSpeechStoppedEvent,\n ): void {\n this.emit('input_speech_stopped', {\n userTranscriptionEnabled: this.oaiRealtimeModel._options.inputAudioTranscription !== null,\n } as llm.InputSpeechStoppedEvent);\n }\n\n private handleResponseCreated(event: api_proto.ResponseCreatedEvent): void {\n if (!event.response.id) {\n throw new Error('response.id is missing');\n }\n\n this.currentGeneration = {\n messageChannel: stream.createStreamChannel<llm.MessageGeneration>(),\n functionChannel: stream.createStreamChannel<llm.FunctionCall>(),\n messages: new Map(),\n _doneFut: new Future(),\n _createdTimestamp: Date.now(),\n };\n\n // Build generation event and resolve client future (if any) before emitting,\n // matching Python behavior.\n const generationEv = {\n messageStream: this.currentGeneration.messageChannel.stream(),\n functionStream: this.currentGeneration.functionChannel.stream(),\n userInitiated: false,\n responseId: event.response.id,\n } as llm.GenerationCreatedEvent;\n\n const clientEventId = event.response.metadata?.client_event_id;\n if (clientEventId) {\n const handle = this.responseCreatedFutures[clientEventId];\n if (handle) {\n delete this.responseCreatedFutures[clientEventId];\n generationEv.userInitiated = true;\n if (!handle.doneFut.done) {\n handle.doneFut.resolve(generationEv);\n }\n }\n }\n\n this.emit('generation_created', generationEv);\n }\n\n private handleResponseOutputItemAdded(event: api_proto.ResponseOutputItemAddedEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n if (!event.item.type) {\n throw new Error('item.type is not set');\n }\n\n const itemType = event.item.type;\n\n if (itemType !== 'message') {\n // non-message items (e.g. function calls) don't need additional handling here\n // the generation event was already emitted in handleResponseCreated\n this.textModeRecoveryRetries = 0;\n return;\n }\n\n const itemId = event.item.id;\n if (!itemId) {\n throw new Error('item.id is not set');\n }\n\n const modalitiesFut = new Future<Modality[]>();\n const itemGeneration: MessageGeneration = {\n messageId: itemId,\n textChannel: stream.createStreamChannel<string | TimedString>(),\n audioChannel: stream.createStreamChannel<AudioFrame>(),\n audioTranscript: '',\n modalities: modalitiesFut,\n };\n\n // If audioOutput is not supported, close audio channel immediately\n if (!this.oaiRealtimeModel.capabilities.audioOutput) {\n itemGeneration.audioChannel.close();\n modalitiesFut.resolve(['text']);\n }\n\n this.currentGeneration.messageChannel.write({\n messageId: itemId,\n textStream: itemGeneration.textChannel.stream(),\n audioStream: itemGeneration.audioChannel.stream(),\n modalities: modalitiesFut.await,\n });\n\n this.currentGeneration.messages.set(itemId, itemGeneration);\n }\n\n private handleConversationItemCreated(event: api_proto.ConversationItemCreatedEvent): void {\n if (!event.item.id) {\n throw new Error('item.id is not set');\n }\n\n try {\n this.remoteChatCtx.insert(event.previous_item_id, openAIItemToLivekitItem(event.item));\n } catch (error) {\n this.#logger.error({ error, itemId: event.item.id }, 'failed to insert conversation item');\n }\n\n const fut = this.itemCreateFutures[event.item.id];\n if (fut) {\n fut.resolve();\n delete this.itemCreateFutures[event.item.id];\n }\n }\n\n private handleConversationItemDeleted(event: api_proto.ConversationItemDeletedEvent): void {\n if (!event.item_id) {\n throw new Error('item_id is not set');\n }\n\n // Clean up audio-capable tracking for deleted items\n this.audioCapableItemIds.delete(event.item_id);\n\n try {\n this.remoteChatCtx.delete(event.item_id);\n } catch (error) {\n this.#logger.error({ error, itemId: event.item_id }, 'failed to delete conversation item');\n }\n\n const fut = this.itemDeleteFutures[event.item_id];\n if (fut) {\n fut.resolve();\n delete this.itemDeleteFutures[event.item_id];\n }\n }\n\n private handleConversationItemInputAudioTranscriptionCompleted(\n event: api_proto.ConversationItemInputAudioTranscriptionCompletedEvent,\n ): void {\n const remoteItem = this.remoteChatCtx.get(event.item_id);\n if (!remoteItem) {\n return;\n }\n\n const item = remoteItem.item;\n if (item instanceof llm.ChatMessage) {\n item.content.push(event.transcript);\n } else {\n throw new Error('item is not a chat message');\n }\n\n this.emit('input_audio_transcription_completed', {\n itemId: event.item_id,\n transcript: event.transcript,\n isFinal: true,\n } as llm.InputTranscriptionCompleted);\n }\n\n private handleConversationItemInputAudioTranscriptionFailed(\n event: api_proto.ConversationItemInputAudioTranscriptionFailedEvent,\n ): void {\n this.#logger.error(\n { error: event.error },\n 'OpenAI Realtime API failed to transcribe input audio',\n );\n }\n\n private handleResponseContentPartAdded(event: api_proto.ResponseContentPartAddedEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n const itemId = event.item_id;\n const itemType = event.part.type;\n\n const itemGeneration = this.currentGeneration.messages.get(itemId);\n if (!itemGeneration) {\n this.#logger.warn(`itemGeneration not found for itemId=${itemId}`);\n return;\n }\n\n const isTextType = itemType === 'text' || itemType === 'output_text';\n if (isTextType && this.oaiRealtimeModel.capabilities.audioOutput) {\n this.#logger.warn('Text response received from OpenAI Realtime API in audio modality.');\n }\n\n if (!itemGeneration.modalities.done) {\n const modalityResult: Modality[] = isTextType ? ['text'] : ['audio', 'text'];\n itemGeneration.modalities.resolve(modalityResult);\n\n // Track items with real server-side audio for truncation eligibility\n if (!isTextType) {\n this.audioCapableItemIds.add(itemId);\n }\n }\n\n if (this.currentGeneration._firstTokenTimestamp === undefined) {\n this.currentGeneration._firstTokenTimestamp = Date.now();\n }\n }\n\n private handleResponseContentPartDone(event: api_proto.ResponseContentPartDoneEvent): void {\n if (!event.part) {\n return;\n }\n if (event.part.type !== 'text') {\n return;\n }\n\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n // TODO(shubhra): handle text mode recovery\n }\n\n private handleResponseTextDelta(event: api_proto.ResponseTextDeltaEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n const itemGeneration = this.currentGeneration.messages.get(event.item_id);\n if (!itemGeneration) {\n throw new Error('itemGeneration is not set');\n }\n\n if (\n !this.oaiRealtimeModel.capabilities.audioOutput &&\n !this.currentGeneration._firstTokenTimestamp\n ) {\n this.currentGeneration._firstTokenTimestamp = Date.now();\n }\n\n itemGeneration.textChannel.write(event.delta);\n itemGeneration.audioTranscript += event.delta;\n }\n\n private handleResponseTextDone(_event: api_proto.ResponseTextDoneEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n }\n\n private handleResponseAudioTranscriptDelta(\n event: api_proto.ResponseAudioTranscriptDeltaEvent,\n ): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n const itemId = event.item_id;\n\n // When start_time is provided, wrap the delta in a TimedString for aligned transcripts\n let delta: string | TimedString = event.delta;\n if (event.start_time !== undefined) {\n delta = createTimedString({ text: event.delta, startTime: event.start_time });\n }\n\n const itemGeneration = this.currentGeneration.messages.get(itemId);\n if (!itemGeneration) {\n throw new Error('itemGeneration is not set');\n } else {\n itemGeneration.textChannel.write(delta);\n itemGeneration.audioTranscript += event.delta;\n }\n }\n\n private handleResponseAudioDelta(event: api_proto.ResponseAudioDeltaEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n const itemGeneration = this.currentGeneration.messages.get(event.item_id);\n if (!itemGeneration) {\n throw new Error('itemGeneration is not set');\n }\n\n if (this.currentGeneration._firstTokenTimestamp === undefined) {\n this.currentGeneration._firstTokenTimestamp = Date.now();\n }\n\n if (!itemGeneration.modalities.done) {\n itemGeneration.modalities.resolve(['audio', 'text']);\n }\n\n const binaryString = atob(event.delta);\n const len = binaryString.length;\n const bytes = new Uint8Array(len);\n for (let i = 0; i < len; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n itemGeneration.audioChannel.write(\n new AudioFrame(\n new Int16Array(bytes.buffer),\n api_proto.SAMPLE_RATE,\n api_proto.NUM_CHANNELS,\n bytes.length / 2,\n ),\n );\n }\n\n private handleResponseAudioTranscriptDone(\n _event: api_proto.ResponseAudioTranscriptDoneEvent,\n ): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n }\n\n private handleResponseAudioDone(_event: api_proto.ResponseAudioDoneEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n }\n\n private handleResponseOutputItemDone(event: api_proto.ResponseOutputItemDoneEvent): void {\n if (!this.currentGeneration) {\n throw new Error('currentGeneration is not set');\n }\n\n const itemId = event.item.id;\n const itemType = event.item.type;\n\n if (itemType === 'function_call') {\n const item = event.item;\n if (!item.call_id || !item.name || !item.arguments) {\n throw new Error('item is not a function call');\n }\n this.currentGeneration.functionChannel.write(\n llm.FunctionCall.create({\n callId: item.call_id,\n name: item.name,\n args: item.arguments,\n }),\n );\n } else if (itemType === 'message') {\n const itemGeneration = this.currentGeneration.messages.get(itemId);\n if (!itemGeneration) {\n return;\n }\n // text response doesn't have itemGeneration\n itemGeneration.textChannel.close();\n itemGeneration.audioChannel.close();\n if (!itemGeneration.modalities.done) {\n // In case message modalities is not set, this shouldn't happen\n itemGeneration.modalities.resolve(this.oaiRealtimeModel._options.modalities);\n }\n }\n }\n\n private handleResponseDone(_event: api_proto.ResponseDoneEvent): void {\n if (!this.currentGeneration) {\n // OpenAI has a race condition where we could receive response.done without any\n // previous response.created (This happens generally during interruption)\n return;\n }\n\n const createdTimestamp = this.currentGeneration._createdTimestamp;\n const firstTokenTimestamp = this.currentGeneration._firstTokenTimestamp;\n\n this.#logger.debug(\n {\n messageCount: this.currentGeneration.messages.size,\n },\n 'Closing generation channels in handleResponseDone',\n );\n\n for (const generation of this.currentGeneration.messages.values()) {\n generation.textChannel.close();\n generation.audioChannel.close();\n if (!generation.modalities.done) {\n generation.modalities.resolve(this.oaiRealtimeModel._options.modalities);\n }\n }\n\n this.currentGeneration.functionChannel.close();\n this.currentGeneration.messageChannel.close();\n\n for (const itemId of this.currentGeneration.messages.keys()) {\n const remoteItem = this.remoteChatCtx.get(itemId);\n if (remoteItem && remoteItem.item instanceof llm.ChatMessage) {\n remoteItem.item.content.push(this.currentGeneration.messages.get(itemId)!.audioTranscript);\n }\n }\n\n this.currentGeneration._doneFut.resolve();\n this.currentGeneration = undefined;\n\n // Calculate and emit metrics\n const usage = _event.response.usage;\n const ttftMs = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;\n const durationMs = Date.now() - createdTimestamp;\n\n const realtimeMetrics: metrics.RealtimeModelMetrics = {\n type: 'realtime_model_metrics',\n timestamp: createdTimestamp,\n requestId: _event.response.id || '',\n ttftMs,\n durationMs,\n cancelled: _event.response.status === 'cancelled',\n label: 'openai_realtime',\n inputTokens: usage?.input_tokens ?? 0,\n outputTokens: usage?.output_tokens ?? 0,\n totalTokens: usage?.total_tokens ?? 0,\n tokensPerSecond: durationMs > 0 ? (usage?.output_tokens ?? 0) / (durationMs / 1000) : 0,\n inputTokenDetails: {\n audioTokens: usage?.input_token_details?.audio_tokens ?? 0,\n textTokens: usage?.input_token_details?.text_tokens ?? 0,\n imageTokens: 0, // Not supported yet\n cachedTokens: usage?.input_token_details?.cached_tokens ?? 0,\n cachedTokensDetails: usage?.input_token_details?.cached_tokens_details\n ? {\n audioTokens: usage?.input_token_details?.cached_tokens_details?.audio_tokens ?? 0,\n textTokens: usage?.input_token_details?.cached_tokens_details?.text_tokens ?? 0,\n imageTokens: usage?.input_token_details?.cached_tokens_details?.image_tokens ?? 0,\n }\n : undefined,\n },\n outputTokenDetails: {\n textTokens: usage?.output_token_details?.text_tokens ?? 0,\n audioTokens: usage?.output_token_details?.audio_tokens ?? 0,\n imageTokens: 0,\n },\n };\n\n this.emit('metrics_collected', realtimeMetrics);\n // TODO(brian): handle response done but not complete\n }\n\n private handleError(event: api_proto.ErrorEvent): void {\n if (event.error.message.startsWith('Cancellation failed')) {\n return;\n }\n\n this.#logger.error({ error: event.error }, 'OpenAI Realtime API returned an error');\n this.emitError({\n error: new APIError(event.error.message, {\n body: event.error,\n retryable: true,\n }),\n recoverable: true,\n });\n\n // TODO(brian): set error for response future if it exists\n }\n\n private emitError({ error, recoverable }: { error: Error; recoverable: boolean }): void {\n // IMPORTANT: only emit error if there are listeners; otherwise emit will throw an error\n this.emit('error', {\n timestamp: Date.now(),\n // TODO(brian): add label\n label: '',\n error,\n recoverable,\n } as llm.RealtimeModelError);\n }\n\n private *resampleAudio(frame: AudioFrame): Generator<AudioFrame> {\n yield frame;\n }\n\n private createResponse({\n userInitiated,\n instructions,\n oldHandle,\n }: {\n userInitiated: boolean;\n instructions?: string;\n oldHandle?: CreateResponseHandle;\n }): CreateResponseHandle {\n const handle = oldHandle || new CreateResponseHandle({ instructions });\n if (oldHandle && instructions) {\n handle.instructions = instructions;\n }\n\n const eventId = shortuuid('response_create_');\n if (userInitiated) {\n this.responseCreatedFutures[eventId] = handle;\n }\n\n const response: api_proto.ResponseCreateEvent['response'] = {};\n if (instructions) response.instructions = instructions;\n if (userInitiated) response.metadata = { client_event_id: eventId };\n\n this.sendEvent({\n type: 'response.create',\n event_id: eventId,\n response: Object.keys(response).length > 0 ? response : undefined,\n });\n\n return handle;\n }\n}\n\n/** @internal Exported for testing purposes */\nexport function livekitItemToOpenAIItem(item: llm.ChatItem): api_proto.ItemResource {\n switch (item.type) {\n case 'function_call':\n return {\n id: item.id,\n type: 'function_call',\n call_id: item.callId,\n name: item.name,\n arguments: item.args,\n } as api_proto.FunctionCallItem;\n case 'function_call_output':\n return {\n id: item.id,\n type: 'function_call_output',\n call_id: item.callId,\n output: item.output,\n } as api_proto.FunctionCallOutputItem;\n case 'message':\n const role = item.role === 'developer' ? 'system' : item.role;\n const contentList: api_proto.Content[] = [];\n for (const c of item.content) {\n if (typeof c === 'string') {\n contentList.push({\n type: role === 'assistant' ? 'output_text' : 'input_text',\n text: c,\n } as api_proto.InputTextContent | api_proto.OutputTextContent);\n } else if (c.type === 'image_content') {\n // not supported for now\n continue;\n } else if (c.type === 'audio_content') {\n if (role === 'user') {\n const encodedAudio = Buffer.from(combineAudioFrames(c.frame).data).toString('base64');\n contentList.push({\n type: 'input_audio',\n audio: encodedAudio,\n } as api_proto.InputAudioContent);\n }\n }\n }\n return {\n id: item.id,\n type: 'message',\n role,\n content: contentList,\n } as api_proto.UserItem | api_proto.AssistantItem | api_proto.SystemItem;\n default:\n throw new Error(`Unsupported item type: ${(item as any).type}`);\n }\n}\n\nfunction openAIItemToLivekitItem(item: api_proto.ItemResource): llm.ChatItem {\n if (!item.id) {\n throw new Error('item.id is not set');\n }\n\n switch (item.type) {\n case 'function_call':\n return llm.FunctionCall.create({\n id: item.id,\n callId: item.call_id,\n name: item.name,\n args: item.arguments,\n });\n case 'function_call_output':\n return llm.FunctionCallOutput.create({\n id: item.id,\n callId: item.call_id,\n output: item.output,\n isError: false,\n });\n case 'message':\n const content: llm.ChatContent[] = [];\n // item.content can be a single object or an array; normalize to array\n const contents = Array.isArray(item.content) ? item.content : [item.content];\n for (const c of contents) {\n if (c.type === 'text' || c.type === 'input_text') {\n content.push(c.text);\n }\n }\n return llm.ChatMessage.create({\n id: item.id,\n role: item.role,\n content,\n });\n }\n}\n\nfunction createMockAudioItem(durationSeconds: number = 2): llm.ChatMessage {\n const audioData = Buffer.alloc(durationSeconds * SAMPLE_RATE);\n return llm.ChatMessage.create({\n id: shortuuid(MOCK_AUDIO_ID_PREFIX),\n role: 'user',\n content: [\n {\n type: 'audio_content',\n frame: [\n new AudioFrame(\n new Int16Array(audioData.buffer),\n SAMPLE_RATE,\n NUM_CHANNELS,\n audioData.length / 2,\n ),\n ],\n } as llm.AudioContent,\n ],\n });\n}\n\nfunction toOaiToolChoice(toolChoice?: llm.ToolChoice): api_proto.ToolChoice {\n if (typeof toolChoice === 'string') {\n return toolChoice;\n }\n\n if (toolChoice?.type === 'function') {\n return toolChoice.function.name;\n }\n\n return 'auto';\n}\n"],"mappings":"AAGA;AAAA,EAEE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,OACK;AACP,SAAS,aAAa;AAEtB,SAAS,YAAY,0BAA0B;AAC/C,SAA4B,iBAAiB;AAC7C,YAAY,eAAe;AAG3B,MAAM,aAAa,QAAQ,IAAI,kBAAkB,OAAO,QAAQ,IAAI,eAAe,IAAI;AAEvF,MAAM,cAAc;AACpB,MAAM,eAAe;AACrB,MAAM,WAAW;AAEjB,MAAM,uBAAuB;AA+C7B,MAAM,qBAAqB;AAAA,EACzB;AAAA,EACA;AAAA;AAAA,EAEA,YAAY,EAAE,aAAa,GAA8B;AACvD,SAAK,eAAe;AACpB,SAAK,UAAU,IAAI,OAAO;AAAA,EAC5B;AACF;AAEA,MAAM,kCAAkC;AACxC,MAAM,yBAAsD;AAAA,EAC1D,MAAM;AAAA,EACN,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,oBAAoB;AACtB;AACA,MAAM,oCAAuE;AAAA,EAC3E,OAAO;AACT;AACA,MAAM,sBAAsC;AAC5C,MAAM,qCAAqD;AAE3D,MAAM,0CAA6E;AAAA,EACjF,OAAO;AACT;AAEA,MAAM,+BAA4D;AAAA,EAChE,MAAM;AAAA,EACN,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,iBAAiB;AACnB;AAEA,MAAM,+BAA+B,KAAK,KAAK;AAE/C,MAAM,iCAAiC;AAAA,EACrC,OAAO;AAAA,EACP,OAAO;AAAA,EACP,yBAAyB;AAAA,EACzB,0BAA0B;AAAA,EAC1B,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,yBAAyB;AAAA,EACzB,oBAAoB;AAAA,EACpB,aAAa;AAAA,EACb,YAAY,CAAC,QAAQ,OAAO;AAAA,EAC5B,SAAS;AACX;AACO,MAAM,sBAAsB,IAAI,cAAc;AAAA,EACnD,aAAa,UAAU;AAAA,EACvB,cAAc,UAAU;AAAA,EACxB,cAAc,UAAU;AAAA,EACxB,eAAe,UAAU;AAAA;AAAA,EAGzB;AAAA,EAEA,IAAI,QAAgB;AAClB,WAAO,KAAK,SAAS;AAAA,EACvB;AAAA,EAEA,YACE,UAmBI,CAAC,GACL;AACA,UAAM,aAAc,QAAQ,cAC1B,+BAA+B;AAEjC,UAAM;AAAA,MACJ,mBAAmB;AAAA,MACnB,eAAe,QAAQ,kBAAkB;AAAA,MACzC,mBAAmB,QAAQ,4BAA4B;AAAA,MACvD,yBAAyB;AAAA,MACzB,aAAa,WAAW,SAAS,OAAO;AAAA,IAC1C,CAAC;AAED,UAAM,UAAU,CAAC,EAAE,QAAQ,cAAc,QAAQ,cAAc,QAAQ;AAEvE,QAAI,QAAQ,WAAW,MAAM,CAAC,SAAS;AACrC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,SAAS,QAAQ,UAAU,QAAQ,IAAI;AAE7C,QAAI,CAAC,UAAU,CAAC,SAAS;AACvB,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,QAAQ,WAAW,SAAS;AAC/B,YAAM,gBAAgB,QAAQ,IAAI;AAClC,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,cAAQ,UAAU,GAAG,cAAc,QAAQ,OAAO,EAAE,CAAC;AAAA,IACvD;AAEA,UAAM,EAAE,YAAY,GAAG,GAAG,yBAAyB,IAAI;AACvD,SAAK,WAAW;AAAA,MACd,GAAG;AAAA,MACH,GAAG;AAAA,MACH,SAAS,QAAQ,WAAW;AAAA,MAC5B;AAAA,MACA;AAAA,MACA,OAAO,QAAQ,SAAS,+BAA+B;AAAA,MACvD;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAsBA,OAAO,UAAU;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,IACR;AAAA;AAAA,IACA,0BAA0B;AAAA,IAC1B;AAAA,IACA,gBAAgB;AAAA,IAChB;AAAA,IACA;AAAA,EACF,GAeG;AACD,aAAS,UAAU,QAAQ,IAAI;AAC/B,QAAI,CAAC,UAAU,CAAC,YAAY;AAC1B,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,iBAAa,cAAc,QAAQ,IAAI;AACvC,QAAI,CAAC,YAAY;AACf,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,SAAS;AACZ,sBAAgB,iBAAiB,QAAQ,IAAI;AAC7C,UAAI,CAAC,eAAe;AAClB,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,gBAAU,GAAG,cAAc,QAAQ,OAAO,EAAE,CAAC;AAAA,IAC/C;AAEA,WAAO,IAAI,cAAc;AAAA,MACvB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,WAAO,IAAI,gBAAgB,IAAI;AAAA,EACjC;AAAA,EAEA,MAAM,QAAQ;AACZ;AAAA,EACF;AACF;AAEA,SAAS,eAAe;AAAA,EACtB;AAAA,EACA;AAAA,EACA,UAAU;AAAA,EACV;AAAA,EACA;AACF,GAMW;AACT,QAAM,MAAM,IAAI,IAAI,CAAC,SAAS,UAAU,EAAE,KAAK,GAAG,CAAC;AAEnD,MAAI,IAAI,aAAa,UAAU;AAC7B,QAAI,WAAW;AAAA,EACjB;AAGA,MAAI,CAAC,IAAI,YAAY,CAAC,IAAI,OAAO,SAAS,EAAE,SAAS,IAAI,SAAS,QAAQ,OAAO,EAAE,CAAC,GAAG;AACrF,QAAI,WAAW,IAAI,SAAS,QAAQ,OAAO,EAAE,IAAI;AAAA,EACnD,OAAO;AACL,QAAI,WAAW,IAAI,SAAS,QAAQ,OAAO,EAAE;AAAA,EAC/C;AAEA,QAAM,cAAsC,CAAC;AAC7C,MAAI,SAAS;AACX,QAAI,YAAY;AACd,kBAAY,aAAa,IAAI;AAAA,IAC/B;AACA,QAAI,iBAAiB;AACnB,kBAAY,YAAY,IAAI;AAAA,IAC9B;AAAA,EACF,OAAO;AACL,gBAAY,OAAO,IAAI;AAAA,EACzB;AAEA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,WAAW,GAAG;AACtD,QAAI,aAAa,IAAI,KAAK,KAAK;AAAA,EACjC;AAEA,SAAO,IAAI,SAAS;AACtB;AAYO,MAAM,wBAAwB,IAAI,gBAAgB;AAAA,EAC/C,SAA0B,CAAC;AAAA,EAC3B,gBAAuC,IAAI,IAAI,kBAAkB;AAAA,EACjE,iBAAiB,IAAI,MAA6B;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,yBAAiE,CAAC;AAAA,EAElE,0BAAkC;AAAA,EAElC,oBAA8C,CAAC;AAAA,EAC/C,oBAA8C,CAAC;AAAA;AAAA;AAAA,EAI/C,sBAAmC,oBAAI,IAAI;AAAA,EAE3C,oBAAoB,IAAI,MAAM;AAAA,EAC9B,oBAAoB,IAAI,MAAM;AAAA;AAAA,EAG9B,UAAU,IAAI,gBAAgB,aAAa,cAAc,cAAc,EAAE;AAAA,EAEzE,mBAA2B;AAAA,EAEnC,UAAU,IAAI;AAAA,EACd;AAAA,EACA,UAAU;AAAA,EAEV,YAAY,eAA8B;AACxC,UAAM,aAAa;AAEnB,SAAK,mBAAmB;AAExB,SAAK,QAAQ,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,KAAK,UAAU,MAAM,CAAC;AAE7D,SAAK,UAAU,KAAK,yBAAyB,CAAC;AAAA,EAChD;AAAA,EAEA,UAAU,SAAsC;AAC9C,SAAK,eAAe,IAAI,OAAO;AAAA,EACjC;AAAA,EAEQ,2BAAyD;AAC/D,UAAM,cAAqC,EAAE,MAAM,aAAa,MAAM,YAAY;AAElF,UAAM,WAAqB,KAAK,iBAAiB,SAAS,WAAW,SAAS,OAAO,IACjF,UACA;AAEJ,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,MAAM;AAAA,QACN,OAAO,KAAK,iBAAiB,SAAS;AAAA,QACtC,mBAAmB,CAAC,QAAQ;AAAA,QAC5B,OAAO;AAAA,UACL,OAAO;AAAA,YACL,QAAQ;AAAA,YACR,iBAAiB,KAAK,iBAAiB,SAAS;AAAA,YAChD,eAAe,KAAK,iBAAiB,SAAS;AAAA,YAC9C,gBAAgB,KAAK,iBAAiB,SAAS;AAAA,UACjD;AAAA,UACA,QAAQ;AAAA,YACN,QAAQ;AAAA,YACR,OAAO,KAAK,iBAAiB,SAAS;AAAA,YACtC,OAAO,KAAK,iBAAiB,SAAS;AAAA,UACxC;AAAA,QACF;AAAA,QACA,mBACE,KAAK,iBAAiB,SAAS,4BAA4B,WACvD,QACA,KAAK,iBAAiB,SAAS;AAAA,QACrC,aAAa,gBAAgB,KAAK,iBAAiB,SAAS,UAAU;AAAA,QACtE,SAAS,KAAK,iBAAiB,SAAS;AAAA,QACxC,cAAc,KAAK;AAAA,MACrB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,IAAI,UAAU;AACZ,WAAO,KAAK,cAAc,UAAU;AAAA,EACtC;AAAA,EAEA,IAAI,QAAQ;AACV,WAAO,EAAE,GAAG,KAAK,OAAO;AAAA,EAC1B;AAAA,EAEA,MAAM,cAAc,UAA0C;AAC5D,UAAM,SAAS,MAAM,KAAK,kBAAkB,KAAK;AACjD,UAAM,SAAS,KAAK,0BAA0B,QAAQ;AACtD,UAAM,UAA0B,CAAC;AAEjC,eAAW,SAAS,QAAQ;AAC1B,YAAM,SAAS,IAAI,OAAa;AAChC,cAAQ,KAAK,MAAM;AAEnB,UAAI,MAAM,SAAS,4BAA4B;AAC7C,aAAK,kBAAkB,MAAM,KAAK,EAAE,IAAI;AAAA,MAC1C,WAAW,MAAM,QAAQ,4BAA4B;AACnD,aAAK,kBAAkB,MAAM,OAAO,IAAI;AAAA,MAC1C;AAEA,WAAK,UAAU,KAAK;AAAA,IACtB;AAEA,QAAI,QAAQ,WAAW,GAAG;AACxB,aAAO;AACP;AAAA,IACF;AAEA,QAAI;AAEF,YAAM,QAAQ,KAAK;AAAA,QACjB,QAAQ,IAAI,OAAO;AAAA,QACnB,MAAM,GAAI,EAAE,KAAK,MAAM;AACrB,gBAAM,IAAI,MAAM,kCAAkC;AAAA,QACpD,CAAC;AAAA,MACH,CAAC;AAAA,IACH,SAAS,GAAG;AACV,WAAK,QAAQ,MAAO,EAAY,OAAO;AACvC,YAAM;AAAA,IACR,UAAE;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEQ,0BACN,SACA,eAAwB,OAC2D;AACnF,UAAM,aAAa,QAAQ,KAAK;AAChC,QAAI,cAAc;AAChB,iBAAW,MAAM,KAAK,oBAAoB,CAAC;AAAA,IAC7C,OAAO;AAEL,iBAAW,QAAQ,WAAW,MAAM;AAAA,QAClC,CAAC,SAAS,CAAC,KAAK,GAAG,WAAW,oBAAoB;AAAA,MACpD;AAAA,IACF;AAEA,UAAM,SAGA,CAAC;AAEP,UAAM,UAAU,IAAI,mBAAmB,KAAK,SAAS,UAAU;AAC/D,eAAW,MAAM,QAAQ,UAAU;AACjC,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,SAAS;AAAA,QACT,UAAU,UAAU,kBAAkB;AAAA,MACxC,CAA0C;AAAA,IAC5C;AAEA,eAAW,CAAC,YAAY,EAAE,KAAK,QAAQ,UAAU;AAC/C,YAAM,WAAW,WAAW,QAAQ,EAAE;AACtC,UAAI,CAAC,UAAU;AACb,cAAM,IAAI,MAAM,aAAa,EAAE,YAAY;AAAA,MAC7C;AACA,aAAO,KAAK;AAAA,QACV,MAAM;AAAA,QACN,MAAM,wBAAwB,QAAQ;AAAA,QACtC,kBAAkB,cAAc;AAAA,QAChC,UAAU,UAAU,kBAAkB;AAAA,MACxC,CAA0C;AAAA,IAC5C;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,YAAY,QAAwC;AACxD,UAAM,SAAS,MAAM,KAAK,kBAAkB,KAAK;AACjD,UAAM,KAAK,KAAK,uBAAuB,MAAM;AAC7C,SAAK,UAAU,EAAE;AAEjB,QAAI,CAAC,GAAG,QAAQ,OAAO;AACrB,YAAM,IAAI,MAAM,+CAA+C;AAAA,IACjE;AAIA,UAAM,oBAAoB,IAAI,IAAI,GAAG,QAAQ,MAAM,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC;AAC3E,UAAM,gBAAgB,OAAO;AAAA,MAC3B,OAAO,QAAQ,MAAM,EAAE;AAAA,QACrB,CAAC,CAAC,MAAM,IAAI,MAAM,IAAI,eAAe,IAAI,KAAK,kBAAkB,IAAI,IAAI;AAAA,MAC1E;AAAA,IACF;AAEA,SAAK,SAAS;AAEd,WAAO;AAAA,EACT;AAAA,EAEQ,uBAAuB,QAAuD;AACpF,UAAM,WAA6B,CAAC;AAEpC,eAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,MAAM,GAAG;AACjD,UAAI,CAAC,IAAI,eAAe,IAAI,GAAG;AAC7B,aAAK,QAAQ,MAAM,EAAE,MAAM,KAAK,GAAG,oDAAoD;AACvF;AAAA,MACF;AAEA,YAAM,EAAE,YAAY,gBAAgB,YAAY,IAAI;AACpD,UAAI;AACF,cAAM,aAAa,IAAI;AAAA,UACrB;AAAA,QACF;AAEA,iBAAS,KAAK;AAAA,UACZ;AAAA,UACA;AAAA,UACA;AAAA,UACA,MAAM;AAAA,QACR,CAAC;AAAA,MACH,SAAS,GAAG;AACV,aAAK,QAAQ,MAAM,EAAE,MAAM,KAAK,GAAG,oDAAoD;AACvF;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,MAAM;AAAA,QACN,OAAO,KAAK,iBAAiB,SAAS;AAAA,QACtC,OAAO;AAAA,MACT;AAAA,MACA,UAAU,UAAU,eAAe;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,MAAM,mBAAmB,eAAsC;AAC7D,UAAM,UAAU,UAAU,sBAAsB;AAChD,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,QACP,MAAM;AAAA,QACN,cAAc;AAAA,MAChB;AAAA,MACA,UAAU;AAAA,IACZ,CAAiC;AACjC,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,cAAc,EAAE,WAAW,GAA0C;AACnE,UAAM,UAAmD;AAAA,MACvD,MAAM;AAAA,IACR;AAEA,SAAK,iBAAiB,SAAS,aAAa;AAC5C,YAAQ,cAAc,gBAAgB,UAAU;AAIhD,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,SAAS;AAAA,MACT,UAAU,UAAU,iBAAiB;AAAA,IACvC,CAAC;AAAA,EACH;AAAA,EAEA,UAAU,OAAyB;AACjC,eAAW,KAAK,KAAK,cAAc,KAAK,GAAG;AACzC,iBAAW,MAAM,KAAK,QAAQ,MAAM,EAAE,KAAK,MAAqB,GAAG;AACjE,aAAK,UAAU;AAAA,UACb,MAAM;AAAA,UACN,OAAO,OAAO,KAAK,GAAG,KAAK,MAAM,EAAE,SAAS,QAAQ;AAAA,QACtD,CAA0C;AAE1C,aAAK,oBAAqB,GAAG,oBAAoB,GAAG,aAAc;AAAA,MACpE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,cAA6B;AACjC,QAAI,KAAK,mBAAmB,KAAK;AAE/B,WAAK,UAAU;AAAA,QACb,MAAM;AAAA,MACR,CAA0C;AAC1C,WAAK,mBAAmB;AAAA,IAC1B;AAAA,EACF;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,IACR,CAAyC;AACzC,SAAK,mBAAmB;AAAA,EAC1B;AAAA,EAEA,MAAM,cAAc,cAA4D;AAC9E,UAAM,SAAS,KAAK,eAAe,EAAE,cAAc,eAAe,KAAK,CAAC;AACxE,SAAK,0BAA0B;AAC/B,WAAO,OAAO,QAAQ;AAAA,EACxB;AAAA,EAEA,MAAM,YAA2B;AAC/B,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,IACR,CAAkC;AAAA,EACpC;AAAA,EAEA,MAAM,SAAS,UAKG;AAGhB,UAAM,mBAAmB,CAAC,SAAS,cAAc,SAAS,WAAW,SAAS,OAAO;AACrF,UAAM,qBAAqB,KAAK,oBAAoB,IAAI,SAAS,SAAS;AAE1E,QAAI,oBAAoB,oBAAoB;AAC1C,WAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,eAAe;AAAA,QACf,SAAS,SAAS;AAAA,QAClB,cAAc,SAAS;AAAA,MACzB,CAA4C;AAAA,IAC9C,WAAW,SAAS,oBAAoB,QAAW;AAEjD,YAAM,UAAU,KAAK,QAAQ,KAAK;AAClC,YAAM,MAAM,QAAQ,UAAU,SAAS,SAAS;AAChD,UAAI,QAAQ,QAAW;AACrB,cAAM,OAAO,QAAQ,MAAM,GAAG;AAC9B,YAAI,QAAQ,KAAK,SAAS,WAAW;AACnC,gBAAM,UAAU,IAAI,YAAY,OAAO;AAAA,YACrC,GAAG;AAAA,YACH,SAAS,CAAC,SAAS,eAAe;AAAA,UACpC,CAAC;AACD,kBAAQ,MAAM,GAAG,IAAI;AACrB,gBAAM,SAAS,KAAK,0BAA0B,OAAO;AACrD,qBAAW,MAAM,QAAQ;AACvB,iBAAK,UAAU,EAAE;AAAA,UACnB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,cACN,OACyB;AACzB,UAAM,eAAwC,CAAC;AAC/C,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAI,UAAU,QAAW;AACvB,qBAAa,GAAG,IAAI;AAAA,MACtB;AAAA,IACF;AAEA,QAAI,aAAa,SAAS,OAAO,aAAa,UAAU,UAAU;AAChE,aAAO,EAAE,GAAG,cAAc,OAAO,MAAM;AAAA,IACzC;AACA,QACE,aAAa,SACb,OAAO,aAAa,UAAU,YAC9B,MAAM,SAAS,wBACf;AACA,aAAO,EAAE,GAAG,cAAc,OAAO,MAAM;AAAA,IACzC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,eAAmC;AAC/C,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,IAChB;AAEA,QAAI,KAAK,iBAAiB,SAAS,SAAS;AAI1C,UAAI,KAAK,iBAAiB,SAAS,YAAY;AAC7C,gBAAQ,gBAAgB,UAAU,KAAK,iBAAiB,SAAS,UAAU;AAAA,MAC7E,WAAW,KAAK,iBAAiB,SAAS,QAAQ;AAChD,gBAAQ,SAAS,IAAI,KAAK,iBAAiB,SAAS;AAAA,MACtD,OAAO;AACL,cAAM,IAAI,MAAM,6CAA6C;AAAA,MAC/D;AAAA,IACF,OAAO;AACL,UAAI,CAAC,KAAK,iBAAiB,SAAS,QAAQ;AAC1C,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,cAAQ,gBAAgB,UAAU,KAAK,iBAAiB,SAAS,MAAM;AAAA,IACzE;AAEA,UAAM,MAAM,eAAe;AAAA,MACzB,SAAS,KAAK,iBAAiB,SAAS;AAAA,MACxC,OAAO,KAAK,iBAAiB,SAAS;AAAA,MACtC,SAAS,KAAK,iBAAiB,SAAS;AAAA,MACxC,YAAY,KAAK,iBAAiB,SAAS;AAAA,MAC3C,iBAAiB,KAAK,iBAAiB,SAAS;AAAA,IAClD,CAAC;AAED,QAAI,YAAY;AACd,WAAK,QAAQ,MAAM,wCAAwC,GAAG,EAAE;AAAA,IAClE;AAEA,WAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,YAAM,KAAK,IAAI,UAAU,KAAK,EAAE,QAAQ,CAAC;AACzC,UAAI,UAAU;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,WAAG,MAAM;AACT,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,iBAAiB,SAAS,YAAY,SAAS;AAEvD,SAAG,KAAK,QAAQ,MAAM;AACpB,YAAI,CAAC,QAAS;AACd,kBAAU;AACV,qBAAa,OAAO;AACpB,gBAAQ,EAAE;AAAA,MACZ,CAAC;AAED,SAAG,KAAK,SAAS,MAAM;AACrB,YAAI,CAAC,QAAS;AACd,kBAAU;AACV,qBAAa,OAAO;AACpB,eAAO,IAAI,MAAM,uCAAuC,CAAC;AAAA,MAC3D,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,QAAoC;AAClD,QAAI,eAAe;AACnB,QAAI,aAAa;AACjB,QAAI,SAA2B;AAC/B,UAAM,aAAa,KAAK,iBAAiB,SAAS,YAAY;AAE9D,UAAM,YAAY,YAAY;AAC5B,WAAK,QAAQ;AAAA,QACX;AAAA,UACE,oBAAoB,KAAK,iBAAiB,SAAS;AAAA,QACrD;AAAA,QACA;AAAA,MACF;AAGA,iBAAW,OAAO,OAAO,OAAO,KAAK,iBAAiB,GAAG;AACvD,YAAI,CAAC,IAAI,KAAM,KAAI,OAAO,IAAI,MAAM,qBAAqB,CAAC;AAAA,MAC5D;AACA,WAAK,oBAAoB,CAAC;AAE1B,iBAAW,OAAO,OAAO,OAAO,KAAK,iBAAiB,GAAG;AACvD,YAAI,CAAC,IAAI,KAAM,KAAI,OAAO,IAAI,MAAM,qBAAqB,CAAC;AAAA,MAC5D;AACA,WAAK,oBAAoB,CAAC;AAG1B,WAAK,oBAAoB,MAAM;AAE/B,YAAM,SAAkC,CAAC;AAGzC,aAAO,KAAK,KAAK,yBAAyB,CAAC;AAG3C,UAAI,OAAO,KAAK,KAAK,MAAM,EAAE,SAAS,GAAG;AACvC,eAAO,KAAK,KAAK,uBAAuB,KAAK,MAAM,CAAC;AAAA,MACtD;AAGA,YAAM,UAAU,KAAK,QAAQ,KAAK;AAAA,QAChC,qBAAqB;AAAA,QACrB,qBAAqB;AAAA,QACrB,qBAAqB;AAAA,MACvB,CAAC;AAED,YAAM,aAAa,KAAK;AACxB,WAAK,gBAAgB,IAAI,IAAI,kBAAkB;AAC/C,aAAO,KAAK,GAAG,KAAK,0BAA0B,OAAO,CAAC;AAEtD,UAAI;AACF,mBAAW,MAAM,QAAQ;AACvB,eAAK,KAAK,8BAA8B,EAAE;AAC1C,iBAAQ,KAAK,KAAK,UAAU,EAAE,CAAC;AAAA,QACjC;AAAA,MACF,SAAS,OAAO;AACd,aAAK,gBAAgB;AACrB,cAAM,IAAI,mBAAmB;AAAA,UAC3B,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AAEA,WAAK,QAAQ,MAAM,oCAAoC;AAEvD,WAAK,KAAK,uBAAuB,CAAC,CAAwC;AAAA,IAC5E;AAEA,mBAAe;AACf,WAAO,CAAC,KAAK,WAAW,CAAC,OAAO,SAAS;AACvC,WAAK,QAAQ,MAAM,sDAAsD;AACzE,eAAS,MAAM,KAAK,aAAa;AACjC,UAAI,OAAO,QAAS;AAEpB,UAAI;AACF,YAAI,cAAc;AAChB,gBAAM,UAAU;AAChB,cAAI,OAAO,QAAS;AACpB,uBAAa;AAAA,QACf;AAEA,cAAM,KAAK,MAAM,MAAM;AACvB,YAAI,OAAO,QAAS;AAAA,MACtB,SAAS,OAAO;AACd,YAAI,CAAC,WAAW,KAAK,GAAG;AACtB,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAEA,YAAI,eAAe,KAAK,CAAC,MAAM,WAAW;AACxC,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAEA,YAAI,eAAe,YAAY;AAC7B,eAAK,UAAU,EAAE,OAAuB,aAAa,MAAM,CAAC;AAC5D,gBAAM,IAAI,mBAAmB;AAAA,YAC3B,SAAS,+CAA+C,UAAU;AAAA,YAClE,SAAS;AAAA,cACP,MAAM;AAAA,cACN,WAAW;AAAA,YACb;AAAA,UACF,CAAC;AAAA,QACH;AAEA,aAAK,UAAU,EAAE,OAAuB,aAAa,KAAK,CAAC;AAC3D,cAAM,gBACJ,eAAe,IACX,kCACA,KAAK,iBAAiB,SAAS,YAAY;AACjD,aAAK,QAAQ;AAAA,UACX;AAAA,YACE,SAAS;AAAA,YACT;AAAA,YACA;AAAA,UACF;AAAA,UACA,sDAAsD,gBAAgB,GAAI;AAAA,QAC5E;AAEA,cAAM,MAAM,aAAa;AACzB;AAAA,MACF;AAEA,qBAAe;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,MAAc,MAAM,QAAkC;AACpD,UAAM,gBAAgB,OAAO,WAAuC;AAClE,YAAM,cAAc,IAAI,OAAa;AACrC,aAAO,iBAAiB,SAAS,MAAM,YAAY,QAAQ,CAAC;AAE5D,aAAO,CAAC,KAAK,WAAW,OAAO,eAAe,UAAU,QAAQ,CAAC,OAAO,SAAS;AAC/E,YAAI;AACF,gBAAM,QAAQ,MAAM,QAAQ,KAAK,CAAC,KAAK,eAAe,IAAI,GAAG,YAAY,KAAK,CAAC;AAC/E,cAAI,OAAO,WAAW,YAAY,QAAQ,UAAU,QAAW;AAC7D;AAAA,UACF;AAEA,cAAI,YAAY;AACd,iBAAK,QAAQ,MAAM,KAAK,cAAc,KAAK,GAAG,eAAe,MAAM,IAAI,EAAE;AAAA,UAC3E;AAEA,eAAK,KAAK,8BAA8B,KAAK;AAC7C,iBAAO,KAAK,KAAK,UAAU,KAAK,CAAC;AAAA,QACnC,SAAS,OAAO;AACd;AAAA,QACF;AAAA,MACF;AAEA,aAAO,MAAM;AAAA,IACf;AAEA,UAAM,gBAAgB,IAAI,OAAqB;AAE/C,WAAO,UAAU,CAAC,UAAU;AAC1B,oBAAc,QAAQ,IAAI,mBAAmB,EAAE,SAAS,MAAM,QAAQ,CAAC,CAAC;AAAA,IAC1E;AACA,WAAO,UAAU,MAAM;AACrB,oBAAc,QAAQ;AAAA,IACxB;AAEA,WAAO,YAAY,CAAC,YAA0B;AAE5C,YAAM,QAAa,KAAK,MAAM,QAAQ,IAAc;AAEpD,WAAK,KAAK,gCAAgC,KAAK;AAC/C,UAAI,YAAY;AACd,aAAK,QAAQ,MAAM,KAAK,cAAc,KAAK,GAAG,eAAe,MAAM,IAAI,EAAE;AAAA,MAC3E;AAEA,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH,eAAK,oCAAoC,KAAK;AAC9C;AAAA,QACF,KAAK;AACH,eAAK,oCAAoC,KAAK;AAC9C;AAAA,QACF,KAAK;AACH,eAAK,sBAAsB,KAAK;AAChC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AACH,eAAK,uDAAuD,KAAK;AACjE;AAAA,QACF,KAAK;AACH,eAAK,oDAAoD,KAAK;AAC9D;AAAA,QACF,KAAK;AACH,eAAK,+BAA+B,KAAK;AACzC;AAAA,QACF,KAAK;AACH,eAAK,8BAA8B,KAAK;AACxC;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,wBAAwB,KAAK;AAClC;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,uBAAuB,KAAK;AACjC;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,mCAAmC,KAAK;AAC7C;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,yBAAyB,KAAK;AACnC;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,kCAAkC,KAAK;AAC5C;AAAA,QACF,KAAK;AAAA,QACL,KAAK;AACH,eAAK,wBAAwB,KAAK;AAClC;AAAA,QACF,KAAK;AACH,eAAK,6BAA6B,KAAK;AACvC;AAAA,QACF,KAAK;AACH,eAAK,mBAAmB,KAAK;AAC7B;AAAA,QACF,KAAK;AACH,eAAK,YAAY,KAAK;AACtB;AAAA,QACF;AACE,cAAI,YAAY;AACd,iBAAK,QAAQ,MAAM,oBAAoB,MAAM,IAAI,EAAE;AAAA,UACrD;AACA;AAAA,MACJ;AAAA,IACF;AAEA,UAAM,WAAW,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM,cAAc,MAAM,CAAC;AAEhE,UAAM,SAAS,KAAK,KAAK,CAAC,EAAE,OAAO,MAAM;AACvC,YAAM,eAAe,IAAI,QAAc,CAAC,YAAY;AAClD,eAAO,iBAAiB,SAAS,MAAM;AACrC,kBAAQ;AAAA,QACV,CAAC;AAAA,MACH,CAAC;AAED,aAAO,QAAQ,KAAK,CAAC,cAAc,OAAO,YAAY,CAAC;AAAA,IACzD,CAAC;AAED,UAAM,oBAAoB,KAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AACxD,YAAM,MAAM,KAAK,iBAAiB,SAAS,oBAAoB,EAAE,OAAO,CAAC;AACzE,aAAO,IAAI,mBAAmB;AAAA,QAC5B,SAAS;AAAA,MACX,CAAC;AAAA,IACH,CAAC;AAED,QAAI;AACF,YAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,OAAO,QAAQ,SAAS,QAAQ,kBAAkB,MAAM,CAAC;AAE5F,UAAI,kBAAkB,QAAQ,KAAK,mBAAmB;AACpD,cAAM,KAAK,kBAAkB,SAAS;AAAA,MACxC;AAEA,UAAI,kBAAkB,OAAO;AAC3B,cAAM;AAAA,MACR;AAAA,IACF,UAAE;AACA,YAAM,cAAc,CAAC,QAAQ,UAAU,iBAAiB,GAAG,GAAI;AAC/D,aAAO,MAAM;AAAA,IACf;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,MAAM;AACZ,SAAK,UAAU;AACf,UAAM,KAAK;AAGX,eAAW,UAAU,OAAO,OAAO,KAAK,sBAAsB,GAAG;AAC/D,UAAI,CAAC,OAAO,QAAQ,MAAM;AACxB,eAAO,QAAQ,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,MACnD;AAAA,IACF;AACA,SAAK,yBAAyB,CAAC;AAE/B,eAAW,OAAO,OAAO,OAAO,KAAK,iBAAiB,GAAG;AACvD,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,MACxC;AAAA,IACF;AACA,SAAK,oBAAoB,CAAC;AAE1B,eAAW,OAAO,OAAO,OAAO,KAAK,iBAAiB,GAAG;AACvD,UAAI,CAAC,IAAI,MAAM;AACb,YAAI,OAAO,IAAI,MAAM,gBAAgB,CAAC;AAAA,MACxC;AAAA,IACF;AACA,SAAK,oBAAoB,CAAC;AAG1B,QAAI,KAAK,mBAAmB;AAC1B,iBAAW,OAAO,KAAK,kBAAkB,SAAS,OAAO,GAAG;AAC1D,YAAI,YAAY,MAAM;AACtB,YAAI,aAAa,MAAM;AACvB,YAAI,CAAC,IAAI,WAAW,MAAM;AACxB,cAAI,WAAW,QAAQ,KAAK,iBAAiB,SAAS,UAAU;AAAA,QAClE;AAAA,MACF;AACA,WAAK,kBAAkB,SAAS,MAAM;AACtC,WAAK,kBAAkB,eAAe,MAAM;AAC5C,WAAK,kBAAkB,gBAAgB,MAAM;AAC7C,UAAI,CAAC,KAAK,kBAAkB,SAAS,MAAM;AACzC,aAAK,kBAAkB,SAAS,QAAQ;AAAA,MAC1C;AACA,WAAK,oBAAoB;AAAA,IAC3B;AAGA,SAAK,eAAe,MAAM,SAAS;AAAA,EACrC;AAAA,EAEQ,oCACN,QACM;AACN,SAAK,KAAK,wBAAwB,CAAC,CAAgC;AAAA,EACrE;AAAA,EAEQ,oCACN,QACM;AACN,SAAK,KAAK,wBAAwB;AAAA,MAChC,0BAA0B,KAAK,iBAAiB,SAAS,4BAA4B;AAAA,IACvF,CAAgC;AAAA,EAClC;AAAA,EAEQ,sBAAsB,OAA6C;AAnnC7E;AAonCI,QAAI,CAAC,MAAM,SAAS,IAAI;AACtB,YAAM,IAAI,MAAM,wBAAwB;AAAA,IAC1C;AAEA,SAAK,oBAAoB;AAAA,MACvB,gBAAgB,OAAO,oBAA2C;AAAA,MAClE,iBAAiB,OAAO,oBAAsC;AAAA,MAC9D,UAAU,oBAAI,IAAI;AAAA,MAClB,UAAU,IAAI,OAAO;AAAA,MACrB,mBAAmB,KAAK,IAAI;AAAA,IAC9B;AAIA,UAAM,eAAe;AAAA,MACnB,eAAe,KAAK,kBAAkB,eAAe,OAAO;AAAA,MAC5D,gBAAgB,KAAK,kBAAkB,gBAAgB,OAAO;AAAA,MAC9D,eAAe;AAAA,MACf,YAAY,MAAM,SAAS;AAAA,IAC7B;AAEA,UAAM,iBAAgB,WAAM,SAAS,aAAf,mBAAyB;AAC/C,QAAI,eAAe;AACjB,YAAM,SAAS,KAAK,uBAAuB,aAAa;AACxD,UAAI,QAAQ;AACV,eAAO,KAAK,uBAAuB,aAAa;AAChD,qBAAa,gBAAgB;AAC7B,YAAI,CAAC,OAAO,QAAQ,MAAM;AACxB,iBAAO,QAAQ,QAAQ,YAAY;AAAA,QACrC;AAAA,MACF;AAAA,IACF;AAEA,SAAK,KAAK,sBAAsB,YAAY;AAAA,EAC9C;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,QAAI,CAAC,MAAM,KAAK,MAAM;AACpB,YAAM,IAAI,MAAM,sBAAsB;AAAA,IACxC;AAEA,UAAM,WAAW,MAAM,KAAK;AAE5B,QAAI,aAAa,WAAW;AAG1B,WAAK,0BAA0B;AAC/B;AAAA,IACF;AAEA,UAAM,SAAS,MAAM,KAAK;AAC1B,QAAI,CAAC,QAAQ;AACX,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,UAAM,gBAAgB,IAAI,OAAmB;AAC7C,UAAM,iBAAoC;AAAA,MACxC,WAAW;AAAA,MACX,aAAa,OAAO,oBAA0C;AAAA,MAC9D,cAAc,OAAO,oBAAgC;AAAA,MACrD,iBAAiB;AAAA,MACjB,YAAY;AAAA,IACd;AAGA,QAAI,CAAC,KAAK,iBAAiB,aAAa,aAAa;AACnD,qBAAe,aAAa,MAAM;AAClC,oBAAc,QAAQ,CAAC,MAAM,CAAC;AAAA,IAChC;AAEA,SAAK,kBAAkB,eAAe,MAAM;AAAA,MAC1C,WAAW;AAAA,MACX,YAAY,eAAe,YAAY,OAAO;AAAA,MAC9C,aAAa,eAAe,aAAa,OAAO;AAAA,MAChD,YAAY,cAAc;AAAA,IAC5B,CAAC;AAED,SAAK,kBAAkB,SAAS,IAAI,QAAQ,cAAc;AAAA,EAC5D;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,KAAK,IAAI;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAEA,QAAI;AACF,WAAK,cAAc,OAAO,MAAM,kBAAkB,wBAAwB,MAAM,IAAI,CAAC;AAAA,IACvF,SAAS,OAAO;AACd,WAAK,QAAQ,MAAM,EAAE,OAAO,QAAQ,MAAM,KAAK,GAAG,GAAG,oCAAoC;AAAA,IAC3F;AAEA,UAAM,MAAM,KAAK,kBAAkB,MAAM,KAAK,EAAE;AAChD,QAAI,KAAK;AACP,UAAI,QAAQ;AACZ,aAAO,KAAK,kBAAkB,MAAM,KAAK,EAAE;AAAA,IAC7C;AAAA,EACF;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,SAAS;AAClB,YAAM,IAAI,MAAM,oBAAoB;AAAA,IACtC;AAGA,SAAK,oBAAoB,OAAO,MAAM,OAAO;AAE7C,QAAI;AACF,WAAK,cAAc,OAAO,MAAM,OAAO;AAAA,IACzC,SAAS,OAAO;AACd,WAAK,QAAQ,MAAM,EAAE,OAAO,QAAQ,MAAM,QAAQ,GAAG,oCAAoC;AAAA,IAC3F;AAEA,UAAM,MAAM,KAAK,kBAAkB,MAAM,OAAO;AAChD,QAAI,KAAK;AACP,UAAI,QAAQ;AACZ,aAAO,KAAK,kBAAkB,MAAM,OAAO;AAAA,IAC7C;AAAA,EACF;AAAA,EAEQ,uDACN,OACM;AACN,UAAM,aAAa,KAAK,cAAc,IAAI,MAAM,OAAO;AACvD,QAAI,CAAC,YAAY;AACf;AAAA,IACF;AAEA,UAAM,OAAO,WAAW;AACxB,QAAI,gBAAgB,IAAI,aAAa;AACnC,WAAK,QAAQ,KAAK,MAAM,UAAU;AAAA,IACpC,OAAO;AACL,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,SAAK,KAAK,uCAAuC;AAAA,MAC/C,QAAQ,MAAM;AAAA,MACd,YAAY,MAAM;AAAA,MAClB,SAAS;AAAA,IACX,CAAoC;AAAA,EACtC;AAAA,EAEQ,oDACN,OACM;AACN,SAAK,QAAQ;AAAA,MACX,EAAE,OAAO,MAAM,MAAM;AAAA,MACrB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,+BAA+B,OAAsD;AAC3F,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM;AACrB,UAAM,WAAW,MAAM,KAAK;AAE5B,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,QAAI,CAAC,gBAAgB;AACnB,WAAK,QAAQ,KAAK,uCAAuC,MAAM,EAAE;AACjE;AAAA,IACF;AAEA,UAAM,aAAa,aAAa,UAAU,aAAa;AACvD,QAAI,cAAc,KAAK,iBAAiB,aAAa,aAAa;AAChE,WAAK,QAAQ,KAAK,oEAAoE;AAAA,IACxF;AAEA,QAAI,CAAC,eAAe,WAAW,MAAM;AACnC,YAAM,iBAA6B,aAAa,CAAC,MAAM,IAAI,CAAC,SAAS,MAAM;AAC3E,qBAAe,WAAW,QAAQ,cAAc;AAGhD,UAAI,CAAC,YAAY;AACf,aAAK,oBAAoB,IAAI,MAAM;AAAA,MACrC;AAAA,IACF;AAEA,QAAI,KAAK,kBAAkB,yBAAyB,QAAW;AAC7D,WAAK,kBAAkB,uBAAuB,KAAK,IAAI;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,8BAA8B,OAAqD;AACzF,QAAI,CAAC,MAAM,MAAM;AACf;AAAA,IACF;AACA,QAAI,MAAM,KAAK,SAAS,QAAQ;AAC9B;AAAA,IACF;AAEA,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EAGF;AAAA,EAEQ,wBAAwB,OAA+C;AAC7E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM,OAAO;AACxE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AAEA,QACE,CAAC,KAAK,iBAAiB,aAAa,eACpC,CAAC,KAAK,kBAAkB,sBACxB;AACA,WAAK,kBAAkB,uBAAuB,KAAK,IAAI;AAAA,IACzD;AAEA,mBAAe,YAAY,MAAM,MAAM,KAAK;AAC5C,mBAAe,mBAAmB,MAAM;AAAA,EAC1C;AAAA,EAEQ,uBAAuB,QAA+C;AAC5E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,mCACN,OACM;AACN,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM;AAGrB,QAAI,QAA8B,MAAM;AACxC,QAAI,MAAM,eAAe,QAAW;AAClC,cAAQ,kBAAkB,EAAE,MAAM,MAAM,OAAO,WAAW,MAAM,WAAW,CAAC;AAAA,IAC9E;AAEA,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C,OAAO;AACL,qBAAe,YAAY,MAAM,KAAK;AACtC,qBAAe,mBAAmB,MAAM;AAAA,IAC1C;AAAA,EACF;AAAA,EAEQ,yBAAyB,OAAgD;AAC/E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM,OAAO;AACxE,QAAI,CAAC,gBAAgB;AACnB,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AAEA,QAAI,KAAK,kBAAkB,yBAAyB,QAAW;AAC7D,WAAK,kBAAkB,uBAAuB,KAAK,IAAI;AAAA,IACzD;AAEA,QAAI,CAAC,eAAe,WAAW,MAAM;AACnC,qBAAe,WAAW,QAAQ,CAAC,SAAS,MAAM,CAAC;AAAA,IACrD;AAEA,UAAM,eAAe,KAAK,MAAM,KAAK;AACrC,UAAM,MAAM,aAAa;AACzB,UAAM,QAAQ,IAAI,WAAW,GAAG;AAChC,aAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,YAAM,CAAC,IAAI,aAAa,WAAW,CAAC;AAAA,IACtC;AAEA,mBAAe,aAAa;AAAA,MAC1B,IAAI;AAAA,QACF,IAAI,WAAW,MAAM,MAAM;AAAA,QAC3B,UAAU;AAAA,QACV,UAAU;AAAA,QACV,MAAM,SAAS;AAAA,MACjB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,kCACN,QACM;AACN,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,wBAAwB,QAAgD;AAC9E,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAAA,EACF;AAAA,EAEQ,6BAA6B,OAAoD;AACvF,QAAI,CAAC,KAAK,mBAAmB;AAC3B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,UAAM,SAAS,MAAM,KAAK;AAC1B,UAAM,WAAW,MAAM,KAAK;AAE5B,QAAI,aAAa,iBAAiB;AAChC,YAAM,OAAO,MAAM;AACnB,UAAI,CAAC,KAAK,WAAW,CAAC,KAAK,QAAQ,CAAC,KAAK,WAAW;AAClD,cAAM,IAAI,MAAM,6BAA6B;AAAA,MAC/C;AACA,WAAK,kBAAkB,gBAAgB;AAAA,QACrC,IAAI,aAAa,OAAO;AAAA,UACtB,QAAQ,KAAK;AAAA,UACb,MAAM,KAAK;AAAA,UACX,MAAM,KAAK;AAAA,QACb,CAAC;AAAA,MACH;AAAA,IACF,WAAW,aAAa,WAAW;AACjC,YAAM,iBAAiB,KAAK,kBAAkB,SAAS,IAAI,MAAM;AACjE,UAAI,CAAC,gBAAgB;AACnB;AAAA,MACF;AAEA,qBAAe,YAAY,MAAM;AACjC,qBAAe,aAAa,MAAM;AAClC,UAAI,CAAC,eAAe,WAAW,MAAM;AAEnC,uBAAe,WAAW,QAAQ,KAAK,iBAAiB,SAAS,UAAU;AAAA,MAC7E;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,mBAAmB,QAA2C;AAt8CxE;AAu8CI,QAAI,CAAC,KAAK,mBAAmB;AAG3B;AAAA,IACF;AAEA,UAAM,mBAAmB,KAAK,kBAAkB;AAChD,UAAM,sBAAsB,KAAK,kBAAkB;AAEnD,SAAK,QAAQ;AAAA,MACX;AAAA,QACE,cAAc,KAAK,kBAAkB,SAAS;AAAA,MAChD;AAAA,MACA;AAAA,IACF;AAEA,eAAW,cAAc,KAAK,kBAAkB,SAAS,OAAO,GAAG;AACjE,iBAAW,YAAY,MAAM;AAC7B,iBAAW,aAAa,MAAM;AAC9B,UAAI,CAAC,WAAW,WAAW,MAAM;AAC/B,mBAAW,WAAW,QAAQ,KAAK,iBAAiB,SAAS,UAAU;AAAA,MACzE;AAAA,IACF;AAEA,SAAK,kBAAkB,gBAAgB,MAAM;AAC7C,SAAK,kBAAkB,eAAe,MAAM;AAE5C,eAAW,UAAU,KAAK,kBAAkB,SAAS,KAAK,GAAG;AAC3D,YAAM,aAAa,KAAK,cAAc,IAAI,MAAM;AAChD,UAAI,cAAc,WAAW,gBAAgB,IAAI,aAAa;AAC5D,mBAAW,KAAK,QAAQ,KAAK,KAAK,kBAAkB,SAAS,IAAI,MAAM,EAAG,eAAe;AAAA,MAC3F;AAAA,IACF;AAEA,SAAK,kBAAkB,SAAS,QAAQ;AACxC,SAAK,oBAAoB;AAGzB,UAAM,QAAQ,OAAO,SAAS;AAC9B,UAAM,SAAS,sBAAsB,sBAAsB,mBAAmB;AAC9E,UAAM,aAAa,KAAK,IAAI,IAAI;AAEhC,UAAM,kBAAgD;AAAA,MACpD,MAAM;AAAA,MACN,WAAW;AAAA,MACX,WAAW,OAAO,SAAS,MAAM;AAAA,MACjC;AAAA,MACA;AAAA,MACA,WAAW,OAAO,SAAS,WAAW;AAAA,MACtC,OAAO;AAAA,MACP,cAAa,+BAAO,iBAAgB;AAAA,MACpC,eAAc,+BAAO,kBAAiB;AAAA,MACtC,cAAa,+BAAO,iBAAgB;AAAA,MACpC,iBAAiB,aAAa,MAAK,+BAAO,kBAAiB,MAAM,aAAa,OAAQ;AAAA,MACtF,mBAAmB;AAAA,QACjB,eAAa,oCAAO,wBAAP,mBAA4B,iBAAgB;AAAA,QACzD,cAAY,oCAAO,wBAAP,mBAA4B,gBAAe;AAAA,QACvD,aAAa;AAAA;AAAA,QACb,gBAAc,oCAAO,wBAAP,mBAA4B,kBAAiB;AAAA,QAC3D,uBAAqB,oCAAO,wBAAP,mBAA4B,yBAC7C;AAAA,UACE,eAAa,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,iBAAgB;AAAA,UAChF,cAAY,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,gBAAe;AAAA,UAC9E,eAAa,0CAAO,wBAAP,mBAA4B,0BAA5B,mBAAmD,iBAAgB;AAAA,QAClF,IACA;AAAA,MACN;AAAA,MACA,oBAAoB;AAAA,QAClB,cAAY,oCAAO,yBAAP,mBAA6B,gBAAe;AAAA,QACxD,eAAa,oCAAO,yBAAP,mBAA6B,iBAAgB;AAAA,QAC1D,aAAa;AAAA,MACf;AAAA,IACF;AAEA,SAAK,KAAK,qBAAqB,eAAe;AAAA,EAEhD;AAAA,EAEQ,YAAY,OAAmC;AACrD,QAAI,MAAM,MAAM,QAAQ,WAAW,qBAAqB,GAAG;AACzD;AAAA,IACF;AAEA,SAAK,QAAQ,MAAM,EAAE,OAAO,MAAM,MAAM,GAAG,uCAAuC;AAClF,SAAK,UAAU;AAAA,MACb,OAAO,IAAI,SAAS,MAAM,MAAM,SAAS;AAAA,QACvC,MAAM,MAAM;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AAAA,MACD,aAAa;AAAA,IACf,CAAC;AAAA,EAGH;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAAiD;AAEtF,SAAK,KAAK,SAAS;AAAA,MACjB,WAAW,KAAK,IAAI;AAAA;AAAA,MAEpB,OAAO;AAAA,MACP;AAAA,MACA;AAAA,IACF,CAA2B;AAAA,EAC7B;AAAA,EAEA,CAAS,cAAc,OAA0C;AAC/D,UAAM;AAAA,EACR;AAAA,EAEQ,eAAe;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAIyB;AACvB,UAAM,SAAS,aAAa,IAAI,qBAAqB,EAAE,aAAa,CAAC;AACrE,QAAI,aAAa,cAAc;AAC7B,aAAO,eAAe;AAAA,IACxB;AAEA,UAAM,UAAU,UAAU,kBAAkB;AAC5C,QAAI,eAAe;AACjB,WAAK,uBAAuB,OAAO,IAAI;AAAA,IACzC;AAEA,UAAM,WAAsD,CAAC;AAC7D,QAAI,aAAc,UAAS,eAAe;AAC1C,QAAI,cAAe,UAAS,WAAW,EAAE,iBAAiB,QAAQ;AAElE,SAAK,UAAU;AAAA,MACb,MAAM;AAAA,MACN,UAAU;AAAA,MACV,UAAU,OAAO,KAAK,QAAQ,EAAE,SAAS,IAAI,WAAW;AAAA,IAC1D,CAAC;AAED,WAAO;AAAA,EACT;AACF;AAGO,SAAS,wBAAwB,MAA4C;AAClF,UAAQ,KAAK,MAAM;AAAA,IACjB,KAAK;AACH,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN,SAAS,KAAK;AAAA,QACd,MAAM,KAAK;AAAA,QACX,WAAW,KAAK;AAAA,MAClB;AAAA,IACF,KAAK;AACH,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN,SAAS,KAAK;AAAA,QACd,QAAQ,KAAK;AAAA,MACf;AAAA,IACF,KAAK;AACH,YAAM,OAAO,KAAK,SAAS,cAAc,WAAW,KAAK;AACzD,YAAM,cAAmC,CAAC;AAC1C,iBAAW,KAAK,KAAK,SAAS;AAC5B,YAAI,OAAO,MAAM,UAAU;AACzB,sBAAY,KAAK;AAAA,YACf,MAAM,SAAS,cAAc,gBAAgB;AAAA,YAC7C,MAAM;AAAA,UACR,CAA6D;AAAA,QAC/D,WAAW,EAAE,SAAS,iBAAiB;AAErC;AAAA,QACF,WAAW,EAAE,SAAS,iBAAiB;AACrC,cAAI,SAAS,QAAQ;AACnB,kBAAM,eAAe,OAAO,KAAK,mBAAmB,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,QAAQ;AACpF,wBAAY,KAAK;AAAA,cACf,MAAM;AAAA,cACN,OAAO;AAAA,YACT,CAAgC;AAAA,UAClC;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,QACL,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN;AAAA,QACA,SAAS;AAAA,MACX;AAAA,IACF;AACE,YAAM,IAAI,MAAM,0BAA2B,KAAa,IAAI,EAAE;AAAA,EAClE;AACF;AAEA,SAAS,wBAAwB,MAA4C;AAC3E,MAAI,CAAC,KAAK,IAAI;AACZ,UAAM,IAAI,MAAM,oBAAoB;AAAA,EACtC;AAEA,UAAQ,KAAK,MAAM;AAAA,IACjB,KAAK;AACH,aAAO,IAAI,aAAa,OAAO;AAAA,QAC7B,IAAI,KAAK;AAAA,QACT,QAAQ,KAAK;AAAA,QACb,MAAM,KAAK;AAAA,QACX,MAAM,KAAK;AAAA,MACb,CAAC;AAAA,IACH,KAAK;AACH,aAAO,IAAI,mBAAmB,OAAO;AAAA,QACnC,IAAI,KAAK;AAAA,QACT,QAAQ,KAAK;AAAA,QACb,QAAQ,KAAK;AAAA,QACb,SAAS;AAAA,MACX,CAAC;AAAA,IACH,KAAK;AACH,YAAM,UAA6B,CAAC;AAEpC,YAAM,WAAW,MAAM,QAAQ,KAAK,OAAO,IAAI,KAAK,UAAU,CAAC,KAAK,OAAO;AAC3E,iBAAW,KAAK,UAAU;AACxB,YAAI,EAAE,SAAS,UAAU,EAAE,SAAS,cAAc;AAChD,kBAAQ,KAAK,EAAE,IAAI;AAAA,QACrB;AAAA,MACF;AACA,aAAO,IAAI,YAAY,OAAO;AAAA,QAC5B,IAAI,KAAK;AAAA,QACT,MAAM,KAAK;AAAA,QACX;AAAA,MACF,CAAC;AAAA,EACL;AACF;AAEA,SAAS,oBAAoB,kBAA0B,GAAoB;AACzE,QAAM,YAAY,OAAO,MAAM,kBAAkB,WAAW;AAC5D,SAAO,IAAI,YAAY,OAAO;AAAA,IAC5B,IAAI,UAAU,oBAAoB;AAAA,IAClC,MAAM;AAAA,IACN,SAAS;AAAA,MACP;AAAA,QACE,MAAM;AAAA,QACN,OAAO;AAAA,UACL,IAAI;AAAA,YACF,IAAI,WAAW,UAAU,MAAM;AAAA,YAC/B;AAAA,YACA;AAAA,YACA,UAAU,SAAS;AAAA,UACrB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,SAAS,gBAAgB,YAAmD;AAC1E,MAAI,OAAO,eAAe,UAAU;AAClC,WAAO;AAAA,EACT;AAEA,OAAI,yCAAY,UAAS,YAAY;AACnC,WAAO,WAAW,SAAS;AAAA,EAC7B;AAEA,SAAO;AACT;","names":[]}
|
|
@@ -963,13 +963,16 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
|
|
|
963
963
|
throw new Error("currentGeneration is not set");
|
|
964
964
|
}
|
|
965
965
|
const itemId = event.item_id;
|
|
966
|
-
|
|
966
|
+
let delta = event.delta;
|
|
967
|
+
if (event.start_time !== void 0) {
|
|
968
|
+
delta = (0, import_agents.createTimedString)({ text: event.delta, startTime: event.start_time });
|
|
969
|
+
}
|
|
967
970
|
const itemGeneration = this.currentGeneration.messages.get(itemId);
|
|
968
971
|
if (!itemGeneration) {
|
|
969
972
|
throw new Error("itemGeneration is not set");
|
|
970
973
|
} else {
|
|
971
974
|
itemGeneration.textChannel.write(delta);
|
|
972
|
-
itemGeneration.audioTranscript += delta;
|
|
975
|
+
itemGeneration.audioTranscript += event.delta;
|
|
973
976
|
}
|
|
974
977
|
}
|
|
975
978
|
handleResponseAudioDelta(event) {
|