@livekit/agents-plugin-openai 0.9.2 → 1.0.0-next.0

package/dist/realtime/realtime_model.js

@@ -1,380 +1,455 @@
 import {
-  AsyncIterableQueue,
+  APIConnectionError,
+  APIError,
+  AudioByteStream,
+  DEFAULT_API_CONNECT_OPTIONS,
   Future,
   Queue,
+  Task,
+  cancelAndWait,
+  isAPIError,
   llm,
   log,
-  mergeFrames,
-  metrics,
-  multimodal
+  shortuuid,
+  stream
 } from "@livekit/agents";
-import { AudioFrame } from "@livekit/rtc-node";
-import { once } from "node:events";
+import { Mutex } from "@livekit/mutex";
+import { AudioFrame, combineAudioFrames } from "@livekit/rtc-node";
+import { delay } from "@std/async";
 import { WebSocket } from "ws";
 import * as api_proto from "./api_proto.js";
-class InputAudioBuffer {
-  #session;
-  constructor(session) {
-    this.#session = session;
-  }
-  append(frame) {
-    this.#session.queueMsg({
-      type: "input_audio_buffer.append",
-      audio: Buffer.from(frame.data.buffer).toString("base64")
-    });
-  }
-  clear() {
-    this.#session.queueMsg({
-      type: "input_audio_buffer.clear"
-    });
-  }
-  commit() {
-    this.#session.queueMsg({
-      type: "input_audio_buffer.commit"
-    });
+const SAMPLE_RATE = 24e3;
+const NUM_CHANNELS = 1;
+const BASE_URL = "https://api.openai.com/v1";
+const MOCK_AUDIO_ID_PREFIX = "lk_mock_audio_item_";
+class CreateResponseHandle {
+  instructions;
+  doneFut;
+  // TODO(shubhra): add timeout
+  constructor({ instructions }) {
+    this.instructions = instructions;
+    this.doneFut = new Future();
   }
 }
-class ConversationItem {
-  #session;
-  #logger = log();
-  constructor(session) {
-    this.#session = session;
-  }
-  truncate(itemId, contentIndex, audioEnd) {
-    this.#session.queueMsg({
-      type: "conversation.item.truncate",
-      item_id: itemId,
-      content_index: contentIndex,
-      audio_end_ms: audioEnd
-    });
-  }
-  delete(itemId) {
-    this.#session.queueMsg({
-      type: "conversation.item.delete",
-      item_id: itemId
+const DEFAULT_FIRST_RETRY_INTERVAL_MS = 100;
+const DEFAULT_TEMPERATURE = 0.8;
+const DEFAULT_TURN_DETECTION = {
+  type: "server_vad",
+  threshold: 0.5,
+  prefix_padding_ms: 300,
+  silence_duration_ms: 200,
+  create_response: true,
+  interrupt_response: true
+};
+const DEFAULT_INPUT_AUDIO_TRANSCRIPTION = {
+  model: "gpt-4o-mini-transcribe"
+};
+const DEFAULT_TOOL_CHOICE = "auto";
+const DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS = "inf";
+const AZURE_DEFAULT_INPUT_AUDIO_TRANSCRIPTION = {
+  model: "whisper-1"
+};
+const AZURE_DEFAULT_TURN_DETECTION = {
+  type: "server_vad",
+  threshold: 0.5,
+  prefix_padding_ms: 300,
+  silence_duration_ms: 200,
+  create_response: true
+};
+const DEFAULT_MAX_SESSION_DURATION = 20 * 60 * 1e3;
+const DEFAULT_REALTIME_MODEL_OPTIONS = {
+  model: "gpt-4o-realtime-preview",
+  voice: "alloy",
+  temperature: DEFAULT_TEMPERATURE,
+  inputAudioTranscription: DEFAULT_INPUT_AUDIO_TRANSCRIPTION,
+  turnDetection: DEFAULT_TURN_DETECTION,
+  toolChoice: DEFAULT_TOOL_CHOICE,
+  maxResponseOutputTokens: DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS,
+  maxSessionDuration: DEFAULT_MAX_SESSION_DURATION,
+  connOptions: DEFAULT_API_CONNECT_OPTIONS
+};
+class RealtimeModel extends llm.RealtimeModel {
+  sampleRate = api_proto.SAMPLE_RATE;
+  numChannels = api_proto.NUM_CHANNELS;
+  inFrameSize = api_proto.IN_FRAME_SIZE;
+  outFrameSize = api_proto.OUT_FRAME_SIZE;
+  /* @internal */
+  _options;
+  constructor(options = {}) {
+    super({
+      messageTruncation: true,
+      turnDetection: options.turnDetection !== null,
+      userTranscription: options.inputAudioTranscription !== null,
+      autoToolReplyGeneration: false
     });
-  }
-  create(message, previousItemId) {
-    if (!message.content) {
-      return;
+    const isAzure = !!(options.apiVersion || options.entraToken || options.azureDeployment);
+    if (options.apiKey === "" && !isAzure) {
+      throw new Error(
+        "OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable"
+      );
     }
-    let event;
-    if (message.toolCallId) {
-      if (typeof message.content !== "string") {
-        throw new TypeError("message.content must be a string");
-      }
-      event = {
-        type: "conversation.item.create",
-        previous_item_id: previousItemId,
-        item: {
-          type: "function_call_output",
-          call_id: message.toolCallId,
-          output: message.content
-        }
-      };
-    } else {
-      let content = message.content;
-      if (!Array.isArray(content)) {
-        content = [content];
-      }
-      if (message.role === llm.ChatRole.USER) {
-        const contents = [];
-        for (const c of content) {
-          if (typeof c === "string") {
-            contents.push({
-              type: "input_text",
-              text: c
-            });
-          } else if (
-            // typescript type guard for determining ChatAudio vs ChatImage
-            ((c2) => {
-              return c2.frame !== void 0;
-            })(c)
-          ) {
-            contents.push({
-              type: "input_audio",
-              audio: Buffer.from(mergeFrames(c.frame).data.buffer).toString("base64")
-            });
-          }
-        }
-        event = {
-          type: "conversation.item.create",
-          previous_item_id: previousItemId,
-          item: {
-            type: "message",
-            role: "user",
-            content: contents
-          }
-        };
-      } else if (message.role === llm.ChatRole.ASSISTANT) {
-        const contents = [];
-        for (const c of content) {
-          if (typeof c === "string") {
-            contents.push({
-              type: "text",
-              text: c
-            });
-          } else if (
-            // typescript type guard for determining ChatAudio vs ChatImage
-            ((c2) => {
-              return c2.frame !== void 0;
-            })(c)
-          ) {
-            this.#logger.warn("audio content in assistant message is not supported");
-          }
-        }
-        event = {
-          type: "conversation.item.create",
-          previous_item_id: previousItemId,
-          item: {
-            type: "message",
-            role: "assistant",
-            content: contents
-          }
-        };
-      } else if (message.role === llm.ChatRole.SYSTEM) {
-        const contents = [];
-        for (const c of content) {
-          if (typeof c === "string") {
-            contents.push({
-              type: "input_text",
-              text: c
-            });
-          } else if (
-            // typescript type guard for determining ChatAudio vs ChatImage
-            ((c2) => {
-              return c2.frame !== void 0;
-            })(c)
-          ) {
-            this.#logger.warn("audio content in system message is not supported");
-          }
-        }
-        event = {
-          type: "conversation.item.create",
-          previous_item_id: previousItemId,
-          item: {
-            type: "message",
-            role: "system",
-            content: contents
-          }
-        };
-      } else {
-        this.#logger.child({ message }).warn("chat message is not supported inside the realtime API");
-        return;
+    const apiKey = options.apiKey || process.env.OPENAI_API_KEY;
+    if (!apiKey && !isAzure) {
+      throw new Error(
+        "OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable"
+      );
+    }
+    if (!options.baseURL && isAzure) {
+      const azureEndpoint = process.env.AZURE_OPENAI_ENDPOINT;
+      if (!azureEndpoint) {
+        throw new Error(
+          "Missing Azure endpoint. Please pass base_url or set AZURE_OPENAI_ENDPOINT environment variable."
+        );
       }
+      options.baseURL = `${azureEndpoint.replace(/\/$/, "")}/openai`;
     }
-    this.#session.queueMsg(event);
-  }
-}
-class Conversation {
-  #session;
-  constructor(session) {
-    this.#session = session;
-  }
-  get item() {
-    return new ConversationItem(this.#session);
-  }
-}
-class Response {
-  #session;
-  constructor(session) {
-    this.#session = session;
-  }
-  create() {
-    this.#session.queueMsg({
-      type: "response.create"
-    });
-  }
-  cancel() {
-    this.#session.queueMsg({
-      type: "response.cancel"
-    });
+    this._options = {
+      ...DEFAULT_REALTIME_MODEL_OPTIONS,
+      ...options,
+      baseURL: options.baseURL || BASE_URL,
+      apiKey,
+      isAzure,
+      model: options.model || DEFAULT_REALTIME_MODEL_OPTIONS.model
+    };
   }
-}
-class RealtimeModel extends multimodal.RealtimeModel {
-  sampleRate = api_proto.SAMPLE_RATE;
-  numChannels = api_proto.NUM_CHANNELS;
-  inFrameSize = api_proto.IN_FRAME_SIZE;
-  outFrameSize = api_proto.OUT_FRAME_SIZE;
-  #defaultOpts;
-  #sessions = [];
+  /**
+   * Create a RealtimeModel instance configured for Azure OpenAI Service.
+   *
+   * @param azureDeployment - The name of your Azure OpenAI deployment.
+   * @param azureEndpoint - The endpoint URL for your Azure OpenAI resource. If undefined, will attempt to read from the environment variable AZURE_OPENAI_ENDPOINT.
+   * @param apiVersion - API version to use with Azure OpenAI Service. If undefined, will attempt to read from the environment variable OPENAI_API_VERSION.
+   * @param apiKey - Azure OpenAI API key. If undefined, will attempt to read from the environment variable AZURE_OPENAI_API_KEY.
+   * @param entraToken - Azure Entra authentication token. Required if not using API key authentication.
+   * @param baseURL - Base URL for the API endpoint. If undefined, constructed from the azure_endpoint.
+   * @param voice - Voice setting for audio outputs. Defaults to "alloy".
+   * @param inputAudioTranscription - Options for transcribing input audio. Defaults to @see DEFAULT_INPUT_AUDIO_TRANSCRIPTION.
+   * @param turnDetection - Options for server-based voice activity detection (VAD). Defaults to @see DEFAULT_SERVER_VAD_OPTIONS.
+   * @param temperature - Sampling temperature for response generation. Defaults to @see DEFAULT_TEMPERATURE.
+   * @param speed - Speed of the audio output. Defaults to 1.0.
+   * @param maxResponseOutputTokens - Maximum number of tokens in the response. Defaults to @see DEFAULT_MAX_RESPONSE_OUTPUT_TOKENS.
+   * @param maxSessionDuration - Maximum duration of the session in milliseconds. Defaults to @see DEFAULT_MAX_SESSION_DURATION.
+   *
+   * @returns A RealtimeModel instance configured for Azure OpenAI Service.
+   *
+   * @throws Error if required Azure parameters are missing or invalid.
+   */
   static withAzure({
-    baseURL,
     azureDeployment,
-    apiVersion = "2024-10-01-preview",
-    apiKey = void 0,
-    entraToken = void 0,
-    instructions = "",
-    modalities = ["text", "audio"],
-    voice = "alloy",
-    inputAudioFormat = "pcm16",
-    outputAudioFormat = "pcm16",
-    inputAudioTranscription = { model: "whisper-1" },
-    turnDetection = { type: "server_vad" },
-    temperature = 0.8,
-    maxResponseOutputTokens = Infinity
-  }) {
-    return new RealtimeModel({
-      isAzure: true,
-      baseURL: new URL("openai", baseURL).toString(),
-      model: azureDeployment,
-      apiVersion,
-      apiKey,
-      entraToken,
-      instructions,
-      modalities,
-      voice,
-      inputAudioFormat,
-      outputAudioFormat,
-      inputAudioTranscription,
-      turnDetection,
-      temperature,
-      maxResponseOutputTokens
-    });
-  }
-  constructor({
-    modalities = ["text", "audio"],
-    instructions = "",
+    azureEndpoint,
+    apiVersion,
+    apiKey,
+    entraToken,
+    baseURL,
     voice = "alloy",
-    inputAudioFormat = "pcm16",
-    outputAudioFormat = "pcm16",
-    inputAudioTranscription = { model: "whisper-1" },
-    turnDetection = { type: "server_vad" },
+    inputAudioTranscription = AZURE_DEFAULT_INPUT_AUDIO_TRANSCRIPTION,
+    turnDetection = AZURE_DEFAULT_TURN_DETECTION,
     temperature = 0.8,
-    maxResponseOutputTokens = Infinity,
-    model = "gpt-4o-realtime-preview-2024-10-01",
-    apiKey = process.env.OPENAI_API_KEY || "",
-    baseURL = api_proto.BASE_URL,
-    // used for microsoft
-    isAzure = false,
-    apiVersion = void 0,
-    entraToken = void 0
+    speed
   }) {
-    super();
-    if (apiKey === "" && !(isAzure && entraToken)) {
+    apiKey = apiKey || process.env.AZURE_OPENAI_API_KEY;
+    if (!apiKey && !entraToken) {
       throw new Error(
-        "OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable"
+        "Missing credentials. Please pass one of `apiKey`, `entraToken`, or the `AZURE_OPENAI_API_KEY` environment variable."
       );
     }
-    this.#defaultOpts = {
-      modalities,
-      instructions,
+    apiVersion = apiVersion || process.env.OPENAI_API_VERSION;
+    if (!apiVersion) {
+      throw new Error(
+        "Must provide either the `apiVersion` argument or the `OPENAI_API_VERSION` environment variable"
+      );
+    }
+    if (!baseURL) {
+      azureEndpoint = azureEndpoint || process.env.AZURE_OPENAI_ENDPOINT;
+      if (!azureEndpoint) {
+        throw new Error(
+          "Missing Azure endpoint. Please pass the `azure_endpoint` parameter or set the `AZURE_OPENAI_ENDPOINT` environment variable."
+        );
+      }
+      baseURL = `${azureEndpoint.replace(/\/$/, "")}/openai`;
+    }
+    return new RealtimeModel({
       voice,
-      inputAudioFormat,
-      outputAudioFormat,
       inputAudioTranscription,
       turnDetection,
       temperature,
-      maxResponseOutputTokens,
-      model,
+      speed,
       apiKey,
-      baseURL,
-      isAzure,
+      azureDeployment,
       apiVersion,
-      entraToken
-    };
-  }
-  get sessions() {
-    return this.#sessions;
-  }
-  session({
-    fncCtx,
-    chatCtx,
-    modalities = this.#defaultOpts.modalities,
-    instructions = this.#defaultOpts.instructions,
-    voice = this.#defaultOpts.voice,
-    inputAudioFormat = this.#defaultOpts.inputAudioFormat,
-    outputAudioFormat = this.#defaultOpts.outputAudioFormat,
-    inputAudioTranscription = this.#defaultOpts.inputAudioTranscription,
-    turnDetection = this.#defaultOpts.turnDetection,
-    temperature = this.#defaultOpts.temperature,
-    maxResponseOutputTokens = this.#defaultOpts.maxResponseOutputTokens
-  }) {
-    const opts = {
-      modalities,
-      instructions,
-      voice,
-      inputAudioFormat,
-      outputAudioFormat,
-      inputAudioTranscription,
-      turnDetection,
-      temperature,
-      maxResponseOutputTokens,
-      model: this.#defaultOpts.model,
-      apiKey: this.#defaultOpts.apiKey,
-      baseURL: this.#defaultOpts.baseURL,
-      isAzure: this.#defaultOpts.isAzure,
-      apiVersion: this.#defaultOpts.apiVersion,
-      entraToken: this.#defaultOpts.entraToken
-    };
-    const newSession = new RealtimeSession(opts, {
-      chatCtx: chatCtx || new llm.ChatContext(),
-      fncCtx
+      entraToken,
+      baseURL
     });
-    this.#sessions.push(newSession);
-    return newSession;
+  }
+  session() {
+    return new RealtimeSession(this);
   }
   async close() {
-    await Promise.allSettled(this.#sessions.map((session) => session.close()));
+    return;
+  }
+}
+function processBaseURL({
+  baseURL,
+  model,
+  isAzure = false,
+  azureDeployment,
+  apiVersion
+}) {
+  const url = new URL([baseURL, "realtime"].join("/"));
+  if (url.protocol === "https:") {
+    url.protocol = "wss:";
+  }
+  if (!url.pathname || ["", "/v1", "/openai"].includes(url.pathname.replace(/\/$/, ""))) {
+    url.pathname = url.pathname.replace(/\/$/, "") + "/realtime";
+  } else {
+    url.pathname = url.pathname.replace(/\/$/, "");
+  }
+  const queryParams = {};
+  if (isAzure) {
+    if (apiVersion) {
+      queryParams["api-version"] = apiVersion;
+    }
+    if (azureDeployment) {
+      queryParams["deployment"] = azureDeployment;
+    }
+  } else {
+    queryParams["model"] = model;
+  }
+  for (const [key, value] of Object.entries(queryParams)) {
+    url.searchParams.set(key, value);
   }
+  return url.toString();
 }
-class RealtimeSession extends multimodal.RealtimeSession {
-  #chatCtx = void 0;
-  #fncCtx = void 0;
-  #opts;
-  #pendingResponses = {};
-  #sessionId = "not-connected";
-  #ws = null;
-  #expiresAt = null;
+class RealtimeSession extends llm.RealtimeSession {
+  _tools = {};
+  remoteChatCtx = new llm.RemoteChatContext();
+  messageChannel = new Queue();
+  inputResampler;
+  instructions;
+  oaiRealtimeModel;
+  currentGeneration;
+  responseCreatedFutures = {};
+  textModeRecoveryRetries = 0;
+  itemCreateFutures = {};
+  itemDeleteFutures = {};
+  updateChatCtxLock = new Mutex();
+  updateFuncCtxLock = new Mutex();
+  // 100ms chunks
+  bstream = new AudioByteStream(SAMPLE_RATE, NUM_CHANNELS, SAMPLE_RATE / 10);
+  pushedDurationMs = 0;
   #logger = log();
   #task;
-  #closing = true;
-  #sendQueue = new Queue();
-  constructor(opts, { fncCtx, chatCtx }) {
-    super();
-    this.#opts = opts;
-    this.#chatCtx = chatCtx;
-    this.#fncCtx = fncCtx;
-    this.#task = this.#start();
-    this.sessionUpdate({
-      modalities: this.#opts.modalities,
-      instructions: this.#opts.instructions,
-      voice: this.#opts.voice,
-      inputAudioFormat: this.#opts.inputAudioFormat,
-      outputAudioFormat: this.#opts.outputAudioFormat,
-      inputAudioTranscription: this.#opts.inputAudioTranscription,
-      turnDetection: this.#opts.turnDetection,
-      temperature: this.#opts.temperature,
-      maxResponseOutputTokens: this.#opts.maxResponseOutputTokens,
-      toolChoice: "auto"
-    });
+  #closed = false;
+  constructor(realtimeModel) {
+    super(realtimeModel);
+    this.oaiRealtimeModel = realtimeModel;
+    this.#task = this.#mainTask();
+    this.sendEvent(this.createSessionUpdateEvent());
+  }
+  sendEvent(command) {
+    this.messageChannel.put(command);
+  }
+  createSessionUpdateEvent() {
+    return {
+      type: "session.update",
+      session: {
+        model: this.oaiRealtimeModel._options.model,
+        voice: this.oaiRealtimeModel._options.voice,
+        input_audio_format: "pcm16",
+        output_audio_format: "pcm16",
+        modalities: ["text", "audio"],
+        turn_detection: this.oaiRealtimeModel._options.turnDetection,
+        input_audio_transcription: this.oaiRealtimeModel._options.inputAudioTranscription,
+        // TODO(shubhra): add inputAudioNoiseReduction
+        temperature: this.oaiRealtimeModel._options.temperature,
+        tool_choice: toOaiToolChoice(this.oaiRealtimeModel._options.toolChoice),
+        max_response_output_tokens: this.oaiRealtimeModel._options.maxResponseOutputTokens === Infinity ? "inf" : this.oaiRealtimeModel._options.maxResponseOutputTokens,
+        // TODO(shubhra): add tracing options
+        instructions: this.instructions,
+        speed: this.oaiRealtimeModel._options.speed
+      }
+    };
   }
   get chatCtx() {
-    return this.#chatCtx;
+    return this.remoteChatCtx.toChatCtx();
   }
-  get fncCtx() {
-    return this.#fncCtx;
+  get tools() {
+    return { ...this._tools };
   }
-  set fncCtx(ctx) {
-    this.#fncCtx = ctx;
+  async updateChatCtx(_chatCtx) {
+    const unlock = await this.updateChatCtxLock.lock();
+    const events = this.createChatCtxUpdateEvents(_chatCtx);
+    const futures = [];
+    for (const event of events) {
+      const future = new Future();
+      futures.push(future);
+      if (event.type === "conversation.item.create") {
+        this.itemCreateFutures[event.item.id] = future;
+      } else if (event.type == "conversation.item.delete") {
+        this.itemDeleteFutures[event.item_id] = future;
+      }
+      this.sendEvent(event);
+    }
+    if (futures.length === 0) {
+      unlock();
+      return;
+    }
+    try {
+      await Promise.race([
+        Promise.all(futures),
+        delay(5e3).then(() => {
+          throw new Error("Chat ctx update events timed out");
+        })
+      ]);
+    } catch (e) {
+      this.#logger.error(e.message);
+      throw e;
+    } finally {
+      unlock();
+    }
   }
-  get conversation() {
-    return new Conversation(this);
+  createChatCtxUpdateEvents(chatCtx, addMockAudio = false) {
+    const newChatCtx = chatCtx.copy();
+    if (addMockAudio) {
+      newChatCtx.items.push(createMockAudioItem());
+    } else {
+      newChatCtx.items = newChatCtx.items.filter(
+        (item) => !item.id.startsWith(MOCK_AUDIO_ID_PREFIX)
+      );
+    }
+    const events = [];
+    const diffOps = llm.computeChatCtxDiff(this.chatCtx, newChatCtx);
+    for (const op of diffOps.toRemove) {
+      events.push({
+        type: "conversation.item.delete",
+        item_id: op,
+        event_id: shortuuid("chat_ctx_delete_")
+      });
+    }
+    for (const [previousId, id] of diffOps.toCreate) {
+      const chatItem = newChatCtx.getById(id);
+      if (!chatItem) {
+        throw new Error(`Chat item ${id} not found`);
+      }
+      events.push({
+        type: "conversation.item.create",
+        item: livekitItemToOpenAIItem(chatItem),
+        previous_item_id: previousId ?? void 0,
+        event_id: shortuuid("chat_ctx_create_")
+      });
+    }
+    return events;
   }
-  get inputAudioBuffer() {
-    return new InputAudioBuffer(this);
+  async updateTools(_tools) {
+    const unlock = await this.updateFuncCtxLock.lock();
+    const ev = this.createToolsUpdateEvent(_tools);
+    this.sendEvent(ev);
+    if (!ev.session.tools) {
+      throw new Error("Tools are missing in the session update event");
+    }
+    const retainedToolNames = new Set(ev.session.tools.map((tool) => tool.name));
+    const retainedTools = Object.fromEntries(
+      Object.entries(_tools).filter(
+        ([name, tool]) => llm.isFunctionTool(tool) && retainedToolNames.has(name)
+      )
+    );
+    this._tools = retainedTools;
+    unlock();
   }
-  get response() {
-    return new Response(this);
+  createToolsUpdateEvent(_tools) {
+    const oaiTools = [];
+    for (const [name, tool] of Object.entries(_tools)) {
+      if (!llm.isFunctionTool(tool)) {
+        this.#logger.error({ name, tool }, "OpenAI Realtime API doesn't support this tool type");
+        continue;
+      }
+      const { parameters: toolParameters, description } = tool;
+      try {
+        const parameters = llm.toJsonSchema(
+          toolParameters
+        );
+        oaiTools.push({
+          name,
+          description,
+          parameters,
+          type: "function"
+        });
+      } catch (e) {
+        this.#logger.error({ name, tool }, "OpenAI Realtime API doesn't support this tool type");
+        continue;
+      }
+    }
+    return {
+      type: "session.update",
+      session: {
+        model: this.oaiRealtimeModel._options.model,
+        tools: oaiTools
+      },
+      event_id: shortuuid("tools_update_")
+    };
+  }
+  async updateInstructions(_instructions) {
+    const eventId = shortuuid("instructions_update_");
+    this.sendEvent({
+      type: "session.update",
+      session: {
+        instructions: _instructions
+      },
+      event_id: eventId
+    });
+    this.instructions = _instructions;
+  }
+  updateOptions({ toolChoice }) {
+    const options = {};
+    this.oaiRealtimeModel._options.toolChoice = toolChoice;
+    options.tool_choice = toOaiToolChoice(toolChoice);
+    this.sendEvent({
+      type: "session.update",
+      session: options,
+      event_id: shortuuid("options_update_")
+    });
+  }
+  pushAudio(frame) {
+    for (const f of this.resampleAudio(frame)) {
+      for (const nf of this.bstream.write(f.data.buffer)) {
+        this.sendEvent({
+          type: "input_audio_buffer.append",
+          audio: Buffer.from(nf.data.buffer).toString("base64")
+        });
+        this.pushedDurationMs += nf.samplesPerChannel / nf.sampleRate * 1e3;
+      }
+    }
   }
-  get expiration() {
-    if (!this.#expiresAt) {
-      throw new Error("session not started");
+  async commitAudio() {
+    if (this.pushedDurationMs > 100) {
+      this.sendEvent({
+        type: "input_audio_buffer.commit"
+      });
+      this.pushedDurationMs = 0;
     }
-    return this.#expiresAt * 1e3;
   }
-  queueMsg(command) {
-    this.#sendQueue.put(command);
+  async clearAudio() {
+    this.sendEvent({
+      type: "input_audio_buffer.clear"
+    });
+    this.pushedDurationMs = 0;
+  }
+  async generateReply(instructions) {
+    const handle = this.createResponse({ instructions, userInitiated: true });
+    this.textModeRecoveryRetries = 0;
+    return handle.doneFut.await;
+  }
+  async interrupt() {
+    this.sendEvent({
+      type: "response.cancel"
+    });
+  }
+  async truncate(_options) {
+    this.sendEvent({
+      type: "conversation.item.truncate",
+      content_index: 0,
+      item_id: _options.messageId,
+      audio_end_ms: _options.audioEndMs
+    });
   }
   /// Truncates the data field of the event to the specified maxLength to avoid overwhelming logs
   /// with large amounts of base64 audio data.
@@ -395,549 +470,716 @@ class RealtimeSession extends multimodal.RealtimeSession {
     }
     return untypedEvent;
   }
-  sessionUpdate({
-    modalities = this.#opts.modalities,
-    instructions = this.#opts.instructions,
-    voice = this.#opts.voice,
-    inputAudioFormat = this.#opts.inputAudioFormat,
-    outputAudioFormat = this.#opts.outputAudioFormat,
-    inputAudioTranscription = this.#opts.inputAudioTranscription,
-    turnDetection = this.#opts.turnDetection,
-    temperature = this.#opts.temperature,
-    maxResponseOutputTokens = this.#opts.maxResponseOutputTokens,
-    toolChoice = "auto",
-    selectedTools = Object.keys(this.#fncCtx || {})
-  }) {
-    this.#opts = {
-      modalities,
-      instructions,
-      voice,
-      inputAudioFormat,
-      outputAudioFormat,
-      inputAudioTranscription,
-      turnDetection,
-      temperature,
-      maxResponseOutputTokens,
-      model: this.#opts.model,
-      apiKey: this.#opts.apiKey,
-      baseURL: this.#opts.baseURL,
-      isAzure: this.#opts.isAzure,
-      apiVersion: this.#opts.apiVersion,
-      entraToken: this.#opts.entraToken
-    };
-    const tools = this.#fncCtx ? Object.entries(this.#fncCtx).filter(([name]) => selectedTools.includes(name)).map(([name, func]) => ({
-      type: "function",
-      name,
-      description: func.description,
-      parameters: (
-        // don't format parameters if they are raw openai params
-        func.parameters.type == "object" ? func.parameters : llm.oaiParams(func.parameters)
-      )
-    })) : [];
-    const sessionUpdateEvent = {
-      type: "session.update",
-      session: {
-        modalities: this.#opts.modalities,
-        instructions: this.#opts.instructions,
-        voice: this.#opts.voice,
-        input_audio_format: this.#opts.inputAudioFormat,
-        output_audio_format: this.#opts.outputAudioFormat,
-        input_audio_transcription: this.#opts.inputAudioTranscription,
-        turn_detection: this.#opts.turnDetection,
-        temperature: this.#opts.temperature,
-        max_response_output_tokens: this.#opts.maxResponseOutputTokens === Infinity ? "inf" : this.#opts.maxResponseOutputTokens,
-        tools,
-        tool_choice: toolChoice
-      }
+  async createWsConn() {
+    const headers = {
+      "User-Agent": "LiveKit-Agents-JS"
     };
-    if (this.#opts.isAzure && this.#opts.maxResponseOutputTokens === Infinity) {
-      sessionUpdateEvent.session.max_response_output_tokens = void 0;
-    }
-    this.queueMsg(sessionUpdateEvent);
-  }
-  /** Create an empty audio message with the given duration. */
-  #createEmptyUserAudioMessage(duration) {
-    const samples = duration * api_proto.SAMPLE_RATE;
-    return new llm.ChatMessage({
-      role: llm.ChatRole.USER,
-      content: {
-        frame: new AudioFrame(
-          new Int16Array(samples * api_proto.NUM_CHANNELS),
-          api_proto.SAMPLE_RATE,
-          api_proto.NUM_CHANNELS,
-          samples
-        )
+    if (this.oaiRealtimeModel._options.isAzure) {
+      if (this.oaiRealtimeModel._options.entraToken) {
+        headers.Authorization = `Bearer ${this.oaiRealtimeModel._options.entraToken}`;
+      } else if (this.oaiRealtimeModel._options.apiKey) {
+        headers["api-key"] = this.oaiRealtimeModel._options.apiKey;
+      } else {
+        throw new Error("Microsoft API key or entraToken is required");
       }
-    });
-  }
-  /**
-   * Try to recover from a text response to audio mode.
-   *
-   * @remarks
-   * Sometimes the OpenAI Realtime API returns text instead of audio responses.
-   * This method tries to recover from this by requesting a new response after deleting the text
-   * response and creating an empty user audio message.
-   */
-  recoverFromTextResponse(itemId) {
-    if (itemId) {
-      this.conversation.item.delete(itemId);
+    } else {
+      headers.Authorization = `Bearer ${this.oaiRealtimeModel._options.apiKey}`;
+      headers["OpenAI-Beta"] = "realtime=v1";
     }
-    this.conversation.item.create(this.#createEmptyUserAudioMessage(1));
-    this.response.create();
+    const url = processBaseURL({
+      baseURL: this.oaiRealtimeModel._options.baseURL,
+      model: this.oaiRealtimeModel._options.model,
+      isAzure: this.oaiRealtimeModel._options.isAzure,
+      apiVersion: this.oaiRealtimeModel._options.apiVersion,
+      azureDeployment: this.oaiRealtimeModel._options.azureDeployment
+    });
+    this.#logger.debug(`Connecting to OpenAI Realtime API at ${url}`);
+    return new Promise((resolve, reject) => {
+      const ws = new WebSocket(url, { headers });
+      let waiting = true;
+      const timeout = setTimeout(() => {
+        ws.close();
+        reject(new Error("WebSocket connection timeout"));
+      }, this.oaiRealtimeModel._options.connOptions.timeoutMs);
+      ws.once("open", () => {
+        if (!waiting) return;
+        waiting = false;
+        clearTimeout(timeout);
+        resolve(ws);
+      });
+      ws.once("close", () => {
+        if (!waiting) return;
+        waiting = false;
+        clearTimeout(timeout);
+        reject(new Error("OpenAI Realtime API connection closed"));
+      });
+    });
   }
-  #start() {
-    return new Promise(async (resolve, reject) => {
-      const headers = {
-        "User-Agent": "LiveKit-Agents-JS"
-      };
-      if (this.#opts.isAzure) {
-        if (this.#opts.entraToken) {
-          headers.Authorization = `Bearer ${this.#opts.entraToken}`;
-        } else if (this.#opts.apiKey) {
-          headers["api-key"] = this.#opts.apiKey;
-        } else {
-          reject(new Error("Microsoft API key or entraToken is required"));
-          return;
-        }
-      } else {
-        headers.Authorization = `Bearer ${this.#opts.apiKey}`;
-        headers["OpenAI-Beta"] = "realtime=v1";
-      }
-      const url = new URL([this.#opts.baseURL, "realtime"].join("/"));
-      if (url.protocol === "https:") {
-        url.protocol = "wss:";
-      }
-      const queryParams = {};
-      if (this.#opts.isAzure) {
-        queryParams["api-version"] = this.#opts.apiVersion ?? "2024-10-01-preview";
-        queryParams["deployment"] = this.#opts.model;
-      } else {
-        queryParams["model"] = this.#opts.model;
-      }
-      for (const [key, value] of Object.entries(queryParams)) {
-        url.searchParams.set(key, value);
+  async #mainTask() {
+    let reconnecting = false;
+    let numRetries = 0;
+    let wsConn = null;
+    const maxRetries = this.oaiRealtimeModel._options.connOptions.maxRetry;
+    const reconnect = async () => {
+      this.#logger.debug(
+        {
+          maxSessionDuration: this.oaiRealtimeModel._options.maxSessionDuration
+        },
+        "Reconnecting to OpenAI Realtime API"
+      );
+      const events = [];
+      events.push(this.createSessionUpdateEvent());
+      if (Object.keys(this._tools).length > 0) {
+        events.push(this.createToolsUpdateEvent(this._tools));
       }
-      console.debug("Connecting to OpenAI Realtime API at ", url.toString());
-      this.#ws = new WebSocket(url.toString(), {
-        headers
+      const chatCtx = this.chatCtx.copy({
+        excludeFunctionCall: true,
+        excludeInstructions: true,
+        excludeEmptyMessage: true
       });
-      this.#ws.onerror = (error) => {
-        reject(new Error("OpenAI Realtime WebSocket error: " + error.message));
-      };
-      await once(this.#ws, "open");
-      this.#closing = false;
-      this.#ws.onmessage = (message) => {
-        const event = JSON.parse(message.data);
-        this.#logger.debug(`<- ${JSON.stringify(this.#loggableEvent(event))}`);
-        switch (event.type) {
-          case "error":
-            this.#handleError(event);
-            break;
-          case "session.created":
-            this.#handleSessionCreated(event);
-            break;
-          case "session.updated":
-            this.#handleSessionUpdated(event);
-            break;
-          case "conversation.created":
-            this.#handleConversationCreated(event);
-            break;
-          case "input_audio_buffer.committed":
-            this.#handleInputAudioBufferCommitted(event);
-            break;
-          case "input_audio_buffer.cleared":
-            this.#handleInputAudioBufferCleared(event);
-            break;
-          case "input_audio_buffer.speech_started":
-            this.#handleInputAudioBufferSpeechStarted(event);
-            break;
-          case "input_audio_buffer.speech_stopped":
-            this.#handleInputAudioBufferSpeechStopped(event);
-            break;
-          case "conversation.item.created":
-            this.#handleConversationItemCreated(event);
-            break;
-          case "conversation.item.input_audio_transcription.completed":
-            this.#handleConversationItemInputAudioTranscriptionCompleted(event);
-            break;
-          case "conversation.item.input_audio_transcription.failed":
-            this.#handleConversationItemInputAudioTranscriptionFailed(event);
-            break;
-          case "conversation.item.truncated":
-            this.#handleConversationItemTruncated(event);
-            break;
-          case "conversation.item.deleted":
-            this.#handleConversationItemDeleted(event);
-            break;
-          case "response.created":
-            this.#handleResponseCreated(event);
-            break;
-          case "response.done":
-            this.#handleResponseDone(event);
-            break;
-          case "response.output_item.added":
-            this.#handleResponseOutputItemAdded(event);
-            break;
-          case "response.output_item.done":
-            this.#handleResponseOutputItemDone(event);
-            break;
-          case "response.content_part.added":
-            this.#handleResponseContentPartAdded(event);
-            break;
-          case "response.content_part.done":
-            this.#handleResponseContentPartDone(event);
-            break;
-          case "response.text.delta":
-            this.#handleResponseTextDelta(event);
-            break;
-          case "response.text.done":
-            this.#handleResponseTextDone(event);
-            break;
-          case "response.audio_transcript.delta":
-            this.#handleResponseAudioTranscriptDelta(event);
-            break;
-          case "response.audio_transcript.done":
-            this.#handleResponseAudioTranscriptDone(event);
-            break;
-          case "response.audio.delta":
-            this.#handleResponseAudioDelta(event);
-            break;
-          case "response.audio.done":
-            this.#handleResponseAudioDone(event);
-            break;
-          case "response.function_call_arguments.delta":
-            this.#handleResponseFunctionCallArgumentsDelta(event);
-            break;
-          case "response.function_call_arguments.done":
-            this.#handleResponseFunctionCallArgumentsDone(event);
-            break;
-          case "rate_limits.updated":
-            this.#handleRateLimitsUpdated(event);
-            break;
+      const oldChatCtx = this.remoteChatCtx;
+      this.remoteChatCtx = new llm.RemoteChatContext();
+      events.push(...this.createChatCtxUpdateEvents(chatCtx));
+      try {
+        for (const ev of events) {
+          this.emit("openai_client_event_queued", ev);
+          wsConn.send(JSON.stringify(ev));
         }
-      };
-      const sendTask = async () => {
-        while (this.#ws && !this.#closing && this.#ws.readyState === WebSocket.OPEN) {
-          try {
-            const event = await this.#sendQueue.get();
-            if (event.type !== "input_audio_buffer.append") {
-              this.#logger.debug(`-> ${JSON.stringify(this.#loggableEvent(event))}`);
-            }
-            this.#ws.send(JSON.stringify(event));
-          } catch (error) {
-            this.#logger.error("Error sending event:", error);
-          }
+      } catch (error) {
+        this.remoteChatCtx = oldChatCtx;
+        throw new APIConnectionError({
+          message: "Failed to send message to OpenAI Realtime API during session re-connection"
+        });
+      }
+      this.#logger.debug("Reconnected to OpenAI Realtime API");
+      this.emit("session_reconnected", {});
+    };
+    reconnecting = false;
+    while (!this.#closed) {
+      this.#logger.debug("Creating WebSocket connection to OpenAI Realtime API");
+      wsConn = await this.createWsConn();
+      try {
+        if (reconnecting) {
+          await reconnect();
+          numRetries = 0;
         }
-      };
-      sendTask();
-      this.#ws.onclose = () => {
-        if (this.#expiresAt && Date.now() >= this.#expiresAt * 1e3) {
-          this.#closing = true;
+        await this.runWs(wsConn);
+      } catch (error) {
+        if (!isAPIError(error)) {
+          this.emitError({ error, recoverable: false });
+          throw error;
         }
-        if (!this.#closing) {
-          reject(new Error("OpenAI Realtime connection closed unexpectedly"));
+        if (maxRetries === 0 || !error.retryable) {
+          this.emitError({ error, recoverable: false });
+          throw error;
         }
-        this.#ws = null;
-        resolve();
-      };
+        if (numRetries === maxRetries) {
+          this.emitError({ error, recoverable: false });
+          throw new APIConnectionError({
+            message: `OpenAI Realtime API connection failed after ${numRetries} attempts`,
+            options: {
+              body: error,
+              retryable: false
+            }
+          });
+        }
+        this.emitError({ error, recoverable: true });
+        const retryInterval = numRetries === 0 ? DEFAULT_FIRST_RETRY_INTERVAL_MS : this.oaiRealtimeModel._options.connOptions.retryIntervalMs;
+        this.#logger.warn(
+          {
+            attempt: numRetries,
+            maxRetries,
+            error
+          },
+          `OpenAI Realtime API connection failed, retrying in ${retryInterval / 1e3}s`
+        );
+        await delay(retryInterval);
+        numRetries++;
+      }
+      reconnecting = true;
+    }
+  }
+  async runWs(wsConn) {
+    const forwardEvents = async (signal) => {
+      while (!this.#closed && wsConn.readyState === WebSocket.OPEN && !signal.aborted) {
+        try {
+          const event = await this.messageChannel.get();
+          if (signal.aborted) {
+            break;
+          }
+          if (event.type !== "input_audio_buffer.append") {
+            this.#logger.debug(`(client) -> ${JSON.stringify(this.#loggableEvent(event))}`);
+          }
+          this.emit("openai_client_event_queued", event);
+          wsConn.send(JSON.stringify(event));
+        } catch (error) {
+          break;
+        }
+      }
+      wsConn.close();
+    };
+    const wsCloseFuture = new Future();
+    wsConn.onerror = (error) => {
+      wsCloseFuture.resolve(new APIConnectionError({ message: error.message }));
+    };
+    wsConn.onclose = () => {
+      wsCloseFuture.resolve();
+    };
+    wsConn.onmessage = (message) => {
+      const event = JSON.parse(message.data);
+      this.emit("openai_server_event_received", event);
+      this.#logger.debug(`(server) <- ${JSON.stringify(this.#loggableEvent(event))}`);
+      switch (event.type) {
+        case "input_audio_buffer.speech_started":
+          this.handleInputAudioBufferSpeechStarted(event);
+          break;
+        case "input_audio_buffer.speech_stopped":
+          this.handleInputAudioBufferSpeechStopped(event);
+          break;
+        case "response.created":
+          this.handleResponseCreated(event);
+          break;
+        case "response.output_item.added":
+          this.handleResponseOutputItemAdded(event);
+          break;
+        case "conversation.item.created":
+          this.handleConversationItemCreated(event);
+          break;
+        case "conversation.item.deleted":
+          this.handleConversationItemDeleted(event);
+          break;
+        case "conversation.item.input_audio_transcription.completed":
+          this.handleConversationItemInputAudioTranscriptionCompleted(event);
+          break;
+        case "conversation.item.input_audio_transcription.failed":
+          this.handleConversationItemInputAudioTranscriptionFailed(event);
+          break;
+        case "response.content_part.added":
+          this.handleResponseContentPartAdded(event);
+          break;
+        case "response.content_part.done":
+          this.handleResponseContentPartDone(event);
+          break;
+        case "response.audio_transcript.delta":
+          this.handleResponseAudioTranscriptDelta(event);
+          break;
+        case "response.audio.delta":
+          this.handleResponseAudioDelta(event);
+          break;
+        case "response.audio_transcript.done":
+          this.handleResponseAudioTranscriptDone(event);
+          break;
+        case "response.audio.done":
+          this.handleResponseAudioDone(event);
+          break;
+        case "response.output_item.done":
+          this.handleResponseOutputItemDone(event);
+          break;
+        case "response.done":
+          this.handleResponseDone(event);
+          break;
+        case "error":
+          this.handleError(event);
+          break;
+        default:
+          this.#logger.debug(`unhandled event: ${event.type}`);
+          break;
+      }
+    };
+    const sendTask = Task.from(({ signal }) => forwardEvents(signal));
+    const wsTask = Task.from(({ signal }) => {
+      const abortPromise = new Promise((resolve) => {
+        signal.addEventListener("abort", () => {
+          resolve();
+        });
+      });
+      return Promise.race([wsCloseFuture.await, abortPromise]);
     });
+    const waitReconnectTask = Task.from(async ({ signal }) => {
+      await delay(this.oaiRealtimeModel._options.maxSessionDuration, { signal });
+      return new APIConnectionError({
+        message: "OpenAI Realtime API connection timeout"
+      });
+    });
+    try {
+      const result = await Promise.race([wsTask.result, sendTask.result, waitReconnectTask.result]);
+      if (waitReconnectTask.done && this.currentGeneration) {
+        await this.currentGeneration._doneFut.await;
+      }
+      if (result instanceof Error) {
+        throw result;
+      }
+    } finally {
+      await cancelAndWait([wsTask, sendTask, waitReconnectTask], 2e3);
+      wsConn.close();
+    }
   }
   async close() {
-    if (!this.#ws) return;
-    this.#closing = true;
-    this.#ws.close();
+    super.close();
+    this.#closed = true;
     await this.#task;
   }
-  #getContent(ptr) {
-    const response = this.#pendingResponses[ptr.response_id];
-    const output = response.output[ptr.output_index];
-    const content = output.content[ptr.content_index];
-    return content;
+  handleInputAudioBufferSpeechStarted(_event) {
+    this.emit("input_speech_started", {});
   }
-  #handleError(event) {
-    this.#logger.error(`OpenAI Realtime error ${JSON.stringify(event.error)}`);
+  handleInputAudioBufferSpeechStopped(_event) {
+    this.emit("input_speech_stopped", {
+      userTranscriptionEnabled: this.oaiRealtimeModel._options.inputAudioTranscription !== null
+    });
   }
-  #handleSessionCreated(event) {
-    this.#sessionId = event.session.id;
-    this.#expiresAt = event.session.expires_at;
-    this.#logger = this.#logger.child({ sessionId: this.#sessionId });
+  handleResponseCreated(event) {
+    if (!event.response.id) {
+      throw new Error("response.id is missing");
+    }
+    this.currentGeneration = {
+      messageChannel: stream.createStreamChannel(),
+      functionChannel: stream.createStreamChannel(),
+      messages: /* @__PURE__ */ new Map(),
+      _doneFut: new Future(),
+      _createdTimestamp: Date.now()
+    };
+    if (!event.response.metadata || !event.response.metadata.client_event_id) return;
+    const handle = this.responseCreatedFutures[event.response.metadata.client_event_id];
+    if (handle) {
+      delete this.responseCreatedFutures[event.response.metadata.client_event_id];
+      this.responseCreatedFutures[event.response.id] = handle;
+    }
+    this.emit("generation_created", {
+      messageStream: this.currentGeneration.messageChannel.stream(),
+      functionStream: this.currentGeneration.functionChannel.stream(),
+      userInitiated: false
+    });
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  #handleSessionUpdated(event) {
+  handleResponseOutputItemAdded(event) {
+    if (!this.currentGeneration) {
+      throw new Error("currentGeneration is not set");
+    }
+    if (!event.item.type) {
+      throw new Error("item.type is not set");
+    }
+    if (!event.response_id) {
+      throw new Error("response_id is not set");
+    }
+    const itemType = event.item.type;
+    const responseId = event.response_id;
+    if (itemType !== "message") {
+      this.emitGenerationEvent(responseId);
+      this.textModeRecoveryRetries = 0;
+      return;
+    }
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  #handleConversationCreated(event) {
+  handleConversationItemCreated(event) {
+    if (!event.item.id) {
+      throw new Error("item.id is not set");
+    }
+    try {
+      this.remoteChatCtx.insert(event.previous_item_id, openAIItemToLivekitItem(event.item));
+    } catch (error) {
+      this.#logger.error({ error, itemId: event.item.id }, "failed to insert conversation item");
+    }
+    const fut = this.itemCreateFutures[event.item.id];
+    if (fut) {
+      fut.resolve();
+      delete this.itemCreateFutures[event.item.id];
+    }
   }
-  #handleInputAudioBufferCommitted(event) {
-    this.emit("input_speech_committed", {
-      itemId: event.item_id
+  handleConversationItemDeleted(event) {
+    if (!event.item_id) {
+      throw new Error("item_id is not set");
+    }
+    try {
+      this.remoteChatCtx.delete(event.item_id);
+    } catch (error) {
+      this.#logger.error({ error, itemId: event.item_id }, "failed to delete conversation item");
+    }
+    const fut = this.itemDeleteFutures[event.item_id];
+    if (fut) {
+      fut.resolve();
+      delete this.itemDeleteFutures[event.item_id];
+    }
+  }
+  handleConversationItemInputAudioTranscriptionCompleted(event) {
+    const remoteItem = this.remoteChatCtx.get(event.item_id);
+    if (!remoteItem) {
+      return;
+    }
+    const item = remoteItem.item;
+    if (item instanceof llm.ChatMessage) {
+      item.content.push(event.transcript);
+    } else {
+      throw new Error("item is not a chat message");
+    }
+    this.emit("input_audio_transcription_completed", {
+      itemId: event.item_id,
+      transcript: event.transcript,
+      isFinal: true
     });
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  #handleInputAudioBufferCleared(event) {
+  handleConversationItemInputAudioTranscriptionFailed(event) {
+    this.#logger.error(
+      { error: event.error },
+      "OpenAI Realtime API failed to transcribe input audio"
+    );
   }
-  #handleInputAudioBufferSpeechStarted(event) {
-    this.emit("input_speech_started", {
-      itemId: event.item_id
-    });
+  handleResponseContentPartAdded(event) {
+    if (!this.currentGeneration) {
+      throw new Error("currentGeneration is not set");
+    }
+    const itemId = event.item_id;
+    const itemType = event.part.type;
+    const responseId = event.response_id;
+    if (itemType === "audio") {
+      this.emitGenerationEvent(responseId);
+      if (this.textModeRecoveryRetries > 0) {
+        this.#logger.info(
+          { retries: this.textModeRecoveryRetries },
+          "recovered from text-only response"
+        );
+        this.textModeRecoveryRetries = 0;
+      }
+      const itemGeneration = {
+        messageId: itemId,
+        textChannel: stream.createStreamChannel(),
+        audioChannel: stream.createStreamChannel(),
+        audioTranscript: ""
+      };
+      this.currentGeneration.messageChannel.write({
+        messageId: itemId,
+        textStream: itemGeneration.textChannel.stream(),
+        audioStream: itemGeneration.audioChannel.stream()
+      });
+      this.currentGeneration.messages.set(itemId, itemGeneration);
+      this.currentGeneration._firstTokenTimestamp = Date.now();
+      return;
+    } else {
+      this.interrupt();
+      if (this.textModeRecoveryRetries === 0) {
+        this.#logger.warn({ responseId }, "received text-only response from OpenAI Realtime API");
+      }
+    }
   }
-  #handleInputAudioBufferSpeechStopped(event) {
-    this.emit("input_speech_stopped");
+  handleResponseContentPartDone(event) {
+    if (event.part.type !== "text") {
+      return;
+    }
+    if (!this.currentGeneration) {
+      throw new Error("currentGeneration is not set");
+    }
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  #handleConversationItemCreated(event) {
+  handleResponseAudioTranscriptDelta(event) {
+    if (!this.currentGeneration) {
+      throw new Error("currentGeneration is not set");
+    }
+    const itemId = event.item_id;
+    const delta = event.delta;
+    const itemGeneration = this.currentGeneration.messages.get(itemId);
+    if (!itemGeneration) {
+      throw new Error("itemGeneration is not set");
+    } else {
+      itemGeneration.textChannel.write(delta);
+      itemGeneration.audioTranscript += delta;
+    }
   }
-  #handleConversationItemInputAudioTranscriptionCompleted(event) {
-    const transcript = event.transcript;
-    this.emit("input_speech_transcription_completed", {
-      itemId: event.item_id,
-      transcript
-    });
+  handleResponseAudioDelta(event) {
+    if (!this.currentGeneration) {
+      throw new Error("currentGeneration is not set");
+    }
+    const itemGeneration = this.currentGeneration.messages.get(event.item_id);
+    if (!itemGeneration) {
+      throw new Error("itemGeneration is not set");
+    }
+    const binaryString = atob(event.delta);
+    const len = binaryString.length;
+    const bytes = new Uint8Array(len);
+    for (let i = 0; i < len; i++) {
+      bytes[i] = binaryString.charCodeAt(i);
+    }
+    itemGeneration.audioChannel.write(
+      new AudioFrame(
+        new Int16Array(bytes.buffer),
+        api_proto.SAMPLE_RATE,
+        api_proto.NUM_CHANNELS,
+        bytes.length / 2
+      )
+    );
   }
-  #handleConversationItemInputAudioTranscriptionFailed(event) {
-    const error = event.error;
-    this.#logger.error(`OpenAI Realtime failed to transcribe input audio: ${error.message}`);
-    this.emit("input_speech_transcription_failed", {
-      itemId: event.item_id,
-      message: error.message
-    });
+  handleResponseAudioTranscriptDone(_event) {
+    if (!this.currentGeneration) {
+      throw new Error("currentGeneration is not set");
+    }
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  #handleConversationItemTruncated(event) {
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  #handleConversationItemDeleted(event) {
-  }
-  #handleResponseCreated(responseCreated) {
-    const response = responseCreated.response;
-    const doneFut = new Future();
-    const newResponse = {
-      id: response.id,
-      status: response.status,
-      statusDetails: response.status_details,
-      usage: null,
-      output: [],
-      doneFut,
-      createdTimestamp: Date.now()
-    };
-    this.#pendingResponses[newResponse.id] = newResponse;
-    this.emit("response_created", newResponse);
-  }
-  #handleResponseDone(event) {
-    const responseData = event.response;
-    const responseId = responseData.id;
-    const response = this.#pendingResponses[responseId];
-    response.status = responseData.status;
-    response.statusDetails = responseData.status_details;
-    response.usage = responseData.usage ?? null;
-    this.#pendingResponses[responseId] = response;
-    response.doneFut.resolve();
-    let metricsError;
-    let cancelled = false;
-    switch (response.status) {
-      case "failed": {
-        if (response.statusDetails.type !== "failed") break;
-        const err = response.statusDetails.error;
-        metricsError = new metrics.MultimodalLLMError({
-          type: response.statusDetails.type,
-          code: err == null ? void 0 : err.code,
-          message: err == null ? void 0 : err.message
-        });
-        this.#logger.child({ code: err == null ? void 0 : err.code, error: err == null ? void 0 : err.message }).error("response generation failed");
-        break;
-      }
-      case "incomplete": {
-        if (response.statusDetails.type !== "incomplete") break;
-        const reason = response.statusDetails.reason;
-        metricsError = new metrics.MultimodalLLMError({
-          type: response.statusDetails.type,
-          reason
-        });
-        this.#logger.child({ reason }).error("response generation incomplete");
-        break;
+  handleResponseAudioDone(_event) {
+    if (!this.currentGeneration) {
+      throw new Error("currentGeneration is not set");
+    }
+  }
+  handleResponseOutputItemDone(event) {
+    if (!this.currentGeneration) {
+      throw new Error("currentGeneration is not set");
+    }
+    const itemId = event.item.id;
+    const itemType = event.item.type;
+    if (itemType === "function_call") {
+      const item = event.item;
+      if (!item.call_id || !item.name || !item.arguments) {
+        throw new Error("item is not a function call");
       }
-      case "cancelled": {
-        cancelled = true;
-        break;
+      this.currentGeneration.functionChannel.write({
+        callId: item.call_id,
+        name: item.name,
+        args: item.arguments
+      });
+    } else if (itemType === "message") {
+      const itemGeneration = this.currentGeneration.messages.get(itemId);
+      if (!itemGeneration) {
+        return;
       }
+      itemGeneration.textChannel.close();
+      itemGeneration.audioChannel.close();
     }
-    this.emit("response_done", response);
-    let ttft;
-    if (response.firstTokenTimestamp) {
-      ttft = response.firstTokenTimestamp - response.createdTimestamp;
+  }
+  handleResponseDone(_event) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l;
+    if (!this.currentGeneration) {
+      return;
     }
-    const duration = Date.now() - response.createdTimestamp;
-    const usage = response.usage;
-    const metric = {
-      timestamp: response.createdTimestamp,
-      requestId: response.id,
+    const createdTimestamp = this.currentGeneration._createdTimestamp;
+    const firstTokenTimestamp = this.currentGeneration._firstTokenTimestamp;
+    this.#logger.debug(
+      {
+        messageCount: this.currentGeneration.messages.size
+      },
+      "Closing generation channels in handleResponseDone"
+    );
+    for (const generation of this.currentGeneration.messages.values()) {
+      generation.textChannel.close();
+      generation.audioChannel.close();
+    }
+    this.currentGeneration.functionChannel.close();
+    this.currentGeneration.messageChannel.close();
+    for (const itemId of this.currentGeneration.messages.keys()) {
+      const remoteItem = this.remoteChatCtx.get(itemId);
+      if (remoteItem && remoteItem.item instanceof llm.ChatMessage) {
+        remoteItem.item.content.push(this.currentGeneration.messages.get(itemId).audioTranscript);
+      }
+    }
+    this.currentGeneration._doneFut.resolve();
+    this.currentGeneration = void 0;
+    const usage = _event.response.usage;
+    const ttft = firstTokenTimestamp ? firstTokenTimestamp - createdTimestamp : -1;
+    const duration = (Date.now() - createdTimestamp) / 1e3;
+    const realtimeMetrics = {
+      type: "realtime_model_metrics",
+      timestamp: createdTimestamp / 1e3,
+      // Convert to seconds
+      requestId: _event.response.id || "",
       ttft,
       duration,
-      cancelled,
-      label: this.constructor.name,
-      completionTokens: (usage == null ? void 0 : usage.output_tokens) || 0,
-      promptTokens: (usage == null ? void 0 : usage.input_tokens) || 0,
-      totalTokens: (usage == null ? void 0 : usage.total_tokens) || 0,
-      tokensPerSecond: ((usage == null ? void 0 : usage.output_tokens) || 0) / duration * 1e3,
-      error: metricsError,
+      cancelled: _event.response.status === "cancelled",
+      label: "openai_realtime",
+      inputTokens: (usage == null ? void 0 : usage.input_tokens) ?? 0,
+      outputTokens: (usage == null ? void 0 : usage.output_tokens) ?? 0,
+      totalTokens: (usage == null ? void 0 : usage.total_tokens) ?? 0,
+      tokensPerSecond: duration > 0 ? ((usage == null ? void 0 : usage.output_tokens) ?? 0) / duration : 0,
       inputTokenDetails: {
-        cachedTokens: (usage == null ? void 0 : usage.input_token_details.cached_tokens) || 0,
-        textTokens: (usage == null ? void 0 : usage.input_token_details.text_tokens) || 0,
-        audioTokens: (usage == null ? void 0 : usage.input_token_details.audio_tokens) || 0
+        audioTokens: ((_a = usage == null ? void 0 : usage.input_token_details) == null ? void 0 : _a.audio_tokens) ?? 0,
+        textTokens: ((_b = usage == null ? void 0 : usage.input_token_details) == null ? void 0 : _b.text_tokens) ?? 0,
+        imageTokens: 0,
+        // Not supported yet
+        cachedTokens: ((_c = usage == null ? void 0 : usage.input_token_details) == null ? void 0 : _c.cached_tokens) ?? 0,
+        cachedTokensDetails: ((_d = usage == null ? void 0 : usage.input_token_details) == null ? void 0 : _d.cached_tokens_details) ? {
+          audioTokens: ((_f = (_e = usage == null ? void 0 : usage.input_token_details) == null ? void 0 : _e.cached_tokens_details) == null ? void 0 : _f.audio_tokens) ?? 0,
+          textTokens: ((_h = (_g = usage == null ? void 0 : usage.input_token_details) == null ? void 0 : _g.cached_tokens_details) == null ? void 0 : _h.text_tokens) ?? 0,
+          imageTokens: ((_j = (_i = usage == null ? void 0 : usage.input_token_details) == null ? void 0 : _i.cached_tokens_details) == null ? void 0 : _j.image_tokens) ?? 0
+        } : void 0
       },
       outputTokenDetails: {
-        textTokens: (usage == null ? void 0 : usage.output_token_details.text_tokens) || 0,
-        audioTokens: (usage == null ? void 0 : usage.output_token_details.audio_tokens) || 0
+        textTokens: ((_k = usage == null ? void 0 : usage.output_token_details) == null ? void 0 : _k.text_tokens) ?? 0,
+        audioTokens: ((_l = usage == null ? void 0 : usage.output_token_details) == null ? void 0 : _l.audio_tokens) ?? 0,
+        imageTokens: 0
       }
     };
-    this.emit("metrics_collected", metric);
+    this.emit("metrics_collected", realtimeMetrics);
   }
-  #handleResponseOutputItemAdded(event) {
-    const responseId = event.response_id;
-    const response = this.#pendingResponses[responseId];
-    const itemData = event.item;
-    if (itemData.type !== "message" && itemData.type !== "function_call") {
-      throw new Error(`Unexpected item type: ${itemData.type}`);
-    }
-    let role;
-    if (itemData.type === "function_call") {
-      role = "assistant";
-    } else {
-      role = itemData.role;
-    }
-    const newOutput = {
-      responseId,
-      itemId: itemData.id,
-      outputIndex: event.output_index,
-      type: itemData.type,
-      role,
-      content: [],
-      doneFut: new Future()
-    };
-    response == null ? void 0 : response.output.push(newOutput);
-    this.emit("response_output_added", newOutput);
+  handleError(event) {
+    if (event.error.message.startsWith("Cancellation failed")) {
+      return;
+    }
+    this.#logger.error({ error: event.error }, "OpenAI Realtime API returned an error");
+    this.emitError({
+      error: new APIError(event.error.message, {
+        body: event.error,
+        retryable: true
+      }),
+      recoverable: true
+    });
   }
-  #handleResponseOutputItemDone(event) {
-    const responseId = event.response_id;
-    const response = this.#pendingResponses[responseId];
-    const outputIndex = event.output_index;
-    const output = response.output[outputIndex];
-    if ((output == null ? void 0 : output.type) === "function_call") {
-      if (!this.#fncCtx) {
-        this.#logger.error("function call received but no fncCtx is available");
-        return;
-      }
-      const item = event.item;
-      if (item.type !== "function_call") {
-        throw new Error("Expected function_call item");
-      }
-      const func = this.#fncCtx[item.name];
-      if (!func) {
-        this.#logger.error(`no function with name ${item.name} in fncCtx`);
-        return;
-      }
-      this.emit("function_call_started", {
-        callId: item.call_id
-      });
-      const parsedArgs = JSON.parse(item.arguments);
-      this.#logger.debug(
-        `[Function Call ${item.call_id}] Executing ${item.name} with arguments ${parsedArgs}`
-      );
-      func.execute(parsedArgs).then(
-        (content) => {
-          this.#logger.debug(`[Function Call ${item.call_id}] ${item.name} returned ${content}`);
-          this.emit("function_call_completed", {
-            callId: item.call_id
-          });
-          this.conversation.item.create(
-            llm.ChatMessage.createToolFromFunctionResult({
-              name: item.name,
-              toolCallId: item.call_id,
-              result: content
-            }),
-            output.itemId
-          );
-          this.response.create();
-        },
-        (error) => {
-          this.#logger.error(`[Function Call ${item.call_id}] ${item.name} failed with ${error}`);
-          this.emit("function_call_failed", {
-            callId: item.call_id
-          });
-        }
-      );
+  emitError({ error, recoverable }) {
+    this.emit("error", {
+      timestamp: Date.now(),
+      // TODO(brian): add label
+      label: "",
+      error,
+      recoverable
+    });
+  }
+  *resampleAudio(frame) {
+    yield frame;
+  }
+  createResponse({
+    userInitiated,
+    instructions,
+    oldHandle
+  }) {
+    const handle = oldHandle || new CreateResponseHandle({ instructions });
+    if (oldHandle && instructions) {
+      handle.instructions = instructions;
     }
-    output == null ? void 0 : output.doneFut.resolve();
-    this.emit("response_output_done", output);
+    const eventId = shortuuid("response_create_");
+    if (userInitiated) {
+      this.responseCreatedFutures[eventId] = handle;
+    }
+    const response = {};
+    if (instructions) response.instructions = instructions;
+    if (userInitiated) response.metadata = { client_event_id: eventId };
+    this.sendEvent({
+      type: "response.create",
+      event_id: eventId,
+      response: Object.keys(response).length > 0 ? response : void 0
+    });
+    return handle;
   }
-  #handleResponseContentPartAdded(event) {
-    const responseId = event.response_id;
-    const response = this.#pendingResponses[responseId];
-    const outputIndex = event.output_index;
-    const output = response.output[outputIndex];
-    const textStream = new AsyncIterableQueue();
-    const audioStream = new AsyncIterableQueue();
-    const newContent = {
-      responseId,
-      itemId: event.item_id,
-      outputIndex,
-      contentIndex: event.content_index,
-      text: "",
-      audio: [],
-      textStream,
-      audioStream,
-      toolCalls: [],
-      contentType: event.part.type
+  emitGenerationEvent(responseId) {
+    if (!this.currentGeneration) {
+      throw new Error("currentGeneration is not set");
+    }
+    const generation_ev = {
+      messageStream: this.currentGeneration.messageChannel.stream(),
+      functionStream: this.currentGeneration.functionChannel.stream(),
+      userInitiated: false
     };
-    output == null ? void 0 : output.content.push(newContent);
-    response.firstTokenTimestamp = Date.now();
-    this.emit("response_content_added", newContent);
-  }
-  #handleResponseContentPartDone(event) {
-    const content = this.#getContent(event);
-    this.emit("response_content_done", content);
-  }
-  #handleResponseTextDelta(event) {
-    this.emit("response_text_delta", event);
-  }
-  #handleResponseTextDone(event) {
-    const content = this.#getContent(event);
-    content.text = event.text;
-    this.emit("response_text_done", event);
-  }
-  #handleResponseAudioTranscriptDelta(event) {
-    const content = this.#getContent(event);
-    const transcript = event.delta;
-    content.text += transcript;
-    content.textStream.put(transcript);
-  }
-  #handleResponseAudioTranscriptDone(event) {
-    const content = this.#getContent(event);
-    content.textStream.close();
-  }
-  #handleResponseAudioDelta(event) {
-    const content = this.#getContent(event);
-    const data = Buffer.from(event.delta, "base64");
-    const audio = new AudioFrame(
-      new Int16Array(data.buffer),
-      api_proto.SAMPLE_RATE,
-      api_proto.NUM_CHANNELS,
-      data.length / 2
-    );
-    content.audio.push(audio);
-    content.audioStream.put(audio);
+    const handle = this.responseCreatedFutures[responseId];
+    if (handle) {
+      delete this.responseCreatedFutures[responseId];
+      generation_ev.userInitiated = true;
+      if (handle.doneFut.done) {
+        this.#logger.warn({ responseId }, "response received after timeout");
+      } else {
+        handle.doneFut.resolve(generation_ev);
+      }
+    }
+    this.#logger.debug({ responseId }, "Emitting generation_created event");
+    this.emit("generation_created", generation_ev);
   }
-  #handleResponseAudioDone(event) {
-    const content = this.#getContent(event);
-    content.audioStream.close();
+}
+function livekitItemToOpenAIItem(item) {
+  switch (item.type) {
+    case "function_call":
+      return {
+        id: item.id,
+        type: "function_call",
+        call_id: item.callId,
+        name: item.name,
+        arguments: item.args
+      };
+    case "function_call_output":
+      return {
+        id: item.id,
+        type: "function_call_output",
+        call_id: item.callId,
+        output: item.output
+      };
+    case "message":
+      const role = item.role === "developer" ? "system" : item.role;
+      const contentList = [];
+      for (const c of item.content) {
+        if (typeof c === "string") {
+          contentList.push({
+            type: role === "assistant" ? "text" : "input_text",
+            text: c
+          });
+        } else if (c.type === "image_content") {
+          continue;
+        } else if (c.type === "audio_content") {
+          if (role === "user") {
+            const encodedAudio = Buffer.from(combineAudioFrames(c.frame).data).toString("base64");
+            contentList.push({
+              type: "input_audio",
+              audio: encodedAudio
+            });
+          }
+        }
+      }
+      return {
+        id: item.id,
+        type: "message",
+        role,
+        content: contentList
+      };
   }
-  #handleResponseFunctionCallArgumentsDelta(event) {
+}
+function openAIItemToLivekitItem(item) {
+  if (!item.id) {
+    throw new Error("item.id is not set");
   }
-  #handleResponseFunctionCallArgumentsDone(event) {
+  switch (item.type) {
+    case "function_call":
+      return llm.FunctionCall.create({
+        id: item.id,
+        callId: item.call_id,
+        name: item.name,
+        args: item.arguments
+      });
+    case "function_call_output":
+      return llm.FunctionCallOutput.create({
+        id: item.id,
+        callId: item.call_id,
+        output: item.output,
+        isError: false
+      });
+    case "message":
+      const content = [];
+      const contents = Array.isArray(item.content) ? item.content : [item.content];
+      for (const c of contents) {
+        if (c.type === "text" || c.type === "input_text") {
+          content.push(c.text);
+        }
+      }
+      return llm.ChatMessage.create({
+        id: item.id,
+        role: item.role,
+        content
+      });
+  }
+}
+function createMockAudioItem(durationSeconds = 2) {
+  const audioData = Buffer.alloc(durationSeconds * SAMPLE_RATE);
+  return llm.ChatMessage.create({
+    id: shortuuid(MOCK_AUDIO_ID_PREFIX),
+    role: "user",
+    content: [
+      {
+        type: "audio_content",
+        frame: [
+          new AudioFrame(
+            new Int16Array(audioData.buffer),
+            SAMPLE_RATE,
+            NUM_CHANNELS,
+            audioData.length / 2
+          )
+        ]
+      }
+    ]
+  });
+}
+function toOaiToolChoice(toolChoice) {
+  if (typeof toolChoice === "string") {
+    return toolChoice;
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  #handleRateLimitsUpdated(event) {
+  if ((toolChoice == null ? void 0 : toolChoice.type) === "function") {
+    return toolChoice.function.name;
   }
+  return "auto";
 }
 export {
   RealtimeModel,