@livekit/agents-plugin-openai 0.6.1 → 0.7.1

package/dist/realtime/realtime_model.cjs

@@ -0,0 +1,878 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var realtime_model_exports = {};
+__export(realtime_model_exports, {
+  RealtimeModel: () => RealtimeModel,
+  RealtimeSession: () => RealtimeSession
+});
+module.exports = __toCommonJS(realtime_model_exports);
+var import_agents = require("@livekit/agents");
+var import_rtc_node = require("@livekit/rtc-node");
+var import_node_events = require("node:events");
+var import_ws = require("ws");
+var api_proto = __toESM(require("./api_proto.cjs"), 1);
+class InputAudioBuffer {
+  #session;
+  constructor(session) {
+    this.#session = session;
+  }
+  append(frame) {
+    this.#session.queueMsg({
+      type: "input_audio_buffer.append",
+      audio: Buffer.from(frame.data.buffer).toString("base64")
+    });
+  }
+  clear() {
+    this.#session.queueMsg({
+      type: "input_audio_buffer.clear"
+    });
+  }
+  commit() {
+    this.#session.queueMsg({
+      type: "input_audio_buffer.commit"
+    });
+  }
+}
+class ConversationItem {
+  #session;
+  #logger = (0, import_agents.log)();
+  constructor(session) {
+    this.#session = session;
+  }
+  truncate(itemId, contentIndex, audioEnd) {
+    this.#session.queueMsg({
+      type: "conversation.item.truncate",
+      item_id: itemId,
+      content_index: contentIndex,
+      audio_end_ms: audioEnd
+    });
+  }
+  delete(itemId) {
+    this.#session.queueMsg({
+      type: "conversation.item.delete",
+      item_id: itemId
+    });
+  }
+  create(message, previousItemId) {
+    if (!message.content) {
+      return;
+    }
+    let event;
+    if (message.toolCallId) {
+      if (typeof message.content !== "string") {
+        throw new TypeError("message.content must be a string");
+      }
+      event = {
+        type: "conversation.item.create",
+        previous_item_id: previousItemId,
+        item: {
+          type: "function_call_output",
+          call_id: message.toolCallId,
+          output: message.content
+        }
+      };
+    } else {
+      let content = message.content;
+      if (!Array.isArray(content)) {
+        content = [content];
+      }
+      if (message.role === import_agents.llm.ChatRole.USER) {
+        const contents = [];
+        for (const c of content) {
+          if (typeof c === "string") {
+            contents.push({
+              type: "input_text",
+              text: c
+            });
+          } else if (
+            // typescript type guard for determining ChatAudio vs ChatImage
+            ((c2) => {
+              return c2.frame !== void 0;
+            })(c)
+          ) {
+            contents.push({
+              type: "input_audio",
+              audio: Buffer.from((0, import_agents.mergeFrames)(c.frame).data.buffer).toString("base64")
+            });
+          }
+        }
+        event = {
+          type: "conversation.item.create",
+          previous_item_id: previousItemId,
+          item: {
+            type: "message",
+            role: "user",
+            content: contents
+          }
+        };
+      } else if (message.role === import_agents.llm.ChatRole.ASSISTANT) {
+        const contents = [];
+        for (const c of content) {
+          if (typeof c === "string") {
+            contents.push({
+              type: "text",
+              text: c
+            });
+          } else if (
+            // typescript type guard for determining ChatAudio vs ChatImage
+            ((c2) => {
+              return c2.frame !== void 0;
+            })(c)
+          ) {
+            this.#logger.warn("audio content in assistant message is not supported");
+          }
+        }
+        event = {
+          type: "conversation.item.create",
+          previous_item_id: previousItemId,
+          item: {
+            type: "message",
+            role: "assistant",
+            content: contents
+          }
+        };
+      } else if (message.role === import_agents.llm.ChatRole.SYSTEM) {
+        const contents = [];
+        for (const c of content) {
+          if (typeof c === "string") {
+            contents.push({
+              type: "input_text",
+              text: c
+            });
+          } else if (
+            // typescript type guard for determining ChatAudio vs ChatImage
+            ((c2) => {
+              return c2.frame !== void 0;
+            })(c)
+          ) {
+            this.#logger.warn("audio content in system message is not supported");
+          }
+        }
+        event = {
+          type: "conversation.item.create",
+          previous_item_id: previousItemId,
+          item: {
+            type: "message",
+            role: "system",
+            content: contents
+          }
+        };
+      } else {
+        this.#logger.child({ message }).warn("chat message is not supported inside the realtime API");
+        return;
+      }
+    }
+    this.#session.queueMsg(event);
+  }
+}
+class Conversation {
+  #session;
+  constructor(session) {
+    this.#session = session;
+  }
+  get item() {
+    return new ConversationItem(this.#session);
+  }
+}
+class Response {
+  #session;
+  constructor(session) {
+    this.#session = session;
+  }
+  create() {
+    this.#session.queueMsg({
+      type: "response.create"
+    });
+  }
+  cancel() {
+    this.#session.queueMsg({
+      type: "response.cancel"
+    });
+  }
+}
+class RealtimeModel extends import_agents.multimodal.RealtimeModel {
+  sampleRate = api_proto.SAMPLE_RATE;
+  numChannels = api_proto.NUM_CHANNELS;
+  inFrameSize = api_proto.IN_FRAME_SIZE;
+  outFrameSize = api_proto.OUT_FRAME_SIZE;
+  #defaultOpts;
+  #sessions = [];
+  static withAzure({
+    baseURL,
+    azureDeployment,
+    apiVersion = "2024-10-01-preview",
+    apiKey = void 0,
+    entraToken = void 0,
+    instructions = "",
+    modalities = ["text", "audio"],
+    voice = "alloy",
+    inputAudioFormat = "pcm16",
+    outputAudioFormat = "pcm16",
+    inputAudioTranscription = { model: "whisper-1" },
+    turnDetection = { type: "server_vad" },
+    temperature = 0.8,
+    maxResponseOutputTokens = Infinity
+  }) {
+    return new RealtimeModel({
+      isAzure: true,
+      baseURL: new URL("openai", baseURL).toString(),
+      model: azureDeployment,
+      apiVersion,
+      apiKey,
+      entraToken,
+      instructions,
+      modalities,
+      voice,
+      inputAudioFormat,
+      outputAudioFormat,
+      inputAudioTranscription,
+      turnDetection,
+      temperature,
+      maxResponseOutputTokens
+    });
+  }
+  constructor({
+    modalities = ["text", "audio"],
+    instructions = "",
+    voice = "alloy",
+    inputAudioFormat = "pcm16",
+    outputAudioFormat = "pcm16",
+    inputAudioTranscription = { model: "whisper-1" },
+    turnDetection = { type: "server_vad" },
+    temperature = 0.8,
+    maxResponseOutputTokens = Infinity,
+    model = "gpt-4o-realtime-preview-2024-10-01",
+    apiKey = process.env.OPENAI_API_KEY || "",
+    baseURL = api_proto.BASE_URL,
+    // used for microsoft
+    isAzure = false,
+    apiVersion = void 0,
+    entraToken = void 0
+  }) {
+    super();
+    if (apiKey === "") {
+      throw new Error(
+        "OpenAI API key is required, either using the argument or by setting the OPENAI_API_KEY environmental variable"
+      );
+    }
+    this.#defaultOpts = {
+      modalities,
+      instructions,
+      voice,
+      inputAudioFormat,
+      outputAudioFormat,
+      inputAudioTranscription,
+      turnDetection,
+      temperature,
+      maxResponseOutputTokens,
+      model,
+      apiKey,
+      baseURL,
+      isAzure,
+      apiVersion,
+      entraToken
+    };
+  }
+  get sessions() {
+    return this.#sessions;
+  }
+  session({
+    fncCtx,
+    chatCtx,
+    modalities = this.#defaultOpts.modalities,
+    instructions = this.#defaultOpts.instructions,
+    voice = this.#defaultOpts.voice,
+    inputAudioFormat = this.#defaultOpts.inputAudioFormat,
+    outputAudioFormat = this.#defaultOpts.outputAudioFormat,
+    inputAudioTranscription = this.#defaultOpts.inputAudioTranscription,
+    turnDetection = this.#defaultOpts.turnDetection,
+    temperature = this.#defaultOpts.temperature,
+    maxResponseOutputTokens = this.#defaultOpts.maxResponseOutputTokens
+  }) {
+    const opts = {
+      modalities,
+      instructions,
+      voice,
+      inputAudioFormat,
+      outputAudioFormat,
+      inputAudioTranscription,
+      turnDetection,
+      temperature,
+      maxResponseOutputTokens,
+      model: this.#defaultOpts.model,
+      apiKey: this.#defaultOpts.apiKey,
+      baseURL: this.#defaultOpts.baseURL,
+      isAzure: this.#defaultOpts.isAzure,
+      apiVersion: this.#defaultOpts.apiVersion,
+      entraToken: this.#defaultOpts.entraToken
+    };
+    const newSession = new RealtimeSession(opts, {
+      chatCtx: chatCtx || new import_agents.llm.ChatContext(),
+      fncCtx
+    });
+    this.#sessions.push(newSession);
+    return newSession;
+  }
+  async close() {
+    await Promise.allSettled(this.#sessions.map((session) => session.close()));
+  }
+}
+class RealtimeSession extends import_agents.multimodal.RealtimeSession {
+  #chatCtx = void 0;
+  #fncCtx = void 0;
+  #opts;
+  #pendingResponses = {};
+  #sessionId = "not-connected";
+  #ws = null;
+  #expiresAt = null;
+  #logger = (0, import_agents.log)();
+  #task;
+  #closing = true;
+  #sendQueue = new import_agents.Queue();
+  constructor(opts, { fncCtx, chatCtx }) {
+    super();
+    this.#opts = opts;
+    this.#chatCtx = chatCtx;
+    this.#fncCtx = fncCtx;
+    this.#task = this.#start();
+    this.sessionUpdate({
+      modalities: this.#opts.modalities,
+      instructions: this.#opts.instructions,
+      voice: this.#opts.voice,
+      inputAudioFormat: this.#opts.inputAudioFormat,
+      outputAudioFormat: this.#opts.outputAudioFormat,
+      inputAudioTranscription: this.#opts.inputAudioTranscription,
+      turnDetection: this.#opts.turnDetection,
+      temperature: this.#opts.temperature,
+      maxResponseOutputTokens: this.#opts.maxResponseOutputTokens,
+      toolChoice: "auto"
+    });
+  }
+  get chatCtx() {
+    return this.#chatCtx;
+  }
+  get fncCtx() {
+    return this.#fncCtx;
+  }
+  set fncCtx(ctx) {
+    this.#fncCtx = ctx;
+  }
+  get conversation() {
+    return new Conversation(this);
+  }
+  get inputAudioBuffer() {
+    return new InputAudioBuffer(this);
+  }
+  get response() {
+    return new Response(this);
+  }
+  get expiration() {
+    if (!this.#expiresAt) {
+      throw new Error("session not started");
+    }
+    return this.#expiresAt * 1e3;
+  }
+  queueMsg(command) {
+    this.#sendQueue.put(command);
+  }
+  /// Truncates the data field of the event to the specified maxLength to avoid overwhelming logs
+  /// with large amounts of base64 audio data.
+  #loggableEvent(event, maxLength = 30) {
+    const untypedEvent = {};
+    for (const [key, value] of Object.entries(event)) {
+      if (value !== void 0) {
+        untypedEvent[key] = value;
+      }
+    }
+    if (untypedEvent.audio && typeof untypedEvent.audio === "string") {
+      const truncatedData = untypedEvent.audio.slice(0, maxLength) + (untypedEvent.audio.length > maxLength ? "\u2026" : "");
+      return { ...untypedEvent, audio: truncatedData };
+    }
+    if (untypedEvent.delta && typeof untypedEvent.delta === "string" && event.type === "response.audio.delta") {
+      const truncatedDelta = untypedEvent.delta.slice(0, maxLength) + (untypedEvent.delta.length > maxLength ? "\u2026" : "");
+      return { ...untypedEvent, delta: truncatedDelta };
+    }
+    return untypedEvent;
+  }
+  sessionUpdate({
+    modalities = this.#opts.modalities,
+    instructions = this.#opts.instructions,
+    voice = this.#opts.voice,
+    inputAudioFormat = this.#opts.inputAudioFormat,
+    outputAudioFormat = this.#opts.outputAudioFormat,
+    inputAudioTranscription = this.#opts.inputAudioTranscription,
+    turnDetection = this.#opts.turnDetection,
+    temperature = this.#opts.temperature,
+    maxResponseOutputTokens = this.#opts.maxResponseOutputTokens,
+    toolChoice = "auto"
+  }) {
+    this.#opts = {
+      modalities,
+      instructions,
+      voice,
+      inputAudioFormat,
+      outputAudioFormat,
+      inputAudioTranscription,
+      turnDetection,
+      temperature,
+      maxResponseOutputTokens,
+      model: this.#opts.model,
+      apiKey: this.#opts.apiKey,
+      baseURL: this.#opts.baseURL,
+      isAzure: this.#opts.isAzure,
+      apiVersion: this.#opts.apiVersion,
+      entraToken: this.#opts.entraToken
+    };
+    const tools = this.#fncCtx ? Object.entries(this.#fncCtx).map(([name, func]) => ({
+      type: "function",
+      name,
+      description: func.description,
+      parameters: (
+        // don't format parameters if they are raw openai params
+        func.parameters.type == "object" ? func.parameters : import_agents.llm.oaiParams(func.parameters)
+      )
+    })) : [];
+    const sessionUpdateEvent = {
+      type: "session.update",
+      session: {
+        modalities: this.#opts.modalities,
+        instructions: this.#opts.instructions,
+        voice: this.#opts.voice,
+        input_audio_format: this.#opts.inputAudioFormat,
+        output_audio_format: this.#opts.outputAudioFormat,
+        input_audio_transcription: this.#opts.inputAudioTranscription,
+        turn_detection: this.#opts.turnDetection,
+        temperature: this.#opts.temperature,
+        max_response_output_tokens: this.#opts.maxResponseOutputTokens === Infinity ? "inf" : this.#opts.maxResponseOutputTokens,
+        tools,
+        tool_choice: toolChoice
+      }
+    };
+    if (this.#opts.isAzure && this.#opts.maxResponseOutputTokens === Infinity) {
+      sessionUpdateEvent.session.max_response_output_tokens = void 0;
+    }
+    this.queueMsg(sessionUpdateEvent);
+  }
+  #start() {
+    return new Promise(async (resolve, reject) => {
+      const headers = {
+        "User-Agent": "LiveKit-Agents-JS"
+      };
+      if (this.#opts.isAzure) {
+        if (this.#opts.entraToken) {
+          headers.Authorization = `Bearer ${this.#opts.entraToken}`;
+        } else if (this.#opts.apiKey) {
+          headers["api-key"] = this.#opts.apiKey;
+        } else {
+          reject(new Error("Microsoft API key or entraToken is required"));
+          return;
+        }
+      } else {
+        headers.Authorization = `Bearer ${this.#opts.apiKey}`;
+        headers["OpenAI-Beta"] = "realtime=v1";
+      }
+      const url = new URL([this.#opts.baseURL, "realtime"].join("/"));
+      if (url.protocol === "https:") {
+        url.protocol = "wss:";
+      }
+      const queryParams = {};
+      if (this.#opts.isAzure) {
+        queryParams["api-version"] = "2024-10-01-preview";
+        queryParams["deployment"] = this.#opts.model;
+      } else {
+        queryParams["model"] = this.#opts.model;
+      }
+      for (const [key, value] of Object.entries(queryParams)) {
+        url.searchParams.set(key, value);
+      }
+      console.debug("Connecting to OpenAI Realtime API at ", url.toString());
+      this.#ws = new import_ws.WebSocket(url.toString(), {
+        headers
+      });
+      this.#ws.onerror = (error) => {
+        reject(new Error("OpenAI Realtime WebSocket error: " + error.message));
+      };
+      await (0, import_node_events.once)(this.#ws, "open");
+      this.#closing = false;
+      this.#ws.onmessage = (message) => {
+        const event = JSON.parse(message.data);
+        this.#logger.debug(`<- ${JSON.stringify(this.#loggableEvent(event))}`);
+        switch (event.type) {
+          case "error":
+            this.#handleError(event);
+            break;
+          case "session.created":
+            this.#handleSessionCreated(event);
+            break;
+          case "session.updated":
+            this.#handleSessionUpdated(event);
+            break;
+          case "conversation.created":
+            this.#handleConversationCreated(event);
+            break;
+          case "input_audio_buffer.committed":
+            this.#handleInputAudioBufferCommitted(event);
+            break;
+          case "input_audio_buffer.cleared":
+            this.#handleInputAudioBufferCleared(event);
+            break;
+          case "input_audio_buffer.speech_started":
+            this.#handleInputAudioBufferSpeechStarted(event);
+            break;
+          case "input_audio_buffer.speech_stopped":
+            this.#handleInputAudioBufferSpeechStopped(event);
+            break;
+          case "conversation.item.created":
+            this.#handleConversationItemCreated(event);
+            break;
+          case "conversation.item.input_audio_transcription.completed":
+            this.#handleConversationItemInputAudioTranscriptionCompleted(event);
+            break;
+          case "conversation.item.input_audio_transcription.failed":
+            this.#handleConversationItemInputAudioTranscriptionFailed(event);
+            break;
+          case "conversation.item.truncated":
+            this.#handleConversationItemTruncated(event);
+            break;
+          case "conversation.item.deleted":
+            this.#handleConversationItemDeleted(event);
+            break;
+          case "response.created":
+            this.#handleResponseCreated(event);
+            break;
+          case "response.done":
+            this.#handleResponseDone(event);
+            break;
+          case "response.output_item.added":
+            this.#handleResponseOutputItemAdded(event);
+            break;
+          case "response.output_item.done":
+            this.#handleResponseOutputItemDone(event);
+            break;
+          case "response.content_part.added":
+            this.#handleResponseContentPartAdded(event);
+            break;
+          case "response.content_part.done":
+            this.#handleResponseContentPartDone(event);
+            break;
+          case "response.text.delta":
+            this.#handleResponseTextDelta(event);
+            break;
+          case "response.text.done":
+            this.#handleResponseTextDone(event);
+            break;
+          case "response.audio_transcript.delta":
+            this.#handleResponseAudioTranscriptDelta(event);
+            break;
+          case "response.audio_transcript.done":
+            this.#handleResponseAudioTranscriptDone(event);
+            break;
+          case "response.audio.delta":
+            this.#handleResponseAudioDelta(event);
+            break;
+          case "response.audio.done":
+            this.#handleResponseAudioDone(event);
+            break;
+          case "response.function_call_arguments.delta":
+            this.#handleResponseFunctionCallArgumentsDelta(event);
+            break;
+          case "response.function_call_arguments.done":
+            this.#handleResponseFunctionCallArgumentsDone(event);
+            break;
+          case "rate_limits.updated":
+            this.#handleRateLimitsUpdated(event);
+            break;
+        }
+      };
+      const sendTask = async () => {
+        while (this.#ws && !this.#closing && this.#ws.readyState === import_ws.WebSocket.OPEN) {
+          try {
+            const event = await this.#sendQueue.get();
+            if (event.type !== "input_audio_buffer.append") {
+              this.#logger.debug(`-> ${JSON.stringify(this.#loggableEvent(event))}`);
+            }
+            this.#ws.send(JSON.stringify(event));
+          } catch (error) {
+            this.#logger.error("Error sending event:", error);
+          }
+        }
+      };
+      sendTask();
+      this.#ws.onclose = () => {
+        if (this.#expiresAt && Date.now() >= this.#expiresAt * 1e3) {
+          this.#closing = true;
+        }
+        if (!this.#closing) {
+          reject(new Error("OpenAI Realtime connection closed unexpectedly"));
+        }
+        this.#ws = null;
+        resolve();
+      };
+    });
+  }
+  async close() {
+    if (!this.#ws) return;
+    this.#closing = true;
+    this.#ws.close();
+    await this.#task;
+  }
+  #getContent(ptr) {
+    const response = this.#pendingResponses[ptr.response_id];
+    const output = response.output[ptr.output_index];
+    const content = output.content[ptr.content_index];
+    return content;
+  }
+  #handleError(event) {
+    this.#logger.error(`OpenAI Realtime error ${JSON.stringify(event.error)}`);
+  }
+  #handleSessionCreated(event) {
+    this.#sessionId = event.session.id;
+    this.#expiresAt = event.session.expires_at;
+    this.#logger = this.#logger.child({ sessionId: this.#sessionId });
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  #handleSessionUpdated(event) {
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  #handleConversationCreated(event) {
+  }
+  #handleInputAudioBufferCommitted(event) {
+    this.emit("input_speech_committed", {
+      itemId: event.item_id
+    });
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  #handleInputAudioBufferCleared(event) {
+  }
+  #handleInputAudioBufferSpeechStarted(event) {
+    this.emit("input_speech_started", {
+      itemId: event.item_id
+    });
+  }
+  #handleInputAudioBufferSpeechStopped(event) {
+    this.emit("input_speech_stopped");
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  #handleConversationItemCreated(event) {
+  }
+  #handleConversationItemInputAudioTranscriptionCompleted(event) {
+    const transcript = event.transcript;
+    this.emit("input_speech_transcription_completed", {
+      itemId: event.item_id,
+      transcript
+    });
+  }
+  #handleConversationItemInputAudioTranscriptionFailed(event) {
+    const error = event.error;
+    this.#logger.error(`OpenAI Realtime failed to transcribe input audio: ${error.message}`);
+    this.emit("input_speech_transcription_failed", {
+      itemId: event.item_id,
+      message: error.message
+    });
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  #handleConversationItemTruncated(event) {
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  #handleConversationItemDeleted(event) {
+  }
+  #handleResponseCreated(responseCreated) {
+    const response = responseCreated.response;
+    const doneFut = new import_agents.Future();
+    const newResponse = {
+      id: response.id,
+      status: response.status,
+      statusDetails: response.status_details,
+      usage: null,
+      output: [],
+      doneFut
+    };
+    this.#pendingResponses[newResponse.id] = newResponse;
+    this.emit("response_created", newResponse);
+  }
+  #handleResponseDone(event) {
+    const responseData = event.response;
+    const responseId = responseData.id;
+    const response = this.#pendingResponses[responseId];
+    response.status = responseData.status;
+    response.statusDetails = responseData.status_details;
+    response.usage = responseData.usage ?? null;
+    this.#pendingResponses[responseId] = response;
+    response.doneFut.resolve();
+    this.emit("response_done", response);
+  }
+  #handleResponseOutputItemAdded(event) {
+    const responseId = event.response_id;
+    const response = this.#pendingResponses[responseId];
+    const itemData = event.item;
+    if (itemData.type !== "message" && itemData.type !== "function_call") {
+      throw new Error(`Unexpected item type: ${itemData.type}`);
+    }
+    let role;
+    if (itemData.type === "function_call") {
+      role = "assistant";
+    } else {
+      role = itemData.role;
+    }
+    const newOutput = {
+      responseId,
+      itemId: itemData.id,
+      outputIndex: event.output_index,
+      type: itemData.type,
+      role,
+      content: [],
+      doneFut: new import_agents.Future()
+    };
+    response == null ? void 0 : response.output.push(newOutput);
+    this.emit("response_output_added", newOutput);
+  }
+  #handleResponseOutputItemDone(event) {
+    const responseId = event.response_id;
+    const response = this.#pendingResponses[responseId];
+    const outputIndex = event.output_index;
+    const output = response.output[outputIndex];
+    if ((output == null ? void 0 : output.type) === "function_call") {
+      if (!this.#fncCtx) {
+        this.#logger.error("function call received but no fncCtx is available");
+        return;
+      }
+      const item = event.item;
+      if (item.type !== "function_call") {
+        throw new Error("Expected function_call item");
+      }
+      const func = this.#fncCtx[item.name];
+      if (!func) {
+        this.#logger.error(`no function with name ${item.name} in fncCtx`);
+        return;
+      }
+      this.emit("function_call_started", {
+        callId: item.call_id
+      });
+      const parsedArgs = JSON.parse(item.arguments);
+      this.#logger.debug(
+        `[Function Call ${item.call_id}] Executing ${item.name} with arguments ${parsedArgs}`
+      );
+      func.execute(parsedArgs).then(
+        (content) => {
+          this.#logger.debug(`[Function Call ${item.call_id}] ${item.name} returned ${content}`);
+          this.emit("function_call_completed", {
+            callId: item.call_id
+          });
+          this.conversation.item.create(
+            import_agents.llm.ChatMessage.createToolFromFunctionResult({
+              name: item.name,
+              toolCallId: item.call_id,
+              result: content
+            }),
+            output.itemId
+          );
+          this.response.create();
+        },
+        (error) => {
+          this.#logger.error(`[Function Call ${item.call_id}] ${item.name} failed with ${error}`);
+          this.emit("function_call_failed", {
+            callId: item.call_id
+          });
+        }
+      );
+    }
+    output == null ? void 0 : output.doneFut.resolve();
+    this.emit("response_output_done", output);
+  }
+  #handleResponseContentPartAdded(event) {
+    const responseId = event.response_id;
+    const response = this.#pendingResponses[responseId];
+    const outputIndex = event.output_index;
+    const output = response.output[outputIndex];
+    const textStream = new import_agents.AsyncIterableQueue();
+    const audioStream = new import_agents.AsyncIterableQueue();
+    const newContent = {
+      responseId,
+      itemId: event.item_id,
+      outputIndex,
+      contentIndex: event.content_index,
+      text: "",
+      audio: [],
+      textStream,
+      audioStream,
+      toolCalls: []
+    };
+    output == null ? void 0 : output.content.push(newContent);
+    this.emit("response_content_added", newContent);
+  }
+  #handleResponseContentPartDone(event) {
+    const content = this.#getContent(event);
+    this.emit("response_content_done", content);
+  }
+  #handleResponseTextDelta(event) {
+    this.emit("response_text_delta", event);
+  }
+  #handleResponseTextDone(event) {
+    this.emit("response_text_done", event);
+  }
+  #handleResponseAudioTranscriptDelta(event) {
+    const content = this.#getContent(event);
+    const transcript = event.delta;
+    content.text += transcript;
+    content.textStream.put(transcript);
+  }
+  #handleResponseAudioTranscriptDone(event) {
+    const content = this.#getContent(event);
+    content.textStream.close();
+  }
+  #handleResponseAudioDelta(event) {
+    const content = this.#getContent(event);
+    const data = Buffer.from(event.delta, "base64");
+    const audio = new import_rtc_node.AudioFrame(
+      new Int16Array(data.buffer),
+      api_proto.SAMPLE_RATE,
+      api_proto.NUM_CHANNELS,
+      data.length / 2
+    );
+    content.audio.push(audio);
+    content.audioStream.put(audio);
+  }
+  #handleResponseAudioDone(event) {
+    const content = this.#getContent(event);
+    content.audioStream.close();
+  }
+  #handleResponseFunctionCallArgumentsDelta(event) {
+  }
+  #handleResponseFunctionCallArgumentsDone(event) {
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  #handleRateLimitsUpdated(event) {
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  RealtimeModel,
+  RealtimeSession
+});
+//# sourceMappingURL=realtime_model.cjs.map