@livekit/agents-plugin-openai 1.0.51 → 1.2.0

package/dist/realtime/realtime_model.test.cjs

@@ -4,13 +4,13 @@ var import_vitest = require("vitest");
 var import_realtime_model = require("./realtime_model.cjs");
 (0, import_vitest.describe)("livekitItemToOpenAIItem", () => {
   (0, import_vitest.describe)("message items", () => {
-    (0, import_vitest.it)("should use output_text type for assistant messages", () => {
+    (0, import_vitest.it)("should use output_text type for assistant messages", async () => {
       const assistantMessage = new import_agents.llm.ChatMessage({
         role: "assistant",
         content: "Hello, how can I help you?",
         id: "test-assistant-msg"
       });
-      const result = (0, import_realtime_model.livekitItemToOpenAIItem)(assistantMessage);
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(assistantMessage);
       (0, import_vitest.expect)(result.type).toBe("message");
       (0, import_vitest.expect)(result.role).toBe("assistant");
       (0, import_vitest.expect)(result.content).toHaveLength(1);
@@ -18,13 +18,13 @@ var import_realtime_model = require("./realtime_model.cjs");
       (0, import_vitest.expect)(content.type).toBe("output_text");
       (0, import_vitest.expect)(content.text).toBe("Hello, how can I help you?");
     });
-    (0, import_vitest.it)("should use input_text type for user messages", () => {
+    (0, import_vitest.it)("should use input_text type for user messages", async () => {
       const userMessage = new import_agents.llm.ChatMessage({
         role: "user",
         content: "What is the weather like?",
         id: "test-user-msg"
       });
-      const result = (0, import_realtime_model.livekitItemToOpenAIItem)(userMessage);
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(userMessage);
       (0, import_vitest.expect)(result.type).toBe("message");
       (0, import_vitest.expect)(result.role).toBe("user");
       (0, import_vitest.expect)(result.content).toHaveLength(1);
@@ -32,54 +32,142 @@ var import_realtime_model = require("./realtime_model.cjs");
       (0, import_vitest.expect)(content.type).toBe("input_text");
       (0, import_vitest.expect)(content.text).toBe("What is the weather like?");
     });
-    (0, import_vitest.it)("should use input_text type for system messages", () => {
+    (0, import_vitest.it)("should use input_text type for system messages", async () => {
       const systemMessage = new import_agents.llm.ChatMessage({
         role: "system",
         content: "You are a helpful assistant.",
         id: "test-system-msg"
       });
-      const result = (0, import_realtime_model.livekitItemToOpenAIItem)(systemMessage);
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(systemMessage);
       (0, import_vitest.expect)(result.type).toBe("message");
       (0, import_vitest.expect)(result.role).toBe("system");
       (0, import_vitest.expect)(result.content).toHaveLength(1);
       const content = result.content[0];
       (0, import_vitest.expect)(content.type).toBe("input_text");
     });
-    (0, import_vitest.it)("should convert developer role to system role", () => {
+    (0, import_vitest.it)("should convert developer role to system role", async () => {
       const developerMessage = new import_agents.llm.ChatMessage({
         role: "developer",
         content: "System instructions.",
         id: "test-developer-msg"
       });
-      const result = (0, import_realtime_model.livekitItemToOpenAIItem)(developerMessage);
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(developerMessage);
       (0, import_vitest.expect)(result.type).toBe("message");
       (0, import_vitest.expect)(result.role).toBe("system");
       const content = result.content[0];
       (0, import_vitest.expect)(content.type).toBe("input_text");
     });
-    (0, import_vitest.it)("should handle multiple content items for assistant", () => {
+    (0, import_vitest.it)("should handle multiple content items for assistant", async () => {
       const multiContentMessage = new import_agents.llm.ChatMessage({
         role: "assistant",
         content: ["First part.", "Second part."],
         id: "test-multi-msg"
       });
-      const result = (0, import_realtime_model.livekitItemToOpenAIItem)(multiContentMessage);
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(
+        multiContentMessage
+      );
       (0, import_vitest.expect)(result.content).toHaveLength(2);
       const content0 = result.content[0];
       const content1 = result.content[1];
       (0, import_vitest.expect)(content0.type).toBe("output_text");
       (0, import_vitest.expect)(content1.type).toBe("output_text");
     });
+    (0, import_vitest.it)("should convert image content to input_image for user messages", async () => {
+      const base64Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+      const imageContent = import_agents.llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: "image/png"
+      });
+      const userMessage = new import_agents.llm.ChatMessage({
+        role: "user",
+        content: [imageContent],
+        id: "test-image-msg"
+      });
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(userMessage);
+      (0, import_vitest.expect)(result.type).toBe("message");
+      (0, import_vitest.expect)(result.role).toBe("user");
+      (0, import_vitest.expect)(result.content).toHaveLength(1);
+      const content = result.content[0];
+      (0, import_vitest.expect)(content.type).toBe("input_image");
+      (0, import_vitest.expect)(content.image_url).toBe(
+        `data:image/png;base64,${base64Data}`
+      );
+    });
+    (0, import_vitest.it)("should ignore image content for assistant messages", async () => {
+      const base64Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+      const imageContent = import_agents.llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: "image/png"
+      });
+      const assistantMessage = new import_agents.llm.ChatMessage({
+        role: "assistant",
+        content: [imageContent],
+        id: "test-assistant-image-msg"
+      });
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(assistantMessage);
+      (0, import_vitest.expect)(result.type).toBe("message");
+      (0, import_vitest.expect)(result.role).toBe("assistant");
+      (0, import_vitest.expect)(result.content).toHaveLength(0);
+    });
+    (0, import_vitest.it)("should ignore image content for system messages", async () => {
+      const base64Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+      const imageContent = import_agents.llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: "image/png"
+      });
+      const systemMessage = new import_agents.llm.ChatMessage({
+        role: "system",
+        content: [imageContent],
+        id: "test-system-image-msg"
+      });
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(systemMessage);
+      (0, import_vitest.expect)(result.type).toBe("message");
+      (0, import_vitest.expect)(result.role).toBe("system");
+      (0, import_vitest.expect)(result.content).toHaveLength(0);
+    });
+    (0, import_vitest.it)("should ignore image content for developer messages mapped to system", async () => {
+      const base64Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+      const imageContent = import_agents.llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: "image/png"
+      });
+      const developerMessage = new import_agents.llm.ChatMessage({
+        role: "developer",
+        content: [imageContent],
+        id: "test-developer-image-msg"
+      });
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(developerMessage);
+      (0, import_vitest.expect)(result.type).toBe("message");
+      (0, import_vitest.expect)(result.role).toBe("system");
+      (0, import_vitest.expect)(result.content).toHaveLength(0);
+    });
+    (0, import_vitest.it)("should handle mixed text and image content", async () => {
+      const base64Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+      const imageContent = import_agents.llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: "image/png"
+      });
+      const userMessage = new import_agents.llm.ChatMessage({
+        role: "user",
+        content: ["Describe this image:", imageContent],
+        id: "test-mixed-msg"
+      });
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(userMessage);
+      (0, import_vitest.expect)(result.type).toBe("message");
+      (0, import_vitest.expect)(result.content).toHaveLength(2);
+      (0, import_vitest.expect)(result.content[0].type).toBe("input_text");
+      (0, import_vitest.expect)(result.content[1].type).toBe("input_image");
+    });
   });
   (0, import_vitest.describe)("function_call items", () => {
-    (0, import_vitest.it)("should convert function call items correctly", () => {
+    (0, import_vitest.it)("should convert function call items correctly", async () => {
       const functionCall = new import_agents.llm.FunctionCall({
         callId: "call-123",
         name: "get_weather",
         args: '{"location": "San Francisco"}',
         id: "test-func-call"
       });
-      const result = (0, import_realtime_model.livekitItemToOpenAIItem)(functionCall);
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(functionCall);
       (0, import_vitest.expect)(result.type).toBe("function_call");
       (0, import_vitest.expect)(result.id).toBe("test-func-call");
       (0, import_vitest.expect)(result.call_id).toBe("call-123");
@@ -88,14 +176,16 @@ var import_realtime_model = require("./realtime_model.cjs");
     });
   });
   (0, import_vitest.describe)("function_call_output items", () => {
-    (0, import_vitest.it)("should convert function call output items correctly", () => {
+    (0, import_vitest.it)("should convert function call output items correctly", async () => {
       const functionOutput = new import_agents.llm.FunctionCallOutput({
         callId: "call-123",
         output: "The weather in San Francisco is sunny.",
         isError: false,
         id: "test-func-output"
       });
-      const result = (0, import_realtime_model.livekitItemToOpenAIItem)(functionOutput);
+      const result = await (0, import_realtime_model.livekitItemToOpenAIItem)(
+        functionOutput
+      );
       (0, import_vitest.expect)(result.type).toBe("function_call_output");
       (0, import_vitest.expect)(result.id).toBe("test-func-output");
       (0, import_vitest.expect)(result.call_id).toBe("call-123");

package/dist/realtime/realtime_model.test.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/realtime/realtime_model.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm } from '@livekit/agents';\nimport { describe, expect, it } from 'vitest';\nimport type * as api_proto from './api_proto.js';\nimport { livekitItemToOpenAIItem } from './realtime_model.js';\n\ndescribe('livekitItemToOpenAIItem', () => {\n  describe('message items', () => {\n    it('should use output_text type for assistant messages', () => {\n      const assistantMessage = new llm.ChatMessage({\n        role: 'assistant',\n        content: 'Hello, how can I help you?',\n        id: 'test-assistant-msg',\n      });\n\n      const result = livekitItemToOpenAIItem(assistantMessage) as api_proto.AssistantItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('assistant');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('output_text');\n      expect((content as api_proto.OutputTextContent).text).toBe('Hello, how can I help you?');\n    });\n\n    it('should use input_text type for user messages', () => {\n      const userMessage = new llm.ChatMessage({\n        role: 'user',\n        content: 'What is the weather like?',\n        id: 'test-user-msg',\n      });\n\n      const result = livekitItemToOpenAIItem(userMessage) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('user');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n      expect((content as api_proto.InputTextContent).text).toBe('What is the weather like?');\n    });\n\n    it('should use input_text type for system messages', () => {\n      const systemMessage = new llm.ChatMessage({\n        role: 'system',\n        content: 'You are a helpful assistant.',\n        id: 'test-system-msg',\n      });\n\n      const result = livekitItemToOpenAIItem(systemMessage) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n    });\n\n    it('should convert developer role to system role', () => {\n      const developerMessage = new llm.ChatMessage({\n        role: 'developer',\n        content: 'System instructions.',\n        id: 'test-developer-msg',\n      });\n\n      const result = livekitItemToOpenAIItem(developerMessage) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n    });\n\n    it('should handle multiple content items for assistant', () => {\n      const multiContentMessage = new llm.ChatMessage({\n        role: 'assistant',\n        content: ['First part.', 'Second part.'],\n        id: 'test-multi-msg',\n      });\n\n      const result = livekitItemToOpenAIItem(multiContentMessage) as api_proto.AssistantItem;\n\n      expect(result.content).toHaveLength(2);\n      const content0 = result.content[0]!;\n      const content1 = result.content[1]!;\n      expect(content0.type).toBe('output_text');\n      expect(content1.type).toBe('output_text');\n    });\n  });\n\n  describe('function_call items', () => {\n    it('should convert function call items correctly', () => {\n      const functionCall = new llm.FunctionCall({\n        callId: 'call-123',\n        name: 'get_weather',\n        args: '{\"location\": \"San Francisco\"}',\n        id: 'test-func-call',\n      });\n\n      const result = livekitItemToOpenAIItem(functionCall) as api_proto.FunctionCallItem;\n\n      expect(result.type).toBe('function_call');\n      expect(result.id).toBe('test-func-call');\n      expect(result.call_id).toBe('call-123');\n      expect(result.name).toBe('get_weather');\n      expect(result.arguments).toBe('{\"location\": \"San Francisco\"}');\n    });\n  });\n\n  describe('function_call_output items', () => {\n    it('should convert function call output items correctly', () => {\n      const functionOutput = new llm.FunctionCallOutput({\n        callId: 'call-123',\n        output: 'The weather in San Francisco is sunny.',\n        isError: false,\n        id: 'test-func-output',\n      });\n\n      const result = livekitItemToOpenAIItem(functionOutput) as api_proto.FunctionCallOutputItem;\n\n      expect(result.type).toBe('function_call_output');\n      expect(result.id).toBe('test-func-output');\n      expect(result.call_id).toBe('call-123');\n      expect(result.output).toBe('The weather in San Francisco is sunny.');\n    });\n  });\n});\n"],"mappings":";AAGA,oBAAoB;AACpB,oBAAqC;AAErC,4BAAwC;AAAA,IAExC,wBAAS,2BAA2B,MAAM;AACxC,8BAAS,iBAAiB,MAAM;AAC9B,0BAAG,sDAAsD,MAAM;AAC7D,YAAM,mBAAmB,IAAI,kBAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,aAAS,+CAAwB,gBAAgB;AAEvD,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,WAAW;AACpC,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,gCAAO,QAAQ,IAAI,EAAE,KAAK,aAAa;AACvC,gCAAQ,QAAwC,IAAI,EAAE,KAAK,4BAA4B;AAAA,IACzF,CAAC;AAED,0BAAG,gDAAgD,MAAM;AACvD,YAAM,cAAc,IAAI,kBAAI,YAAY;AAAA,QACtC,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,aAAS,+CAAwB,WAAW;AAElD,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,MAAM;AAC/B,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,gCAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AACtC,gCAAQ,QAAuC,IAAI,EAAE,KAAK,2BAA2B;AAAA,IACvF,CAAC;AAED,0BAAG,kDAAkD,MAAM;AACzD,YAAM,gBAAgB,IAAI,kBAAI,YAAY;AAAA,QACxC,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,aAAS,+CAAwB,aAAa;AAEpD,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,gCAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AAAA,IACxC,CAAC;AAED,0BAAG,gDAAgD,MAAM;AACvD,YAAM,mBAAmB,IAAI,kBAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,aAAS,+CAAwB,gBAAgB;AAEvD,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,gCAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AAAA,IACxC,CAAC;AAED,0BAAG,sDAAsD,MAAM;AAC7D,YAAM,sBAAsB,IAAI,kBAAI,YAAY;AAAA,QAC9C,MAAM;AAAA,QACN,SAAS,CAAC,eAAe,cAAc;AAAA,QACvC,IAAI;AAAA,MACN,CAAC;AAED,YAAM,aAAS,+CAAwB,mBAAmB;AAE1D,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,WAAW,OAAO,QAAQ,CAAC;AACjC,YAAM,WAAW,OAAO,QAAQ,CAAC;AACjC,gCAAO,SAAS,IAAI,EAAE,KAAK,aAAa;AACxC,gCAAO,SAAS,IAAI,EAAE,KAAK,aAAa;AAAA,IAC1C,CAAC;AAAA,EACH,CAAC;AAED,8BAAS,uBAAuB,MAAM;AACpC,0BAAG,gDAAgD,MAAM;AACvD,YAAM,eAAe,IAAI,kBAAI,aAAa;AAAA,QACxC,QAAQ;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,QACN,IAAI;AAAA,MACN,CAAC;AAED,YAAM,aAAS,+CAAwB,YAAY;AAEnD,gCAAO,OAAO,IAAI,EAAE,KAAK,eAAe;AACxC,gCAAO,OAAO,EAAE,EAAE,KAAK,gBAAgB;AACvC,gCAAO,OAAO,OAAO,EAAE,KAAK,UAAU;AACtC,gCAAO,OAAO,IAAI,EAAE,KAAK,aAAa;AACtC,gCAAO,OAAO,SAAS,EAAE,KAAK,+BAA+B;AAAA,IAC/D,CAAC;AAAA,EACH,CAAC;AAED,8BAAS,8BAA8B,MAAM;AAC3C,0BAAG,uDAAuD,MAAM;AAC9D,YAAM,iBAAiB,IAAI,kBAAI,mBAAmB;AAAA,QAChD,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,aAAS,+CAAwB,cAAc;AAErD,gCAAO,OAAO,IAAI,EAAE,KAAK,sBAAsB;AAC/C,gCAAO,OAAO,EAAE,EAAE,KAAK,kBAAkB;AACzC,gCAAO,OAAO,OAAO,EAAE,KAAK,UAAU;AACtC,gCAAO,OAAO,MAAM,EAAE,KAAK,wCAAwC;AAAA,IACrE,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}
+{"version":3,"sources":["../../src/realtime/realtime_model.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm } from '@livekit/agents';\nimport { describe, expect, it } from 'vitest';\nimport type * as api_proto from './api_proto.js';\nimport { livekitItemToOpenAIItem } from './realtime_model.js';\n\ndescribe('livekitItemToOpenAIItem', () => {\n  describe('message items', () => {\n    it('should use output_text type for assistant messages', async () => {\n      const assistantMessage = new llm.ChatMessage({\n        role: 'assistant',\n        content: 'Hello, how can I help you?',\n        id: 'test-assistant-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(assistantMessage)) as api_proto.AssistantItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('assistant');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('output_text');\n      expect((content as api_proto.OutputTextContent).text).toBe('Hello, how can I help you?');\n    });\n\n    it('should use input_text type for user messages', async () => {\n      const userMessage = new llm.ChatMessage({\n        role: 'user',\n        content: 'What is the weather like?',\n        id: 'test-user-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('user');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n      expect((content as api_proto.InputTextContent).text).toBe('What is the weather like?');\n    });\n\n    it('should use input_text type for system messages', async () => {\n      const systemMessage = new llm.ChatMessage({\n        role: 'system',\n        content: 'You are a helpful assistant.',\n        id: 'test-system-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(systemMessage)) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n    });\n\n    it('should convert developer role to system role', async () => {\n      const developerMessage = new llm.ChatMessage({\n        role: 'developer',\n        content: 'System instructions.',\n        id: 'test-developer-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(developerMessage)) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n    });\n\n    it('should handle multiple content items for assistant', async () => {\n      const multiContentMessage = new llm.ChatMessage({\n        role: 'assistant',\n        content: ['First part.', 'Second part.'],\n        id: 'test-multi-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(\n        multiContentMessage,\n      )) as api_proto.AssistantItem;\n\n      expect(result.content).toHaveLength(2);\n      const content0 = result.content[0]!;\n      const content1 = result.content[1]!;\n      expect(content0.type).toBe('output_text');\n      expect(content1.type).toBe('output_text');\n    });\n\n    it('should convert image content to input_image for user messages', async () => {\n      const base64Data =\n        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';\n      const imageContent = llm.createImageContent({\n        image: `data:image/png;base64,${base64Data}`,\n        mimeType: 'image/png',\n      });\n\n      const userMessage = new llm.ChatMessage({\n        role: 'user',\n        content: [imageContent],\n        id: 'test-image-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('user');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_image');\n      expect((content as api_proto.InputImageContent).image_url).toBe(\n        `data:image/png;base64,${base64Data}`,\n      );\n    });\n\n    it('should ignore image content for assistant messages', async () => {\n      const base64Data =\n        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';\n      const imageContent = llm.createImageContent({\n        image: `data:image/png;base64,${base64Data}`,\n        mimeType: 'image/png',\n      });\n\n      const assistantMessage = new llm.ChatMessage({\n        role: 'assistant',\n        content: [imageContent],\n        id: 'test-assistant-image-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(assistantMessage)) as api_proto.AssistantItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('assistant');\n      expect(result.content).toHaveLength(0);\n    });\n\n    it('should ignore image content for system messages', async () => {\n      const base64Data =\n        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';\n      const imageContent = llm.createImageContent({\n        image: `data:image/png;base64,${base64Data}`,\n        mimeType: 'image/png',\n      });\n\n      const systemMessage = new llm.ChatMessage({\n        role: 'system',\n        content: [imageContent],\n        id: 'test-system-image-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(systemMessage)) as api_proto.SystemItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      expect(result.content).toHaveLength(0);\n    });\n\n    it('should ignore image content for developer messages mapped to system', async () => {\n      const base64Data =\n        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';\n      const imageContent = llm.createImageContent({\n        image: `data:image/png;base64,${base64Data}`,\n        mimeType: 'image/png',\n      });\n\n      const developerMessage = new llm.ChatMessage({\n        role: 'developer',\n        content: [imageContent],\n        id: 'test-developer-image-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(developerMessage)) as api_proto.SystemItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      expect(result.content).toHaveLength(0);\n    });\n\n    it('should handle mixed text and image content', async () => {\n      const base64Data =\n        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';\n      const imageContent = llm.createImageContent({\n        image: `data:image/png;base64,${base64Data}`,\n        mimeType: 'image/png',\n      });\n\n      const userMessage = new llm.ChatMessage({\n        role: 'user',\n        content: ['Describe this image:', imageContent],\n        id: 'test-mixed-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.content).toHaveLength(2);\n      expect(result.content[0]!.type).toBe('input_text');\n      expect(result.content[1]!.type).toBe('input_image');\n    });\n  });\n\n  describe('function_call items', () => {\n    it('should convert function call items correctly', async () => {\n      const functionCall = new llm.FunctionCall({\n        callId: 'call-123',\n        name: 'get_weather',\n        args: '{\"location\": \"San Francisco\"}',\n        id: 'test-func-call',\n      });\n\n      const result = (await livekitItemToOpenAIItem(functionCall)) as api_proto.FunctionCallItem;\n\n      expect(result.type).toBe('function_call');\n      expect(result.id).toBe('test-func-call');\n      expect(result.call_id).toBe('call-123');\n      expect(result.name).toBe('get_weather');\n      expect(result.arguments).toBe('{\"location\": \"San Francisco\"}');\n    });\n  });\n\n  describe('function_call_output items', () => {\n    it('should convert function call output items correctly', async () => {\n      const functionOutput = new llm.FunctionCallOutput({\n        callId: 'call-123',\n        output: 'The weather in San Francisco is sunny.',\n        isError: false,\n        id: 'test-func-output',\n      });\n\n      const result = (await livekitItemToOpenAIItem(\n        functionOutput,\n      )) as api_proto.FunctionCallOutputItem;\n\n      expect(result.type).toBe('function_call_output');\n      expect(result.id).toBe('test-func-output');\n      expect(result.call_id).toBe('call-123');\n      expect(result.output).toBe('The weather in San Francisco is sunny.');\n    });\n  });\n});\n"],"mappings":";AAGA,oBAAoB;AACpB,oBAAqC;AAErC,4BAAwC;AAAA,IAExC,wBAAS,2BAA2B,MAAM;AACxC,8BAAS,iBAAiB,MAAM;AAC9B,0BAAG,sDAAsD,YAAY;AACnE,YAAM,mBAAmB,IAAI,kBAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM,+CAAwB,gBAAgB;AAE9D,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,WAAW;AACpC,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,gCAAO,QAAQ,IAAI,EAAE,KAAK,aAAa;AACvC,gCAAQ,QAAwC,IAAI,EAAE,KAAK,4BAA4B;AAAA,IACzF,CAAC;AAED,0BAAG,gDAAgD,YAAY;AAC7D,YAAM,cAAc,IAAI,kBAAI,YAAY;AAAA,QACtC,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM,+CAAwB,WAAW;AAEzD,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,MAAM;AAC/B,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,gCAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AACtC,gCAAQ,QAAuC,IAAI,EAAE,KAAK,2BAA2B;AAAA,IACvF,CAAC;AAED,0BAAG,kDAAkD,YAAY;AAC/D,YAAM,gBAAgB,IAAI,kBAAI,YAAY;AAAA,QACxC,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM,+CAAwB,aAAa;AAE3D,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,gCAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AAAA,IACxC,CAAC;AAED,0BAAG,gDAAgD,YAAY;AAC7D,YAAM,mBAAmB,IAAI,kBAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM,+CAAwB,gBAAgB;AAE9D,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,gCAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AAAA,IACxC,CAAC;AAED,0BAAG,sDAAsD,YAAY;AACnE,YAAM,sBAAsB,IAAI,kBAAI,YAAY;AAAA,QAC9C,MAAM;AAAA,QACN,SAAS,CAAC,eAAe,cAAc;AAAA,QACvC,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM;AAAA,QACpB;AAAA,MACF;AAEA,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,WAAW,OAAO,QAAQ,CAAC;AACjC,YAAM,WAAW,OAAO,QAAQ,CAAC;AACjC,gCAAO,SAAS,IAAI,EAAE,KAAK,aAAa;AACxC,gCAAO,SAAS,IAAI,EAAE,KAAK,aAAa;AAAA,IAC1C,CAAC;AAED,0BAAG,iEAAiE,YAAY;AAC9E,YAAM,aACJ;AACF,YAAM,eAAe,kBAAI,mBAAmB;AAAA,QAC1C,OAAO,yBAAyB,UAAU;AAAA,QAC1C,UAAU;AAAA,MACZ,CAAC;AAED,YAAM,cAAc,IAAI,kBAAI,YAAY;AAAA,QACtC,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM,+CAAwB,WAAW;AAEzD,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,MAAM;AAC/B,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,gCAAO,QAAQ,IAAI,EAAE,KAAK,aAAa;AACvC,gCAAQ,QAAwC,SAAS,EAAE;AAAA,QACzD,yBAAyB,UAAU;AAAA,MACrC;AAAA,IACF,CAAC;AAED,0BAAG,sDAAsD,YAAY;AACnE,YAAM,aACJ;AACF,YAAM,eAAe,kBAAI,mBAAmB;AAAA,QAC1C,OAAO,yBAAyB,UAAU;AAAA,QAC1C,UAAU;AAAA,MACZ,CAAC;AAED,YAAM,mBAAmB,IAAI,kBAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM,+CAAwB,gBAAgB;AAE9D,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,WAAW;AACpC,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AAAA,IACvC,CAAC;AAED,0BAAG,mDAAmD,YAAY;AAChE,YAAM,aACJ;AACF,YAAM,eAAe,kBAAI,mBAAmB;AAAA,QAC1C,OAAO,yBAAyB,UAAU;AAAA,QAC1C,UAAU;AAAA,MACZ,CAAC;AAED,YAAM,gBAAgB,IAAI,kBAAI,YAAY;AAAA,QACxC,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM,+CAAwB,aAAa;AAE3D,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AAAA,IACvC,CAAC;AAED,0BAAG,uEAAuE,YAAY;AACpF,YAAM,aACJ;AACF,YAAM,eAAe,kBAAI,mBAAmB;AAAA,QAC1C,OAAO,yBAAyB,UAAU;AAAA,QAC1C,UAAU;AAAA,MACZ,CAAC;AAED,YAAM,mBAAmB,IAAI,kBAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM,+CAAwB,gBAAgB;AAE9D,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AAAA,IACvC,CAAC;AAED,0BAAG,8CAA8C,YAAY;AAC3D,YAAM,aACJ;AACF,YAAM,eAAe,kBAAI,mBAAmB;AAAA,QAC1C,OAAO,yBAAyB,UAAU;AAAA,QAC1C,UAAU;AAAA,MACZ,CAAC;AAED,YAAM,cAAc,IAAI,kBAAI,YAAY;AAAA,QACtC,MAAM;AAAA,QACN,SAAS,CAAC,wBAAwB,YAAY;AAAA,QAC9C,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM,+CAAwB,WAAW;AAEzD,gCAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,gCAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,gCAAO,OAAO,QAAQ,CAAC,EAAG,IAAI,EAAE,KAAK,YAAY;AACjD,gCAAO,OAAO,QAAQ,CAAC,EAAG,IAAI,EAAE,KAAK,aAAa;AAAA,IACpD,CAAC;AAAA,EACH,CAAC;AAED,8BAAS,uBAAuB,MAAM;AACpC,0BAAG,gDAAgD,YAAY;AAC7D,YAAM,eAAe,IAAI,kBAAI,aAAa;AAAA,QACxC,QAAQ;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,QACN,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM,+CAAwB,YAAY;AAE1D,gCAAO,OAAO,IAAI,EAAE,KAAK,eAAe;AACxC,gCAAO,OAAO,EAAE,EAAE,KAAK,gBAAgB;AACvC,gCAAO,OAAO,OAAO,EAAE,KAAK,UAAU;AACtC,gCAAO,OAAO,IAAI,EAAE,KAAK,aAAa;AACtC,gCAAO,OAAO,SAAS,EAAE,KAAK,+BAA+B;AAAA,IAC/D,CAAC;AAAA,EACH,CAAC;AAED,8BAAS,8BAA8B,MAAM;AAC3C,0BAAG,uDAAuD,YAAY;AACpE,YAAM,iBAAiB,IAAI,kBAAI,mBAAmB;AAAA,QAChD,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,UAAM;AAAA,QACpB;AAAA,MACF;AAEA,gCAAO,OAAO,IAAI,EAAE,KAAK,sBAAsB;AAC/C,gCAAO,OAAO,EAAE,EAAE,KAAK,kBAAkB;AACzC,gCAAO,OAAO,OAAO,EAAE,KAAK,UAAU;AACtC,gCAAO,OAAO,MAAM,EAAE,KAAK,wCAAwC;AAAA,IACrE,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}

package/dist/realtime/realtime_model.test.js

@@ -3,13 +3,13 @@ import { describe, expect, it } from "vitest";
 import { livekitItemToOpenAIItem } from "./realtime_model.js";
 describe("livekitItemToOpenAIItem", () => {
   describe("message items", () => {
-    it("should use output_text type for assistant messages", () => {
+    it("should use output_text type for assistant messages", async () => {
       const assistantMessage = new llm.ChatMessage({
         role: "assistant",
         content: "Hello, how can I help you?",
         id: "test-assistant-msg"
       });
-      const result = livekitItemToOpenAIItem(assistantMessage);
+      const result = await livekitItemToOpenAIItem(assistantMessage);
       expect(result.type).toBe("message");
       expect(result.role).toBe("assistant");
       expect(result.content).toHaveLength(1);
@@ -17,13 +17,13 @@ describe("livekitItemToOpenAIItem", () => {
       expect(content.type).toBe("output_text");
       expect(content.text).toBe("Hello, how can I help you?");
     });
-    it("should use input_text type for user messages", () => {
+    it("should use input_text type for user messages", async () => {
       const userMessage = new llm.ChatMessage({
         role: "user",
         content: "What is the weather like?",
         id: "test-user-msg"
       });
-      const result = livekitItemToOpenAIItem(userMessage);
+      const result = await livekitItemToOpenAIItem(userMessage);
       expect(result.type).toBe("message");
       expect(result.role).toBe("user");
       expect(result.content).toHaveLength(1);
@@ -31,54 +31,142 @@ describe("livekitItemToOpenAIItem", () => {
       expect(content.type).toBe("input_text");
       expect(content.text).toBe("What is the weather like?");
     });
-    it("should use input_text type for system messages", () => {
+    it("should use input_text type for system messages", async () => {
       const systemMessage = new llm.ChatMessage({
         role: "system",
         content: "You are a helpful assistant.",
         id: "test-system-msg"
       });
-      const result = livekitItemToOpenAIItem(systemMessage);
+      const result = await livekitItemToOpenAIItem(systemMessage);
       expect(result.type).toBe("message");
       expect(result.role).toBe("system");
       expect(result.content).toHaveLength(1);
       const content = result.content[0];
       expect(content.type).toBe("input_text");
     });
-    it("should convert developer role to system role", () => {
+    it("should convert developer role to system role", async () => {
       const developerMessage = new llm.ChatMessage({
         role: "developer",
         content: "System instructions.",
         id: "test-developer-msg"
       });
-      const result = livekitItemToOpenAIItem(developerMessage);
+      const result = await livekitItemToOpenAIItem(developerMessage);
       expect(result.type).toBe("message");
       expect(result.role).toBe("system");
       const content = result.content[0];
       expect(content.type).toBe("input_text");
     });
-    it("should handle multiple content items for assistant", () => {
+    it("should handle multiple content items for assistant", async () => {
       const multiContentMessage = new llm.ChatMessage({
         role: "assistant",
         content: ["First part.", "Second part."],
         id: "test-multi-msg"
       });
-      const result = livekitItemToOpenAIItem(multiContentMessage);
+      const result = await livekitItemToOpenAIItem(
+        multiContentMessage
+      );
       expect(result.content).toHaveLength(2);
       const content0 = result.content[0];
       const content1 = result.content[1];
       expect(content0.type).toBe("output_text");
       expect(content1.type).toBe("output_text");
     });
+    it("should convert image content to input_image for user messages", async () => {
+      const base64Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+      const imageContent = llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: "image/png"
+      });
+      const userMessage = new llm.ChatMessage({
+        role: "user",
+        content: [imageContent],
+        id: "test-image-msg"
+      });
+      const result = await livekitItemToOpenAIItem(userMessage);
+      expect(result.type).toBe("message");
+      expect(result.role).toBe("user");
+      expect(result.content).toHaveLength(1);
+      const content = result.content[0];
+      expect(content.type).toBe("input_image");
+      expect(content.image_url).toBe(
+        `data:image/png;base64,${base64Data}`
+      );
+    });
+    it("should ignore image content for assistant messages", async () => {
+      const base64Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+      const imageContent = llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: "image/png"
+      });
+      const assistantMessage = new llm.ChatMessage({
+        role: "assistant",
+        content: [imageContent],
+        id: "test-assistant-image-msg"
+      });
+      const result = await livekitItemToOpenAIItem(assistantMessage);
+      expect(result.type).toBe("message");
+      expect(result.role).toBe("assistant");
+      expect(result.content).toHaveLength(0);
+    });
+    it("should ignore image content for system messages", async () => {
+      const base64Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+      const imageContent = llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: "image/png"
+      });
+      const systemMessage = new llm.ChatMessage({
+        role: "system",
+        content: [imageContent],
+        id: "test-system-image-msg"
+      });
+      const result = await livekitItemToOpenAIItem(systemMessage);
+      expect(result.type).toBe("message");
+      expect(result.role).toBe("system");
+      expect(result.content).toHaveLength(0);
+    });
+    it("should ignore image content for developer messages mapped to system", async () => {
+      const base64Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+      const imageContent = llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: "image/png"
+      });
+      const developerMessage = new llm.ChatMessage({
+        role: "developer",
+        content: [imageContent],
+        id: "test-developer-image-msg"
+      });
+      const result = await livekitItemToOpenAIItem(developerMessage);
+      expect(result.type).toBe("message");
+      expect(result.role).toBe("system");
+      expect(result.content).toHaveLength(0);
+    });
+    it("should handle mixed text and image content", async () => {
+      const base64Data = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+      const imageContent = llm.createImageContent({
+        image: `data:image/png;base64,${base64Data}`,
+        mimeType: "image/png"
+      });
+      const userMessage = new llm.ChatMessage({
+        role: "user",
+        content: ["Describe this image:", imageContent],
+        id: "test-mixed-msg"
+      });
+      const result = await livekitItemToOpenAIItem(userMessage);
+      expect(result.type).toBe("message");
+      expect(result.content).toHaveLength(2);
+      expect(result.content[0].type).toBe("input_text");
+      expect(result.content[1].type).toBe("input_image");
+    });
   });
   describe("function_call items", () => {
-    it("should convert function call items correctly", () => {
+    it("should convert function call items correctly", async () => {
       const functionCall = new llm.FunctionCall({
         callId: "call-123",
         name: "get_weather",
         args: '{"location": "San Francisco"}',
         id: "test-func-call"
       });
-      const result = livekitItemToOpenAIItem(functionCall);
+      const result = await livekitItemToOpenAIItem(functionCall);
       expect(result.type).toBe("function_call");
       expect(result.id).toBe("test-func-call");
       expect(result.call_id).toBe("call-123");
@@ -87,14 +175,16 @@ describe("livekitItemToOpenAIItem", () => {
     });
   });
   describe("function_call_output items", () => {
-    it("should convert function call output items correctly", () => {
+    it("should convert function call output items correctly", async () => {
       const functionOutput = new llm.FunctionCallOutput({
         callId: "call-123",
         output: "The weather in San Francisco is sunny.",
         isError: false,
         id: "test-func-output"
       });
-      const result = livekitItemToOpenAIItem(functionOutput);
+      const result = await livekitItemToOpenAIItem(
+        functionOutput
+      );
       expect(result.type).toBe("function_call_output");
       expect(result.id).toBe("test-func-output");
       expect(result.call_id).toBe("call-123");

package/dist/realtime/realtime_model.test.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/realtime/realtime_model.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm } from '@livekit/agents';\nimport { describe, expect, it } from 'vitest';\nimport type * as api_proto from './api_proto.js';\nimport { livekitItemToOpenAIItem } from './realtime_model.js';\n\ndescribe('livekitItemToOpenAIItem', () => {\n  describe('message items', () => {\n    it('should use output_text type for assistant messages', () => {\n      const assistantMessage = new llm.ChatMessage({\n        role: 'assistant',\n        content: 'Hello, how can I help you?',\n        id: 'test-assistant-msg',\n      });\n\n      const result = livekitItemToOpenAIItem(assistantMessage) as api_proto.AssistantItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('assistant');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('output_text');\n      expect((content as api_proto.OutputTextContent).text).toBe('Hello, how can I help you?');\n    });\n\n    it('should use input_text type for user messages', () => {\n      const userMessage = new llm.ChatMessage({\n        role: 'user',\n        content: 'What is the weather like?',\n        id: 'test-user-msg',\n      });\n\n      const result = livekitItemToOpenAIItem(userMessage) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('user');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n      expect((content as api_proto.InputTextContent).text).toBe('What is the weather like?');\n    });\n\n    it('should use input_text type for system messages', () => {\n      const systemMessage = new llm.ChatMessage({\n        role: 'system',\n        content: 'You are a helpful assistant.',\n        id: 'test-system-msg',\n      });\n\n      const result = livekitItemToOpenAIItem(systemMessage) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n    });\n\n    it('should convert developer role to system role', () => {\n      const developerMessage = new llm.ChatMessage({\n        role: 'developer',\n        content: 'System instructions.',\n        id: 'test-developer-msg',\n      });\n\n      const result = livekitItemToOpenAIItem(developerMessage) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n    });\n\n    it('should handle multiple content items for assistant', () => {\n      const multiContentMessage = new llm.ChatMessage({\n        role: 'assistant',\n        content: ['First part.', 'Second part.'],\n        id: 'test-multi-msg',\n      });\n\n      const result = livekitItemToOpenAIItem(multiContentMessage) as api_proto.AssistantItem;\n\n      expect(result.content).toHaveLength(2);\n      const content0 = result.content[0]!;\n      const content1 = result.content[1]!;\n      expect(content0.type).toBe('output_text');\n      expect(content1.type).toBe('output_text');\n    });\n  });\n\n  describe('function_call items', () => {\n    it('should convert function call items correctly', () => {\n      const functionCall = new llm.FunctionCall({\n        callId: 'call-123',\n        name: 'get_weather',\n        args: '{\"location\": \"San Francisco\"}',\n        id: 'test-func-call',\n      });\n\n      const result = livekitItemToOpenAIItem(functionCall) as api_proto.FunctionCallItem;\n\n      expect(result.type).toBe('function_call');\n      expect(result.id).toBe('test-func-call');\n      expect(result.call_id).toBe('call-123');\n      expect(result.name).toBe('get_weather');\n      expect(result.arguments).toBe('{\"location\": \"San Francisco\"}');\n    });\n  });\n\n  describe('function_call_output items', () => {\n    it('should convert function call output items correctly', () => {\n      const functionOutput = new llm.FunctionCallOutput({\n        callId: 'call-123',\n        output: 'The weather in San Francisco is sunny.',\n        isError: false,\n        id: 'test-func-output',\n      });\n\n      const result = livekitItemToOpenAIItem(functionOutput) as api_proto.FunctionCallOutputItem;\n\n      expect(result.type).toBe('function_call_output');\n      expect(result.id).toBe('test-func-output');\n      expect(result.call_id).toBe('call-123');\n      expect(result.output).toBe('The weather in San Francisco is sunny.');\n    });\n  });\n});\n"],"mappings":"AAGA,SAAS,WAAW;AACpB,SAAS,UAAU,QAAQ,UAAU;AAErC,SAAS,+BAA+B;AAExC,SAAS,2BAA2B,MAAM;AACxC,WAAS,iBAAiB,MAAM;AAC9B,OAAG,sDAAsD,MAAM;AAC7D,YAAM,mBAAmB,IAAI,IAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAS,wBAAwB,gBAAgB;AAEvD,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,WAAW;AACpC,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,aAAO,QAAQ,IAAI,EAAE,KAAK,aAAa;AACvC,aAAQ,QAAwC,IAAI,EAAE,KAAK,4BAA4B;AAAA,IACzF,CAAC;AAED,OAAG,gDAAgD,MAAM;AACvD,YAAM,cAAc,IAAI,IAAI,YAAY;AAAA,QACtC,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAS,wBAAwB,WAAW;AAElD,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,MAAM;AAC/B,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,aAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AACtC,aAAQ,QAAuC,IAAI,EAAE,KAAK,2BAA2B;AAAA,IACvF,CAAC;AAED,OAAG,kDAAkD,MAAM;AACzD,YAAM,gBAAgB,IAAI,IAAI,YAAY;AAAA,QACxC,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAS,wBAAwB,aAAa;AAEpD,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,aAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AAAA,IACxC,CAAC;AAED,OAAG,gDAAgD,MAAM;AACvD,YAAM,mBAAmB,IAAI,IAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAS,wBAAwB,gBAAgB;AAEvD,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,aAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AAAA,IACxC,CAAC;AAED,OAAG,sDAAsD,MAAM;AAC7D,YAAM,sBAAsB,IAAI,IAAI,YAAY;AAAA,QAC9C,MAAM;AAAA,QACN,SAAS,CAAC,eAAe,cAAc;AAAA,QACvC,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAS,wBAAwB,mBAAmB;AAE1D,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,WAAW,OAAO,QAAQ,CAAC;AACjC,YAAM,WAAW,OAAO,QAAQ,CAAC;AACjC,aAAO,SAAS,IAAI,EAAE,KAAK,aAAa;AACxC,aAAO,SAAS,IAAI,EAAE,KAAK,aAAa;AAAA,IAC1C,CAAC;AAAA,EACH,CAAC;AAED,WAAS,uBAAuB,MAAM;AACpC,OAAG,gDAAgD,MAAM;AACvD,YAAM,eAAe,IAAI,IAAI,aAAa;AAAA,QACxC,QAAQ;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,QACN,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAS,wBAAwB,YAAY;AAEnD,aAAO,OAAO,IAAI,EAAE,KAAK,eAAe;AACxC,aAAO,OAAO,EAAE,EAAE,KAAK,gBAAgB;AACvC,aAAO,OAAO,OAAO,EAAE,KAAK,UAAU;AACtC,aAAO,OAAO,IAAI,EAAE,KAAK,aAAa;AACtC,aAAO,OAAO,SAAS,EAAE,KAAK,+BAA+B;AAAA,IAC/D,CAAC;AAAA,EACH,CAAC;AAED,WAAS,8BAA8B,MAAM;AAC3C,OAAG,uDAAuD,MAAM;AAC9D,YAAM,iBAAiB,IAAI,IAAI,mBAAmB;AAAA,QAChD,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAS,wBAAwB,cAAc;AAErD,aAAO,OAAO,IAAI,EAAE,KAAK,sBAAsB;AAC/C,aAAO,OAAO,EAAE,EAAE,KAAK,kBAAkB;AACzC,aAAO,OAAO,OAAO,EAAE,KAAK,UAAU;AACtC,aAAO,OAAO,MAAM,EAAE,KAAK,wCAAwC;AAAA,IACrE,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}
+{"version":3,"sources":["../../src/realtime/realtime_model.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { llm } from '@livekit/agents';\nimport { describe, expect, it } from 'vitest';\nimport type * as api_proto from './api_proto.js';\nimport { livekitItemToOpenAIItem } from './realtime_model.js';\n\ndescribe('livekitItemToOpenAIItem', () => {\n  describe('message items', () => {\n    it('should use output_text type for assistant messages', async () => {\n      const assistantMessage = new llm.ChatMessage({\n        role: 'assistant',\n        content: 'Hello, how can I help you?',\n        id: 'test-assistant-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(assistantMessage)) as api_proto.AssistantItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('assistant');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('output_text');\n      expect((content as api_proto.OutputTextContent).text).toBe('Hello, how can I help you?');\n    });\n\n    it('should use input_text type for user messages', async () => {\n      const userMessage = new llm.ChatMessage({\n        role: 'user',\n        content: 'What is the weather like?',\n        id: 'test-user-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('user');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n      expect((content as api_proto.InputTextContent).text).toBe('What is the weather like?');\n    });\n\n    it('should use input_text type for system messages', async () => {\n      const systemMessage = new llm.ChatMessage({\n        role: 'system',\n        content: 'You are a helpful assistant.',\n        id: 'test-system-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(systemMessage)) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n    });\n\n    it('should convert developer role to system role', async () => {\n      const developerMessage = new llm.ChatMessage({\n        role: 'developer',\n        content: 'System instructions.',\n        id: 'test-developer-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(developerMessage)) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_text');\n    });\n\n    it('should handle multiple content items for assistant', async () => {\n      const multiContentMessage = new llm.ChatMessage({\n        role: 'assistant',\n        content: ['First part.', 'Second part.'],\n        id: 'test-multi-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(\n        multiContentMessage,\n      )) as api_proto.AssistantItem;\n\n      expect(result.content).toHaveLength(2);\n      const content0 = result.content[0]!;\n      const content1 = result.content[1]!;\n      expect(content0.type).toBe('output_text');\n      expect(content1.type).toBe('output_text');\n    });\n\n    it('should convert image content to input_image for user messages', async () => {\n      const base64Data =\n        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';\n      const imageContent = llm.createImageContent({\n        image: `data:image/png;base64,${base64Data}`,\n        mimeType: 'image/png',\n      });\n\n      const userMessage = new llm.ChatMessage({\n        role: 'user',\n        content: [imageContent],\n        id: 'test-image-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('user');\n      expect(result.content).toHaveLength(1);\n      const content = result.content[0]!;\n      expect(content.type).toBe('input_image');\n      expect((content as api_proto.InputImageContent).image_url).toBe(\n        `data:image/png;base64,${base64Data}`,\n      );\n    });\n\n    it('should ignore image content for assistant messages', async () => {\n      const base64Data =\n        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';\n      const imageContent = llm.createImageContent({\n        image: `data:image/png;base64,${base64Data}`,\n        mimeType: 'image/png',\n      });\n\n      const assistantMessage = new llm.ChatMessage({\n        role: 'assistant',\n        content: [imageContent],\n        id: 'test-assistant-image-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(assistantMessage)) as api_proto.AssistantItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('assistant');\n      expect(result.content).toHaveLength(0);\n    });\n\n    it('should ignore image content for system messages', async () => {\n      const base64Data =\n        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';\n      const imageContent = llm.createImageContent({\n        image: `data:image/png;base64,${base64Data}`,\n        mimeType: 'image/png',\n      });\n\n      const systemMessage = new llm.ChatMessage({\n        role: 'system',\n        content: [imageContent],\n        id: 'test-system-image-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(systemMessage)) as api_proto.SystemItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      expect(result.content).toHaveLength(0);\n    });\n\n    it('should ignore image content for developer messages mapped to system', async () => {\n      const base64Data =\n        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';\n      const imageContent = llm.createImageContent({\n        image: `data:image/png;base64,${base64Data}`,\n        mimeType: 'image/png',\n      });\n\n      const developerMessage = new llm.ChatMessage({\n        role: 'developer',\n        content: [imageContent],\n        id: 'test-developer-image-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(developerMessage)) as api_proto.SystemItem;\n\n      expect(result.type).toBe('message');\n      expect(result.role).toBe('system');\n      expect(result.content).toHaveLength(0);\n    });\n\n    it('should handle mixed text and image content', async () => {\n      const base64Data =\n        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';\n      const imageContent = llm.createImageContent({\n        image: `data:image/png;base64,${base64Data}`,\n        mimeType: 'image/png',\n      });\n\n      const userMessage = new llm.ChatMessage({\n        role: 'user',\n        content: ['Describe this image:', imageContent],\n        id: 'test-mixed-msg',\n      });\n\n      const result = (await livekitItemToOpenAIItem(userMessage)) as api_proto.UserItem;\n\n      expect(result.type).toBe('message');\n      expect(result.content).toHaveLength(2);\n      expect(result.content[0]!.type).toBe('input_text');\n      expect(result.content[1]!.type).toBe('input_image');\n    });\n  });\n\n  describe('function_call items', () => {\n    it('should convert function call items correctly', async () => {\n      const functionCall = new llm.FunctionCall({\n        callId: 'call-123',\n        name: 'get_weather',\n        args: '{\"location\": \"San Francisco\"}',\n        id: 'test-func-call',\n      });\n\n      const result = (await livekitItemToOpenAIItem(functionCall)) as api_proto.FunctionCallItem;\n\n      expect(result.type).toBe('function_call');\n      expect(result.id).toBe('test-func-call');\n      expect(result.call_id).toBe('call-123');\n      expect(result.name).toBe('get_weather');\n      expect(result.arguments).toBe('{\"location\": \"San Francisco\"}');\n    });\n  });\n\n  describe('function_call_output items', () => {\n    it('should convert function call output items correctly', async () => {\n      const functionOutput = new llm.FunctionCallOutput({\n        callId: 'call-123',\n        output: 'The weather in San Francisco is sunny.',\n        isError: false,\n        id: 'test-func-output',\n      });\n\n      const result = (await livekitItemToOpenAIItem(\n        functionOutput,\n      )) as api_proto.FunctionCallOutputItem;\n\n      expect(result.type).toBe('function_call_output');\n      expect(result.id).toBe('test-func-output');\n      expect(result.call_id).toBe('call-123');\n      expect(result.output).toBe('The weather in San Francisco is sunny.');\n    });\n  });\n});\n"],"mappings":"AAGA,SAAS,WAAW;AACpB,SAAS,UAAU,QAAQ,UAAU;AAErC,SAAS,+BAA+B;AAExC,SAAS,2BAA2B,MAAM;AACxC,WAAS,iBAAiB,MAAM;AAC9B,OAAG,sDAAsD,YAAY;AACnE,YAAM,mBAAmB,IAAI,IAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM,wBAAwB,gBAAgB;AAE9D,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,WAAW;AACpC,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,aAAO,QAAQ,IAAI,EAAE,KAAK,aAAa;AACvC,aAAQ,QAAwC,IAAI,EAAE,KAAK,4BAA4B;AAAA,IACzF,CAAC;AAED,OAAG,gDAAgD,YAAY;AAC7D,YAAM,cAAc,IAAI,IAAI,YAAY;AAAA,QACtC,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM,wBAAwB,WAAW;AAEzD,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,MAAM;AAC/B,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,aAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AACtC,aAAQ,QAAuC,IAAI,EAAE,KAAK,2BAA2B;AAAA,IACvF,CAAC;AAED,OAAG,kDAAkD,YAAY;AAC/D,YAAM,gBAAgB,IAAI,IAAI,YAAY;AAAA,QACxC,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM,wBAAwB,aAAa;AAE3D,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,aAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AAAA,IACxC,CAAC;AAED,OAAG,gDAAgD,YAAY;AAC7D,YAAM,mBAAmB,IAAI,IAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM,wBAAwB,gBAAgB;AAE9D,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,aAAO,QAAQ,IAAI,EAAE,KAAK,YAAY;AAAA,IACxC,CAAC;AAED,OAAG,sDAAsD,YAAY;AACnE,YAAM,sBAAsB,IAAI,IAAI,YAAY;AAAA,QAC9C,MAAM;AAAA,QACN,SAAS,CAAC,eAAe,cAAc;AAAA,QACvC,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM;AAAA,QACpB;AAAA,MACF;AAEA,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,WAAW,OAAO,QAAQ,CAAC;AACjC,YAAM,WAAW,OAAO,QAAQ,CAAC;AACjC,aAAO,SAAS,IAAI,EAAE,KAAK,aAAa;AACxC,aAAO,SAAS,IAAI,EAAE,KAAK,aAAa;AAAA,IAC1C,CAAC;AAED,OAAG,iEAAiE,YAAY;AAC9E,YAAM,aACJ;AACF,YAAM,eAAe,IAAI,mBAAmB;AAAA,QAC1C,OAAO,yBAAyB,UAAU;AAAA,QAC1C,UAAU;AAAA,MACZ,CAAC;AAED,YAAM,cAAc,IAAI,IAAI,YAAY;AAAA,QACtC,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM,wBAAwB,WAAW;AAEzD,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,MAAM;AAC/B,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,YAAM,UAAU,OAAO,QAAQ,CAAC;AAChC,aAAO,QAAQ,IAAI,EAAE,KAAK,aAAa;AACvC,aAAQ,QAAwC,SAAS,EAAE;AAAA,QACzD,yBAAyB,UAAU;AAAA,MACrC;AAAA,IACF,CAAC;AAED,OAAG,sDAAsD,YAAY;AACnE,YAAM,aACJ;AACF,YAAM,eAAe,IAAI,mBAAmB;AAAA,QAC1C,OAAO,yBAAyB,UAAU;AAAA,QAC1C,UAAU;AAAA,MACZ,CAAC;AAED,YAAM,mBAAmB,IAAI,IAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM,wBAAwB,gBAAgB;AAE9D,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,WAAW;AACpC,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AAAA,IACvC,CAAC;AAED,OAAG,mDAAmD,YAAY;AAChE,YAAM,aACJ;AACF,YAAM,eAAe,IAAI,mBAAmB;AAAA,QAC1C,OAAO,yBAAyB,UAAU;AAAA,QAC1C,UAAU;AAAA,MACZ,CAAC;AAED,YAAM,gBAAgB,IAAI,IAAI,YAAY;AAAA,QACxC,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM,wBAAwB,aAAa;AAE3D,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AAAA,IACvC,CAAC;AAED,OAAG,uEAAuE,YAAY;AACpF,YAAM,aACJ;AACF,YAAM,eAAe,IAAI,mBAAmB;AAAA,QAC1C,OAAO,yBAAyB,UAAU;AAAA,QAC1C,UAAU;AAAA,MACZ,CAAC;AAED,YAAM,mBAAmB,IAAI,IAAI,YAAY;AAAA,QAC3C,MAAM;AAAA,QACN,SAAS,CAAC,YAAY;AAAA,QACtB,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM,wBAAwB,gBAAgB;AAE9D,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,IAAI,EAAE,KAAK,QAAQ;AACjC,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AAAA,IACvC,CAAC;AAED,OAAG,8CAA8C,YAAY;AAC3D,YAAM,aACJ;AACF,YAAM,eAAe,IAAI,mBAAmB;AAAA,QAC1C,OAAO,yBAAyB,UAAU;AAAA,QAC1C,UAAU;AAAA,MACZ,CAAC;AAED,YAAM,cAAc,IAAI,IAAI,YAAY;AAAA,QACtC,MAAM;AAAA,QACN,SAAS,CAAC,wBAAwB,YAAY;AAAA,QAC9C,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM,wBAAwB,WAAW;AAEzD,aAAO,OAAO,IAAI,EAAE,KAAK,SAAS;AAClC,aAAO,OAAO,OAAO,EAAE,aAAa,CAAC;AACrC,aAAO,OAAO,QAAQ,CAAC,EAAG,IAAI,EAAE,KAAK,YAAY;AACjD,aAAO,OAAO,QAAQ,CAAC,EAAG,IAAI,EAAE,KAAK,aAAa;AAAA,IACpD,CAAC;AAAA,EACH,CAAC;AAED,WAAS,uBAAuB,MAAM;AACpC,OAAG,gDAAgD,YAAY;AAC7D,YAAM,eAAe,IAAI,IAAI,aAAa;AAAA,QACxC,QAAQ;AAAA,QACR,MAAM;AAAA,QACN,MAAM;AAAA,QACN,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM,wBAAwB,YAAY;AAE1D,aAAO,OAAO,IAAI,EAAE,KAAK,eAAe;AACxC,aAAO,OAAO,EAAE,EAAE,KAAK,gBAAgB;AACvC,aAAO,OAAO,OAAO,EAAE,KAAK,UAAU;AACtC,aAAO,OAAO,IAAI,EAAE,KAAK,aAAa;AACtC,aAAO,OAAO,SAAS,EAAE,KAAK,+BAA+B;AAAA,IAC/D,CAAC;AAAA,EACH,CAAC;AAED,WAAS,8BAA8B,MAAM;AAC3C,OAAG,uDAAuD,YAAY;AACpE,YAAM,iBAAiB,IAAI,IAAI,mBAAmB;AAAA,QAChD,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR,SAAS;AAAA,QACT,IAAI;AAAA,MACN,CAAC;AAED,YAAM,SAAU,MAAM;AAAA,QACpB;AAAA,MACF;AAEA,aAAO,OAAO,IAAI,EAAE,KAAK,sBAAsB;AAC/C,aAAO,OAAO,EAAE,EAAE,KAAK,kBAAkB;AACzC,aAAO,OAAO,OAAO,EAAE,KAAK,UAAU;AACtC,aAAO,OAAO,MAAM,EAAE,KAAK,wCAAwC;AAAA,IACrE,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}

package/dist/realtime/realtime_model_beta.cjs

@@ -305,23 +305,22 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
   }
   async updateChatCtx(_chatCtx) {
     const unlock = await this.updateChatCtxLock.lock();
-    const events = this.createChatCtxUpdateEvents(_chatCtx);
-    const futures = [];
-    for (const event of events) {
-      const future = new import_agents.Future();
-      futures.push(future);
-      if (event.type === "conversation.item.create") {
-        this.itemCreateFutures[event.item.id] = future;
-      } else if (event.type == "conversation.item.delete") {
-        this.itemDeleteFutures[event.item_id] = future;
-      }
-      this.sendEvent(event);
-    }
-    if (futures.length === 0) {
-      unlock();
-      return;
-    }
     try {
+      const events = await this.createChatCtxUpdateEvents(_chatCtx);
+      const futures = [];
+      for (const event of events) {
+        const future = new import_agents.Future();
+        futures.push(future);
+        if (event.type === "conversation.item.create") {
+          this.itemCreateFutures[event.item.id] = future;
+        } else if (event.type == "conversation.item.delete") {
+          this.itemDeleteFutures[event.item_id] = future;
+        }
+        this.sendEvent(event);
+      }
+      if (futures.length === 0) {
+        return;
+      }
       await Promise.race([
         Promise.all(futures),
         (0, import_agents.delay)(5e3).then(() => {
@@ -335,7 +334,7 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
       unlock();
     }
   }
-  createChatCtxUpdateEvents(chatCtx, addMockAudio = false) {
+  async createChatCtxUpdateEvents(chatCtx, addMockAudio = false) {
     const newChatCtx = chatCtx.copy();
     if (addMockAudio) {
       newChatCtx.items.push(createMockAudioItem());
@@ -360,7 +359,7 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
       }
       events.push({
         type: "conversation.item.create",
-        item: livekitItemToOpenAIItem(chatItem),
+        item: await livekitItemToOpenAIItem(chatItem),
         previous_item_id: previousId ?? void 0,
         event_id: (0, import_agents.shortuuid)("chat_ctx_create_")
       });
@@ -490,7 +489,7 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
             content: [_options.audioTranscript]
           });
           chatCtx.items[idx] = newItem;
-          const events = this.createChatCtxUpdateEvents(chatCtx);
+          const events = await this.createChatCtxUpdateEvents(chatCtx);
           for (const ev of events) {
             this.sendEvent(ev);
           }
@@ -584,7 +583,7 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
       });
       const oldChatCtx = this.remoteChatCtx;
       this.remoteChatCtx = new import_agents.llm.RemoteChatContext();
-      events.push(...this.createChatCtxUpdateEvents(chatCtx));
+      events.push(...await this.createChatCtxUpdateEvents(chatCtx));
       try {
         for (const ev of events) {
           this.emit("openai_client_event_queued", ev);
@@ -1159,7 +1158,7 @@ class RealtimeSession extends import_agents.llm.RealtimeSession {
     return handle;
   }
 }
-function livekitItemToOpenAIItem(item) {
+async function livekitItemToOpenAIItem(item) {
   switch (item.type) {
     case "function_call":
       return {
@@ -1186,7 +1185,22 @@ function livekitItemToOpenAIItem(item) {
             text: c
           });
         } else if (c.type === "image_content") {
-          continue;
+          if (role !== "user") {
+            continue;
+          }
+          const serialized = await import_agents.llm.serializeImage(c);
+          if (serialized.externalUrl) {
+            (0, import_agents.log)().warn("External URL is not supported for input_image in realtime API");
+            continue;
+          }
+          if (!serialized.base64Data) {
+            (0, import_agents.log)().warn("Serialized image has no data bytes");
+            continue;
+          }
+          contentList.push({
+            type: "input_image",
+            image_url: `data:${serialized.mimeType};base64,${serialized.base64Data}`
+          });
         } else if (c.type === "audio_content") {
           if (role === "user") {
             const encodedAudio = Buffer.from((0, import_rtc_node.combineAudioFrames)(c.frame).data).toString("base64");
@@ -1232,6 +1246,10 @@ function openAIItemToLivekitItem(item) {
       for (const c of contents) {
         if (c.type === "text" || c.type === "input_text") {
           content.push(c.text);
+        } else if (c.type === "input_image" && c.image_url) {
+          content.push(
+            import_agents.llm.createImageContent({ image: c.image_url })
+          );
         }
       }
       return import_agents.llm.ChatMessage.create({