@aigne/core 1.57.5 → 1.58.1

package/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## [1.58.1](https://github.com/AIGNE-io/aigne-framework/compare/core-v1.58.0...core-v1.58.1) (2025-09-05)
+
+
+### Bug Fixes
+
+* should not return local path from aigne hub service ([#460](https://github.com/AIGNE-io/aigne-framework/issues/460)) ([c959717](https://github.com/AIGNE-io/aigne-framework/commit/c95971774f7e84dbeb3313f60b3e6464e2bb22e4))
+
+## [1.58.0](https://github.com/AIGNE-io/aigne-framework/compare/core-v1.57.5...core-v1.58.0) (2025-09-05)
+
+
+### Features
+
+* add modalities support for chat model ([#454](https://github.com/AIGNE-io/aigne-framework/issues/454)) ([70d1bf6](https://github.com/AIGNE-io/aigne-framework/commit/70d1bf631f4e711235d89c6df8ee210a19179b30))
+
+
+### Bug Fixes
+
+* add disabled observability env ([#453](https://github.com/AIGNE-io/aigne-framework/issues/453)) ([3e01107](https://github.com/AIGNE-io/aigne-framework/commit/3e01107deb07d3e4eb6fbe49a7b39919fa412df1))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/observability-api bumped to 0.10.2
+
 ## [1.57.5](https://github.com/AIGNE-io/aigne-framework/compare/core-v1.57.4...core-v1.57.5) (2025-09-01)
 
 

package/lib/cjs/agents/agent.d.ts

@@ -432,7 +432,7 @@ export declare abstract class Agent<I extends Message = any, O extends Message =
      * @param options Invocation options
      * @returns Final processed output
      */
-    private processAgentOutput;
+    protected processAgentOutput(input: I, output: Exclude<AgentResponse<O>, AgentResponseStream<O>>, options: AgentInvokeOptions): Promise<O>;
     /**
      * Process errors that occur during agent execution
      *

package/lib/cjs/agents/ai-agent.d.ts

@@ -1,9 +1,10 @@
 import { type ZodObject, type ZodType, z } from "zod";
 import { PromptBuilder } from "../prompt/prompt-builder.js";
 import { Agent, type AgentInvokeOptions, type AgentOptions, type AgentProcessAsyncGenerator, type Message } from "./agent.js";
-import type { ChatModel, ChatModelInput } from "./chat-model.js";
+import type { ChatModel, ChatModelInput, FileOutputType } from "./chat-model.js";
 import type { GuideRailAgentOutput } from "./guide-rail-agent.js";
 export declare const DEFAULT_OUTPUT_KEY = "message";
+export declare const DEFAULT_FILE_OUTPUT_KEY = "files";
 /**
  * Configuration options for an AI Agent
  *
@@ -31,12 +32,15 @@ export interface AIAgentOptions<I extends Message = Message, O extends Message =
      * Pick a message from input to use as the user's message
      */
     inputKey?: string;
+    fileInputKey?: string;
     /**
      * Custom key to use for text output in the response
      *
      * Defaults to `message` if not specified
      */
     outputKey?: string;
+    fileOutputKey?: string;
+    fileOutputType?: FileOutputType;
     /**
      * Controls how the agent uses tools during execution
      *
@@ -217,6 +221,7 @@ export declare class AIAgent<I extends Message = any, O extends Message = any> e
      * Pick a message from input to use as the user's message
      */
     inputKey?: string;
+    fileInputKey?: string;
     /**
      * Custom key to use for text output in the response
      *
@@ -225,6 +230,8 @@ export declare class AIAgent<I extends Message = any, O extends Message = any> e
      * {@includeCode ../../test/agents/ai-agent.test.ts#example-ai-agent-custom-output-key}
      */
     outputKey: string;
+    fileOutputKey: string;
+    fileOutputType?: FileOutputType;
     /**
      * Controls how the agent uses tools during execution
      *

package/lib/cjs/agents/ai-agent.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AIAgent = exports.aiAgentOptionsSchema = exports.aiAgentToolChoiceSchema = exports.AIAgentToolChoice = exports.DEFAULT_OUTPUT_KEY = void 0;
+exports.AIAgent = exports.aiAgentOptionsSchema = exports.aiAgentToolChoiceSchema = exports.AIAgentToolChoice = exports.DEFAULT_FILE_OUTPUT_KEY = exports.DEFAULT_OUTPUT_KEY = void 0;
 const zod_1 = require("zod");
 const prompt_builder_js_1 = require("../prompt/prompt-builder.js");
 const structured_stream_instructions_js_1 = require("../prompt/prompts/structured-stream-instructions.js");
@@ -10,6 +10,7 @@ const type_utils_js_1 = require("../utils/type-utils.js");
 const agent_js_1 = require("./agent.js");
 const types_js_1 = require("./types.js");
 exports.DEFAULT_OUTPUT_KEY = "message";
+exports.DEFAULT_FILE_OUTPUT_KEY = "files";
 /**
  * Tool choice options for AI agents
  *
@@ -111,7 +112,10 @@ class AIAgent extends agent_js_1.Agent {
                 ? prompt_builder_js_1.PromptBuilder.from(options.instructions)
                 : (options.instructions ?? new prompt_builder_js_1.PromptBuilder());
         this.inputKey = options.inputKey;
+        this.fileInputKey = options.fileInputKey;
         this.outputKey = options.outputKey || exports.DEFAULT_OUTPUT_KEY;
+        this.fileOutputKey = options.fileOutputKey || exports.DEFAULT_FILE_OUTPUT_KEY;
+        this.fileOutputType = options.fileOutputType;
         this.toolChoice = options.toolChoice;
         this.memoryAgentsAsTools = options.memoryAgentsAsTools;
         this.memoryPromptTemplate = options.memoryPromptTemplate;
@@ -151,6 +155,7 @@ class AIAgent extends agent_js_1.Agent {
      * Pick a message from input to use as the user's message
      */
     inputKey;
+    fileInputKey;
     /**
      * Custom key to use for text output in the response
      *
@@ -159,6 +164,8 @@ class AIAgent extends agent_js_1.Agent {
      * {@includeCode ../../test/agents/ai-agent.test.ts#example-ai-agent-custom-output-key}
      */
     outputKey;
+    fileOutputKey;
+    fileOutputType;
     /**
      * Controls how the agent uses tools during execution
      *
@@ -269,7 +276,7 @@ class AIAgent extends agent_js_1.Agent {
                     }
                 }
             }
-            const { toolCalls, json, text } = modelOutput;
+            const { toolCalls, json, text, files } = modelOutput;
             if (toolCalls?.length) {
                 const executedToolCalls = [];
                 // Execute tools
@@ -308,6 +315,9 @@ class AIAgent extends agent_js_1.Agent {
             if (text) {
                 Object.assign(result, { [outputKey]: text });
             }
+            if (files) {
+                Object.assign(result, { [this.fileOutputKey]: files });
+            }
             if (!(0, type_utils_js_1.isEmpty)(result)) {
                 yield { delta: { json: result } };
             }

package/lib/cjs/agents/chat-model.d.ts

@@ -1,6 +1,6 @@
 import { z } from "zod";
 import { type PromiseOrValue } from "../utils/type-utils.js";
-import { Agent, type AgentInvokeOptions, type AgentOptions, type AgentProcessResult, type Message } from "./agent.js";
+import { Agent, type AgentInvokeOptions, type AgentOptions, type AgentProcessResult, type AgentResponse, type AgentResponseStream, type Message } from "./agent.js";
 export declare class StructuredOutputError extends Error {
 }
 /**
@@ -104,6 +104,11 @@ export declare abstract class ChatModel extends Agent<ChatModelInput, ChatModelO
      * @returns A promise or direct value containing the model's response
      */
     abstract process(input: ChatModelInput, options: AgentInvokeOptions): PromiseOrValue<AgentProcessResult<ChatModelOutput>>;
+    protected processAgentOutput(input: ChatModelInput, output: Exclude<AgentResponse<ChatModelOutput>, AgentResponseStream<ChatModelOutput>>, options: AgentInvokeOptions): Promise<ChatModelOutput>;
+    transformFileOutput(input: ChatModelInput, data: FileUnionContent, options: AgentInvokeOptions): Promise<FileUnionContent>;
+    static getFileExtension(type: string): string | undefined;
+    static getMimeType(filename: string): string | undefined;
+    protected downloadFile(url: string): Promise<Response>;
 }
 /**
  * Input message format for ChatModel
@@ -128,6 +133,7 @@ export interface ChatModelInput extends Message {
      * Specifies the expected response format
      */
     responseFormat?: ChatModelInputResponseFormat;
+    fileOutputType?: FileOutputType;
     /**
      * List of tools available for the model to use
      */
@@ -190,7 +196,7 @@ export interface ChatModelInputMessage {
  *
  * Can be a simple string, or a mixed array of text and image content
  */
-export type ChatModelInputMessageContent = string | (TextContent | ImageUrlContent)[];
+export type ChatModelInputMessageContent = string | UnionContent[];
 /**
  * Text content type
  *
@@ -200,15 +206,304 @@ export type TextContent = {
     type: "text";
     text: string;
 };
+export declare const textContentSchema: z.ZodObject<{
+    type: z.ZodLiteral<"text">;
+    text: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "text";
+    text: string;
+}, {
+    type: "text";
+    text: string;
+}>;
+export interface FileContentBase {
+    filename?: string;
+    mimeType?: string;
+}
+export declare const fileContentBaseSchema: z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+}, "strip", z.ZodTypeAny, {
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>;
 /**
  * Image URL content type
  *
  * Used for image parts of message content, referencing images via URL
  */
-export type ImageUrlContent = {
-    type: "image_url";
+export interface UrlContent extends FileContentBase {
+    type: "url";
     url: string;
-};
+}
+export declare const urlContentSchema: z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"url">;
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>;
+export interface FileContent extends FileContentBase {
+    type: "file";
+    data: string;
+}
+export declare const fileContentSchema: z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"file">;
+    data: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>;
+export interface LocalContent extends FileContentBase {
+    type: "local";
+    path: string;
+}
+export declare const localContentSchema: z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"local">;
+    path: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>;
+export type FileUnionContent = LocalContent | UrlContent | FileContent;
+export declare const fileUnionContentSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"local">;
+    path: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"url">;
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"file">;
+    data: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>]>;
+export declare const fileUnionContentsSchema: z.ZodUnion<[z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"local">;
+    path: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"url">;
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"file">;
+    data: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>]>, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"local">;
+    path: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"url">;
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"file">;
+    data: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>]>, "many">]>;
+export type UnionContent = TextContent | FileUnionContent;
+export declare const unionContentSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    type: z.ZodLiteral<"text">;
+    text: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "text";
+    text: string;
+}, {
+    type: "text";
+    text: string;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"local">;
+    path: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"url">;
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"file">;
+    data: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>]>;
 /**
  * Model response format settings
  *
@@ -277,6 +572,7 @@ export type ChatModelInputToolChoice = "auto" | "none" | "required" | {
         description?: string;
     };
 };
+export type Modality = "text" | "image" | "audio";
 /**
  * Model-specific configuration options
  *
@@ -307,6 +603,7 @@ export interface ChatModelOptions {
      * Whether to allow parallel tool calls
      */
     parallelToolCalls?: boolean;
+    modalities?: Modality[];
 }
 /**
  * Output message format for ChatModel
@@ -342,6 +639,11 @@ export interface ChatModelOutput extends Message {
      * Model name or version used
      */
     model?: string;
+    files?: FileUnionContent[];
+}
+export declare enum FileOutputType {
+    local = "local",
+    file = "file"
 }
 /**
  * Tool call information in model output

package/lib/cjs/agents/chat-model.js

@@ -32,10 +32,17 @@ var __importStar = (this && this.__importStar) || (function () {
         return result;
     };
 })();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.chatModelOutputUsageSchema = exports.ChatModel = exports.StructuredOutputError = void 0;
+exports.chatModelOutputUsageSchema = exports.FileOutputType = exports.unionContentSchema = exports.fileUnionContentsSchema = exports.fileUnionContentSchema = exports.localContentSchema = exports.fileContentSchema = exports.urlContentSchema = exports.fileContentBaseSchema = exports.textContentSchema = exports.ChatModel = exports.StructuredOutputError = void 0;
+const index_js_1 = require("@aigne/platform-helpers/nodejs/index.js");
 const ajv_1 = require("ajv");
+const mime_1 = __importDefault(require("mime"));
+const uuid_1 = require("uuid");
 const zod_1 = require("zod");
+const schema_js_1 = require("../loader/schema.js");
 const type_utils_js_1 = require("../utils/type-utils.js");
 const agent_js_1 = require("./agent.js");
 const CHAT_MODEL_DEFAULT_RETRY_OPTIONS = {
@@ -207,19 +214,111 @@ class ChatModel extends agent_js_1.Agent {
                 options.context.usage.aigneHubCredits += usage.aigneHubCredits;
         }
     }
+    async processAgentOutput(input, output, options) {
+        if (output.files) {
+            const files = zod_1.z.array(exports.fileUnionContentSchema).parse(output.files);
+            output = {
+                ...output,
+                files: await Promise.all(files.map((file) => this.transformFileOutput(input, file, options))),
+            };
+        }
+        return super.processAgentOutput(input, output, options);
+    }
+    async transformFileOutput(input, data, options) {
+        const fileOutputType = input.fileOutputType || FileOutputType.local;
+        if (fileOutputType === data.type)
+            return data;
+        const common = (0, type_utils_js_1.pick)(data, "filename", "mimeType");
+        switch (fileOutputType) {
+            case FileOutputType.local: {
+                const dir = index_js_1.nodejs.path.join(index_js_1.nodejs.os.tmpdir(), options.context.id);
+                await index_js_1.nodejs.fs.mkdir(dir, { recursive: true });
+                const ext = ChatModel.getFileExtension(data.mimeType || data.filename || "");
+                const id = (0, uuid_1.v7)();
+                const filename = ext ? `${id}.${ext}` : id;
+                const path = index_js_1.nodejs.path.join(dir, filename);
+                if (data.type === "file") {
+                    await index_js_1.nodejs.fs.writeFile(path, data.data, "base64");
+                }
+                else if (data.type === "url") {
+                    await this.downloadFile(data.url)
+                        .then((res) => res.body)
+                        .then((body) => body && index_js_1.nodejs.fs.writeFile(path, body));
+                }
+                else {
+                    throw new Error(`Unexpected file type: ${data.type}`);
+                }
+                return { ...common, type: "local", path };
+            }
+            case FileOutputType.file: {
+                let base64;
+                if (data.type === "local") {
+                    base64 = await index_js_1.nodejs.fs.readFile(data.path, "base64");
+                }
+                else if (data.type === "url") {
+                    base64 = Buffer.from(await (await this.downloadFile(data.url)).arrayBuffer()).toString("base64");
+                }
+                else {
+                    throw new Error(`Unexpected file type: ${data.type}`);
+                }
+                return { ...common, type: "file", data: base64 };
+            }
+        }
+    }
+    static getFileExtension(type) {
+        return mime_1.default.getExtension(type) || undefined;
+    }
+    static getMimeType(filename) {
+        return mime_1.default.getType(filename) || undefined;
+    }
+    async downloadFile(url) {
+        const response = await fetch(url);
+        if (!response.ok) {
+            const text = await response.text().catch(() => null);
+            throw new Error(`Failed to download content from ${url}, ${response.status} ${response.statusText} ${text}`);
+        }
+        return response;
+    }
 }
 exports.ChatModel = ChatModel;
+exports.textContentSchema = zod_1.z.object({
+    type: zod_1.z.literal("text"),
+    text: zod_1.z.string(),
+});
+exports.fileContentBaseSchema = zod_1.z.object({
+    filename: (0, schema_js_1.optionalize)(zod_1.z.string()),
+    mimeType: (0, schema_js_1.optionalize)(zod_1.z.string()),
+});
+exports.urlContentSchema = exports.fileContentBaseSchema.extend({
+    type: zod_1.z.literal("url"),
+    url: zod_1.z.string(),
+});
+exports.fileContentSchema = exports.fileContentBaseSchema.extend({
+    type: zod_1.z.literal("file"),
+    data: zod_1.z.string(),
+});
+exports.localContentSchema = exports.fileContentBaseSchema.extend({
+    type: zod_1.z.literal("local"),
+    path: zod_1.z.string(),
+});
+exports.fileUnionContentSchema = zod_1.z.discriminatedUnion("type", [
+    exports.localContentSchema,
+    exports.urlContentSchema,
+    exports.fileContentSchema,
+]);
+exports.fileUnionContentsSchema = zod_1.z.union([
+    exports.fileUnionContentSchema,
+    zod_1.z.array(exports.fileUnionContentSchema),
+]);
+exports.unionContentSchema = zod_1.z.discriminatedUnion("type", [
+    exports.textContentSchema,
+    exports.localContentSchema,
+    exports.urlContentSchema,
+    exports.fileContentSchema,
+]);
 const chatModelInputMessageSchema = zod_1.z.object({
     role: zod_1.z.union([zod_1.z.literal("system"), zod_1.z.literal("user"), zod_1.z.literal("agent"), zod_1.z.literal("tool")]),
-    content: zod_1.z
-        .union([
-        zod_1.z.string(),
-        zod_1.z.array(zod_1.z.union([
-            zod_1.z.object({ type: zod_1.z.literal("text"), text: zod_1.z.string() }),
-            zod_1.z.object({ type: zod_1.z.literal("image_url"), url: zod_1.z.string() }),
-        ])),
-    ])
-        .optional(),
+    content: zod_1.z.union([zod_1.z.string(), zod_1.z.array(exports.unionContentSchema)]).optional(),
     toolCalls: zod_1.z
         .array(zod_1.z.object({
         id: zod_1.z.string(),
@@ -266,6 +365,7 @@ const chatModelOptionsSchema = zod_1.z.object({
     frequencyPenalty: zod_1.z.number().optional(),
     presencePenalty: zod_1.z.number().optional(),
     parallelToolCalls: zod_1.z.boolean().optional().default(true),
+    modalities: zod_1.z.array(zod_1.z.enum(["text", "image", "audio"])).optional(),
 });
 const chatModelInputSchema = zod_1.z.object({
     messages: zod_1.z.array(chatModelInputMessageSchema),
@@ -274,6 +374,11 @@ const chatModelInputSchema = zod_1.z.object({
     toolChoice: chatModelInputToolChoiceSchema.optional(),
     modelOptions: chatModelOptionsSchema.optional(),
 });
+var FileOutputType;
+(function (FileOutputType) {
+    FileOutputType["local"] = "local";
+    FileOutputType["file"] = "file";
+})(FileOutputType || (exports.FileOutputType = FileOutputType = {}));
 const chatModelOutputToolCallSchema = zod_1.z.object({
     id: zod_1.z.string(),
     type: zod_1.z.literal("function"),
@@ -293,4 +398,5 @@ const chatModelOutputSchema = zod_1.z.object({
     toolCalls: zod_1.z.array(chatModelOutputToolCallSchema).optional(),
     usage: exports.chatModelOutputUsageSchema.optional(),
     model: zod_1.z.string().optional(),
+    files: zod_1.z.array(exports.fileUnionContentSchema).optional(),
 });