@aigne/core 1.57.5 → 1.58.1

package/lib/esm/agents/ai-agent.js

@@ -7,6 +7,7 @@ import { checkArguments, isEmpty } from "../utils/type-utils.js";
 import { Agent, agentOptionsSchema, isAgentResponseDelta, } from "./agent.js";
 import { isTransferAgentOutput } from "./types.js";
 export const DEFAULT_OUTPUT_KEY = "message";
+export const DEFAULT_FILE_OUTPUT_KEY = "files";
 /**
  * Tool choice options for AI agents
  *
@@ -108,7 +109,10 @@ export class AIAgent extends Agent {
                 ? PromptBuilder.from(options.instructions)
                 : (options.instructions ?? new PromptBuilder());
         this.inputKey = options.inputKey;
+        this.fileInputKey = options.fileInputKey;
         this.outputKey = options.outputKey || DEFAULT_OUTPUT_KEY;
+        this.fileOutputKey = options.fileOutputKey || DEFAULT_FILE_OUTPUT_KEY;
+        this.fileOutputType = options.fileOutputType;
         this.toolChoice = options.toolChoice;
         this.memoryAgentsAsTools = options.memoryAgentsAsTools;
         this.memoryPromptTemplate = options.memoryPromptTemplate;
@@ -148,6 +152,7 @@ export class AIAgent extends Agent {
      * Pick a message from input to use as the user's message
      */
     inputKey;
+    fileInputKey;
     /**
      * Custom key to use for text output in the response
      *
@@ -156,6 +161,8 @@ export class AIAgent extends Agent {
      * {@includeCode ../../test/agents/ai-agent.test.ts#example-ai-agent-custom-output-key}
      */
     outputKey;
+    fileOutputKey;
+    fileOutputType;
     /**
      * Controls how the agent uses tools during execution
      *
@@ -266,7 +273,7 @@ export class AIAgent extends Agent {
                     }
                 }
             }
-            const { toolCalls, json, text } = modelOutput;
+            const { toolCalls, json, text, files } = modelOutput;
             if (toolCalls?.length) {
                 const executedToolCalls = [];
                 // Execute tools
@@ -305,6 +312,9 @@ export class AIAgent extends Agent {
             if (text) {
                 Object.assign(result, { [outputKey]: text });
             }
+            if (files) {
+                Object.assign(result, { [this.fileOutputKey]: files });
+            }
             if (!isEmpty(result)) {
                 yield { delta: { json: result } };
             }

package/lib/esm/agents/chat-model.d.ts

@@ -1,6 +1,6 @@
 import { z } from "zod";
 import { type PromiseOrValue } from "../utils/type-utils.js";
-import { Agent, type AgentInvokeOptions, type AgentOptions, type AgentProcessResult, type Message } from "./agent.js";
+import { Agent, type AgentInvokeOptions, type AgentOptions, type AgentProcessResult, type AgentResponse, type AgentResponseStream, type Message } from "./agent.js";
 export declare class StructuredOutputError extends Error {
 }
 /**
@@ -104,6 +104,11 @@ export declare abstract class ChatModel extends Agent<ChatModelInput, ChatModelO
      * @returns A promise or direct value containing the model's response
      */
     abstract process(input: ChatModelInput, options: AgentInvokeOptions): PromiseOrValue<AgentProcessResult<ChatModelOutput>>;
+    protected processAgentOutput(input: ChatModelInput, output: Exclude<AgentResponse<ChatModelOutput>, AgentResponseStream<ChatModelOutput>>, options: AgentInvokeOptions): Promise<ChatModelOutput>;
+    transformFileOutput(input: ChatModelInput, data: FileUnionContent, options: AgentInvokeOptions): Promise<FileUnionContent>;
+    static getFileExtension(type: string): string | undefined;
+    static getMimeType(filename: string): string | undefined;
+    protected downloadFile(url: string): Promise<Response>;
 }
 /**
  * Input message format for ChatModel
@@ -128,6 +133,7 @@ export interface ChatModelInput extends Message {
      * Specifies the expected response format
      */
     responseFormat?: ChatModelInputResponseFormat;
+    fileOutputType?: FileOutputType;
     /**
      * List of tools available for the model to use
      */
@@ -190,7 +196,7 @@ export interface ChatModelInputMessage {
  *
  * Can be a simple string, or a mixed array of text and image content
  */
-export type ChatModelInputMessageContent = string | (TextContent | ImageUrlContent)[];
+export type ChatModelInputMessageContent = string | UnionContent[];
 /**
  * Text content type
  *
@@ -200,15 +206,304 @@ export type TextContent = {
     type: "text";
     text: string;
 };
+export declare const textContentSchema: z.ZodObject<{
+    type: z.ZodLiteral<"text">;
+    text: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "text";
+    text: string;
+}, {
+    type: "text";
+    text: string;
+}>;
+export interface FileContentBase {
+    filename?: string;
+    mimeType?: string;
+}
+export declare const fileContentBaseSchema: z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+}, "strip", z.ZodTypeAny, {
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>;
 /**
  * Image URL content type
  *
  * Used for image parts of message content, referencing images via URL
  */
-export type ImageUrlContent = {
-    type: "image_url";
+export interface UrlContent extends FileContentBase {
+    type: "url";
     url: string;
-};
+}
+export declare const urlContentSchema: z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"url">;
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>;
+export interface FileContent extends FileContentBase {
+    type: "file";
+    data: string;
+}
+export declare const fileContentSchema: z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"file">;
+    data: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>;
+export interface LocalContent extends FileContentBase {
+    type: "local";
+    path: string;
+}
+export declare const localContentSchema: z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"local">;
+    path: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>;
+export type FileUnionContent = LocalContent | UrlContent | FileContent;
+export declare const fileUnionContentSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"local">;
+    path: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"url">;
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"file">;
+    data: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>]>;
+export declare const fileUnionContentsSchema: z.ZodUnion<[z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"local">;
+    path: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"url">;
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"file">;
+    data: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>]>, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"local">;
+    path: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"url">;
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"file">;
+    data: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>]>, "many">]>;
+export type UnionContent = TextContent | FileUnionContent;
+export declare const unionContentSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    type: z.ZodLiteral<"text">;
+    text: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "text";
+    text: string;
+}, {
+    type: "text";
+    text: string;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"local">;
+    path: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    path: string;
+    type: "local";
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"url">;
+    url: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "url";
+    url: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>, z.ZodObject<{
+    filename: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+    mimeType: z.ZodType<string | undefined, z.ZodTypeDef, string | undefined>;
+} & {
+    type: z.ZodLiteral<"file">;
+    data: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}, {
+    type: "file";
+    data: string;
+    filename?: string | undefined;
+    mimeType?: string | undefined;
+}>]>;
 /**
  * Model response format settings
  *
@@ -277,6 +572,7 @@ export type ChatModelInputToolChoice = "auto" | "none" | "required" | {
         description?: string;
     };
 };
+export type Modality = "text" | "image" | "audio";
 /**
  * Model-specific configuration options
  *
@@ -307,6 +603,7 @@ export interface ChatModelOptions {
      * Whether to allow parallel tool calls
      */
     parallelToolCalls?: boolean;
+    modalities?: Modality[];
 }
 /**
  * Output message format for ChatModel
@@ -342,6 +639,11 @@ export interface ChatModelOutput extends Message {
      * Model name or version used
      */
     model?: string;
+    files?: FileUnionContent[];
+}
+export declare enum FileOutputType {
+    local = "local",
+    file = "file"
 }
 /**
  * Tool call information in model output

package/lib/esm/agents/chat-model.js

@@ -1,6 +1,10 @@
+import { nodejs } from "@aigne/platform-helpers/nodejs/index.js";
 import { Ajv } from "ajv";
+import mime from "mime";
+import { v7 } from "uuid";
 import { z } from "zod";
-import { checkArguments } from "../utils/type-utils.js";
+import { optionalize } from "../loader/schema.js";
+import { checkArguments, pick } from "../utils/type-utils.js";
 import { Agent, agentOptionsSchema, } from "./agent.js";
 const CHAT_MODEL_DEFAULT_RETRY_OPTIONS = {
     retries: 3,
@@ -170,18 +174,110 @@ export class ChatModel extends Agent {
                 options.context.usage.aigneHubCredits += usage.aigneHubCredits;
         }
     }
+    async processAgentOutput(input, output, options) {
+        if (output.files) {
+            const files = z.array(fileUnionContentSchema).parse(output.files);
+            output = {
+                ...output,
+                files: await Promise.all(files.map((file) => this.transformFileOutput(input, file, options))),
+            };
+        }
+        return super.processAgentOutput(input, output, options);
+    }
+    async transformFileOutput(input, data, options) {
+        const fileOutputType = input.fileOutputType || FileOutputType.local;
+        if (fileOutputType === data.type)
+            return data;
+        const common = pick(data, "filename", "mimeType");
+        switch (fileOutputType) {
+            case FileOutputType.local: {
+                const dir = nodejs.path.join(nodejs.os.tmpdir(), options.context.id);
+                await nodejs.fs.mkdir(dir, { recursive: true });
+                const ext = ChatModel.getFileExtension(data.mimeType || data.filename || "");
+                const id = v7();
+                const filename = ext ? `${id}.${ext}` : id;
+                const path = nodejs.path.join(dir, filename);
+                if (data.type === "file") {
+                    await nodejs.fs.writeFile(path, data.data, "base64");
+                }
+                else if (data.type === "url") {
+                    await this.downloadFile(data.url)
+                        .then((res) => res.body)
+                        .then((body) => body && nodejs.fs.writeFile(path, body));
+                }
+                else {
+                    throw new Error(`Unexpected file type: ${data.type}`);
+                }
+                return { ...common, type: "local", path };
+            }
+            case FileOutputType.file: {
+                let base64;
+                if (data.type === "local") {
+                    base64 = await nodejs.fs.readFile(data.path, "base64");
+                }
+                else if (data.type === "url") {
+                    base64 = Buffer.from(await (await this.downloadFile(data.url)).arrayBuffer()).toString("base64");
+                }
+                else {
+                    throw new Error(`Unexpected file type: ${data.type}`);
+                }
+                return { ...common, type: "file", data: base64 };
+            }
+        }
+    }
+    static getFileExtension(type) {
+        return mime.getExtension(type) || undefined;
+    }
+    static getMimeType(filename) {
+        return mime.getType(filename) || undefined;
+    }
+    async downloadFile(url) {
+        const response = await fetch(url);
+        if (!response.ok) {
+            const text = await response.text().catch(() => null);
+            throw new Error(`Failed to download content from ${url}, ${response.status} ${response.statusText} ${text}`);
+        }
+        return response;
+    }
 }
+export const textContentSchema = z.object({
+    type: z.literal("text"),
+    text: z.string(),
+});
+export const fileContentBaseSchema = z.object({
+    filename: optionalize(z.string()),
+    mimeType: optionalize(z.string()),
+});
+export const urlContentSchema = fileContentBaseSchema.extend({
+    type: z.literal("url"),
+    url: z.string(),
+});
+export const fileContentSchema = fileContentBaseSchema.extend({
+    type: z.literal("file"),
+    data: z.string(),
+});
+export const localContentSchema = fileContentBaseSchema.extend({
+    type: z.literal("local"),
+    path: z.string(),
+});
+export const fileUnionContentSchema = z.discriminatedUnion("type", [
+    localContentSchema,
+    urlContentSchema,
+    fileContentSchema,
+]);
+export const fileUnionContentsSchema = z.union([
+    fileUnionContentSchema,
+    z.array(fileUnionContentSchema),
+]);
+export const unionContentSchema = z.discriminatedUnion("type", [
+    textContentSchema,
+    localContentSchema,
+    urlContentSchema,
+    fileContentSchema,
+]);
 const chatModelInputMessageSchema = z.object({
     role: z.union([z.literal("system"), z.literal("user"), z.literal("agent"), z.literal("tool")]),
-    content: z
-        .union([
-        z.string(),
-        z.array(z.union([
-            z.object({ type: z.literal("text"), text: z.string() }),
-            z.object({ type: z.literal("image_url"), url: z.string() }),
-        ])),
-    ])
-        .optional(),
+    content: z.union([z.string(), z.array(unionContentSchema)]).optional(),
     toolCalls: z
         .array(z.object({
         id: z.string(),
@@ -228,6 +324,7 @@ const chatModelOptionsSchema = z.object({
     frequencyPenalty: z.number().optional(),
     presencePenalty: z.number().optional(),
     parallelToolCalls: z.boolean().optional().default(true),
+    modalities: z.array(z.enum(["text", "image", "audio"])).optional(),
 });
 const chatModelInputSchema = z.object({
     messages: z.array(chatModelInputMessageSchema),
@@ -236,6 +333,11 @@ const chatModelInputSchema = z.object({
     toolChoice: chatModelInputToolChoiceSchema.optional(),
     modelOptions: chatModelOptionsSchema.optional(),
 });
+export var FileOutputType;
+(function (FileOutputType) {
+    FileOutputType["local"] = "local";
+    FileOutputType["file"] = "file";
+})(FileOutputType || (FileOutputType = {}));
 const chatModelOutputToolCallSchema = z.object({
     id: z.string(),
     type: z.literal("function"),
@@ -255,4 +357,5 @@ const chatModelOutputSchema = z.object({
     toolCalls: z.array(chatModelOutputToolCallSchema).optional(),
     usage: chatModelOutputUsageSchema.optional(),
     model: z.string().optional(),
+    files: z.array(fileUnionContentSchema).optional(),
 });

package/lib/esm/agents/image-model.d.ts

@@ -32,13 +32,13 @@ export declare const imageModelInputSchema: z.ZodObject<{
 }, "strip", z.ZodTypeAny, {
     prompt: string;
     model?: string | undefined;
-    responseFormat?: "url" | "base64" | undefined;
+    responseFormat?: "base64" | "url" | undefined;
     image?: string | string[] | undefined;
     n?: number | undefined;
 }, {
     prompt: string;
     model?: string | undefined;
-    responseFormat?: "url" | "base64" | undefined;
+    responseFormat?: "base64" | "url" | undefined;
     image?: string | string[] | undefined;
     n?: number | undefined;
 }>;

package/lib/esm/aigne/context.js

@@ -196,6 +196,8 @@ export class AIGNEContext {
         return this.internal.events.emit(eventName, ...newArgs);
     }
     async trace(eventName, args, b) {
+        if (process.env.AIGNE_OBSERVABILITY_DISABLED)
+            return;
         const span = this.span;
         if (!span)
             return;

package/lib/esm/prompt/prompt-builder.d.ts

@@ -1,7 +1,7 @@
 import type { GetPromptResult } from "@modelcontextprotocol/sdk/types.js";
 import { Agent, type AgentInvokeOptions, type Message } from "../agents/agent.js";
-import type { AIAgent } from "../agents/ai-agent.js";
-import type { ChatModel, ChatModelInput } from "../agents/chat-model.js";
+import { type AIAgent } from "../agents/ai-agent.js";
+import { type ChatModel, type ChatModelInput } from "../agents/chat-model.js";
 import { ChatMessagesTemplate } from "./template.js";
 export interface PromptBuilderOptions {
     instructions?: string | ChatMessagesTemplate;