@aigne/core 1.57.4 → 1.58.0

package/lib/esm/prompt/prompt-builder.js

@@ -3,8 +3,11 @@ import { stringify } from "yaml";
 import { ZodObject } from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
 import { Agent } from "../agents/agent.js";
+import { DEFAULT_FILE_OUTPUT_KEY, DEFAULT_OUTPUT_KEY } from "../agents/ai-agent.js";
+import { fileUnionContentsSchema, } from "../agents/chat-model.js";
+import { optionalize } from "../loader/schema.js";
 import { outputSchemaToResponseFormatSchema } from "../utils/json-schema.js";
-import { isNil, isRecord, unique } from "../utils/type-utils.js";
+import { checkArguments, flat, isRecord, unique } from "../utils/type-utils.js";
 import { MEMORY_MESSAGE_TEMPLATE } from "./prompts/memory-message-template.js";
 import { STRUCTURED_STREAM_INSTRUCTIONS } from "./prompts/structured-stream-instructions.js";
 import { AgentMessageTemplate, ChatMessagesTemplate, PromptTemplate, SystemMessageTemplate, UserMessageTemplate, } from "./template.js";
@@ -34,11 +37,11 @@ export class PromptBuilder {
                         content = resource.text;
                     }
                     else if (typeof resource.blob === "string") {
-                        content = [{ type: "image_url", url: resource.blob }];
+                        content = [{ type: "url", url: resource.blob }];
                     }
                 }
                 else if (i.content.type === "image") {
-                    content = [{ type: "image_url", url: i.content.data }];
+                    content = [{ type: "url", url: i.content.data }];
                 }
                 if (!content)
                     throw new Error(`Unsupported content type ${i.content.type}`);
@@ -62,6 +65,7 @@ export class PromptBuilder {
             responseFormat: options.agent?.structuredStreamMode
                 ? undefined
                 : this.buildResponseFormat(options),
+            fileOutputType: options.agent?.fileOutputType,
             ...this.buildTools(options),
         };
     }
@@ -80,6 +84,10 @@ export class PromptBuilder {
         const messages = (await (typeof this.instructions === "string"
             ? ChatMessagesTemplate.from([SystemMessageTemplate.from(this.instructions)])
             : this.instructions)?.format(options.input, { workingDir: this.workingDir })) ?? [];
+        const fileInputKey = options.agent?.fileInputKey;
+        const files = flat(fileInputKey
+            ? checkArguments("Check input files", optionalize(fileUnionContentsSchema), input?.[fileInputKey])
+            : null);
         const memories = [];
         if (options.agent && options.context) {
             memories.push(...(await options.agent.retrieveMemories({ search: message }, { context: options.context })));
@@ -101,26 +109,68 @@ export class PromptBuilder {
                 },
             })));
         }
-        if (message) {
-            messages.push({
-                role: "user",
-                content: message,
-            });
+        if (message || files.length) {
+            const content = [];
+            if (message)
+                content.push({ type: "text", text: message });
+            if (files.length)
+                content.push(...files);
+            messages.push({ role: "user", content });
         }
         return messages;
     }
     async convertMemoriesToMessages(memories, options) {
         const messages = [];
         const other = [];
+        const inputKey = options.agent?.inputKey;
+        const fileInputKey = options.agent?.fileInputKey;
+        const outputKey = options.agent?.outputKey || DEFAULT_OUTPUT_KEY;
+        const fileOutputKey = options.agent?.fileOutputKey || DEFAULT_FILE_OUTPUT_KEY;
         const stringOrStringify = (value) => typeof value === "string" ? value : stringify(value);
-        for (const { content } of memories) {
-            if (isRecord(content) && "input" in content && "output" in content) {
-                if (!isNil(content.input) && content.input !== "") {
-                    messages.push({ role: "user", content: stringOrStringify(content.input) });
+        const convertMemoryToMessage = async (content) => {
+            const { input, output } = content;
+            if (!input || !output)
+                return [];
+            const result = [];
+            const userMessageContent = [];
+            if (typeof input === "object") {
+                const inputMessage = inputKey ? Reflect.get(input, inputKey) : undefined;
+                const inputFiles = fileInputKey ? Reflect.get(input, fileInputKey) : undefined;
+                if (inputMessage) {
+                    userMessageContent.push({ type: "text", text: stringOrStringify(inputMessage) });
                 }
-                if (!isNil(content.output) && content.output !== "") {
-                    messages.push({ role: "agent", content: stringOrStringify(content.output) });
+                if (inputFiles) {
+                    userMessageContent.push(...flat(checkArguments("Check memory input files", optionalize(fileUnionContentsSchema), inputFiles)));
+                }
+            }
+            if (!userMessageContent.length) {
+                userMessageContent.push({ type: "text", text: stringOrStringify(input) });
+            }
+            result.push({ role: "user", content: userMessageContent });
+            const agentMessageContent = [];
+            if (typeof output === "object") {
+                const outputMessage = Reflect.get(output, outputKey);
+                const outputFiles = Reflect.get(output, fileOutputKey);
+                if (outputMessage) {
+                    agentMessageContent.push({ type: "text", text: stringOrStringify(outputMessage) });
                 }
+                if (outputFiles) {
+                    agentMessageContent.push(...flat(checkArguments("Check memory output files", optionalize(fileUnionContentsSchema), outputFiles)));
+                }
+            }
+            if (!agentMessageContent.length) {
+                agentMessageContent.push({ type: "text", text: stringOrStringify(output) });
+            }
+            result.push({ role: "agent", content: agentMessageContent });
+            return result;
+        };
+        for (const { content } of memories) {
+            if (isRecord(content) &&
+                "input" in content &&
+                content.input &&
+                "output" in content &&
+                content.output) {
+                messages.push(...(await convertMemoryToMessage(content)));
             }
             else {
                 other.push(content);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/core",
-  "version": "1.57.4",
+  "version": "1.58.0",
   "description": "The functional core of agentic AI",
   "publishConfig": {
     "access": "public"
@@ -82,6 +82,7 @@
     "is-network-error": "^1.1.0",
     "jaison": "^2.0.2",
     "jsonata": "^2.0.6",
+    "mime": "^4.0.7",
     "nunjucks": "^3.2.4",
     "p-retry": "^6.2.1",
     "raw-body": "^3.0.0",
@@ -91,7 +92,7 @@
     "yaml": "^2.8.0",
     "zod": "^3.25.67",
     "zod-to-json-schema": "^3.24.6",
-    "@aigne/observability-api": "^0.10.1",
+    "@aigne/observability-api": "^0.10.2",
     "@aigne/platform-helpers": "^0.6.2"
   },
   "devDependencies": {