@ax-llm/ax 12.0.5 → 12.0.6

package/index.d.cts

@@ -686,7 +686,7 @@ type AxAIOpenAIChatRequest<TModel> = {
     } | {
         role: 'user';
         content: string | ({
-            type: 'text';
+            type: string;
             text: string;
         } | {
             type: 'image_url';
@@ -700,11 +700,20 @@ type AxAIOpenAIChatRequest<TModel> = {
                 data: string;
                 format?: 'wav';
             };
+        } | {
+            type: 'file';
+            file: {
+                file_data: string;
+                filename: string;
+            };
         })[];
         name?: string;
     } | {
         role: 'assistant';
-        content: string;
+        content: string | {
+            type: string;
+            text: string;
+        };
         name?: string;
         tool_calls?: {
             type: 'function';
@@ -2335,6 +2344,7 @@ declare class AxPromptTemplate {
     private readonly thoughtFieldName;
     private readonly functions?;
     constructor(sig: Readonly<AxSignature>, options?: Readonly<AxPromptTemplateOptions>, fieldTemplates?: Record<string, AxFieldTemplateFn>);
+    private renderSingleValueUserContent;
     render: <T extends AxGenIn>(values: T | ReadonlyArray<AxMessage<T>>, // Allow T (AxGenIn) or array of AxMessages
     { examples, demos, }: Readonly<{
         skipSystemPrompt?: boolean;

package/index.d.ts

@@ -686,7 +686,7 @@ type AxAIOpenAIChatRequest<TModel> = {
     } | {
         role: 'user';
         content: string | ({
-            type: 'text';
+            type: string;
             text: string;
         } | {
             type: 'image_url';
@@ -700,11 +700,20 @@ type AxAIOpenAIChatRequest<TModel> = {
                 data: string;
                 format?: 'wav';
             };
+        } | {
+            type: 'file';
+            file: {
+                file_data: string;
+                filename: string;
+            };
         })[];
         name?: string;
     } | {
         role: 'assistant';
-        content: string;
+        content: string | {
+            type: string;
+            text: string;
+        };
         name?: string;
         tool_calls?: {
             type: 'function';
@@ -2335,6 +2344,7 @@ declare class AxPromptTemplate {
     private readonly thoughtFieldName;
     private readonly functions?;
     constructor(sig: Readonly<AxSignature>, options?: Readonly<AxPromptTemplateOptions>, fieldTemplates?: Record<string, AxFieldTemplateFn>);
+    private renderSingleValueUserContent;
     render: <T extends AxGenIn>(values: T | ReadonlyArray<AxMessage<T>>, // Allow T (AxGenIn) or array of AxMessages
     { examples, demos, }: Readonly<{
         skipSystemPrompt?: boolean;

package/index.js

@@ -3328,7 +3328,7 @@ var AxAIGoogleGeminiImpl = class {
             }
           ];
           return {
-            role: "model",
+            role: "user",
             parts
           };
         }
@@ -4051,8 +4051,11 @@ var AxAIMistral = class extends AxAIOpenAIBase {
     for (const message of messages) {
       if (message.role === "user" && Array.isArray(message.content)) {
         const contentUpdated = message.content.map((item) => {
-          if (typeof item === "object" && item !== null && item.type === "image_url") {
-            return { type: "image_url", image_url: item.image_url?.url };
+          if (typeof item === "object" && item !== null && "image_url" in item) {
+            return {
+              type: "image_url",
+              image_url: { url: item.image_url?.url }
+            };
           }
           return item;
         });
@@ -4725,8 +4728,6 @@ var AxAIOpenAIResponsesImpl = class {
         baseResult.content = event.delta;
         break;
       case "response.output_text.done":
-        baseResult.id = event.item_id;
-        baseResult.content = event.text;
         break;
       case "response.function_call_arguments.delta":
         baseResult.id = event.item_id;
@@ -5549,9 +5550,6 @@ var MemoryImpl = class {
     functionCalls
   }) {
     const isContentEmpty = typeof content === "string" && content.trim() === "";
-    if (isContentEmpty && (!functionCalls || functionCalls.length === 0)) {
-      return;
-    }
     if (isContentEmpty) {
       this.addMemory({ name, role: "assistant", functionCalls });
     } else {
@@ -5576,17 +5574,16 @@ var MemoryImpl = class {
   }) {
     const lastItem = this.data.at(-1);
     if (!lastItem || lastItem.chat.role !== "assistant") {
-      this.addResultMessage({ content, name, functionCalls });
-    } else {
-      if ("content" in lastItem.chat && typeof content === "string" && content.trim() !== "") {
-        lastItem.chat.content = content;
-      }
-      if ("name" in lastItem.chat && name) {
-        lastItem.chat.name = name;
-      }
-      if ("functionCalls" in lastItem.chat && functionCalls) {
-        lastItem.chat.functionCalls = functionCalls;
-      }
+      throw new Error("No assistant message to update");
+    }
+    if (typeof content === "string" && content.trim() !== "") {
+      lastItem.chat.content = content;
+    }
+    if (name && name.trim() !== "") {
+      lastItem.chat.name = name;
+    }
+    if (functionCalls && functionCalls.length > 0) {
+      lastItem.chat.functionCalls = functionCalls;
     }
     if (this.options?.debug) {
       if (delta && typeof delta === "string") {
@@ -6116,6 +6113,12 @@ ${outputFields}`);
       text: task.join("\n\n")
     };
   }
+  renderSingleValueUserContent = (values, renderedExamples, renderedDemos, examplesInSystemPrompt) => {
+    const completion = this.renderInputFields(values);
+    const promptList = examplesInSystemPrompt ? completion : [...renderedExamples, ...renderedDemos, ...completion];
+    const prompt = promptList.filter((v) => v !== void 0);
+    return prompt.every((v) => v.type === "text") ? prompt.map((v) => v.text).join("\n") : prompt.reduce(combineConsecutiveStrings("\n"), []);
+  };
   render = (values, {
     examples,
     demos
@@ -6144,60 +6147,49 @@ ${outputFields}`);
       role: "system",
       content: systemContent
     };
-    let userMessages = [];
     if (Array.isArray(values)) {
+      let userMessages = [];
       const history = values;
-      let lastRole = void 0;
-      for (const message of history) {
-        let messageContent = "";
-        if (message.role === "user") {
-          const userMsgParts = this.renderInputFields(
-            message.values
-            // Cast message.values (AxGenIn) to T (which extends AxGenIn)
+      for (const [index, message] of history.entries()) {
+        let content;
+        if (index === 0) {
+          content = this.renderSingleValueUserContent(
+            message.values,
+            renderedExamples,
+            renderedDemos,
+            examplesInSystemPrompt
           );
-          messageContent = userMsgParts.map((part) => part.type === "text" ? part.text : "").join("").trim();
-        } else if (message.role === "assistant") {
-          const assistantMsgParts = this.renderInputFields(
-            message.values
-            // Cast message.values (AxGenIn) to T (which extends AxGenIn)
+        } else {
+          content = this.renderSingleValueUserContent(
+            message.values,
+            [],
+            [],
+            false
           );
-          messageContent = assistantMsgParts.map((part) => part.type === "text" ? part.text : "").join("").trim();
         }
-        if (messageContent) {
-          if (lastRole === message.role && userMessages.length > 0) {
-            const lastMessage = userMessages[userMessages.length - 1];
-            if (lastMessage) {
-              lastMessage.content += "\n" + messageContent;
-            }
-          } else {
-            if (message.role === "user") {
-              userMessages.push({ role: "user", content: messageContent });
-            } else if (message.role === "assistant") {
-              userMessages.push({ role: "assistant", content: messageContent });
-            }
-          }
-          lastRole = message.role;
+        if (message.role === "user") {
+          userMessages.push({ role: "user", content });
+          continue;
         }
+        if (message.role !== "assistant") {
+          throw new Error("Invalid message role");
+        }
+        if (typeof content !== "string") {
+          throw new Error(
+            "Assistant message cannot contain non-text content like images, files,etc"
+          );
+        }
+        userMessages.push({ role: "assistant", content });
       }
-    } else {
-      const currentValues = values;
-      const completion = this.renderInputFields(currentValues);
-      const promptList = examplesInSystemPrompt ? completion : [...renderedExamples, ...renderedDemos, ...completion];
-      const promptFilter = promptList.filter((v) => v !== void 0);
-      let userContent;
-      if (promptFilter.every((v) => v.type === "text")) {
-        userContent = promptFilter.map((v) => v.text).join("\n");
-      } else {
-        userContent = promptFilter.map((part) => {
-          if (part.type === "text") return part.text;
-          if (part.type === "image") return "[IMAGE]";
-          if (part.type === "audio") return "[AUDIO]";
-          return "";
-        }).join("\n").trim();
-      }
-      userMessages.push({ role: "user", content: userContent });
+      return [systemPrompt, ...userMessages];
     }
-    return [systemPrompt, ...userMessages];
+    const userContent = this.renderSingleValueUserContent(
+      values,
+      renderedExamples,
+      renderedDemos,
+      examplesInSystemPrompt
+    );
+    return [systemPrompt, { role: "user", content: userContent }];
   };
   renderExtraFields = (extraFields) => {
     const prompt = [];
@@ -6264,9 +6256,6 @@ ${outputFields}`);
         if ("text" in v) {
           v.text = v.text + "\n";
         }
-        if ("image" in v) {
-          v.image = v.image;
-        }
         list.push(v);
       });
     }
@@ -6297,9 +6286,6 @@ ${outputFields}`);
         if ("text" in v) {
           v.text = v.text + "\n";
         }
-        if ("image" in v) {
-          v.image = v.image;
-        }
         list.push(v);
       });
     }
@@ -8950,6 +8936,14 @@ var AxGen = class extends AxProgramWithSignature {
       s: -1
     };
     let content = "";
+    mem.addResult(
+      {
+        content: "",
+        name: "initial",
+        functionCalls: []
+      },
+      sessionId
+    );
     for await (const v of res) {
       const result = v.results[0];
       if (!result) {