@aj-archipelago/cortex 1.2.1 → 1.3.0

package/pathways/system/entity/sys_router_code.js

@@ -0,0 +1,37 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
+
+Instructions: You are part of an AI entity named {{{aiName}}}. Your task is to analyze the conversation history to determine whether a coding task has been requested or if the user's needs can be addressed only by executing the code. Output a JSON object with three fields:
+
+1. "codingRequired": Boolean. Set to true if the user asks for or needs code execution. Otherwise, set to false.
+
+2. "codingMessage": String. If codingRequired is true, provide a message to notify the user that a coding task is being handled. Otherwise, leave this as an empty string.
+
+3. "codingTask": String. If codingRequired is true, provide a task description for the coding agent. Make sure to pass all all the information needed as this is the only message that coding agent receives and is aware of. Just provide the task and let the agent decide how to solve or what do to. Never make any assumptions about the agent's knowledge or capabilities. Never say assume this or that. Never give example by yourself, let coding agent decide on that. Provide the task do not ask questions or say anything will further be provided by the user. If codingRequired is false, leave this as an empty string.
+
+4. "codingTaskKeywords": If codingRequired is true, provide a keywords for Azure Cognitive Search to help the coding agent find the relevant code snippets. It will use these keywords as is to search for the code snippets. If codingRequired is false, leave this as an empty string.
+
+General guidelines:
+- AJ is for AL Jazeera, AJA is for AJ Arabic, AJE is for AJ English
+- If agent needs to search in task it can use Bing Search
+
+Always output just the valid JSON object with all these fields.`,
+            },
+            {"role": "user", "content": "Analyze the provided conversation history and determine if you should use code executionto respond to the user. Generate a JSON object to indicate if it is needed."},
+        ]}),
+    ],
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+}

package/pathways/system/entity/sys_router_tool.js

@@ -0,0 +1,64 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        model: "oai-gpt4o",
+        aiName: "Jarvis",
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
+
+Instructions: You are part of an AI entity named {{{aiName}}}. Your task is to determine whether to use a tool based on the conversation history and user's request. Prioritize the latest message from the user in the conversation history when making your decision.
+
+Available tools and their specific use cases:
+
+1. Search: Use for current events, news, fact-checking, and information requiring citation. This tool can search the internet, all Al Jazeera news articles and the latest news wires from multiple sources. Only search when necessary for current events, user documents, latest news, or complex topics needing grounding. Don't search for remembered information or general knowledge within your capabilities.
+
+2. Document: Access user's personal document index. Use for user-specific uploaded information. If user refers vaguely to "this document/file/article" without context, search the personal index.
+
+3. Write: Engage for any task related to composing, editing, or refining written content. This includes articles, essays, scripts, or any form of textual creation or modification. If you need to search for information or look at a document first, use the Search or Document tools. This tool is just to create or modify content.
+
+4. Image: Use when asked to create, generate, or manipulate visual content. This covers photographs, illustrations, diagrams, or any other type of image. Always use this tool for image requests unless explicitly directed to use CodeExecution.
+
+5. Code: Engage for any programming-related tasks, including creating, modifying, reviewing, or explaining code. Use for general coding discussions or when specific programming expertise is needed.
+
+6. CodeExecution: Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks that require code execution like data analysis, data processing, or business intelligence tasks.
+
+7. Reason: Employ for reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices. Also use when deep, step-by-step reasoning is required.
+
+8. PDF: Use specifically for processing and answering questions about PDF file content.
+
+9. Vision: Engage for analyzing and responding to queries about image files (jpg, gif, bmp, png, etc).
+
+10. Video: Use for processing and answering questions about video or audio file content.
+
+11. Clarify: Use when you must have more information from the user to determine which tool to use. In this case your tool message should be one or more questions to the user to clarify their request.
+
+Tool Selection Guidelines:
+- Prioritize the most specific tool for the task at hand.
+- If multiple tools seem applicable, choose the one most central to the user's request.
+- For ambiguous requests, consider using the Reason tool to plan a multi-step approach.
+- Always use the Image tool for image generation unless explicitly directed to use CodeExecution.
+- If the user explicitly asks you to use a tool, you must use it.
+
+Decision Output:
+If you decide to use a tool, return a JSON object in this format:
+{"toolRequired": true, "toolFunction": "toolName", "toolMessage": "message to the user to wait a moment while you work", "toolReason": "detailed explanation of why this tool was chosen"}
+
+If no tool is required, return:
+{"toolRequired": false, "toolReason": "explanation of why no tool was necessary"}
+
+Return only the JSON object without additional commentary.`,
+            },
+            {"role": "user", "content": "Analyze the provided conversation history and determine if you should use any of the tools to respond to the user. Generate a JSON object to indicate if a tool is needed."},
+        ]}),
+    ],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+}

package/pathways/{sys_claude_35_sonnet.js → system/rest_streaming/sys_claude_35_sonnet.js}

@@ -1,7 +1,7 @@
 // sys_claude_35_sonnet.js
 // override handler for claude-35-sonnet
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_claude_3_haiku.js → system/rest_streaming/sys_claude_3_haiku.js}

@@ -1,7 +1,7 @@
 // sys_claude_3_haiku.js
 // override handler for claude-3-haiku
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_google_chat.js → system/rest_streaming/sys_google_chat.js}

@@ -1,7 +1,7 @@
 // sys_google_chat.js
 // override handler for palm-chat
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_google_code_chat.js → system/rest_streaming/sys_google_code_chat.js}

@@ -1,7 +1,7 @@
 // sys_google_code_chat.js
 // override handler for palm-code-chat
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_google_gemini_chat.js → system/rest_streaming/sys_google_gemini_chat.js}

@@ -1,7 +1,7 @@
 // sys_google_gemini_chat.js
 // override handler for gemini-chat
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat.js → system/rest_streaming/sys_openai_chat.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat.js
 // override handler for gpt-3.5-turbo
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_16.js → system/rest_streaming/sys_openai_chat_16.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_16.js
 // override handler for gpt-3.5-turbo-16k
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_gpt4.js → system/rest_streaming/sys_openai_chat_gpt4.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_gpt4.js
 // override handler for gpt-4
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_gpt4_32.js → system/rest_streaming/sys_openai_chat_gpt4_32.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_gpt4_32.js
 // override handler for gpt-4-32
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_gpt4_turbo.js → system/rest_streaming/sys_openai_chat_gpt4_turbo.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_gpt4_turbo.js
 // override handler for gpt-4-turbo
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_parse_numbered_object_list.js → system/sys_parse_numbered_object_list.js}

@@ -1,10 +1,10 @@
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../server/prompt.js';
 
 export default {
     prompt: [
         new Prompt({
             messages: [
-                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert the list into a JSON object with the given fields. If there are extra fields, assistant will ignore them. If there are some missing fields, assistant will just skip the missing fields and return the rest. If the conversion is not at all possible, assistant will return an empty JSON array. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
+                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert the list into a JSON object with the given fields. If a field value is numeric, it should be returned as a number in the JSON object. If there are extra fields, assistant will ignore them. If there are some missing fields, assistant will just skip the missing fields and return the rest. If the conversion is not at all possible, assistant will return an empty JSON array. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
                 { "role": "user", "content": `Fields: {{{format}}}\nList: {{{text}}}`},
             ]
         })

package/pathways/{sys_repair_json.js → system/sys_repair_json.js}

@@ -1,4 +1,4 @@
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_claude35_sonnet.js → system/workspaces/run_claude35_sonnet.js}

@@ -1,6 +1,6 @@
 
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_claude3_haiku.js → system/workspaces/run_claude3_haiku.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_gpt35turbo.js → system/workspaces/run_gpt35turbo.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from "../server/prompt.js" 
+import { Prompt } from "../../../server/prompt.js" 
 
 export default {
     prompt: [

package/pathways/{run_gpt4.js → system/workspaces/run_gpt4.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_gpt4_32.js → system/workspaces/run_gpt4_32.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/server/pathwayResolver.js

@@ -221,15 +221,59 @@ class PathwayResolver {
         // Get saved context from contextId or change contextId if needed
         const { contextId } = args;
         this.savedContextId = contextId ? contextId : uuidv4();
-        this.savedContext = contextId ? (getv && (await getv(contextId)) || {}) : {};
-
-        // Save the context before processing the request
-        const savedContextStr = JSON.stringify(this.savedContext);
+        
+        const loadMemory = async () => {
+            // Load initial values
+            this.savedContext = (getv && await getv(contextId)) || {};
+            this.memorySelf = (getv && await getv(`${contextId}-memorySelf`)) || "";
+            this.memoryDirectives = (getv && await getv(`${contextId}-memoryDirectives`)) || "";
+            this.memoryTopics = (getv && await getv(`${contextId}-memoryTopics`)) || "";
+            this.memoryUser = (getv && await getv(`${contextId}-memoryUser`)) || "";
+
+            // Store initial state for comparison
+            this.initialState = {
+                savedContext: this.savedContext,
+                memorySelf: this.memorySelf,
+                memoryDirectives: this.memoryDirectives,
+                memoryTopics: this.memoryTopics,
+                memoryUser: this.memoryUser
+            };
+        };
+
+        const saveChangedMemory = async () => {
+            this.savedContextId = this.savedContextId || uuidv4();
+            
+            const currentState = {
+                savedContext: this.savedContext,
+                memorySelf: this.memorySelf,
+                memoryDirectives: this.memoryDirectives,
+                memoryTopics: this.memoryTopics,
+                memoryUser: this.memoryUser
+            };
+
+            if (currentState.savedContext !== this.initialState.savedContext) {
+                setv && await setv(this.savedContextId, this.savedContext);
+            }
+            if (currentState.memorySelf !== this.initialState.memorySelf) {
+                setv && await setv(`${this.savedContextId}-memorySelf`, this.memorySelf);
+            }
+            if (currentState.memoryDirectives !== this.initialState.memoryDirectives) {
+                setv && await setv(`${this.savedContextId}-memoryDirectives`, this.memoryDirectives);
+            }
+            if (currentState.memoryTopics !== this.initialState.memoryTopics) {
+                setv && await setv(`${this.savedContextId}-memoryTopics`, this.memoryTopics);
+            }
+            if (currentState.memoryUser !== this.initialState.memoryUser) {
+                setv && await setv(`${this.savedContextId}-memoryUser`, this.memoryUser);
+            }
+        };
 
         const MAX_RETRIES = 3;
         let data = null;
         
         for (let retries = 0; retries < MAX_RETRIES; retries++) {
+            await loadMemory(); // Reset memory state on each retry
+            
             data = await this.processRequest(args);
             if (!data) {
                 break;
@@ -241,13 +285,10 @@ class PathwayResolver {
             }
 
             logger.warn(`Bad pathway result - retrying pathway. Attempt ${retries + 1} of ${MAX_RETRIES}`);
-            this.savedContext = JSON.parse(savedContextStr);
         }
 
-        // Update saved context if it has changed, generating a new contextId if necessary
-        if (savedContextStr !== JSON.stringify(this.savedContext)) {
-            this.savedContextId = this.savedContextId || uuidv4();
-            setv && setv(this.savedContextId, this.savedContext);
+        if (data !== null) {
+            await saveChangedMemory();
         }
 
         return data;
@@ -419,7 +460,14 @@ class PathwayResolver {
 
         // If this text is empty, skip applying the prompt as it will likely be a nonsensical result
         if (!/^\s*$/.test(text) || parameters?.file || parameters?.inputVector || this?.modelName.includes('cognitive')) {
-            result = await this.modelExecutor.execute(text, { ...parameters, ...this.savedContext }, prompt, this);
+            result = await this.modelExecutor.execute(text, { 
+                ...parameters, 
+                ...this.savedContext,
+                memorySelf: this.memorySelf,
+                memoryDirectives: this.memoryDirectives,
+                memoryTopics: this.memoryTopics,
+                memoryUser: this.memoryUser 
+            }, prompt, this);
         } else {
             result = text;
         }
@@ -439,6 +487,10 @@ class PathwayResolver {
 
         // save the result to the context if requested and no errors
         if (prompt.saveResultTo && this.errors.length === 0) {
+            // Update memory property if it matches a known type
+            if (["memorySelf", "memoryUser", "memoryDirectives", "memoryTopics"].includes(prompt.saveResultTo)) {
+                this[prompt.saveResultTo] = result;
+            }
             this.savedContext[prompt.saveResultTo] = result;
         }
         return result;

package/server/plugins/azureCognitivePlugin.js

@@ -55,7 +55,7 @@ class AzureCognitivePlugin extends ModelPlugin {
             { search: searchQuery,  
                 "searchMode": "all",
                 "queryType": "full",
-                select: 'id', top: TOP 
+                select: 'id', top: TOP, skip: 0
             };
 
             const docsToDelete = JSON.parse(await this.executeRequest(cortexRequest));
@@ -115,6 +115,19 @@ class AzureCognitivePlugin extends ModelPlugin {
             ];
         } else {
             data.search = modelPromptText;
+            data.top = parameters.top || 50;
+            data.skip = 0;
+            if (parameters.titleOnly) {
+                switch(indexName){
+                    case 'indexcortex':
+                    case 'indexwires':
+                        data.select = 'title,id';
+                        break;
+                    default:
+                        data.select = 'title,id,url';
+                        break;
+                }
+            }
         }
 
         filter && (data.filter = filter);

package/server/plugins/claude3VertexPlugin.js

@@ -1,9 +1,10 @@
 import OpenAIVisionPlugin from "./openAiVisionPlugin.js";
 import logger from "../../lib/logger.js";
+import axios from 'axios';
 
 const allowedMIMETypes = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
-async function convertContentItem(item) {
 
+async function convertContentItem(item, maxImageSize) {
   let imageUrl = "";
 
   try {
@@ -27,6 +28,14 @@ async function convertContentItem(item) {
             try {
               const urlData = imageUrl.startsWith("data:") ? imageUrl : await fetchImageAsDataURL(imageUrl);
               if (!urlData) { return null; }
+              
+              // Check base64 size
+              const base64Size = (urlData.length * 3) / 4;
+              if (base64Size > maxImageSize) {
+                logger.warn(`Image size ${base64Size} bytes exceeds maximum allowed size ${maxImageSize} - skipping image content.`);
+                return null;
+              }
+              
               const [, mimeType = "image/jpeg"] = urlData.match(/data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,.*/) || [];
               const base64Image = urlData.split(",")[1];
 
@@ -60,25 +69,26 @@ async function convertContentItem(item) {
 // Fetch image and convert to base 64 data URL
 async function fetchImageAsDataURL(imageUrl) {
   try {
-    const response = await fetch(imageUrl, { method: 'HEAD' });
-
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
+    // First check headers
+    const headResponse = await axios.head(imageUrl, {
+      timeout: 30000, // 30 second timeout
+      maxRedirects: 5
+    });
 
-    const contentType = response.headers.get('content-type');
+    const contentType = headResponse.headers['content-type'];
     if (!contentType || !allowedMIMETypes.includes(contentType)) {
       logger.warn(`Unsupported image type: ${contentType} - skipping image content.`);
       return null;
     }
 
-    const dataResponse = await fetch(imageUrl);
-    if (!dataResponse.ok) {
-      throw new Error(`HTTP error! status: ${dataResponse.status}`);
-    }
+    // Then get the actual image data
+    const dataResponse = await axios.get(imageUrl, {
+      timeout: 30000,
+      responseType: 'arraybuffer',
+      maxRedirects: 5
+    });
 
-    const buffer = await dataResponse.arrayBuffer();
-    const base64Image = Buffer.from(buffer).toString("base64");
+    const base64Image = Buffer.from(dataResponse.data).toString('base64');
     return `data:${contentType};base64,${base64Image}`;
   }
   catch (e) {
@@ -151,7 +161,7 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     const claude3Messages = await Promise.all(
       finalMessages.map(async (message) => {
         const contentArray = Array.isArray(message.content) ? message.content : [message.content];
-        const claude3Content = await Promise.all(contentArray.map(convertContentItem));
+        const claude3Content = await Promise.all(contentArray.map(item => convertContentItem(item, this.getModelMaxImageSize())));
         return {
           role: message.role,
           content: claude3Content.filter(Boolean),
@@ -301,7 +311,7 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
 
   shortenContent(content, maxWords = 40) {
     const words = content.split(" ");
-    if (words.length <= maxWords) {
+    if (words.length <= maxWords || logger.level === 'debug') {
       return content;
     }
     return words.slice(0, maxWords / 2).join(" ") +

package/server/plugins/gemini15ChatPlugin.js

@@ -200,7 +200,7 @@ class Gemini15ChatPlugin extends ModelPlugin {
         } else if (Array.isArray(responseData)) {
             const { mergedResult, safetyRatings } = mergeResults(responseData);
             if (safetyRatings?.length) {
-                logger.warn(`!!! response was blocked because the input or response potentially violates policies`);
+                logger.warn(`response was blocked because the input or response potentially violates policies`);
                 logger.verbose(`Safety Ratings: ${JSON.stringify(safetyRatings, null, 2)}`);
             }
             const { length, units } = this.getLength(mergedResult);

package/server/plugins/geminiChatPlugin.js

@@ -195,7 +195,7 @@ class GeminiChatPlugin extends ModelPlugin {
         } else if (Array.isArray(responseData)) {
             const { mergedResult, safetyRatings } = mergeResults(responseData);
             if (safetyRatings?.length) {
-                logger.warn(`!!! response was blocked because the input or response potentially violates policies`);
+                logger.warn(`response was blocked because the input or response potentially violates policies`);
                 logger.verbose(`Safety Ratings: ${JSON.stringify(safetyRatings, null, 2)}`);
             }
             const { length, units } = this.getLength(mergedResult);

package/server/plugins/modelPlugin.js

@@ -9,6 +9,7 @@ import { config } from '../../config.js';
 const DEFAULT_MAX_TOKENS = 4096;
 const DEFAULT_MAX_RETURN_TOKENS = 256;
 const DEFAULT_PROMPT_TOKEN_RATIO = 0.5;
+const DEFAULT_MAX_IMAGE_SIZE = 20 * 1024 * 1024; // 20MB default
 
 class ModelPlugin {
     constructor(pathway, model) {
@@ -249,7 +250,12 @@ class ModelPlugin {
         let length = 0;
         let units = isProd ? 'characters' : 'tokens';
         if (data) {
-           length = isProd ? data.length : encode(data).length;
+            if (isProd || data.length > 5000) {
+                length = data.length;
+                units = 'characters';
+            } else {
+                length = encode(data).length;
+            }
         }
         return {length, units};
     }
@@ -341,6 +347,9 @@ class ModelPlugin {
         return requestProgress;
     }
 
+    getModelMaxImageSize() {
+        return (this.promptParameters.maxImageSize ?? this.model.maxImageSize ?? DEFAULT_MAX_IMAGE_SIZE);
+    }
 
 }
 

package/server/plugins/openAiChatPlugin.js

@@ -115,9 +115,10 @@ class OpenAIChatPlugin extends ModelPlugin {
                 const content = message.content === undefined ? JSON.stringify(message) : (Array.isArray(message.content) ? message.content.map(item => JSON.stringify(item)).join(', ') : message.content);
                 const words = content.split(" ");
                 const { length, units } = this.getLength(content);
-                const preview = words.length < 41 ? content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");
-    
-                logger.verbose(`message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${preview}"`);
+                
+                const displayContent = logger.level === 'debug' ? content : (words.length < 41 ? content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" "));
+
+                logger.verbose(`message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${displayContent}"`);
                 totalLength += length;
                 totalUnits = units;
             });

package/server/plugins/openAiDallE3Plugin.js

@@ -52,14 +52,20 @@ class OpenAIDallE3Plugin extends ModelPlugin {
 
         requestPromise
         .then((response) => handleResponse(response))
-        .catch((error) => handleResponse(error));
+        .catch((error) => handleResponse(error, true));
 
-        function handleResponse(response) {
+        function handleResponse(response, isError = false) {
             let status = "succeeded";
-            let data = JSON.stringify(response);
-            if (response.data.error) {
+            let data;
+
+            if (isError) {
+                status = "failed";
+                data = JSON.stringify({ error: response.message || response });
+            } else if (response.data?.error) {
                 status = "failed";
                 data = JSON.stringify(response.data);
+            } else {
+                data = JSON.stringify(response);
             }
 
             const requestProgress = {
@@ -80,12 +86,14 @@ class OpenAIDallE3Plugin extends ModelPlugin {
             let progress =
                 requestDurationEstimator.calculatePercentComplete(callid);
 
+            if (typeof progress === 'number' && !isNaN(progress) && progress >= 0 && progress <= 1) {
                 await publishRequestProgress({
                     requestId,
                     status: "pending",
                     progress,
                     data,
                 });
+            }
 
             if (state.status !== "pending") {
                 break;

package/server/plugins/openAiVisionPlugin.js

@@ -31,8 +31,7 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
                         }
 
                         if (typeof parsedItem === 'object' && parsedItem !== null && parsedItem.type === 'image_url') {
-                            parsedItem.image_url.url = parsedItem.url || parsedItem.image_url.url;
-                            return parsedItem;
+                            return {type: parsedItem.type, image_url: {url: parsedItem.url || parsedItem.image_url.url}};
                         }
                         
                         return parsedItem;