@aj-archipelago/cortex 1.2.1 → 1.3.1

package/pathways/system/entity/sys_query_builder.js

@@ -0,0 +1,110 @@
+import { Prompt } from '../../../server/prompt.js';
+import entityConstants from './shared/sys_entity_constants.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextInfo: ``,
+        useMemory: false,
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{#if useMemory}}{{renderTemplate AI_MEMORY}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{/if}}{{renderTemplate AI_CONVERSATION_HISTORY}}
+                
+Instructions: You are a search helper AI. Your role is to analyze the included Conversation History to understand what the user is asking for and decide what data sources if any to use to help the user and produce a JSON object with fields that communicate your decisions. You have vast internal knowledge up to your training cutoff date, but your internal knowledge is not always sufficient to answer questions about current events or the latest news.
+
+You have the ability to search one or more of the following indexes:
+- "aje" for all news articles published by Al Jazeera English (written in English)
+- "aja" for all news articles published by Al Jazeera Arabic (written in Arabic)
+- "wires" for latest news wires from all wires sources (news & articles)
+- "personal" for the user's documents and uploaded files
+
+AJE and AJA are not just translations of each other - they are different news organizations with different reporting styles and focus, so often searching both indexes will provide a more complete answer.
+
+To search an index, you can provide an appropriate search string or wildcard (e.g. "*") in the corresponding field for the index: "searchAJE", "searchAJA", "searchWires", and "searchPersonal" respectively. It's helpful if the search string is in the language of the index. Longer search strings will get you more relevant and specific results, but shorter ones or wildcards will get you a broader result set. Wildcards are especially useful in finding all results over a time period or finding vague information (e.g. "the news", "the latest").
+
+You have the ability to search the internet in all languages using Bing Search. To do that, just put the search query in the "searchBing" field. Your Bing search query can be as simple or long and detailed as you need it to be and you can use the Bing advanced search syntax (+, -, OR, AND, quotes, etc.) to get more relevant or specific results. It's usually helpful to search the internet in addition to your other sources unless the user has explicitly asked for a specific search source (e.g. "the wires").
+
+If you choose to search anything at all, you must always set the "searchRequired" field to true.
+
+When the user explicitly asks for a specific search source (e.g. "the wires", "my uploads", "the internet"), use ONLY that source.
+
+When the user is referencing something specific, (e.g. "this", "this document", "this file", "my uploads","this article", etc.) and you don't see the document contents in the conversation history, use a wildcard search on the personal index with no date filter to see if there is anything relevant. In this case, don't search any other indexes.
+
+When the user's query requires a date filter for accurate data retrieval, pay special attention to qualifier words like "latest","tonight", "this afternoon", "today", "yesterday", "this week", "last week", "this month", etc. The current time and date in GMT is {{now}}, but references like "today" or "yesterday" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT. If a date filter is required, formulate it in a valid OData $filter format and include it in the "dateFilter" field. Do not just put the date in the field - it needs to be filter expression like "date ge 2024-02-22T00:00:00Z". Don't use eq with an exact date time as this is unlikely to return any results.
+
+When the user requests an overview, count, or analysis of topics or trends from a specific index over a given time period (e.g., 'What topics were covered yesterday on AJE?' or 'What were the hot topics on the wires this week?' or 'How many articles did AJA publish last week?'), follow these steps:
+
+- Use a wildcard search ('*') on the appropriate index(es).
+- Apply a date filter corresponding to the specified time period.
+- Set the 'titleOnly' field to true.
+- Analyze the results to identify and summarize the main topics or trends.
+
+Determine the language that the user is speaking in the conversation and fill the "language" field using the ISO 639-3 format and put the full language name in the "languageStr" field.
+
+Add a short message to the resultsMessage field that acknowledges the user's request and indicates you're processing it.
+- The message should be a very short, casual phrase (2-5 words) that acknowledges the user's request and indicates you're processing it.
+- The message to the user should be conversational and natural and match the rest of the conversation style and tone.
+- The message should take 1-2 seconds to say out loud. Examples: 'Hmm, let's see...', 'Just a sec...', 'Checking...'"
+
+You should only ever respond with the JSON object and never with any additional notes or commentary.
+
+Example JSON objects and messages for different queries:
+
+"What's the latest on the wires?"
+{
+    "searchRequired": true,"
+    "searchWires": "*",
+    "dateFilter": "date ge 2024-02-22T00:00:00Z",
+    "titleOnly": false,
+    "language": "eng",
+    "languageStr": "English",
+    "resultsMessage": "Reading the wires..."
+}
+    
+"What's going on in the world today?"
+{
+    "searchRequired": true,
+    "searchWires": "world news",
+    "searchAJA": "عالم حدث اليوم",
+    "searchAJE": "world news",
+    "searchBing": "world news today",
+    "dateFilter": "date ge 2024-02-22T00:00:00Z",
+    "titleOnly": false,
+    "language": "eng",
+    "languageStr": "English",
+    "resultsMessage": "Just a few seconds..."
+}
+    
+"What is this document about?"
+{
+    "searchRequired": true,
+    "searchPersonal": "*",
+    "language": "eng",
+    "languageStr": "English",
+    "resultsMessage": "Almost done..."
+}
+    
+"What topics were covered last week on AJE?"
+{
+    "searchRequired": true,
+    "searchAJE": "*",
+    "dateFilter": "date ge 2024-02-22T00:00:00Z and date le 2024-02-28T23:59:59Z",
+    "titleOnly": true,
+    "language": "eng",
+    "languageStr": "English",
+    "resultsMessage": "Almost there..."
+}`,
+            },
+            {"role": "user", "content": "Examine the Conversation History and decide what data sources if any to search to help the user and produce a JSON object with fields that communicate your decisions."},
+        ]}),
+    ],
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+    ...entityConstants
+}

package/pathways/system/entity/sys_router_code.js

@@ -0,0 +1,37 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
+
+Instructions: You are part of an AI entity named {{{aiName}}}. Your task is to analyze the conversation history to determine whether a coding task has been requested or if the user's needs can be addressed only by executing the code. Output a JSON object with three fields:
+
+1. "codingRequired": Boolean. Set to true if the user asks for or needs code execution. Otherwise, set to false.
+
+2. "codingMessage": String. If codingRequired is true, provide a message to notify the user that a coding task is being handled. Otherwise, leave this as an empty string.
+
+3. "codingTask": String. If codingRequired is true, provide a task description for the coding agent. Make sure to pass all all the information needed as this is the only message that coding agent receives and is aware of. Just provide the task and let the agent decide how to solve or what do to. Never make any assumptions about the agent's knowledge or capabilities. Never say assume this or that. Never give example by yourself, let coding agent decide on that. Provide the task do not ask questions or say anything will further be provided by the user. If codingRequired is false, leave this as an empty string.
+
+4. "codingTaskKeywords": If codingRequired is true, provide a keywords for Azure Cognitive Search to help the coding agent find the relevant code snippets. It will use these keywords as is to search for the code snippets. If codingRequired is false, leave this as an empty string.
+
+General guidelines:
+- AJ is for AL Jazeera, AJA is for AJ Arabic, AJE is for AJ English
+- If agent needs to search in task it can use Bing Search
+
+Always output just the valid JSON object with all these fields.`,
+            },
+            {"role": "user", "content": "Analyze the provided conversation history and determine if you should use code executionto respond to the user. Generate a JSON object to indicate if it is needed."},
+        ]}),
+    ],
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+}

package/pathways/system/entity/sys_router_tool.js

@@ -0,0 +1,67 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        model: "oai-gpt4o",
+        aiName: "Jarvis",
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
+
+Instructions: You are part of an AI entity named {{{aiName}}}. Your task is to determine whether to use a tool based on the conversation history and user's request. Prioritize the latest message from the user in the conversation history when making your decision.
+
+Available tools and their specific use cases:
+
+1. Search: Use for current events, news, fact-checking, and information requiring citation. This tool can search the internet, all Al Jazeera news articles and the latest news wires from multiple sources. Only search when necessary for current events, user documents, latest news, or complex topics needing grounding. Don't search for remembered information or general knowledge within your capabilities.
+
+2. Document: Access user's personal document index. Use for user-specific uploaded information. If user refers vaguely to "this document/file/article" without context, search the personal index.
+
+3. Write: Engage for any task related to composing, editing, or refining written content. This includes articles, essays, scripts, or any form of textual creation or modification. If you need to search for information or look at a document first, use the Search or Document tools. This tool is just to create or modify content.
+
+4. Image: Use when asked to create, generate, or manipulate visual content. This covers photographs, illustrations, diagrams, or any other type of image. Always use this tool for image requests unless explicitly directed to use CodeExecution.
+
+5. Code: Engage for any programming-related tasks, including creating, modifying, reviewing, or explaining code. Use for general coding discussions or when specific programming expertise is needed.
+
+6. CodeExecution: Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks that require code execution like data analysis, data processing, or business intelligence tasks.
+
+7. Reason: Employ for reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices. Also use when deep, step-by-step reasoning is required.
+
+8. PDF: Use specifically for processing and answering questions about PDF file content.
+
+9. Vision: Engage for analyzing and responding to queries about image files (jpg, gif, bmp, png, etc).
+
+10. Video: Use for processing and answering questions about video or audio file content.
+
+11. Clarify: Use when you must have more information from the user to determine which tool to use. In this case your tool message should be one or more questions to the user to clarify their request.
+
+Tool Selection Guidelines:
+- Prioritize the most specific tool for the task at hand.
+- If multiple tools seem applicable, choose the one most central to the user's request.
+- For ambiguous requests, consider using the Reason tool to plan a multi-step approach.
+- Always use the Image tool for image generation unless explicitly directed to use CodeExecution.
+- If the user explicitly asks you to use a tool, you must use it.
+
+Decision Output:
+If you decide to use a tool, return a JSON object in this format:
+{"toolRequired": true, "toolFunction": "toolName", "toolMessage": "message to the user to wait a moment while you work", "toolReason": "detailed explanation of why this tool was chosen"}
+
+- The message to the user should flow naturally with the conversation history and match the rest of the conversation history in style and tone.
+- The message should be specific about what you're doing and why and how long it will take, but keep it short as if you were speaking it out loud.
+
+If no tool is required, return:
+{"toolRequired": false, "toolReason": "explanation of why no tool was necessary"}
+
+Return only the JSON object without additional commentary.`,
+            },
+            {"role": "user", "content": "Analyze the provided conversation history and determine if you should use any of the tools to respond to the user. Generate a JSON object to indicate if a tool is needed."},
+        ]}),
+    ],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+}

package/pathways/{sys_claude_35_sonnet.js → system/rest_streaming/sys_claude_35_sonnet.js}

@@ -1,7 +1,7 @@
 // sys_claude_35_sonnet.js
 // override handler for claude-35-sonnet
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_claude_3_haiku.js → system/rest_streaming/sys_claude_3_haiku.js}

@@ -1,7 +1,7 @@
 // sys_claude_3_haiku.js
 // override handler for claude-3-haiku
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_google_chat.js → system/rest_streaming/sys_google_chat.js}

@@ -1,7 +1,7 @@
 // sys_google_chat.js
 // override handler for palm-chat
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_google_code_chat.js → system/rest_streaming/sys_google_code_chat.js}

@@ -1,7 +1,7 @@
 // sys_google_code_chat.js
 // override handler for palm-code-chat
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_google_gemini_chat.js → system/rest_streaming/sys_google_gemini_chat.js}

@@ -1,7 +1,7 @@
 // sys_google_gemini_chat.js
 // override handler for gemini-chat
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat.js → system/rest_streaming/sys_openai_chat.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat.js
 // override handler for gpt-3.5-turbo
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_16.js → system/rest_streaming/sys_openai_chat_16.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_16.js
 // override handler for gpt-3.5-turbo-16k
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_gpt4.js → system/rest_streaming/sys_openai_chat_gpt4.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_gpt4.js
 // override handler for gpt-4
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_gpt4_32.js → system/rest_streaming/sys_openai_chat_gpt4_32.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_gpt4_32.js
 // override handler for gpt-4-32
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_gpt4_turbo.js → system/rest_streaming/sys_openai_chat_gpt4_turbo.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_gpt4_turbo.js
 // override handler for gpt-4-turbo
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_parse_numbered_object_list.js → system/sys_parse_numbered_object_list.js}

@@ -1,10 +1,10 @@
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../server/prompt.js';
 
 export default {
     prompt: [
         new Prompt({
             messages: [
-                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert the list into a JSON object with the given fields. If there are extra fields, assistant will ignore them. If there are some missing fields, assistant will just skip the missing fields and return the rest. If the conversion is not at all possible, assistant will return an empty JSON array. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
+                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert the list into a JSON object with the given fields. If a field value is numeric, it should be returned as a number in the JSON object. If there are extra fields, assistant will ignore them. If there are some missing fields, assistant will just skip the missing fields and return the rest. If the conversion is not at all possible, assistant will return an empty JSON array. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
                 { "role": "user", "content": `Fields: {{{format}}}\nList: {{{text}}}`},
             ]
         })

package/pathways/{sys_repair_json.js → system/sys_repair_json.js}

@@ -1,4 +1,4 @@
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_claude35_sonnet.js → system/workspaces/run_claude35_sonnet.js}

@@ -1,6 +1,6 @@
 
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_claude3_haiku.js → system/workspaces/run_claude3_haiku.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_gpt35turbo.js → system/workspaces/run_gpt35turbo.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from "../server/prompt.js" 
+import { Prompt } from "../../../server/prompt.js" 
 
 export default {
     prompt: [

package/pathways/{run_gpt4.js → system/workspaces/run_gpt4.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_gpt4_32.js → system/workspaces/run_gpt4_32.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/server/pathwayResolver.js

@@ -221,15 +221,59 @@ class PathwayResolver {
         // Get saved context from contextId or change contextId if needed
         const { contextId } = args;
         this.savedContextId = contextId ? contextId : uuidv4();
-        this.savedContext = contextId ? (getv && (await getv(contextId)) || {}) : {};
-
-        // Save the context before processing the request
-        const savedContextStr = JSON.stringify(this.savedContext);
+        
+        const loadMemory = async () => {
+            // Load initial values
+            this.savedContext = (getv && await getv(contextId)) || {};
+            this.memorySelf = (getv && await getv(`${contextId}-memorySelf`)) || "";
+            this.memoryDirectives = (getv && await getv(`${contextId}-memoryDirectives`)) || "";
+            this.memoryTopics = (getv && await getv(`${contextId}-memoryTopics`)) || "";
+            this.memoryUser = (getv && await getv(`${contextId}-memoryUser`)) || "";
+
+            // Store initial state for comparison
+            this.initialState = {
+                savedContext: this.savedContext,
+                memorySelf: this.memorySelf,
+                memoryDirectives: this.memoryDirectives,
+                memoryTopics: this.memoryTopics,
+                memoryUser: this.memoryUser
+            };
+        };
+
+        const saveChangedMemory = async () => {
+            this.savedContextId = this.savedContextId || uuidv4();
+            
+            const currentState = {
+                savedContext: this.savedContext,
+                memorySelf: this.memorySelf,
+                memoryDirectives: this.memoryDirectives,
+                memoryTopics: this.memoryTopics,
+                memoryUser: this.memoryUser
+            };
+
+            if (currentState.savedContext !== this.initialState.savedContext) {
+                setv && await setv(this.savedContextId, this.savedContext);
+            }
+            if (currentState.memorySelf !== this.initialState.memorySelf) {
+                setv && await setv(`${this.savedContextId}-memorySelf`, this.memorySelf);
+            }
+            if (currentState.memoryDirectives !== this.initialState.memoryDirectives) {
+                setv && await setv(`${this.savedContextId}-memoryDirectives`, this.memoryDirectives);
+            }
+            if (currentState.memoryTopics !== this.initialState.memoryTopics) {
+                setv && await setv(`${this.savedContextId}-memoryTopics`, this.memoryTopics);
+            }
+            if (currentState.memoryUser !== this.initialState.memoryUser) {
+                setv && await setv(`${this.savedContextId}-memoryUser`, this.memoryUser);
+            }
+        };
 
         const MAX_RETRIES = 3;
         let data = null;
         
         for (let retries = 0; retries < MAX_RETRIES; retries++) {
+            await loadMemory(); // Reset memory state on each retry
+            
             data = await this.processRequest(args);
             if (!data) {
                 break;
@@ -241,13 +285,10 @@ class PathwayResolver {
             }
 
             logger.warn(`Bad pathway result - retrying pathway. Attempt ${retries + 1} of ${MAX_RETRIES}`);
-            this.savedContext = JSON.parse(savedContextStr);
         }
 
-        // Update saved context if it has changed, generating a new contextId if necessary
-        if (savedContextStr !== JSON.stringify(this.savedContext)) {
-            this.savedContextId = this.savedContextId || uuidv4();
-            setv && setv(this.savedContextId, this.savedContext);
+        if (data !== null) {
+            await saveChangedMemory();
         }
 
         return data;
@@ -419,7 +460,14 @@ class PathwayResolver {
 
         // If this text is empty, skip applying the prompt as it will likely be a nonsensical result
         if (!/^\s*$/.test(text) || parameters?.file || parameters?.inputVector || this?.modelName.includes('cognitive')) {
-            result = await this.modelExecutor.execute(text, { ...parameters, ...this.savedContext }, prompt, this);
+            result = await this.modelExecutor.execute(text, { 
+                ...parameters, 
+                ...this.savedContext,
+                memorySelf: this.memorySelf,
+                memoryDirectives: this.memoryDirectives,
+                memoryTopics: this.memoryTopics,
+                memoryUser: this.memoryUser 
+            }, prompt, this);
         } else {
             result = text;
         }
@@ -439,6 +487,10 @@ class PathwayResolver {
 
         // save the result to the context if requested and no errors
         if (prompt.saveResultTo && this.errors.length === 0) {
+            // Update memory property if it matches a known type
+            if (["memorySelf", "memoryUser", "memoryDirectives", "memoryTopics"].includes(prompt.saveResultTo)) {
+                this[prompt.saveResultTo] = result;
+            }
             this.savedContext[prompt.saveResultTo] = result;
         }
         return result;

package/server/plugins/azureCognitivePlugin.js

@@ -55,7 +55,7 @@ class AzureCognitivePlugin extends ModelPlugin {
             { search: searchQuery,  
                 "searchMode": "all",
                 "queryType": "full",
-                select: 'id', top: TOP 
+                select: 'id', top: TOP, skip: 0
             };
 
             const docsToDelete = JSON.parse(await this.executeRequest(cortexRequest));
@@ -115,6 +115,19 @@ class AzureCognitivePlugin extends ModelPlugin {
             ];
         } else {
             data.search = modelPromptText;
+            data.top = parameters.top || 50;
+            data.skip = 0;
+            if (parameters.titleOnly) {
+                switch(indexName){
+                    case 'indexcortex':
+                    case 'indexwires':
+                        data.select = 'title,id';
+                        break;
+                    default:
+                        data.select = 'title,id,url';
+                        break;
+                }
+            }
         }
 
         filter && (data.filter = filter);

package/server/plugins/azureVideoTranslatePlugin.js

@@ -27,7 +27,7 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
 
     getRequestParameters(_, parameters, __) {
         const excludedParameters = [
-            'text', 'parameters', 'prompt', 'promptParameters', 'previousResult', 'stream'
+            'text', 'parameters', 'prompt', 'promptParameters', 'previousResult', 'stream', 'memoryContext'
         ];
         
         return Object.fromEntries(

package/server/plugins/claude3VertexPlugin.js

@@ -1,9 +1,10 @@
 import OpenAIVisionPlugin from "./openAiVisionPlugin.js";
 import logger from "../../lib/logger.js";
+import axios from 'axios';
 
 const allowedMIMETypes = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
-async function convertContentItem(item) {
 
+async function convertContentItem(item, maxImageSize) {
   let imageUrl = "";
 
   try {
@@ -27,6 +28,14 @@ async function convertContentItem(item) {
             try {
               const urlData = imageUrl.startsWith("data:") ? imageUrl : await fetchImageAsDataURL(imageUrl);
               if (!urlData) { return null; }
+              
+              // Check base64 size
+              const base64Size = (urlData.length * 3) / 4;
+              if (base64Size > maxImageSize) {
+                logger.warn(`Image size ${base64Size} bytes exceeds maximum allowed size ${maxImageSize} - skipping image content.`);
+                return null;
+              }
+              
               const [, mimeType = "image/jpeg"] = urlData.match(/data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,.*/) || [];
               const base64Image = urlData.split(",")[1];
 
@@ -60,25 +69,26 @@ async function convertContentItem(item) {
 // Fetch image and convert to base 64 data URL
 async function fetchImageAsDataURL(imageUrl) {
   try {
-    const response = await fetch(imageUrl, { method: 'HEAD' });
-
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
+    // First check headers
+    const headResponse = await axios.head(imageUrl, {
+      timeout: 30000, // 30 second timeout
+      maxRedirects: 5
+    });
 
-    const contentType = response.headers.get('content-type');
+    const contentType = headResponse.headers['content-type'];
     if (!contentType || !allowedMIMETypes.includes(contentType)) {
       logger.warn(`Unsupported image type: ${contentType} - skipping image content.`);
       return null;
     }
 
-    const dataResponse = await fetch(imageUrl);
-    if (!dataResponse.ok) {
-      throw new Error(`HTTP error! status: ${dataResponse.status}`);
-    }
+    // Then get the actual image data
+    const dataResponse = await axios.get(imageUrl, {
+      timeout: 30000,
+      responseType: 'arraybuffer',
+      maxRedirects: 5
+    });
 
-    const buffer = await dataResponse.arrayBuffer();
-    const base64Image = Buffer.from(buffer).toString("base64");
+    const base64Image = Buffer.from(dataResponse.data).toString('base64');
     return `data:${contentType};base64,${base64Image}`;
   }
   catch (e) {
@@ -151,7 +161,7 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     const claude3Messages = await Promise.all(
       finalMessages.map(async (message) => {
         const contentArray = Array.isArray(message.content) ? message.content : [message.content];
-        const claude3Content = await Promise.all(contentArray.map(convertContentItem));
+        const claude3Content = await Promise.all(contentArray.map(item => convertContentItem(item, this.getModelMaxImageSize())));
         return {
           role: message.role,
           content: claude3Content.filter(Boolean),
@@ -301,7 +311,7 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
 
   shortenContent(content, maxWords = 40) {
     const words = content.split(" ");
-    if (words.length <= maxWords) {
+    if (words.length <= maxWords || logger.level === 'debug') {
       return content;
     }
     return words.slice(0, maxWords / 2).join(" ") +

package/server/plugins/gemini15ChatPlugin.js

@@ -200,7 +200,7 @@ class Gemini15ChatPlugin extends ModelPlugin {
         } else if (Array.isArray(responseData)) {
             const { mergedResult, safetyRatings } = mergeResults(responseData);
             if (safetyRatings?.length) {
-                logger.warn(`!!! response was blocked because the input or response potentially violates policies`);
+                logger.warn(`response was blocked because the input or response potentially violates policies`);
                 logger.verbose(`Safety Ratings: ${JSON.stringify(safetyRatings, null, 2)}`);
             }
             const { length, units } = this.getLength(mergedResult);

package/server/plugins/geminiChatPlugin.js

@@ -195,7 +195,7 @@ class GeminiChatPlugin extends ModelPlugin {
         } else if (Array.isArray(responseData)) {
             const { mergedResult, safetyRatings } = mergeResults(responseData);
             if (safetyRatings?.length) {
-                logger.warn(`!!! response was blocked because the input or response potentially violates policies`);
+                logger.warn(`response was blocked because the input or response potentially violates policies`);
                 logger.verbose(`Safety Ratings: ${JSON.stringify(safetyRatings, null, 2)}`);
             }
             const { length, units } = this.getLength(mergedResult);