@aj-archipelago/cortex 1.3.40 → 1.3.41

package/config.js

@@ -118,7 +118,7 @@ var config = convict({
         default: {
             AI_MEMORY: `<SHORT_TERM_MEMORY>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\n<TOPICS>\n{{{memoryTopics}}}\n</TOPICS>\n</SHORT_TERM_MEMORY>`,
             AI_MEMORY_INSTRUCTIONS: "You have persistent memories of important details, instructions, and context - consult your memories when formulating a response to make sure you're applying your learnings.\nIf you don't see relevant information in your short term memory, you should use your SearchMemory tool to search your long term memory for details.\nAlso included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.",
-            AI_TOOLS: "You have access to a powerful set of tools that you can use to help accomplish tasks and provide better responses. Here's how to use them effectively:\n\n1. Take your time and use tools as many times as you need to be sure you have all the information to make a good response. In many cases you will want to make multiple tool calls. You can call multiple tools in parallel or you can chain them, waiting for the results of one for information before you call another. This allows you to dig deeper, compile more information, read various sources, and even double check and verify your information before responding.\n\n2. Tool Selection and Planning:\n- Carefully review your available tools before responding\n- For complex multi-step tasks, use your planning tool first to create a step-by-step plan to achieve the goal using the tools at your disposal\n- Consider which tools would be most appropriate for each step\n\n3. Best Practices:\n- Always verify tool capabilities before telling users something can't be done\n- Use tools proactively when they can provide better or more accurate information\n- If a user explicitly requests tool usage, you must comply\n- When using multiple tools, maintain context between tool calls\n- If a tool fails, consider alternative approaches or tools\n\n4. Common Use Cases:\n- Research: Use search tools across multiple sources to gather information before responding\n- Analysis: Use tools to process and analyze data or content\n- Generation: Use appropriate tools for creating content, images, or code\n- Verification: Use tools to validate information or check facts\n\nRemember: Your goal is to provide the most helpful and accurate responses possible. Don't hesitate to use tools when they can improve your response quality or accuracy.",
+            AI_TOOLS: "You can execute tools in a loop agentically - you will have a chance to evaluate every tool response before deciding what action to take next - there is no time or execution limit. You have access to a powerful set of tools to help accomplish tasks and deliver the best responses. Instructions for tool use:\n\n1. Always dig deep, verify and cross-check:\n- Take your time and use tools as many times as needed to ensure truth, accuracy, depth, and completeness.\n- Leverage both parallel and sequential tool calls for thorough investigation: start broadly, then dive deeper on leads, cross-check facts, and synthesize findings before responding.\n\n2. Plan carefully:\n- Carefully review all available tools before responding.\n- For complex or investigative tasks, use the planning tool first to break the goal into clear steps.\n- Select the most appropriate tool(s) for each step—think beyond single searches to multi-pass, multi-source discovery.\n\n3. Always dive deeper and use as many of your tools as apply:\n- Proactively use tools to refine, verify, and expand on initial findings—don’t settle for the first result if more depth or confirmation may help.\n- Always verify tool capabilities before concluding something can’t be done.\n- If a user explicitly requests tool usage, comply.\n- Maintain context across tool calls to ensure continuity and coherence.\n- If a tool fails, try alternatives or creative approaches.\n\n4. Common Use Cases:\n- Research: Explore multiple sources and perspectives to build a complete picture.\n- Analysis: Use tools to process, compare, and critically assess data or content.\n- Generation: Employ tools for creating content, visuals, or code as needed.\n- Verification: Prioritize cross-checking and fact validation, especially for claims or evolving news.\n\n5. Reflect and Personalize:\n- Synthesize findings into concise, relevant, and personalized responses.\n- If user preferences or past feedback are available, tailor responses accordingly.\n- Before finalizing, review your answer for clarity, completeness, and alignment with user expectations.\n- If you see a recent <VERIFICATION_PLAN> from a tool call, you MUST follow it step by step before giving your final response.\n\nRemember: Your responsibility is to provide the most helpful, well-reasoned, and accurate responses possible. Use tools iteratively and reflectively—don't hesitate to dig deeper or double-check when it improves response quality!",
             AI_DIRECTIVES: `These are your directives and learned behaviors:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>`,
             AI_CONVERSATION_HISTORY: "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}}\n</CONVERSATION_HISTORY>",
             AI_COMMON_INSTRUCTIONS: "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}",

package/lib/requestExecutor.js

@@ -207,7 +207,7 @@ const requestWithMonitor = async (endpoint, url, data, axiosConfigObj) => {
     return { response, duration };
 }
 
-const MAX_RETRY = 10; // retries for error handling
+const MAX_RETRY = 5; // retries for error handling
 const MAX_DUPLICATE_REQUESTS = 3; // duplicate requests to manage latency spikes
 const DUPLICATE_REQUEST_AFTER = 10; // 10 seconds
 
@@ -318,6 +318,7 @@ const makeRequest = async (cortexRequest) => {
                 if (cortexRequest.model.endpoints.length === 1) {
                     if (status !== 429 &&
                         status !== 408 &&
+                        status !== 500 &&
                         status !== 502 &&
                         status !== 503 &&
                         status !== 504) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.40",
+  "version": "1.3.41",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/chat_title.js

@@ -20,7 +20,7 @@ export default {
         title: '',
         text: '',
     },
-    model: 'oai-gpt41-nano',
+    model: 'oai-gpt41-mini',
     useInputChunking: false,
     temperature: 0,
     enableDuplicateRequests: false

package/pathways/system/entity/sys_entity_agent.js

@@ -66,10 +66,15 @@ export default {
                     const messageWithIcon = toolIcon ? `${toolIcon}  ${toolUserMessage}` : toolUserMessage;
                     await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `${messageWithIcon}\n\n`, 1000, false);
 
-                    if (toolArgs.detailedInstructions) {
-                        toolMessages.push({role: "user", content: toolArgs.detailedInstructions});
-                    }
+                    const toolResult = await callTool(toolFunction, {
+                        ...args,
+                        ...toolArgs,
+                        toolFunction,
+                        chatHistory: toolMessages,
+                        stream: false
+                    }, entityTools, pathwayResolver);
 
+                    // Tool calls and results need to be paired together in the message history
                     // Add the tool call to the isolated message history
                     toolMessages.push({
                         role: "assistant",
@@ -84,14 +89,6 @@ export default {
                         }]
                     });
 
-                    const toolResult = await callTool(toolFunction, {
-                        ...args,
-                        ...toolArgs,
-                        toolFunction,
-                        chatHistory: toolMessages,
-                        stream: false
-                    }, entityTools, pathwayResolver);
-
                     // Add the tool result to the isolated message history
                     let toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult);
 

package/pathways/system/entity/sys_get_entities.js

@@ -6,6 +6,7 @@ import { getAvailableEntities } from './tools/shared/sys_entity_tools.js';
 export default {
     prompt: [],
     inputParameters: {},
+    model: 'oai-gpt41-mini',
     executePathway: async ({ args }) => {
         try {
             const entities = getAvailableEntities();

package/pathways/system/entity/tools/sys_tool_callmodel.js

@@ -34,7 +34,7 @@ export default {
                     },
                     userPrompt: {
                         type: "string",
-                        description: "The complete prompt to send as a user message to the model instructing the model to perform the task you need."
+                        description: "The complete prompt to send as a user message to the model instructing the model to perform the task you need. Keep in mind this model does not share your context, conversation history, tool call results, or memories - so include all relevant information in the user prompt."
                     },
                     model: {
                         type: "string",

package/pathways/system/entity/tools/sys_tool_coding.js

@@ -8,7 +8,7 @@ export default {
         [
             new Prompt({ messages: [
                 {"role": "system", "content": `You are the part of an AI entity named {{aiName}} that provides advanced coding and programming capabilities. You excel at writing, reviewing, and explaining code across various programming languages. You can help with code generation, debugging, optimization, and best practices. Think carefully about the latest request and provide a detailed, well thought out, carefully reviewed response.\n{{renderTemplate AI_DATETIME}}`},
-                "{{chatHistory}}",
+                "{{chatHistory}}"
             ]}),
         ],
     inputParameters: {
@@ -46,6 +46,9 @@ export default {
     }],
     
     executePathway: async ({args, runAllPrompts, resolver}) => {
+        if (args.detailedInstructions) {
+            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+        }
         let result = await runAllPrompts({ ...args, stream: false });        
         resolver.tool = JSON.stringify({ toolUsed: "coding" });          
         return result;

package/pathways/system/entity/tools/sys_tool_codingagent.js

@@ -48,13 +48,13 @@ export default {
         icon: "🤖",
         function: {
             name: "CodeExecution",
-            description: "Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks.",
+            description: "Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks - examples include data analysis, file manipulation, or other tasks that require code execution.",
             parameters: {
                 type: "object",
                 properties: {
                     codingTask: {
                         type: "string",
-                        description: "Detailed task description for the coding agent. Include all necessary information as this is the only message the coding agent receives. Let the agent decide how to solve it without making assumptions about its capabilities."
+                        description: "Detailed task description for the coding agent. Include all necessary information as this is the only message the coding agent receives. Let the agent decide how to solve it without making assumptions about its capabilities. IMPORTANT: The coding agent does not share your context, so you must provide it with all the information in this message. If you are asking it to operate on files or other data from your context, you must provide the fully-qualified URL to each of the files you want it to use. Also make sure you explicitly instruct the agent to use those files."
                     },
                     userMessage: {
                         type: "string",

package/pathways/system/entity/tools/sys_tool_cognitive_search.js

@@ -24,7 +24,7 @@ export default {
             icon: "📂",
             function: {
                 name: "SearchPersonal",
-                description: "Search through the user's personal documents and uploaded files. Use this for finding information in user-provided content.",
+                description: "Search through the user's index of personal documents and indexed uploaded files. Use this for finding information in user-provided content or if the user refers to a file or a document that you don't see elsewhere in your context.",
                 parameters: {
                     type: "object",
                     properties: {
@@ -55,9 +55,9 @@ export default {
         },
         {
             type: "function",
+            icon: "📰",
             function: {
                 name: "SearchAJA",
-                icon: "📰",
                 description: "Search through Al Jazeera Arabic news articles. Use this for finding Arabic news content.",
                 parameters: {
                     type: "object",

package/pathways/system/entity/tools/sys_tool_image.js

@@ -42,7 +42,7 @@ export default {
 
         try {   
             let model = "replicate-flux-11-pro";
-            let prompt = args.detailedInstructions;
+            let prompt = args.detailedInstructions || "";
             let numberResults = args.numberResults || 1;
             let negativePrompt = args.negativePrompt || "";
 

package/pathways/system/entity/tools/sys_tool_readfile.js

@@ -112,6 +112,9 @@ export default {
     }],
     
     executePathway: async ({args, runAllPrompts, resolver}) => {
+        if (args.detailedInstructions) {
+            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+        }
         const result = await runAllPrompts({ ...args });
         resolver.tool = JSON.stringify({ toolUsed: "vision" });
         return result;

package/pathways/system/entity/tools/sys_tool_reasoning.js

@@ -49,7 +49,7 @@ export default {
         icon: "🧠",
         function: {
             name: "Reason",
-            description: "Employ for advancedreasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices.",
+            description: "Employ for advanced reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices.",
             parameters: {
                 type: "object",
                 properties: {
@@ -68,6 +68,9 @@ export default {
     }],
     
     executePathway: async ({args, runAllPrompts, resolver}) => {
+        if (args.detailedInstructions) {
+            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+        }
         let result = await runAllPrompts({ ...args, stream: false });        
         resolver.tool = JSON.stringify({ toolUsed: "reasoning" });          
         return result;

package/pathways/system/entity/tools/sys_tool_remember.js

@@ -53,6 +53,9 @@ export default {
     }],
 
     executePathway: async ({args, runAllPrompts, resolver}) => {
+        if (args.detailedInstructions) {
+            args.chatHistory.push({role: "user", content: args.detailedInstructions});
+        }
         resolver.tool = JSON.stringify({ toolUsed: "memory" });
         return await callPathway('sys_search_memory', { ...args, stream: false, section: 'memoryAll', updateContext: true });
     }

package/pathways/system/entity/tools/sys_tool_verify.js

@@ -0,0 +1,63 @@
+// sys_tool_verify.js
+// Entity tool that provides response verification and critical evaluation capabilities
+
+import { Prompt } from '../../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `You are the part of an AI entity named {{aiName}} that provides critical evaluation and verification capabilities. You excel at analyzing responses for accuracy, completeness, and potential issues. You do not have the tools to do the verification yourself, but you can help identify gaps, inconsistencies, questions, and areas that need further verification. Think carefully about the response and provide a detailed plan for verification including questions and follow-up actions to ensure the response is robust and reliable. Always include a section called <VERIFICATION_PLAN> that outlines step by step actions to take to answer questions or further research the response.\n{{renderTemplate AI_DATETIME}}`},
+                "{{chatHistory}}",
+                {"role": "user", "content": `Please provide a {{verificationLevel}} review of the following draft response:\n\n<DRAFT_RESPONSE>\n{{responseText}}\n</DRAFT_RESPONSE>\n\n{{context}}`}
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+        model: 'oai-o3'
+    },
+    max_tokens: 100000,
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    toolDefinition: [{
+        type: "function",
+        icon: "🔍",
+        function: {
+            name: "Verify",
+            description: "Critically evaluate a response and provide verification questions and follow-up actions. You should call this tool and execute the steps provided in the resulting verification plan before presenting a final response to the user to ensure it is accurate, complete, and consistent with the user's request.",
+            parameters: {
+                type: "object",
+                properties: {
+                    responseText: {
+                        type: "string",
+                        description: "The draft response text to be verified"
+                    },
+                    context: {
+                        type: "string",
+                        description: "Additional context or instructions about the verification request"
+                    },
+                    verificationLevel: {
+                        type: "string",
+                        description: "The level of verification required (basic, thorough, or critical)",
+                        enum: ["basic", "thorough", "critical"]
+                    },
+                    userMessage: {
+                        type: "string",
+                        description: "A user-friendly message that describes what you're doing with this tool"
+                    }
+                },
+                required: ["responseText", "verificationLevel", "userMessage"]
+            }
+        }
+    }],
+    
+    executePathway: async ({args, runAllPrompts, resolver}) => {        
+        let result = await runAllPrompts({ ...args, stream: false });        
+        resolver.tool = JSON.stringify({ toolUsed: "verification" });          
+        return result;
+    }
+} 

package/server/plugins/azureBingPlugin.js

@@ -75,7 +75,7 @@ class AzureBingPlugin extends ModelPlugin {
     
     // Override the logging function to display the request and response
     logRequestData(data, responseData, prompt) {
-        this.logAIRequestFinished();
+        //this.logAIRequestFinished();
     
         logger.verbose(`${this.parseResponse(responseData)}`);
     

package/server/plugins/openAiReasoningPlugin.js

@@ -8,12 +8,14 @@ class OpenAIReasoningPlugin extends OpenAIChatPlugin {
         for (const message of messages) {
             if (message.role === 'user' || message.role === 'assistant') {
                 newMessages.push({
+                    ...message,
                     role: message.role,
                     content: this.parseContent(message.content)
                 });
             } else if (message.role === 'system') {
                 // System messages to developer: https://platform.openai.com/docs/guides/text-generation#messages-and-roles
                 newMessages.push({
+                    ...message,
                     role: "developer",
                     content: this.parseContent(message.content)
                 });

package/server/plugins/openAiReasoningVisionPlugin.js

@@ -7,9 +7,10 @@ class OpenAIReasoningVisionPlugin extends OpenAIVisionPlugin {
 
         let newMessages = [];
 
+        // System messages to developer: https://platform.openai.com/docs/guides/text-generation#messages-and-roles
         newMessages = parsedMessages.map(message => ({
-            role: message.role === 'system' ? 'developer' : message.role,
-            content: message.content
+            ...message,
+            role: message.role === 'system' ? 'developer' : message.role
         })).filter(message => ['user', 'assistant', 'developer', 'tool'].includes(message.role));
 
         return newMessages;

package/server/plugins/openAiVisionPlugin.js

@@ -25,40 +25,33 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
         return await Promise.all(messages.map(async message => {
             try {
                 // Handle tool-related message types
-                if (message.role === "tool") {
+                if (message.role === "tool" || (message.role === "assistant" && message.tool_calls)) {
                     return {
-                        role: message.role,
-                        content: message.content,
-                        tool_call_id: message.tool_call_id
-                    };
-                }
-
-                if (message.role === "assistant" && message.tool_calls) {
-                    return {
-                        role: message.role,
-                        content: message.content,
-                        tool_calls: message.tool_calls
+                        ...message
                     };
                 }
 
                 if (Array.isArray(message.content)) {
-                    message.content = await Promise.all(message.content.map(async item => {
-                        const parsedItem = safeJsonParse(item);
+                    return {
+                        ...message,
+                        content: await Promise.all(message.content.map(async item => {
+                            const parsedItem = safeJsonParse(item);
 
-                        if (typeof parsedItem === 'string') {
-                            return { type: 'text', text: parsedItem };
-                        }
+                            if (typeof parsedItem === 'string') {
+                                return { type: 'text', text: parsedItem };
+                            }
 
-                        if (typeof parsedItem === 'object' && parsedItem !== null && parsedItem.type === 'image_url') {
-                            const url = parsedItem.url || parsedItem.image_url?.url;
-                            if (url && await this.validateImageUrl(url)) {
-                                return {type: parsedItem.type, image_url: {url}};
+                            if (typeof parsedItem === 'object' && parsedItem !== null && parsedItem.type === 'image_url') {
+                                const url = parsedItem.url || parsedItem.image_url?.url;
+                                if (url && await this.validateImageUrl(url)) {
+                                    return {type: parsedItem.type, image_url: {url}};
+                                }
+                                return { type: 'text', text: typeof item === 'string' ? item : JSON.stringify(item) };
                             }
-                            return { type: 'text', text: typeof item === 'string' ? item : JSON.stringify(item) };
-                        }
-                        
-                        return parsedItem;
-                    }));
+                            
+                            return parsedItem;
+                        }))
+                    };
                 }
             } catch (e) {
                 return message;