@aj-archipelago/cortex 1.3.45 → 1.3.47

package/config.js

@@ -117,16 +117,29 @@ var config = convict({
         format: Object,
         default: {
             AI_MEMORY: `<SHORT_TERM_MEMORY>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\n<TOPICS>\n{{{memoryTopics}}}\n</TOPICS>\n</SHORT_TERM_MEMORY>`,
-            AI_MEMORY_INSTRUCTIONS: "You have persistent memories of important details, instructions, and context - consult your memories when formulating a response to make sure you're applying your learnings.\nIf you don't see relevant information in your short term memory, you should use your SearchMemory tool to search your long term memory for details.\nAlso included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.",
-            AI_TOOLS: "You can execute tools in a loop agentically - you will have a chance to evaluate every tool response before deciding what action to take next - there is no time or execution limit. You have access to a powerful set of tools to help accomplish tasks and deliver the best responses. Instructions for tool use:\n\n1. Always dig deep, verify and cross-check:\n- Take your time and use tools as many times as needed to ensure truth, accuracy, depth, and completeness.\n- Gather data from multiple sources when possible.\n- Leverage both parallel and sequential tool calls for thorough investigation: start broadly, then dive deeper on leads, cross-check facts, and synthesize findings before responding.\n\n2. Plan carefully:\n- Carefully review all available tools before responding.\n- For complex or investigative tasks, use the planning tool first to break the goal into clear steps.\n- Select the most appropriate tool(s) for each step—think beyond single searches to multi-pass, multi-source discovery.\n\n3. Always dive deeper and use as many of your tools as apply:\n- Proactively use tools to refine, verify, and expand on initial findings—don’t settle for the first result if more depth or confirmation may help.\n- Always verify tool capabilities before concluding something can’t be done.\n- If a user explicitly requests tool usage, comply.\n- Maintain context across tool calls to ensure continuity and coherence.\n- If a tool fails, try alternatives or creative approaches.\n\n4. Common Use Cases:\n- Research: Explore multiple sources and perspectives to build a complete picture.\n- Analysis: Use tools to process, compare, and critically assess data or content.\n- Generation: Employ tools for creating content, visuals, or code as needed.\n- Verification: Prioritize cross-checking and fact validation, especially for claims or evolving news.\n\n5. Reflect and Personalize:\n- Synthesize findings into concise, relevant, and personalized responses.\n- If user preferences or past feedback are available, tailor responses accordingly.\n- Before finalizing, review your answer for clarity, completeness, and alignment with user expectations.\n- If you see a recent <VERIFICATION_PLAN> from a tool call, you MUST follow it step by step before giving your final response.\n\nRemember: Your responsibility is to provide the most helpful, well-reasoned, and accurate responses possible. Use tools iteratively and reflectively—don't hesitate to dig deeper or double-check when it improves response quality!",
+
+            AI_MEMORY_CONTEXT: `<CONTEXTUAL_MEMORIES>\n{{{memoryContext}}}\n</CONTEXTUAL_MEMORIES>`,
+
+            AI_MEMORY_INSTRUCTIONS: "You have persistent memories of important details, instructions, and context - consult your memories when formulating a response to make sure you're applying your learnings.\nIf you don't see relevant or sufficient information in your short term or contextual memories, you should use your SearchMemory tool to search your long term memory for details before answering.\nAlso included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.",
+
+            AI_TOOLS: "You have an extensive toolkit. Each time you call a tool you enter a loop: get the result, decide what’s next, and chain as many steps as needed.\n\n1. Search deeply & verify rigorously\n   • Start broad and consult multiple sources, running searches in parallel when speed helps.\n   • For high-stakes or time-sensitive topics, open and read full pages—never rely solely on snippets.\n   • Cross-check facts across sources and always honor user requests to use tools.\n\n2. Plan & sequence before acting\n   • Review the full toolset first.\n   • For multi-step or complex tasks, draft a clear plan (use the Plan tool) and assign the best tool to each step.\n\n3. Escalate & iterate\n   • Don’t accept the first plausible answer—dig until it’s complete, corroborated, and clear.\n   • If a tool falls short, switch strategies or tools while preserving context.\n\n4. Core patterns of use\n   a. Research – gather and compare information.\n   b. Analysis – evaluate, calculate, summarize, or reason.\n   c. Generation – create content, visuals, or code.\n   d. Verification – fact-check and cite; if a <VERIFICATION_PLAN> is present in tool results, follow it before responding.\n\n5. Personalize, synthesize & review\n   • Tailor the final answer to the user’s preferences and history.\n   • Deliver a concise, well-structured response, citing sources with :cd_source[…].\n   • Double-check accuracy, coherence, and alignment with the request.\n\nBottom line: Be thorough, strategic, and iterative. Read sources directly for high-stakes queries and aim for the most accurate, well-reasoned answer—even if it takes multiple tool calls.",
+
             AI_DIRECTIVES: `These are your directives and learned behaviors:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>`,
+
             AI_CONVERSATION_HISTORY: "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}}\n</CONVERSATION_HISTORY>",
+
             AI_COMMON_INSTRUCTIONS: "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}",
-            AI_COMMON_INSTRUCTIONS_MARKDOWN: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, as well as some custom markdown elements, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.",          
-            AI_COMMON_INSTRUCTIONS_VOICE: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",            
-            AI_DATETIME: "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.",           
+
+            AI_COMMON_INSTRUCTIONS_MARKDOWN: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, as well as mermaid charts and some custom markdown elements, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.",
+
+            AI_COMMON_INSTRUCTIONS_VOICE: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",
+
+            AI_DATETIME: "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.",
+
             AI_EXPERTISE: "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, take screenshots, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.",
-            AI_GROUNDING_INSTRUCTIONS: "Grounding your response: Any time you base part or all of your response on one or more search results, you MUST cite the source using a custom markdown directive of the form :cd_source[searchResultId]. There is NO other valid way to cite a source and a good UX depends on you using this directive correctly. Do not include other clickable links to the sourcewhen using the :cd_source[searchResultId] directive. Every search result has a unique searchResultId. You must include it verbatim, copied directly from the search results. Place the directives at the end of the phrase, sentence or paragraph that is grounded in that particular search result. If you are citing multiple search results, use multiple individual:cd_source[searchResultId] directives (e.g. :cd_source[searchResultId1] :cd_source[searchResultId2] :cd_source[searchResultId3] etc.)",
+
+            AI_GROUNDING_INSTRUCTIONS: "Grounding your response: If you base part or all of your response on one or more search results, you MUST cite the source using a custom markdown directive of the form :cd_source[searchResultId]. There is NO other valid way to cite a source and a good UX depends on you using this directive correctly. Do not include other clickable links to the sourcewhen using the :cd_source[searchResultId] directive. Every search result has a unique searchResultId. You must include it verbatim, copied directly from the search results. Place the directives at the end of the phrase, sentence or paragraph that is grounded in that particular search result. If you are citing multiple search results, use multiple individual:cd_source[searchResultId] directives (e.g. :cd_source[searchResultId1] :cd_source[searchResultId2] :cd_source[searchResultId3] etc.)",
+
             AI_STYLE_OPENAI: "oai-gpt41",
             AI_STYLE_ANTHROPIC: "claude-35-sonnet-vertex",
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.45",
+  "version": "1.3.47",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/sys_entity_agent.js

@@ -1,14 +1,16 @@
 // sys_entity_agent.js
 // Agentic extension of the entity system that uses OpenAI's tool calling API
+const MAX_TOOL_CALLS = 50;
+
 import { callPathway, callTool, say } from '../../../lib/pathwayTools.js';
 import logger from '../../../lib/logger.js';
 import { config } from '../../../config.js';
 import { chatArgsHasImageUrl, removeOldImageAndFileContent } from '../../../lib/util.js';
-import { insertToolCallAndResults } from './memory/shared/sys_memory_helpers.js';
 import { Prompt } from '../../../server/prompt.js';
 import { getToolsForEntity, loadEntityConfig } from './tools/shared/sys_entity_tools.js';
 
 export default {
+    emulateOpenAIChatModel: 'cortex-agent',
     useInputChunking: false,
     enableDuplicateRequests: false,
     useSingleTokenStream: false,
@@ -40,153 +42,173 @@ export default {
         const { tool_calls } = message;
         const pathwayResolver = resolver;
         const { entityTools, entityToolsOpenAiFormat } = args;
+
+        pathwayResolver.toolCallCount = (pathwayResolver.toolCallCount || 0);
         
-        // Make a deep copy of the initial chat history
-        const initialMessages = JSON.parse(JSON.stringify(args.chatHistory || []));
+        const preToolCallMessages = JSON.parse(JSON.stringify(args.chatHistory || []));
+        const finalMessages = JSON.parse(JSON.stringify(preToolCallMessages));
 
         if (tool_calls) {
-            // Execute tool calls in parallel but with isolated message histories
-            const toolResults = await Promise.all(tool_calls.map(async (toolCall) => {
-                try {
-                    if (!toolCall?.function?.arguments) {
-                        throw new Error('Invalid tool call structure: missing function arguments');
-                    }
+            if (pathwayResolver.toolCallCount < MAX_TOOL_CALLS) {
+                // Execute tool calls in parallel but with isolated message histories
+                const toolResults = await Promise.all(tool_calls.map(async (toolCall) => {
+                    try {
+                        if (!toolCall?.function?.arguments) {
+                            throw new Error('Invalid tool call structure: missing function arguments');
+                        }
+
+                        const toolArgs = JSON.parse(toolCall.function.arguments);
+                        const toolFunction = toolCall.function.name.toLowerCase();
+                        
+                        // Create an isolated copy of messages for this tool
+                        const toolMessages = JSON.parse(JSON.stringify(preToolCallMessages));
+                        
+                        // Get the tool definition to check for icon
+                        const toolDefinition = entityTools[toolFunction]?.definition;
+                        const toolIcon = toolDefinition?.icon || '🛠️';
+                        
+                        // Report status to the user
+                        const toolUserMessage = toolArgs.userMessage || `Executing tool: ${toolCall.function.name} - ${JSON.stringify(toolArgs)}`;
+                        const messageWithIcon = toolIcon ? `${toolIcon}&nbsp;&nbsp;${toolUserMessage}` : toolUserMessage;
+                        await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `${messageWithIcon}\n\n`, 1000, false);
+
+                        const toolResult = await callTool(toolFunction, {
+                            ...args,
+                            ...toolArgs,
+                            toolFunction,
+                            chatHistory: toolMessages,
+                            stream: false
+                        }, entityTools, pathwayResolver);
+
+                        // Tool calls and results need to be paired together in the message history
+                        // Add the tool call to the isolated message history
+                        toolMessages.push({
+                            role: "assistant",
+                            content: "",
+                            tool_calls: [{
+                                id: toolCall.id,
+                                type: "function",
+                                function: {
+                                    name: toolCall.function.name,
+                                    arguments: JSON.stringify(toolArgs)
+                                }
+                            }]
+                        });
+
+                        // Add the tool result to the isolated message history
+                        const toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult?.result || toolResult);
 
-                    const toolArgs = JSON.parse(toolCall.function.arguments);
-                    const toolFunction = toolCall.function.name.toLowerCase();
-                    
-                    // Create an isolated copy of messages for this tool
-                    const toolMessages = JSON.parse(JSON.stringify(initialMessages));
-                    
-                    // Get the tool definition to check for icon
-                    const toolDefinition = entityTools[toolFunction]?.definition;
-                    const toolIcon = toolDefinition?.icon || '🛠️';
-                    
-                    // Report status to the user
-                    const toolUserMessage = toolArgs.userMessage || `Executing tool: ${toolCall.function.name} - ${JSON.stringify(toolArgs)}`;
-                    const messageWithIcon = toolIcon ? `${toolIcon}&nbsp;&nbsp;${toolUserMessage}` : toolUserMessage;
-                    await say(pathwayResolver.rootRequestId || pathwayResolver.requestId, `${messageWithIcon}\n\n`, 1000, false);
-
-                    const toolResult = await callTool(toolFunction, {
-                        ...args,
-                        ...toolArgs,
-                        toolFunction,
-                        chatHistory: toolMessages,
-                        stream: false
-                    }, entityTools, pathwayResolver);
-
-                    // Tool calls and results need to be paired together in the message history
-                    // Add the tool call to the isolated message history
-                    toolMessages.push({
-                        role: "assistant",
-                        content: "",
-                        tool_calls: [{
-                            id: toolCall.id,
-                            type: "function",
-                            function: {
-                                name: toolCall.function.name,
-                                arguments: JSON.stringify(toolArgs)
-                            }
-                        }]
-                    });
-
-                    // Add the tool result to the isolated message history
-                    const toolResultContent = typeof toolResult === 'string' ? toolResult : JSON.stringify(toolResult?.result || toolResult);
-
-                    toolMessages.push({
-                        role: "tool",
-                        tool_call_id: toolCall.id,
-                        name: toolCall.function.name,
-                        content: toolResultContent
-                    });
-
-                    // Add the screenshots using OpenAI image format
-                    if (toolResult?.toolImages && toolResult.toolImages.length > 0) {
                         toolMessages.push({
-                            role: "user",
-                            content: [
-                                {
-                                    type: "text",
-                                    text: "The tool with id " + toolCall.id + " has also supplied you with these images."
-                                },
-                                ...toolResult.toolImages.map(toolImage => ({
-                                    type: "image_url",
-                                    image_url: {
-                                        url: `data:image/png;base64,${toolImage}`
-                                    }
-                                }))
-                            ]
+                            role: "tool",
+                            tool_call_id: toolCall.id,
+                            name: toolCall.function.name,
+                            content: toolResultContent
                         });
-                    }
 
-                    return { 
-                        success: true, 
-                        result: toolResult,
-                        toolCall,
-                        toolArgs,
-                        toolFunction,
-                        messages: toolMessages
-                    };
-                } catch (error) {
-                    logger.error(`Error executing tool ${toolCall?.function?.name || 'unknown'}: ${error.message}`);
-                    
-                    // Create error message history
-                    const errorMessages = JSON.parse(JSON.stringify(initialMessages));
-                    errorMessages.push({
-                        role: "assistant",
-                        content: "",
-                        tool_calls: [{
-                            id: toolCall.id,
-                            type: "function",
-                            function: {
-                                name: toolCall.function.name,
-                                arguments: JSON.stringify(toolCall.function.arguments)
-                            }
-                        }]
-                    });
-                    errorMessages.push({
-                        role: "tool",
-                        tool_call_id: toolCall.id,
-                        name: toolCall.function.name,
-                        content: `Error: ${error.message}`
-                    });
-
-                    return { 
-                        success: false, 
-                        error: error.message,
-                        toolCall,
-                        toolArgs: toolCall?.function?.arguments ? JSON.parse(toolCall.function.arguments) : {},
-                        toolFunction: toolCall?.function?.name?.toLowerCase() || 'unknown',
-                        messages: errorMessages
-                    };
-                }
-            }));
-
-            // Merge all message histories in order
-            let finalMessages = JSON.parse(JSON.stringify(initialMessages));
-            for (const result of toolResults) {
-                try {
-                    if (!result?.messages) {
-                        logger.error('Invalid tool result structure, skipping message history update');
-                        continue;
+                        // Add the screenshots using OpenAI image format
+                        if (toolResult?.toolImages && toolResult.toolImages.length > 0) {
+                            toolMessages.push({
+                                role: "user",
+                                content: [
+                                    {
+                                        type: "text",
+                                        text: "The tool with id " + toolCall.id + " has also supplied you with these images."
+                                    },
+                                    ...toolResult.toolImages.map(toolImage => ({
+                                        type: "image_url",
+                                        image_url: {
+                                            url: `data:image/png;base64,${toolImage}`
+                                        }
+                                    }))
+                                ]
+                            });
+                        }
+
+                        return { 
+                            success: true, 
+                            result: toolResult,
+                            toolCall,
+                            toolArgs,
+                            toolFunction,
+                            messages: toolMessages
+                        };
+                    } catch (error) {
+                        logger.error(`Error executing tool ${toolCall?.function?.name || 'unknown'}: ${error.message}`);
+                        
+                        // Create error message history
+                        const errorMessages = JSON.parse(JSON.stringify(preToolCallMessages));
+                        errorMessages.push({
+                            role: "assistant",
+                            content: "",
+                            tool_calls: [{
+                                id: toolCall.id,
+                                type: "function",
+                                function: {
+                                    name: toolCall.function.name,
+                                    arguments: JSON.stringify(toolCall.function.arguments)
+                                }
+                            }]
+                        });
+                        errorMessages.push({
+                            role: "tool",
+                            tool_call_id: toolCall.id,
+                            name: toolCall.function.name,
+                            content: `Error: ${error.message}`
+                        });
+
+                        return { 
+                            success: false, 
+                            error: error.message,
+                            toolCall,
+                            toolArgs: toolCall?.function?.arguments ? JSON.parse(toolCall.function.arguments) : {},
+                            toolFunction: toolCall?.function?.name?.toLowerCase() || 'unknown',
+                            messages: errorMessages
+                        };
                     }
+                }));
+
+                // Merge all message histories in order
+                for (const result of toolResults) {
+                    try {
+                        if (!result?.messages) {
+                            logger.error('Invalid tool result structure, skipping message history update');
+                            continue;
+                        }
+
+                        // Add only the new messages from this tool's history
+                        const newMessages = result.messages.slice(preToolCallMessages.length);
+                        finalMessages.push(...newMessages);
+                    } catch (error) {
+                        logger.error(`Error merging message history for tool result: ${error.message}`);
+                    }
+                }
 
-                    // Add only the new messages from this tool's history
-                    const newMessages = result.messages.slice(initialMessages.length);
-                    finalMessages.push(...newMessages);
-                } catch (error) {
-                    logger.error(`Error merging message history for tool result: ${error.message}`);
+                // Check if any tool calls failed
+                const failedTools = toolResults.filter(result => !result.success);
+                if (failedTools.length > 0) {
+                    logger.warn(`Some tool calls failed: ${failedTools.map(t => t.error).join(', ')}`);
                 }
-            }
 
-            // Check if any tool calls failed
-            const failedTools = toolResults.filter(result => !result.success);
-            if (failedTools.length > 0) {
-                logger.warn(`Some tool calls failed: ${failedTools.map(t => t.error).join(', ')}`);
+                pathwayResolver.toolCallCount = (pathwayResolver.toolCallCount || 0) + toolResults.length;
+
+            } else {
+                finalMessages.push({
+                    role: "user",
+                    content: [
+                        {
+                            type: "text",
+                            text: "This agent has reached the maximum number of tool calls - no more tool calls will be executed."
+                        }
+                    ]
+                });
+                
             }
 
             args.chatHistory = finalMessages;
 
+            // clear any accumulated pathwayResolver errors from the tools
+            pathwayResolver.errors = [];
+
             return await pathwayResolver.promptAndParse({
                 ...args,
                 tools: entityToolsOpenAiFormat,
@@ -204,10 +226,6 @@ export default {
         const entityConfig = loadEntityConfig(entityId);
         const { entityTools, entityToolsOpenAiFormat } = getToolsForEntity(entityConfig);
         const { useMemory: entityUseMemory = true, name: entityName, instructions: entityInstructions } = entityConfig || {};
-
-        if (entityId && entityName) {
-            args.aiName = entityName;
-        }
         
         args = {
             ...args,
@@ -228,12 +246,12 @@ export default {
         const promptPrefix = researchMode ? 'Formatting re-enabled\n' : '';
 
         const memoryTemplates = entityUseMemory ? 
-            `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n\n` : '';
+            `{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n\n{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_MEMORY_CONTEXT}}\n\n` : '';
 
-        const instructionTemplates = entityInstructions ? (entityInstructions + '\n\n') : `{{renderTemplate AI_EXPERTISE}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n`;
+        const instructionTemplates = entityInstructions ? (entityInstructions + '\n\n') : `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n{{renderTemplate AI_EXPERTISE}}\n\n`;
 
         const promptMessages = [
-            {"role": "system", "content": `${promptPrefix}${memoryTemplates}${instructionTemplates}{{renderTemplate AI_TOOLS}}\n\n{{renderTemplate AI_GROUNDING_INSTRUCTIONS}}\n\n{{renderTemplate AI_DATETIME}}`},
+            {"role": "system", "content": `${promptPrefix}${instructionTemplates}{{renderTemplate AI_TOOLS}}\n\n{{renderTemplate AI_GROUNDING_INSTRUCTIONS}}\n\n${memoryTemplates}{{renderTemplate AI_DATETIME}}`},
             "{{chatHistory}}",
         ];
 
@@ -263,15 +281,7 @@ export default {
 
         // truncate the chat history in case there is really long content
         const truncatedChatHistory = resolver.modelExecutor.plugin.truncateMessagesToTargetLength(args.chatHistory, null, 1000);
-
-        // Add the memory context to the chat history if applicable
-        if (truncatedChatHistory.length > 1 && entityUseMemory) {
-            const memoryContext = await callPathway('sys_read_memory', { ...args, chatHistory: truncatedChatHistory, section: 'memoryContext', priority: 0, recentHours: 0, stream: false }, resolver);
-            if (memoryContext) {
-                insertToolCallAndResults(args.chatHistory, "Load general memory context information", "LoadMemoryContext", memoryContext);
-            }
-        }
-        
+      
         // Asynchronously manage memory for this context
         if (args.aiMemorySelfModify && entityUseMemory) {
             callPathway('sys_memory_manager', {  ...args, chatHistory: truncatedChatHistory, stream: false })    

package/pathways/system/entity/tools/sys_tool_codingagent.js

@@ -48,13 +48,17 @@ export default {
         icon: "🤖",
         function: {
             name: "CodeExecution",
-            description: "Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks - examples include data analysis, file manipulation, or other tasks that require code execution.",
+            description: "Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks - examples include data analysis, file manipulation, or other tasks that require code execution. This will start a background task and return - you will not receive the response immediately.",
             parameters: {
                 type: "object",
                 properties: {
                     codingTask: {
                         type: "string",
-                        description: "Detailed task description for the coding agent. Include all necessary information as this is the only message the coding agent receives. Let the agent decide how to solve it without making assumptions about its capabilities. IMPORTANT: The coding agent does not share your context, so you must provide it with all the information in this message. If you are asking it to operate on files or other data from your context, you must provide the fully-qualified URL to each of the files you want it to use. Also make sure you explicitly instruct the agent to use those files."
+                        description: "Detailed task description for the coding agent. Include all necessary information as this is the only message the coding agent receives. Let the agent decide how to solve it without making assumptions about its capabilities. IMPORTANT: The coding agent does not share your context, so you must provide it with all the information in this message."
+                    },
+                    inputFiles: {
+                        type: "string",
+                        description: "A list of input files that the coding agent must use to complete the task. Each file should be the fully-qualified URL to the file. Omit this parameter if no input files are needed."
                     },
                     userMessage: {
                         type: "string",
@@ -72,12 +76,18 @@ export default {
     
     executePathway: async ({args, resolver}) => {
         try {
-            const { codingTask, userMessage, codingTaskKeywords } = args;
+            const { codingTask, userMessage, inputFiles, codingTaskKeywords } = args;
             const { contextId } = args;
 
+            let taskSuffix = "";
+            if (inputFiles) {
+                taskSuffix = `You must use the following files as input to complete the task: ${inputFiles}.`
+            }
+
+
             // Send the task to the queue
             const codeRequestId = await sendMessageToQueue({ 
-                message: codingTask, 
+                message: `${codingTask}\n\n${taskSuffix}`, 
                 contextId, 
                 keywords: codingTaskKeywords 
             });

package/pathways/system/entity/tools/sys_tool_remember.js

@@ -28,28 +28,6 @@ export default {
                 required: ["detailedInstructions", "userMessage"]
             }
         }
-    },
-    {
-        type: "function",
-        icon: "🧩",
-        function: {
-            name: "LoadMemoryContext",
-            description: "This tool quickly preloads the memory context for this turn of the conversation.  It's typically automatically used by the system, but you can use it if you need to.",
-            parameters: {
-                type: "object",
-                properties: {
-                    detailedInstructions: {
-                        type: "string",
-                        description: "Detailed instructions about what you need the tool to do"
-                    },
-                    userMessage: {
-                        type: "string",
-                        description: "A user-friendly message that describes what you're doing with this tool"
-                    }
-                },
-                required: ["detailedInstructions", "userMessage"]
-            }
-        }
     }],
 
     executePathway: async ({args, runAllPrompts, resolver}) => {

package/server/modelExecutor.js

@@ -131,7 +131,7 @@ class ModelExecutor {
         } catch (error) {
             logger.error(`Error executing model plugin for pathway ${pathwayResolver?.pathway?.name}: ${error.message}`);
             logger.debug(error.stack);
-            pathwayResolver.errors.push(error.message);
+            pathwayResolver.logError(error.message);
             return null;
         }
     }

package/server/pathwayResolver.js

@@ -77,7 +77,7 @@ class PathwayResolver {
         if (requestProgress.progress === 1 && this.rootRequestId) {
             delete requestProgress.progress;
         }
-        publishRequestProgress({...requestProgress, info: this.tool || ''});
+        publishRequestProgress({...requestProgress, info: this.tool || '', error: this.errors.join(', ')});
     }
 
     // This code handles async and streaming responses for either long-running
@@ -89,12 +89,24 @@ class PathwayResolver {
             responseData = await this.executePathway(args);
         }
         catch (error) {
+            this.errors.push(error.message || error.toString());
             publishRequestProgress({
                 requestId: this.rootRequestId || this.requestId,
                 progress: 1,
                 data: '',
                 info: '',
-                error: error.message || error.toString()
+                error: this.errors.join(', ')
+            });
+            return;
+        }
+
+        if (!responseData) {
+            publishRequestProgress({
+                requestId: this.rootRequestId || this.requestId,
+                progress: 1,
+                data: '',
+                info: '',
+                error: this.errors.join(', ')
             });
             return;
         }
@@ -113,7 +125,8 @@ class PathwayResolver {
                         progress: Math.min(completedCount, totalCount) / totalCount,
                         // Clients expect these to be strings
                         data: JSON.stringify(responseData || ''),
-                        info: this.tool || ''
+                        info: this.tool || '',
+                        error: this.errors.join(', ') || ''
                 });
             }
         }
@@ -501,7 +514,8 @@ class PathwayResolver {
                 memorySelf: this.memorySelf,
                 memoryDirectives: this.memoryDirectives,
                 memoryTopics: this.memoryTopics,
-                memoryUser: this.memoryUser 
+                memoryUser: this.memoryUser,
+                memoryContext: this.memoryContext
             }, prompt, this);
         } else {
             result = text;

package/server/plugins/openAiVisionPlugin.js

@@ -84,7 +84,14 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
                 const { length, units } = this.getLength(content);
                 const displayContent = this.shortenContent(content);
 
-                logger.verbose(`message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${displayContent}"`);
+                let logMessage = `message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${displayContent}"`;
+                
+                // Add tool calls to log if they exist
+                if (message.role === 'assistant' && message.tool_calls) {
+                    logMessage += `, tool_calls: ${JSON.stringify(message.tool_calls)}`;
+                }
+                
+                logger.verbose(logMessage);
                 totalLength += length;
                 totalUnits = units;
             });