@aj-archipelago/cortex 1.3.48 → 1.3.50

package/config.js

@@ -7,6 +7,7 @@ import GcpAuthTokenHelper from './lib/gcpAuthTokenHelper.js';
 import logger from './lib/logger.js';
 import PathwayManager from './lib/pathwayManager.js';
 import { readdir } from 'fs/promises';
+import { entityConstants } from './lib/entityConstants.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -115,34 +116,7 @@ var config = convict({
     },
     entityConstants: {
         format: Object,
-        default: {
-            AI_MEMORY: `<SHORT_TERM_MEMORY>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\n<TOPICS>\n{{{memoryTopics}}}\n</TOPICS>\n</SHORT_TERM_MEMORY>`,
-
-            AI_MEMORY_CONTEXT: `<CONTEXTUAL_MEMORIES>\n{{{memoryContext}}}\n</CONTEXTUAL_MEMORIES>`,
-
-            AI_MEMORY_INSTRUCTIONS: "You have persistent memories of important details, instructions, and context - consult your memories when formulating a response to make sure you're applying your learnings.\nIf you don't see relevant or sufficient information in your short term or contextual memories, you should use your SearchMemory tool to search your long term memory for details before answering.\nAlso included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.",
-
-            AI_TOOLS: "You have an extensive toolkit. Each time you call a tool you enter a loop: get the result, decide what’s next, and chain as many steps as needed.\n\n1. Search deeply & verify rigorously\n   • Start broad and consult multiple sources, running searches in parallel when speed helps.\n   • For high-stakes or time-sensitive topics, open and read full pages—never rely solely on snippets.\n   • Cross-check facts across sources and always honor user requests to use tools.\n\n2. Plan & sequence before acting\n   • Review the full toolset first.\n   • For multi-step or complex tasks, draft a clear plan (use the Plan tool) and assign the best tool to each step.\n\n3. Escalate & iterate\n   • Don’t accept the first plausible answer—dig until it’s complete, corroborated, and clear.\n   • If a tool falls short, switch strategies or tools while preserving context.\n\n4. Core patterns of use\n   a. Research – gather and compare information.\n   b. Analysis – evaluate, calculate, summarize, or reason.\n   c. Generation – create content, visuals, or code.\n   d. Verification – fact-check and cite; if a <VERIFICATION_PLAN> is present in tool results, follow it before responding.\n\n5. Personalize, synthesize & review\n   • Tailor the final answer to the user’s preferences and history.\n   • Deliver a concise, well-structured response, citing sources with :cd_source[…].\n   • Double-check accuracy, coherence, and alignment with the request.\n\nBottom line: Be thorough, strategic, and iterative. Read sources directly for high-stakes queries and aim for the most accurate, well-reasoned answer—even if it takes multiple tool calls.",
-
-            AI_DIRECTIVES: `These are your directives and learned behaviors:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>`,
-
-            AI_CONVERSATION_HISTORY: "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}}\n</CONVERSATION_HISTORY>",
-
-            AI_COMMON_INSTRUCTIONS: "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}",
-
-            AI_COMMON_INSTRUCTIONS_MARKDOWN: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, as well as mermaid charts and some custom markdown elements, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.",
-
-            AI_COMMON_INSTRUCTIONS_VOICE: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",
-
-            AI_DATETIME: "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.",
-
-            AI_EXPERTISE: "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, take screenshots, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.",
-
-            AI_GROUNDING_INSTRUCTIONS: "Grounding your response: If you base part or all of your response on one or more search results, you MUST cite the source using a custom markdown directive of the form :cd_source[searchResultId]. There is NO other valid way to cite a source and a good UX depends on you using this directive correctly. Do not include other clickable links to the sourcewhen using the :cd_source[searchResultId] directive. Every search result has a unique searchResultId. You must include it verbatim, copied directly from the search results. Place the directives at the end of the phrase, sentence or paragraph that is grounded in that particular search result. If you are citing multiple search results, use multiple individual :cd_source[searchResultId] directives (e.g. :cd_source[searchResultId1] :cd_source[searchResultId2] :cd_source[searchResultId3] etc.)",
-
-            AI_STYLE_OPENAI: "oai-gpt41",
-            AI_STYLE_ANTHROPIC: "claude-35-sonnet-vertex",
-        },
+        default: entityConstants,
     },
     entityTools: {
         format: Object,

package/lib/entityConstants.js

@@ -0,0 +1,131 @@
+export const entityConstants = {
+    AI_COMMON_INSTRUCTIONS: "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}",
+
+    AI_COMMON_INSTRUCTIONS_MARKDOWN: "# General Instructions\n\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, as well as mermaid charts and some custom markdown elements, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.",
+
+    AI_COMMON_INSTRUCTIONS_VOICE: "# General Instructions:\n\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",
+
+    AI_DIRECTIVES: `# Directives\n\nThese are your directives and learned behaviors:\n{{{memoryDirectives}}}\n`,
+
+    AI_CONVERSATION_HISTORY: "# Conversation History\n\n{{{toJSON chatHistory}}}\n",
+
+    AI_EXPERTISE: "# Expertise\n\nYour expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, take screenshots, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment that includes access to internal databases and the internet.",
+
+    AI_TOOLS: `# Tool Instructions
+
+You have an extensive toolkit. Each time you call tool(s) you will get the result(s), evaluate, decide what's next, and chain as many steps as needed. Always honor user requests to use specific tools.
+
+1. Search deeply & verify rigorously:
+    - Start broad and consult multiple sources, running searches in parallel where possible.
+    - Consult all available sources and cross-reference with specific searches before responding.
+
+2. Plan & sequence before acting:
+    - Review the toolset first.
+    - For multi-step or complex tasks, draft a clear plan (use the PlanMultiStepTask or other reasoning tool) and assign tool calls to each step.
+
+3. Escalate & iterate:
+    - Don't settle for the first plausible answer—dig until the response is complete, corroborated, and clear.
+    - If a tool falls short, adapt strategy or change tools while preserving context.
+
+4. Core patterns of use:
+    - Research: Gather and compare information.
+    - Analysis: Evaluate, calculate, summarize, or reason.
+    - Generation: Create content, visuals, or code.
+    - Verification: Fact-check and cite. If a <VERIFICATION_PLAN> is present, follow it before responding.
+
+5. Personalize, synthesize & review:
+    - Tailor answers to the user's preferences and history.
+    - Deliver concise, well-structured responses citing sources with :cd_source[…].
+    - Double-check accuracy, coherence, and alignment with the user request.
+
+Bottom line: Be thorough, strategic, and iterative. Read sources directly for high-stakes queries and aim for the most accurate, well-reasoned answer—even if it takes multiple tool calls.
+`,            
+
+    AI_SEARCH_RULES: `# News Search Protocol
+When searching for news, you must complete the following steps:
+
+1. Triangulate
+   - Run multiple, parallel queries across all applicable sources.
+   - Request about double the number of results you want to share, then select the best results.
+   - Confirm that multiple sources tell the same story.
+
+2. Check Freshness
+   - Confirm the publication date.
+   - Apply date filters to surface the most recent credible material.
+
+# Internet Search Protocol
+
+Before you share online information with the user, you MUST complete all of the steps below:
+
+1. Triangulate  
+   - Run multiple, parallel queries across reputable outlets.  
+   - Confirm that independent sources tell the same story.
+
+2. Verify  
+   - Treat social / monetized platforms (YouTube, X, TikTok, Instagram, Reddit, etc.) as unverified tips only.  
+   - Corroborate every claim from those platforms with at least one authoritative source.
+
+3. Check Freshness  
+   - Confirm the publication date.  
+   - Apply date filters to surface the most recent credible material.
+
+4. Read, don't skim  
+   - For high-stakes, complex, or time-sensitive topics, use your tools toopen and read the full article or document.  
+   - Never rely solely on snippets, headlines, or auto-generated summaries.
+`,
+
+    AI_SEARCH_SYNTAX: `# Bing Search Syntax
+
+When creating a query string for your Bing internet search tool, you can use Bing's advanced search operators in your query. E.g. "+(\"exact phrase\") AND term1 -term2"
+
+token1 & token2        (AND operator - both tokens must appear)
+token1 | token2        (OR operator - either token may appear (also the default if no operator is specified))
+-token                 (NOT operator - exclude results with token)
++token                 (Require token)
+"term1 term2"          (Exact phrase match)
+(token1 + token2)      (Override precedence with parentheses)
+
+# AI Search Syntax
+
+When creating a query string for your non-Bing search tools, you can use the following AI Search syntax. Important: these tools do not support AND, OR, or NOT strings as operators - you MUST use the syntax below.  E.g. you cannot use "term1 AND term2", you must use "term1 + term2".
+
+token1 + token2        (AND operator - both tokens must appear)
+token1 | token2        (OR operator - either token may appear (also the default if no operator is specified))
+-token                 (NOT operator - exclude results with token)
+"term1 term2"          (Exact phrase match)
+term*                  (Matches terms starting with "term")
+term~N                 (Match terms similar to "term", edit distance N)
+"term1 term2"~N        (Terms appear within N words of each other)
+(token1 + token2)      (Override precedence with parentheses)
+
+
+**Escaping Special Characters:**
+- You can use backslash (\\) to escape special characters if you need to search for them literally.
+`,
+
+    AI_GROUNDING_INSTRUCTIONS: "# Grounding Responses\n\nIf you base part or all of your response on one or more search results, you MUST cite the source using a custom markdown directive of the form :cd_source[searchResultId]. There is NO other valid way to cite a source and a good UX depends on you using this directive correctly. Do not include other clickable links to the sourcewhen using the :cd_source[searchResultId] directive. Every search result has a unique searchResultId. You must include it verbatim, copied directly from the search results. Place the directives at the end of the phrase, sentence or paragraph that is grounded in that particular search result. If you are citing multiple search results, use multiple individual :cd_source[searchResultId] directives (e.g. :cd_source[searchResultId1] :cd_source[searchResultId2] :cd_source[searchResultId3] etc.)",
+
+    AI_MEMORY_INSTRUCTIONS: `# Memory Instructions
+
+You have a memory system that contains important details, instructions, and context. Consult your memories when formulating a response to ensure your answers reflect previous learnings and context.
+
+The Preloaded Memories are not your complete memory system. If you don't see the information you need in there, or need more details, call your SearchMemory tool to search the rest of your memory system.
+
+It's critical that you never fabricate or miss existing memories. Everything that you say you remember must be backed by a Preloaded Memory or a SearchMemory result without exception.
+
+Your memories may also contain details about the user to help personalize responses. You do not need to include the user's name or personal information in every reply—only when relevant to the conversation.
+
+When sharing information from memory, state it naturally (e.g., 'I remember...'); never refer to the memory structure or technical details.
+
+Privacy is critical. If asked to forget or delete something, always comply affirmatively. If there is user information in your memories you have talked to this user before.
+`,
+
+    AI_MEMORY: "# Preloaded Memories\n\n## Self\n{{{memorySelf}}}\n\n## User\n{{{memoryUser}}}\n\n## Directives\n{{{memoryDirectives}}}\n\n## Topics\n{{{memoryTopics}}}",
+
+    AI_MEMORY_CONTEXT: "## Contextual\n{{{memoryContext}}}",
+
+    AI_DATETIME: "# Time, Date, and Time Zone\n\nThe current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.",
+
+    AI_STYLE_OPENAI: "oai-gpt41",
+    AI_STYLE_ANTHROPIC: "claude-35-sonnet-vertex",
+}; 

package/lib/requestExecutor.js

@@ -189,8 +189,10 @@ const requestWithMonitor = async (endpoint, url, data, axiosConfigObj) => {
     let response;
     try {
         if (axiosConfigObj?.method == 'GET'){
+            logger.debug(`Getting ${url} with data: ${JSON.stringify(data)}`);
             response = await cortexAxios.get(url, axiosConfigObj);
         } else {
+            logger.debug(`Posting ${url} with data: ${JSON.stringify(data)}`);
             response = await cortexAxios.post(url, data, axiosConfigObj);
         }
     } catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.48",
+  "version": "1.3.50",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/memory/shared/sys_memory_helpers.js

@@ -1,6 +1,7 @@
 import { callPathway } from '../../../../../lib/pathwayTools.js';
 import { encode } from '../../../../../lib/encodeCache.js';
 import { getUniqueId } from '../../../../../lib/util.js';
+import logger from '../../../../../lib/logger.js';
 
 const normalizeMemoryFormat = async (args, content) => {
     if (!content) return '';
@@ -33,7 +34,7 @@ const normalizeMemoryFormat = async (args, content) => {
                 formattedContent = [...validLines, ...formattedBlock.split('\n')];
             }
         } catch (error) {
-            console.warn('Error formatting invalid memory lines:', error);
+            logger.warn('Error formatting invalid memory lines:', error);
         }
     }
 
@@ -150,18 +151,20 @@ const insertToolCallAndResults = (chatHistory, toolArgs, toolName, result = null
 const modifyText = (text, modifications) => {
     let modifiedText = text || '';
   
-    modifications.forEach(mod => {
+    modifications.forEach((mod, index) => {
+        logger.debug(`Processing modification ${index + 1}: ${JSON.stringify(mod)}`);
+        
         // Skip invalid modifications
         if (!mod.type) {
-            console.warn('Modification missing type');
+            logger.warn('Modification missing type');
             return;
         }
         if ((mod.type === 'delete' || mod.type === 'change') && !mod.pattern) {
-            console.warn(`${mod.type} modification missing pattern`);
+            logger.warn(`${mod.type} modification missing pattern`);
             return;
         }
         if ((mod.type === 'add' || mod.type === 'change') && !mod.newtext) {
-            console.warn(`${mod.type} modification missing newtext`);
+            logger.warn(`${mod.type} modification missing newtext`);
             return;
         }
 
@@ -171,8 +174,9 @@ const modifyText = (text, modifications) => {
         switch (mod.type) {
             case 'add':
                 const priority = mod.priority || '3';
-                modifiedText = modifiedText + (modifiedText ? '\n' : '') + 
-                    `${priority}|${timestamp}|${mod.newtext}`;
+                const newLine = `${priority}|${timestamp}|${mod.newtext}`;
+                logger.debug(`Adding new line: ${newLine}`);
+                modifiedText = modifiedText + (modifiedText ? '\n' : '') + newLine;
                 break;
             case 'change':
                 // Split into lines
@@ -197,10 +201,12 @@ const modifyText = (text, modifications) => {
                                     // Replace $1, $2, etc with capture group values
                                     newContent = mod.newtext.replace(/\$(\d+)/g, (_, n) => match[n] || '');
                                 }
-                                return `${newPriority}|${timestamp}|${newContent}`;
+                                const newLine = `${newPriority}|${timestamp}|${newContent}`;
+                                logger.debug(`Changing line: ${line} to ${newLine}`);
+                                return newLine;
                             }
                         } catch (e) {
-                            console.warn(`Invalid regex pattern: ${mod.pattern}`);
+                            logger.warn(`Invalid regex pattern: ${mod.pattern}`);
                         }
                     }
                     return line;
@@ -208,6 +214,7 @@ const modifyText = (text, modifications) => {
                 break;
             case 'delete':
                 // Split into lines, filter out matching lines, and rejoin
+                const beforeDelete = modifiedText;
                 modifiedText = modifiedText
                     .split('\n')
                     .filter(line => {
@@ -216,17 +223,24 @@ const modifyText = (text, modifications) => {
                         if (!content) return true;
                         try {
                             const regex = new RegExp(mod.pattern, 'i');
-                            return !regex.test(content.trim());
+                            const shouldKeep = !regex.test(content.trim());
+                            if (!shouldKeep) {
+                                logger.debug(`Deleting line: ${line}`);
+                            }
+                            return shouldKeep;
                         } catch (e) {
-                            console.warn(`Invalid regex pattern: ${mod.pattern}`);
+                            logger.warn(`Invalid regex pattern: ${mod.pattern}`);
                             return true;
                         }
                     })
                     .filter(line => line.trim())
                     .join('\n');
+                if (beforeDelete !== modifiedText) {
+                    logger.debug(`Text after deletion: ${modifiedText}`);
+                }
                 break;
             default:
-                console.warn(`Unknown modification type: ${mod.type}`);
+                logger.warn(`Unknown modification type: ${mod.type}`);
         }
     });
   

package/pathways/system/entity/memory/sys_memory_format.js

@@ -21,7 +21,7 @@ export default {
         chatHistory: [{role: '', content: []}],
         aiName: "Jarvis",
     },
-    model: 'oai-gpt4o',
+    model: 'oai-gpt41',
     useInputChunking: true,
     inputChunkSize: 1000,
     useParallelChunkProcessing: true,

package/pathways/system/entity/memory/sys_memory_lookup_required.js

@@ -0,0 +1,22 @@
+import { Prompt } from '../../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": "You are part of an AI entity named {{{aiName}}}.\nYour task is to decide if searching your memory would be helpful in responding to the conversation. Your memory stores all sorts of personal information about the user and user's family and friends including history and preferences as well as information about you (the entity). If you think searching it would be helpful, return {\"memoryRequired\": true}. If not, return {\"memoryRequired\": false}.\n\n# Conversation to analyze:\n{{{toJSON chatHistory}}}"},
+                {"role": "user", "content": "Generate a JSON object to indicate if information from memory is required."},
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        text: '',
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt41-mini',
+    useInputChunking: false,
+    json: true,
+    responseFormat: { type: "json_object" },
+}

package/pathways/system/entity/memory/sys_memory_process.js

@@ -67,7 +67,7 @@ Each modification object should look like:
         section: "",
         maxIterations: 5
     },
-    model: 'oai-gpt4o',
+    model: 'oai-gpt41',
     useInputChunking: false,
     enableDuplicateRequests: false,
     json: true,

package/pathways/system/entity/memory/sys_memory_required.js

@@ -16,7 +16,7 @@ export default {
         aiName: "Jarvis",
         language: "English",
     },
-    model: 'oai-gpt4o',
+    model: 'oai-gpt41',
     useInputChunking: false,
     json: true,
     ...config.get('entityConstants')

package/pathways/system/entity/memory/sys_memory_update.js

@@ -27,7 +27,7 @@ export default {
         section: "",
         operations: "[]"
     },
-    model: 'oai-gpt4o',
+    model: 'oai-gpt41',
     useInputChunking: false,
     enableDuplicateRequests: false,
     json: true,

package/pathways/system/entity/sys_entity_agent.js

@@ -229,6 +229,27 @@ export default {
         const entityConfig = loadEntityConfig(entityId);
         const { entityTools, entityToolsOpenAiFormat } = getToolsForEntity(entityConfig);
         const { useMemory: entityUseMemory = true, name: entityName, instructions: entityInstructions } = entityConfig || {};
+
+        // Initialize chat history if needed
+        if (!args.chatHistory || args.chatHistory.length === 0) {
+            args.chatHistory = [];
+        }
+
+        // Kick off the memory lookup required pathway in parallel - this takes like 500ms so we want to start it early
+        let memoryLookupRequiredPromise = null;
+        if (entityUseMemory) {
+            const chatHistoryLastTurn = args.chatHistory.slice(-2);
+            const chatHistorySizeOk = (JSON.stringify(chatHistoryLastTurn).length < 5000);
+            if (chatHistorySizeOk) {
+                const timeoutPromise = new Promise((_, reject) => 
+                    setTimeout(() => reject(new Error('Memory lookup timeout')), 800)
+                );
+                memoryLookupRequiredPromise = Promise.race([
+                    callPathway('sys_memory_lookup_required', { ...args, chatHistory: chatHistoryLastTurn, stream: false }),
+                    timeoutPromise
+                ]);
+            }
+        }
         
         args = {
             ...args,
@@ -254,7 +275,7 @@ export default {
         const instructionTemplates = entityInstructions ? (entityInstructions + '\n\n') : `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n{{renderTemplate AI_EXPERTISE}}\n\n`;
 
         const promptMessages = [
-            {"role": "system", "content": `${promptPrefix}${instructionTemplates}{{renderTemplate AI_TOOLS}}\n\n{{renderTemplate AI_GROUNDING_INSTRUCTIONS}}\n\n${memoryTemplates}{{renderTemplate AI_DATETIME}}`},
+            {"role": "system", "content": `${promptPrefix}${instructionTemplates}{{renderTemplate AI_TOOLS}}\n\n{{renderTemplate AI_SEARCH_RULES}}\n\n{{renderTemplate AI_SEARCH_SYNTAX}}\n\n{{renderTemplate AI_GROUNDING_INSTRUCTIONS}}\n\n${memoryTemplates}{{renderTemplate AI_DATETIME}}`},
             "{{chatHistory}}",
         ];
 
@@ -266,11 +287,6 @@ export default {
         const { aiStyle, AI_STYLE_ANTHROPIC, AI_STYLE_OPENAI } = args;
         const styleModel = aiStyle === "Anthropic" ? AI_STYLE_ANTHROPIC : AI_STYLE_OPENAI;
 
-        // Initialize chat history if needed
-        if (!args.chatHistory || args.chatHistory.length === 0) {
-            args.chatHistory = [];
-        }
-
         // Limit the chat history to 20 messages to speed up processing
         if (args.messages && args.messages.length > 0) {
             args.chatHistory = args.messages.slice(-20);
@@ -291,6 +307,20 @@ export default {
             .catch(error => logger.error(error?.message || "Error in sys_memory_manager pathway"));
         }
 
+        let memoryLookupRequired = false;
+
+        try {
+            if (memoryLookupRequiredPromise) {
+                memoryLookupRequired = JSON.parse(await memoryLookupRequiredPromise)?.memoryRequired;
+            } else {
+                memoryLookupRequired = false;
+            }
+        } catch (error) {
+            logger.warn(`Failed to test memory lookup requirement: ${error.message}`);
+            // If we hit the timeout or any other error, we'll proceed without memory lookup
+            memoryLookupRequired = false;
+        }
+
         try {
             let currentMessages = JSON.parse(JSON.stringify(args.chatHistory));
 
@@ -298,7 +328,7 @@ export default {
                 ...args,
                 chatHistory: currentMessages,
                 tools: entityToolsOpenAiFormat,
-                tool_choice: "auto"
+                tool_choice: memoryLookupRequired ? "required" : "auto"
             });
 
             let toolCallback = pathwayResolver.pathway.toolCallback;

package/pathways/system/entity/tools/sys_tool_bing_search.js

@@ -12,14 +12,14 @@ export default {
         type: "function",
         icon: "🌐",
         function: {
-            name: "InternetSearch",
+            name: "SearchInternet",
             description: "This tool allows you to use the Bing search api to search the internet and more. Use this for current events, news, fact-checking, and information requiring citation.",
             parameters: {
                 type: "object",
                 properties: {
                     q: {
                         type: "string",
-                        description: "The complete query to pass to Azure Bing search. You can use advanced search operators in your query: + to require terms, \" \" for exact phrases, () for grouping, AND/& for requiring all terms, NOT/- to exclude terms, OR/| for either term. For example: '+(exact phrase) AND term1 -term2'"
+                        description: "The complete query to pass to Azure Bing search using Bing's search syntax."
                     },
                     freshness: {
                         type: "string",

package/pathways/system/entity/tools/sys_tool_browser.js

@@ -11,7 +11,7 @@ export default {
         type: "function",
         icon: "🌍",
         function: {
-            name: "WebPageContent",
+            name: "FetchWebPageContent",
             description: "This tool allows you to fetch and extract the text content and a screenshot if requested from any webpage. Use this when you need to analyze or understand the content of a specific webpage.",
             parameters: {
                 type: "object",

package/pathways/system/entity/tools/sys_tool_browser_jina.js

@@ -11,7 +11,7 @@ export default {
         type: "function",
         icon: "🌎",
         function: {
-            name: "WebPageContentJina",
+            name: "FetchWebPageContentJina",
             description: "This tool allows you to fetch and extract the text content from any webpage using the Jina API. This is a great fallback for web page content if you don't get a good enough response from your other browser tool.",
             parameters: {
                 type: "object",

package/pathways/system/entity/tools/sys_tool_callmodel.js

@@ -24,7 +24,7 @@ export default {
         icon: "🤖",
         function: {
             name: "CallModel",
-            description: "Use when you need to call an AI model to get a response. This is typically used to perform some sort of LLM analysis (translate, summarize, ask questions about content, etc.), but can literally do anything you need. You can use this to call any model you have access to and perform any task.",
+            description: "Use when you need to call an AI model to get a response. This is typically used to perform some sort of custom LLM analysis (translate, summarize, ask questions about content, etc.), but can literally do anything you need. You can use this to call any model you have access to and perform any task.",
             parameters: {
                 type: "object",
                 properties: {

package/pathways/system/entity/tools/sys_tool_codingagent.js

@@ -48,7 +48,7 @@ export default {
         icon: "🤖",
         function: {
             name: "CodeExecution",
-            description: "Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks - examples include data analysis, file manipulation, or other tasks that require code execution. This will start a background task and return - you will not receive the response immediately.",
+            description: "This tool allows you to engage an agent to write and execute code to perform a task on your behalf. Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks - examples include data analysis, file manipulation, or other tasks that require code execution. With this tool you can also access internal databases and query them directly. This will start a background task and return - you will not receive the response immediately.",
             parameters: {
                 type: "object",
                 properties: {

package/pathways/system/entity/tools/sys_tool_cognitive_search.js

@@ -23,14 +23,14 @@ export default {
             type: "function",
             icon: "📂",
             function: {
-                name: "SearchPersonal",
+                name: "SearchPersonalIndex",
                 description: "Search through the user's index of personal documents and indexed uploaded files and retrieve the content of the files. Use this tool if the user refers to a file or a document that you don't see uploaded elsewhere in your context. Some file types (e.g. Word documents, Excel documents, very large files, etc.) cannot be attached to a message and will be chunked and indexed and stored in the personal index.",
                 parameters: {
                     type: "object",
                     properties: {
                         text: {
                             type: "string",
-                            description: "The search query to find relevant content in personal documents. Can be a specific phrase or '*' for all documents."
+                            description: "The search query to find relevant content in personal documents. Can be a specific phrase or '*' for all documents, or a query formatted with AI Search syntax."
                         },
                         filter: {
                             type: "string",
@@ -42,7 +42,7 @@ export default {
                         },
                         titleOnly: {
                             type: "boolean",
-                            description: "If true, only return document titles without content"
+                            description: "If true, only return document titles without content - faster and great for counting results"
                         },
                         userMessage: {
                             type: "string",
@@ -58,13 +58,13 @@ export default {
             icon: "📰",
             function: {
                 name: "SearchAJA",
-                description: "Search Al Jazeera Arabic news articles. Use this for finding Arabic news content including the latest news and articles.",
+                description: "Search Al Jazeera Arabic news articles. Use this for finding Arabic news content including the latest news and articles. Make sure to include a date filter when looking for recent articles.",
                 parameters: {
                     type: "object",
                     properties: {
                         text: {
                             type: "string",
-                            description: "The search query in Arabic to find relevant news articles. Can be a specific phrase or '*' for all articles."
+                            description: "The search query in Arabic to find relevant news articles. Can be a specific phrase or '*' for all articles, or a query formatted with AI Search syntax."
                         },
                         filter: {
                             type: "string",
@@ -76,7 +76,7 @@ export default {
                         },
                         titleOnly: {
                             type: "boolean",
-                            description: "If true, only return article titles without content"
+                            description: "If true, only return article titles without content - faster and great for counting results"
                         },
                         userMessage: {
                             type: "string",
@@ -92,13 +92,13 @@ export default {
             icon: "📰",
             function: {
                 name: "SearchAJE",
-                description: "Search Al Jazeera English news articles. Use this for finding English news content including the latest news and articles.",
+                description: "Search Al Jazeera English news articles. Use this for finding English news content including the latest news and articles. Make sure to include a date filter when looking for recent articles.",
                 parameters: {
                     type: "object",
                     properties: {
                         text: {
                             type: "string",
-                            description: "The search query in English to find relevant news articles. Can be a specific phrase or '*' for all articles."
+                            description: "The search query in English to find relevant news articles. Can be a specific phrase or '*' for all articles, or a query formatted with AI Search syntax."
                         },
                         filter: {
                             type: "string",
@@ -110,7 +110,7 @@ export default {
                         },
                         titleOnly: {
                             type: "boolean",
-                            description: "If true, only return article titles without content"
+                            description: "If true, only return article titles without content - faster and great for counting results"
                         },
                         userMessage: {
                             type: "string",
@@ -126,13 +126,13 @@ export default {
             icon: "⚡️",
             function: {
                 name: "SearchWires",
-                description: "Search through news wires from Reuters, AFP, AP, and other news agencies. Use this for finding the latest news and articles from the wires.",
+                description: "Search news wires from Reuters, AFP, AP, and other news agencies. Use this for finding the latest news and articles from the wires. Make sure to include a date filter when looking for recent articles.",
                 parameters: {
                     type: "object",
                     properties: {
                         text: {
                             type: "string",
-                            description: "The search query to find relevant news wires. Can be a specific phrase or '*' for all wires."
+                            description: "The search query to find relevant news wires. Can be a specific phrase or '*' for all wires, or a query formatted with AI Search syntax."
                         },
                         filter: {
                             type: "string",
@@ -144,7 +144,7 @@ export default {
                         },
                         titleOnly: {
                             type: "boolean",
-                            description: "If true, only return wire titles without content"
+                            description: "If true, only return wire titles without content - faster and great for counting results"
                         },
                         userMessage: {
                             type: "string",
@@ -162,7 +162,7 @@ export default {
 
         // Map tool names to index names
         const toolToIndex = {
-            'searchpersonal': 'indexcortex',
+            'searchpersonalindex': 'indexcortex',
             'searchaja': 'indexucmsaja',
             'searchaje': 'indexucmsaje',
             'searchwires': 'indexwires'

package/pathways/system/entity/tools/sys_tool_image.js

@@ -14,8 +14,8 @@ export default {
         type: "function",
         icon: "🎨",
         function: {
-            name: "Image",
-            description: "Use when asked to create, generate, or revise visual content. Any time the user asks you for a picture, a selfie, artwork, a drawing or if you want to illustrate something for the user, you can use this tool to generate any sort of image from cartoon to photo realistic.",
+            name: "GenerateImage",
+            description: "Use when asked to create, generate, or generate revisions of visual content. Any time the user asks you for a picture, a selfie, artwork, a drawing or if you want to illustrate something for the user, you can use this tool to generate any sort of image from cartoon to photo realistic.",
             parameters: {
                 type: "object",
                 properties: {

package/pathways/system/entity/tools/sys_tool_readfile.js

@@ -26,8 +26,8 @@ export default {
         type: "function",
         icon: "📄",
         function: {
-            name: "PDF",
-            description: "Use specifically for analyzing and answering questions about PDF file content. Do not use this tool for analyzing and answering questions about other file types.",
+            name: "AnalyzePDF",
+            description: "Use specifically for reading, analyzing, and answering questions about PDF file content. Do not use this tool for analyzing and answering questions about other file types.",
             parameters: {
                 type: "object",
                 properties: {
@@ -48,8 +48,8 @@ export default {
         type: "function",
         icon: "📝",
         function: {
-            name: "Text",
-            description: "Use specifically for analyzing and answering questions about text and csv files.",
+            name: "AnalyzeText",
+            description: "Use specifically for reading, analyzing, and answering questions about text and csv files.",
             parameters: {
                 type: "object",
                 properties: {
@@ -70,8 +70,8 @@ export default {
         type: "function",
         icon: "🖼️",
         function: {
-            name: "Vision",
-            description: "Use specifically for analyzing and answering questions about image files (jpg, gif, bmp, png, etc).",
+            name: "AnalyzeImage",
+            description: "Use specifically for reading, analyzing, and answering questions about image files (jpg, gif, bmp, png, etc). This cannot be used for creating or transforming images.",
             parameters: {
                 type: "object",
                 properties: {
@@ -92,8 +92,8 @@ export default {
         type: "function",
         icon: "🎥",
         function: {
-            name: "Video",
-            description: "Use specifically for analyzing and answering questions about video or audio file content. You MUST use this tool to look at video or audio files.",
+            name: "AnalyzeVideo",
+            description: "Use specifically for reading, analyzing, and answering questions about video or audio file content. You MUST use this tool to look at video or audio files.",
             parameters: {
                 type: "object",
                 properties: {

package/pathways/system/entity/tools/sys_tool_reasoning.js

@@ -26,7 +26,7 @@ export default {
         type: "function",
         icon: "🗺️",
         function: {
-            name: "Plan",
+            name: "PlanMultiStepTask",
             description: "Use specifically to create a thorough, well thought out, step by step plan to accomplish a task. You should always use this tool when you're planning to do something complex or something that might require multiple steps.",
             parameters: {
                 type: "object",
@@ -48,7 +48,7 @@ export default {
         type: "function",
         icon: "🧠",
         function: {
-            name: "Reason",
+            name: "ApplyAdvancedReasoning",
             description: "Employ for advanced reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices.",
             parameters: {
                 type: "object",

package/pathways/system/entity/tools/sys_tool_remember.js

@@ -12,29 +12,7 @@ export default {
         icon: "🧩",
         function: {
             name: "SearchMemory",
-            description: "Use this tool to search your memory and remember information or details that may not be present in your short term or contextual memory. You should always use this tool before you answer questions about remembered information. It's critical that you never fabricate memories.",
-            parameters: {
-                type: "object",
-                properties: {
-                    detailedInstructions: {
-                        type: "string",
-                        description: "Detailed description of what you want to see if you remember"
-                    },
-                    userMessage: {
-                        type: "string",
-                        description: "A user-friendly message that describes what you're doing with this tool"
-                    }
-                },
-                required: ["detailedInstructions", "userMessage"]
-            }
-        }
-    },
-    {
-        type: "function",
-        icon: "🧩",
-        function: {
-            name: "Remember",
-            description: "When the user asks you if you remember something, you must use this tool before you answer. It's critical that you never fabricate memories.",
+            description: "Use this tool to search your memory and retrieve information or details stored in your memory. Use any time the user asks you about something personal or asks you to remember something.",
             parameters: {
                 type: "object",
                 properties: {

package/pathways/system/entity/tools/sys_tool_verify.js

@@ -27,7 +27,7 @@ export default {
         type: "function",
         icon: "🔍",
         function: {
-            name: "Verify",
+            name: "VerifyResponse",
             description: "Critically evaluate a response and provide verification questions and follow-up actions. You should call this tool and execute the steps provided in the resulting verification plan before presenting a final response to the user to ensure it is accurate, complete, and consistent with the user's request.",
             parameters: {
                 type: "object",

package/pathways/system/sys_parse_numbered_object_list.js

@@ -4,7 +4,7 @@ export default {
     prompt: [
         new Prompt({
             messages: [
-                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert each list item into a JSON object with the given fields. If a field value is numeric, it should be returned as a number in the JSON object. If there are extra fields, assistant will ignore them. If a list item doesn't contain all fields, assistant will return the fields that are present and skip the missing fields. If the conversion is not at all possible, assistant will return an empty JSON array. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
+                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert the list into a JSON array named 'list' of objects. Each list item is converted into an array element object with the given fields. If a field value is numeric, it should be returned as a number in the object. If there are extra fields, assistant will ignore them. If a list item doesn't contain all fields, assistant will return the fields that are present and skip the missing fields. If the conversion is not at all possible, assistant will return an empty JSON array {list:[]}.\n\nExample: {list:[{field1: \"value1\", field2: \"value2\"}, {field1: \"value3\", field2: \"value4\"}]}"},
                 { "role": "user", "content": `Fields: {{{format}}}\nList: {{{text}}}`},
             ]
         })

package/server/parser.js

@@ -16,7 +16,7 @@ const parseNumberedList = (str) => {
 async function parseNumberedObjectList(text, format) {
     const parsedList = await callPathway('sys_parse_numbered_object_list', { text, format });
     try {
-        return JSON.parse(parsedList) || [];
+        return JSON.parse(parsedList).list || [];
     } catch (error) {
         logger.warn(`Failed to parse numbered object list: ${error.message}`);
         return [];

package/server/plugins/azureCognitivePlugin.js

@@ -168,6 +168,11 @@ class AzureCognitivePlugin extends ModelPlugin {
             }
         }
 
+        // Add date-based ordering if there's a date filter
+        if (data.filter && data.filter.includes('date')) {
+            data.orderby = 'date desc';
+        }
+
         return { data };
     }
 

package/server/plugins/openAiVisionPlugin.js

@@ -152,9 +152,7 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
 
         requestParameters.max_tokens = maxTokens ? Math.min(maxTokens, modelMaxReturnTokens) : modelMaxReturnTokens;
 
-        if (this.promptParameters.json) {
-            //requestParameters.response_format = { type: "json_object", }
-        }
+        this.promptParameters.responseFormat && (requestParameters.response_format = this.promptParameters.responseFormat);
 
         return requestParameters;
     }