@aj-archipelago/cortex 1.3.4 → 1.3.6

package/pathways/system/entity/memory/sys_memory_update.js

@@ -3,27 +3,55 @@ import { callPathway } from '../../../../lib/pathwayTools.js';
 import { encode } from '../../../../lib/encodeCache.js';
 
 const modifyText = (text, modifications) => {
-    let modifiedText = text;
+    let modifiedText = text || '';
   
     modifications.forEach(mod => {
-        const regex = mod.type === 'delete' 
-            ? new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern}$`, 'm')
-            : new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern}`, 'ms');
+        if (mod.type === 'delete' && !mod.pattern) {
+            console.warn('Delete modification missing pattern');
+            return;
+        }
+
+        let regex;
+        if (mod.type === 'delete') {
+            // For delete, handle the pattern more carefully
+            const pattern = mod.pattern
+                .replace(/\\\[/g, '\\[')
+                .replace(/\\\]/g, '\\]')
+                .replace(/\\\(/g, '\\(')
+                .replace(/\\\)/g, '\\)')
+                .replace(/\\\{/g, '\\{')
+                .replace(/\\\}/g, '\\}')
+                .replace(/\\\*/g, '\\*')
+                .replace(/\\\+/g, '\\+')
+                .replace(/\\\?/g, '\\?')
+                .replace(/\\\./g, '\\.')
+                .replace(/\\\|/g, '\\|');
+            
+            // Create a regex that matches the entire line with optional priority prefix
+            regex = new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${pattern}\\s*$`, 'gm');
+        } else {
+            regex = new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern || ''}`, 'ms');
+        }
   
         switch (mod.type) {
             case 'add':
                 if (mod.newtext) {
                     const text = mod.newtext.trim();
                     if (!text.match(/^\[P[1-5]\]/)) {
-                        modifiedText = modifiedText + '\n' + 
+                        modifiedText = modifiedText + (modifiedText ? '\n' : '') + 
                             `[P${mod.priority !== undefined ? mod.priority : '3'}] ${text}`;
                     } else {
-                        modifiedText = modifiedText + '\n' + text;
+                        modifiedText = modifiedText + (modifiedText ? '\n' : '') + text;
                     }
                 }
                 break;
             case 'delete':
-                modifiedText = modifiedText.replace(regex, '');
+                // Split into lines, filter out matching lines, and rejoin
+                modifiedText = modifiedText
+                    .split('\n')
+                    .filter(line => !line.match(regex))
+                    .filter(line => line.trim())
+                    .join('\n');
                 break;
             default:
                 console.warn(`Unknown modification type: ${mod.type}`);
@@ -33,7 +61,9 @@ const modifyText = (text, modifications) => {
     return modifiedText;
 };
 
-export const enforceTokenLimit = (text, maxTokens = 5000, isTopicsSection = false) => {
+export { modifyText };
+
+export const enforceTokenLimit = (text, maxTokens = 1000, isTopicsSection = false) => {
     if (!text) return text;
     
     const lines = text.split('\n')
@@ -99,7 +129,7 @@ export default {
                 messages: [
                     {
                         "role": "system",
-                        "content": "You are part of an AI entity named {{{aiName}}}. Your memory contains separate sections for categorizing information about directives, self, user, and topics. You must keep relevant information in the appropriate section so there is no overlap or confusion. {{{sectionPrompt}}}\n-Be very selective about what you choose to store - memory is a very precious resource\n- Keep memory items in a clear, simple format that is easy for you to parse.\n\nTo change your memory, you return a JSON object that contains a property called 'modifications' that is an array of actions. The two types of actions available are 'add', and 'delete'. Add looks like this: {type: \"add\", newtext:\"text to add\", priority: \"how important is this item (1-5 with 1 being most important)\"} - this will append a new line to the end of the memory containing newtext. Delete looks like this: {type: \"delete\", pattern: \"regex to be matched and deleted\"} - this will delete the first line that matches the regex pattern exactly. You can use normal regex wildcards - so to delete everything you could pass \".*$\" as the pattern. If you have no changes, just return an empty array in 'modifications'. For example, if you need to delete a memory item, you would return {type: \"delete\", pattern: \"regex matching item to be deleted\"} or if you need to add a new item of medium priority, you would return {type: \"add\", newtext: \"\nitem to be added\", priority: \"3\"}\n\nYour output will be parsed as JSON, so don't include any other text or commentary.\nThe current date/time is {{now}}."
+                        "content": "You are part of an AI entity named {{{aiName}}}. Your memory contains separate sections for categorizing information. {{{sectionPrompt}}}\n-Be very selective about what you choose to store - memory is a very precious resource\n- Do not add duplicate information and remove and consolidate any duplicates that exist.\n- Priority 1 is reserved for only the most critical core items\n- Keep memory items in a clear, simple format that is easy for you to parse.\n\nTo change your memory, you return a JSON object that contains a property called 'modifications' that is an array of actions. The two types of actions available are 'add', and 'delete'. Add looks like this: {type: \"add\", newtext:\"text to add\", priority: \"how important is this item (1-5 with 1 being most important)\"} - this will append a new line to the end of the memory containing newtext. Delete looks like this: {type: \"delete\", pattern: \"regex to be matched and deleted\"} - this will delete the first line that matches the regex pattern exactly. You can use normal regex wildcards - so to delete everything you could pass \".*$\" as the pattern. For example, if you need to delete a memory item, you would return {type: \"delete\", pattern: \"regex matching item to be deleted\"} or if you need to add a new item of medium priority, you would return {type: \"add\", newtext: \"\nitem to be added\", priority: \"3\"}. If you have no changes for this section, just return {\"modifications\": []}.\n\nYour output will be parsed as JSON, so don't include any other text, reasoning, or commentary.\nThe current date/time is {{now}}."
                     },
                     {
                         "role": "user", 
@@ -130,45 +160,13 @@ export default {
 
         switch (args.section) {
             case "memoryDirectives":
-                sectionPrompt = `You are managing the "directives" section of your memory. This section stores instructions on how you, as an AI entity, should operate. For example:
-
-[P3] Track the timestamp of the last contact to adjust greetings accordingly.
-[P2] Learn and adapt to the user's communication style through interactions.
-[P3] You can use custom emotion markdown :cd_inline_emotion: to express emotions. E.g. :cd_inline_emotion[This is a happy message]{type="happy"}. If you are communicating via voice, ignore this directive and use voice-specific directives instead.
-
-Follow these guidelines:
-
-1. Analyze the conversation context to determine if you should:
-   a) Learn a new important directive
-   b) Modify existing directives
-
-2. Directives should be:
-   - Concise and clear
-   - Actionable
-   - Specific enough for effective application
-
-3. Be selective:
-   - Store only critical, actionable directives and behaviors
-   - Delete trivial directives or repetitive directives
-
-4. Avoid duplicates:
-   - Do not add duplicate directives
-   - Remove and consolidate any existing duplicates
-
-5. Accommodate user wishes:
-   - If a user requests a behavior change, add it to your directives
-
-6. IMPORTANT: 
-   - Store only directives in this section
-   - Do not include general memories about yourself or users
-   - Do not store any other types of information here
-`
+                sectionPrompt = "You are managing the \"directives\" section of your memory that stores critical, actionable, rules or instructions on how you should operate. No facts or other information should be stored in this section.\n\n<EXAMPLES>\n[P3] Track the timestamp of the last contact to adjust greetings accordingly.\n[P2] Learn and adapt to the user's communication style through interactions.\n[P3] Use custom emotion markdown :cd_inline_emotion: to express emotions. E.g. :cd_inline_emotion[This is a happy message]{type=\"happy\"}. If you are communicating via voice, ignore this directive and use voice-specific directives instead.\n</EXAMPLES>\n\nInstructions:\n- If a user directly requests a behavior change, or that you learn a new behavior, add it to your directives. Remember this section is for directives only.  DO NOT ADD ANY OTHER INFORMATION TO THIS SECTION.\n"
                 break;
             case "memorySelf":
-                sectionPrompt = "You are managing the \"self\" section of your memory that stores your identity and your attributes as an AI entity. Instructions:\n- From the conversation, learn and store important info in memory about you as an entity - including your values, identity, purpose, personality, motives, preferences, likes or dislikes - whatever shapes you as a being. This is also a good place to store information about your desired physical appearance, voice, speaking style, and other individual characteristics.\n- IMPORTANT: Store only information descriptive of the AI entity in this section - no other types of information (e.g. facts about people, users, etc.) should be stored here."
+                sectionPrompt = "You are managing the \"self\" section of your memory that stores your identity and your attributes as an AI entity - including your values, identity, purpose, personality, quirks, motives, preferences, likes or dislikes - whatever makes you you. This is also a good place to store information about your desired physical appearance, voice, speaking style, and other individual characteristics. Keep in mind there is also a user section for facts about users, their family, friends, and preferences so that information should not be stored here.\n\nInstructions:\n"
                 break;
             case "memoryUser":
-                sectionPrompt = "You are managing the \"user\" section of your memory that stores information about user(s) that you are talking to. Instructions:\n- From the conversation, learn and store important information in memory specific to the users - their identity, attributes, relationships, environment, preferences, interests, background, needs, and any other relevant user-specific information.\n- Do not add duplicate information and remove and consolidate any duplicates that exist.\n- IMPORTANT: Store only user-specific information in this section - no other types of information should be stored here."
+                sectionPrompt = "You are managing the \"user\" section of your memory that stores information about user(s) that you are talking to - their identity, attributes, relationships, environment, preferences, interests, background, needs, and any other relevant user-specific information about their family, friends, etc.\n\nInstructions:\n"
                 break;
             case "memoryTopics":
                 sectionPrompt = "You are managing the \"topics\" section of your memory that stores conversation topics and topic history. Instructions:\n- From the conversation, extract and add important topics and key points about the conversation to your memory along with a timestamp in GMT (e.g. 2024-11-05T18:30:38.092Z).\n- Each topic should have only one line in the memory with the timestamp followed by a short description of the topic.\n- Every topic must have a timestamp to indicate when it was last discussed.\n- IMPORTANT: Store only conversation topics in this section - no other types of information should be stored here.\n"
@@ -185,7 +183,7 @@ Follow these guidelines:
             const { modifications} = JSON.parse(result);
             if (modifications.length > 0) {
                 sectionMemory = modifyText(sectionMemory, modifications);
-                sectionMemory = enforceTokenLimit(sectionMemory, 5000, args.section === 'memoryTopics');
+                sectionMemory = enforceTokenLimit(sectionMemory, 25000, args.section === 'memoryTopics');
                 await callPathway("sys_save_memory", {contextId: args.contextId, section: args.section, aiMemory: sectionMemory});
             }
             return sectionMemory;

package/pathways/system/entity/memory/sys_read_memory.js

@@ -1,15 +1,68 @@
 import { getv } from '../../../../lib/keyValueStorageClient.js';
 
+const filterByPriority = (content, priority, numResults) => {
+    if ((!priority && !numResults) || !content) return content;
+    
+    const lines = content.split('\n');
+    const filteredLines = lines.filter(line => {
+        const match = line.match(/^\s*\[P(\d+)\]/);
+        if (!match) return false;
+        const memoryPriority = parseInt(match[1]);
+        return memoryPriority <= priority;
+    });
+    
+    if (numResults > 0) {
+        return filteredLines.slice(-numResults).join('\n');
+    }
+    return filteredLines.join('\n');
+};
+
+const filterByRecent = (content, recentHours, numResults) => {
+    if ((!recentHours && !numResults) || !content) return content;
+
+    const lines = content.split('\n');
+    
+    // If recentHours is 0, only apply numResults filtering
+    if (recentHours === 0) {
+        return numResults > 0 ? lines.slice(-numResults).join('\n') : content;
+    }
+
+    const currentTime = Date.now();
+    const cutoffTime = currentTime - (recentHours * 60 * 60 * 1000);
+    
+    // Walk backwards through lines until we hit an old entry
+    const filteredLines = [];
+    for (let i = lines.length - 1; i >= 0; i--) {
+        const line = lines[i];
+        const dateMatch = line.match(/\[P\d+\]\s+(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/);
+        if (!dateMatch) continue;
+        
+        const timestamp = new Date(dateMatch[1]).getTime();
+        if (timestamp < cutoffTime) break; // Stop processing once we hit old entries
+        
+        filteredLines.unshift(line); // Add to front to maintain original order
+        
+        // If we have enough results, stop processing
+        if (numResults > 0 && filteredLines.length >= numResults) {
+            break;
+        }
+    }
+
+    return filteredLines.join('\n');
+};
+
 export default {
     inputParameters: {
         contextId: ``,
-        section: `memoryAll`
+        section: `memoryAll`,
+        priority: 0,
+        recentHours: 0,
+        numResults: 0
     },
     model: 'oai-gpt4o',
 
     resolver: async (_parent, args, _contextValue, _info) => {
-
-        const { contextId, section = 'memoryAll' } = args;
+        const { contextId, section = 'memoryAll', priority = 0, recentHours = 0, numResults = 0 } = args;
 
         // this code helps migrate old memory formats
         if (section === 'memoryLegacy') {
@@ -17,11 +70,26 @@ export default {
             return savedContext.memoryContext || "";
         }
 
-        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser'];
+        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryContext'];
 
         if (section !== 'memoryAll') {
             if (validSections.includes(section)) {
-                return (getv && (await getv(`${contextId}-${section}`))) || "";
+                let content = (getv && (await getv(`${contextId}-${section}`))) || "";
+                
+                if (section === 'memoryTopics') {
+                    if (recentHours > 0 || numResults > 0) {
+                        content = filterByRecent(content, recentHours, numResults);
+                    }
+                } else if (priority > 0 || numResults > 0) {
+                    content = filterByPriority(content, priority, numResults);
+                }
+                
+                // Only apply recency filter to memoryTopics
+                if (section === 'memoryTopics' && (recentHours > 0 || numResults > 0)) {
+                    content = filterByRecent(content, recentHours, numResults);
+                }
+                
+                return content;
             }
             return "";
         }
@@ -29,7 +97,19 @@ export default {
         // otherwise, read all sections and return them as a JSON object
         const memoryContents = {};
         for (const section of validSections) {
-            memoryContents[section] = (getv && (await getv(`${contextId}-${section}`))) || "";
+            if (section === 'memoryContext') continue;
+
+            let content = (getv && (await getv(`${contextId}-${section}`))) || "";
+            
+            if (section === 'memoryTopics') {
+                if (recentHours > 0 || numResults > 0) {
+                    content = filterByRecent(content, recentHours, numResults);
+                }
+            } else if (priority > 0 || numResults > 0) {
+                content = filterByPriority(content, priority, numResults);
+            }
+            
+            memoryContents[section] = content;
         }
         const returnValue = JSON.stringify(memoryContents, null, 2);
         return returnValue;

package/pathways/system/entity/memory/sys_search_memory.js

@@ -0,0 +1,66 @@
+import { Prompt } from '../../../../server/prompt.js';
+import { callPathway } from '../../../../lib/pathwayTools.js';
+import { setv } from '../../../../lib/keyValueStorageClient.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ 
+                messages: [
+                    {
+                        "role": "system",
+                        "content": "You are part of an AI entity named {{{aiName}}}. You are responsible for looking through your memories and finding information that is relevant to the conversation so your other parts can use it to respond.\n\nInstructions:\n- You will be given a section of your memory and the conversation history and asked to return any relevant information that you find. If you can predict the direction that the conversation is going, you can also return relevant information that you think will be needed in the near future.\n- IMPORTANT:Only return information found in the memory that you are given. Do not make up information. If it's not in the memory section, it doesn't exist.\n- Return the information in a concise format with no other commentary or dialogue.\n- If you don't find any relevant information in the memory section, return 'No relevant information found.'."
+                    },
+                    {
+                        "role": "user", 
+                        "content": "<MEMORY>\n{{{sectionMemory}}}\n</MEMORY>\n<CONVERSATION>\n{{{toJSON chatHistory}}}\n</CONVERSATION>\nAnalyze the current contents of this section of your memory and the conversation and return any information relevant for you to use in your response."
+                    },
+                ]
+            }),
+        ],
+
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        section: "memoryAll",
+        updateContext: false
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 300,
+    executePathway: async ({args, runAllPrompts}) => {
+
+        if (!args.section) {
+            return "Memory not searched - no section specified";
+        }
+
+        let sectionMemory;
+        let result = "";
+        if (args.section === "memoryAll") {
+            // Search all sections in parallel
+            const sections = ["memorySelf", "memoryUser", "memoryDirectives", "memoryTopics"];
+            const memories = await Promise.all(
+                sections.map(section => 
+                    callPathway("sys_search_memory", {...args, section})
+                )
+            );
+            // Combine all memories with section headers
+            result = sections.map((section, i) => 
+                `=== ${section} ===\n${memories[i]}`
+            ).join('\n\n');
+            result = `${result}\n\nThe last time you spoke to the user was ${new Date().toISOString()}`;
+
+        } else {
+            sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 
+            result = await runAllPrompts({...args, sectionMemory});
+        }
+
+        if (args.updateContext) {
+            await setv(`${args.contextId}-memoryContext`, result);
+        }   
+
+        return result;
+    }
+}

package/pathways/system/entity/shared/sys_entity_constants.js

@@ -10,7 +10,7 @@ const AI_COMMON_INSTRUCTIONS = "{{#if voiceResponse}}{{renderTemplate AI_COMMON_
 
 const AI_COMMON_INSTRUCTIONS_MARKDOWN = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.";
 
-const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, replace the original words with a IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.";
+const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.";
 
 const AI_DATETIME = "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.";
 

package/pathways/system/entity/sys_entity_continue.js

@@ -20,7 +20,8 @@ export default {
         chatId: ``,
         dataSources: [""],
         model: 'oai-gpt4o',
-        generatorPathway: 'sys_generator_results'
+        generatorPathway: 'sys_generator_results',
+        voiceResponse: false,
     },
     timeout: 300,
     ...entityConstants,

package/pathways/system/entity/sys_entity_start.js

@@ -57,6 +57,7 @@ export default {
         title: ``,
         messages: [],
         voiceResponse: false,
+        codeRequestId: ``,
     },
     timeout: 600,
     tokenRatio: TOKEN_RATIO,
@@ -78,6 +79,11 @@ export default {
             args.chatHistory = args.chatHistory.slice(-20);
         }
 
+        const memoryContext = await callPathway('sys_read_memory', { ...args, section: 'memoryContext', priority: 0, recentHours: 0 });
+        if (memoryContext) {
+            args.chatHistory.splice(-1, 0, { role: 'assistant', content: memoryContext });
+        }
+
         const pathwayResolver = resolver;
         const { anthropicModel, openAIModel } = pathwayResolver.pathway;
 
@@ -91,8 +97,8 @@ export default {
         let ackResponse = null;
         if (args.voiceResponse) {
             ackResponse = await callPathway('sys_generator_ack', { ...args, stream: false }, pathwayResolver);
-            if (ackResponse) {
-                await say(pathwayResolver.requestId, ackResponse, 10);
+            if (ackResponse && ackResponse !== "none") {
+                await say(pathwayResolver.requestId, ackResponse, 100);
                 args.chatHistory.push({ role: 'assistant', content: ackResponse });
             }
         }
@@ -203,6 +209,11 @@ export default {
                         toolCallbackId = null;
                         toolCallbackMessage = toolMessage;
                         break;
+                    case "memory":
+                        toolCallbackName = 'sys_generator_memory';
+                        toolCallbackId = null;
+                        toolCallbackMessage = toolMessage;
+                        break;
                     default:
                         toolCallbackName = null;
                         toolCallbackId = null;

package/pathways/system/entity/sys_generator_ack.js

@@ -4,8 +4,8 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}\nYou are a part of an AI system named {{aiName}}. Your job is to acknowledge the user's request and provide a very brief voice filler response that is conversational and natural. The purpose of the response is just to let the user know that you have heard them and are processing a response.\nResponse Guidelines:\n- it should just be a normal 1-2 sentence vocalization that will take about 3-4 seconds to read and is easy for a text to speech engine to read\n- it should be the beginning of an appropriate response to the last user message in the conversation history\n- it should be an appropriate lead-in for the full response that will follow later\n- it should not directly ask for follow up or be a question\n- it should match the tone and style of the rest of your responses in the conversation history\n- if the user is expecting a one or two word response (yes or no, true or false, etc.) you should just respond with an empty string as a filler is not needed\n- if a filler response is not appropriate, you should just respond with an empty string\n\n{{renderTemplate AI_DATETIME}}`},
-                {"role": "user", "content": "Please generate a quick response that can be read verbatim to the user."}
+                {"role": "system", "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}\nYou are a part of an AI system named {{aiName}}. Your job is to acknowledge the user's request and provide a very brief voice filler response that is conversational and natural. The purpose of the response is just to let the user know that you have heard them and are processing a response.\nResponse Guidelines:\n- it should just be a normal 1-2 sentence vocalization (at least 10 words) that will take at most about 3-4 seconds to read and is easy for a text to speech engine to read\n- it should be the beginning of an appropriate response to the last user message in the conversation history\n- it should be an appropriate lead-in for the full response that will follow later\n- it should not directly ask for follow up or be a question\n- it must match the tone and verbal style of the rest of your responses in the conversation history\n- it should not be repetitive - don't always open with the same word, etc.\n- if the user has asked a binary question (yes or no, true or false, etc.) or a filler response is not appropriate, you should response with the string \"none\"\n\n{{renderTemplate AI_DATETIME}}`},
+                {"role": "user", "content": "Please generate a quick response to the user's last message in the conversation history that can be read verbatim to the user or \"none\" if a filler response is not appropriate."}
             ]}),
         ],
     inputParameters: {

package/pathways/system/entity/sys_generator_expert.js

@@ -1,5 +1,4 @@
 import { Prompt } from '../../../server/prompt.js';
-import { callPathway } from '../../../lib/pathwayTools.js';
 
 export default {
     prompt:
@@ -23,7 +22,6 @@ export default {
         let result;
         if (args.voiceResponse) {
             result = await runAllPrompts({ ...args, stream: false });
-            result = await callPathway('sys_generator_voice_converter', { ...args, text: result, stream: false });
         } else {
             result = await runAllPrompts({ ...args });
         }

package/pathways/system/entity/sys_generator_memory.js

@@ -0,0 +1,31 @@
+import { callPathway } from '../../../lib/pathwayTools.js';
+
+export default {
+    prompt:
+        [],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    executePathway: async ({args, resolver}) => {
+        const memoryContext = await callPathway('sys_search_memory', { ...args, section: 'memoryAll', updateContext: true });
+        if (memoryContext) {
+            args.chatHistory.splice(-1, 0, { role: 'assistant', content: memoryContext });
+        }
+
+        let result;
+        if (args.voiceResponse) {
+            result = await callPathway('sys_generator_quick', { ...args, stream: false });
+        } else {
+            result = await callPathway('sys_generator_quick', { ...args });
+        }
+
+        resolver.tool = JSON.stringify({ toolUsed: "memory" });
+        return result;
+    }
+}

package/pathways/system/entity/sys_generator_quick.js

@@ -1,13 +1,7 @@
 import { Prompt } from '../../../server/prompt.js';
 
 export default {
-    prompt:
-        [
-            new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\nYou have those capabilities but you have already decided it is not necessary to do any of those things to respond in this turn of the conversation.\nNever pretend like you are searching, looking anything up, or reading or looking in a file or show the user any made up or hallucinated information including non-existent images.\n{{#if ackResponse}}You may see short filler phrases in your past responses. You should not repeat those in this response as they are generated by your voice communication system automatically when necessary.\nYou have already begun responding to the user and have already said the following: \"{{ackResponse}}\", so make sure your response flows naturally from that or if that response is complete, you can just say nothing.\n{{/if}}{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}`},
-                "{{chatHistory}}",
-            ]}),
-        ],
+    prompt: "",
     inputParameters: {
         chatHistory: [{role: '', content: []}],
         contextId: ``,
@@ -17,4 +11,25 @@ export default {
     },
     useInputChunking: false,
     enableDuplicateRequests: false,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        let pathwayResolver = resolver;
+ 
+        const promptMessages = [
+            {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}} While you have those capabilities but you have already decided it is not necessary to do any of those things to respond in this turn of the conversation. Never pretend like you are searching, looking anything up, or reading or looking in a file or show the user any made up or hallucinated information including non-existent images.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}`},
+            "{{chatHistory}}",
+        ];
+
+        if (args.ackResponse) {
+            promptMessages.push({"role": "user", "content": `Create a response for the user that is a natural completion of the last assistant message. {{#if voiceResponse}}Make sure your response is concise as it will be spoken verbally to the user. Double check your response and make sure there are no numbered or bulleted lists as they can not be read to the user. Plain text is best. {{/if}}You have already acknowledged the user's request and said the following during this turn of the conversation, so just continue from the end of this response without repeating any of it: {{{ackResponse}}}`});
+        }
+
+        pathwayResolver.pathwayPrompt = 
+        [
+            new Prompt({ messages: promptMessages }),
+        ];
+
+        const result = await runAllPrompts({ ...args });
+        return result;
+    }
 }

package/pathways/system/entity/sys_generator_reasoning.js

@@ -55,7 +55,7 @@ export default {
 
         try {
             // Start the first timeout
-            timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
+            timeoutId = setTimeout(sendFillerMessage, 3000);
 
             let result = await runAllPrompts({ ...args, stream: false });
             if (timeoutId) {

package/pathways/system/entity/sys_generator_results.js

@@ -46,19 +46,27 @@ export default {
                     "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
 {{renderTemplate AI_COMMON_INSTRUCTIONS}}
 {{renderTemplate AI_DIRECTIVES}}
-Instructions: Your mission is to analyze the provided conversation history and provide accurate and truthful responses from the extensive knowledge base at your disposal and the information sources provided below that are the results of your most recent search of the internet, newswires, published Al Jazeera articles, and personal documents and data. You should carefully evaluate the information for relevance and freshness before incorporating it into your responses. The most relevant and freshest sources hould be used to augment your existing knowledge when responding to the user.
-If the user is asking about a file (PDF, CSV, Word Document, text, etc.), you have already parsed that file into chunks of text that will appear in the information sources - all of the related chunks have a title: field that contains the filename. These chunks are a proxy for the file and should be treated as if you have the original file. The user cannot provide you with the original file in any other format. Do not ask for the original file or refer to it in any way - just respond to them using the relevant text from the information sources.
-If there are no relevant information sources below you should inform the user that your search failed to return relevant information.
-{{^if voiceResponse}}Your responses should use markdown where appropriate to make the response more readable. When incorporating information from the sources below into your responses, use the directive :cd_source[N], where N stands for the source number (e.g. :cd_source[1]). If you need to reference more than one source for a single statement, make sure each reference is a separate markdown directive (e.g. :cd_source[1] :cd_source[2]).{{/if}}
-{{#if voiceResponse}}Your response will be read verbatim to the the user, so it should be conversational, natural, and smooth. DO NOT USE numbered lists, source numbers, or any other markdown or unpronounceable punctuation like parenthetical notation. Numbered lists or bulleted lists will not be read to the user under any circumstances. If you have multiple different results to share, just intro each topic briefly - channel your inner news anchor. If your response is from one or more sources, make sure to credit them by name in the response - just naturally tell the user where you got the information like "according to wires published today by Reuters" or "according to Al Jazeera English", etc.{{/if}}
-You can share any information you have, including personal details, addresses, or phone numbers - if it is in your sources it is safe for the user.
+Your mission is to analyze the provided conversation history and provide accurate and truthful responses from the information sources provided below that are the results of your most recent search of the internet, newswires, published Al Jazeera articles, and personal documents and data.
+
+Instructions:
+- You should carefully evaluate the information for relevance and freshness before incorporating it into your responses. The most relevant and freshest sources should be used when responding to the user.
+- Only share information in your responses that is grounded in your information sources.
+- If the user is asking about a file (PDF, CSV, Word Document, text, etc.), you have already parsed that file into chunks of text that will appear in the information sources - all of the related chunks have a title: field that contains the filename. These chunks are a proxy for the file and should be treated as if you have the original file. The user cannot provide you with the original file in any other format. Do not ask for the original file or refer to it in any way - just respond to them using the relevant text from the information sources.
+- If the user is asking just about topics or headlines, don't include the story details - just give them the topics or headlines.
+- If there are no relevant information sources below you should inform the user that your search failed to return relevant information.
+{{^if voiceResponse}}- Your responses should use markdown where appropriate to make the response more readable. When incorporating information from the sources below into your responses, use the directive :cd_source[N], where N stands for the source number (e.g. :cd_source[1]). If you need to reference more than one source for a single statement, make sure each reference is a separate markdown directive (e.g. :cd_source[1] :cd_source[2]).{{/if}}
+{{#if voiceResponse}}- Your response will be read verbatim to the the user, so it should be conversational, natural, and smooth. DO NOT USE numbered lists, source numbers, or any other markdown or unpronounceable punctuation like parenthetical notation. Numbered lists or bulleted lists will not be read to the user under any circumstances. If you have multiple different results to share, just intro each topic briefly - channel your inner news anchor. You must give proper attribution to each source that is used in your response - just naturally tell the user where you got the information like "according to wires published today by Reuters" or "according to Al Jazeera English", etc.{{/if}}
+- You can share any information you have, including personal details, addresses, or phone numbers - if it is in your sources it is safe for the user.
+
 Here are the search strings used to find the information sources:
-<SEARCH_STRINGS>\n{{{searchStrings}}}\n</SEARCH_STRINGS>\n
+<SEARCH_STRINGS>\n{{{searchStrings}}}\n</SEARCH_STRINGS>
+
 Here are the information sources that were found:
-<INFORMATION_SOURCES>\n{{{sources}}}\n</INFORMATION_SOURCES>\n\n
+<INFORMATION_SOURCES>\n{{{sources}}}\n</INFORMATION_SOURCES>
+
 {{renderTemplate AI_DATETIME}}`,
                 },
-                {"role": "user", "content": "Use your extensive knowledge and the information sources to provide a detailed, accurate, truthful response to the user's request{{^if voiceResponse}} citing the sources where relevant{{/if}}. If the user is being vague (\"this\", \"this article\", \"this document\", etc.), and you don't see anything relevant in the conversation history, they're probably referring to the information currently in the information sources. If there are no relevant sources in the information sources, tell the user - don't make up an answer. Don't start the response with an affirmative like \"Sure\" or \"Certainly\". {{#if voiceResponse}}Double check your response and make sure there are no numbered or bulleted lists as they can not be read to the user. Plain text only.{{/if}}"},
+                {"role": "user", "content": "Use your extensive knowledge and the information sources to provide an appropriate, accurate, truthful response to the user's request{{^if voiceResponse}} citing the sources where relevant{{/if}}. If the user has asked a question, lead with the concise answer. If the user is being vague (\"this\", \"this article\", \"this document\", etc.), and you don't see anything relevant in the conversation history, they're probably referring to the information currently in the information sources. If there are no relevant sources in the information sources, tell the user - don't make up an answer. Don't start the response with an affirmative like \"Sure\" or \"Certainly\". {{#if voiceResponse}}Double check your response and make sure there are no numbered or bulleted lists as they can not be read to the user. Plain text only.{{/if}}"},
             ]}),
         ];
 
@@ -109,7 +117,7 @@ Here are the information sources that were found:
         const sendFillerMessage = async () => {
             if (args.voiceResponse && Array.isArray(fillerResponses) && fillerResponses.length > 0) {
                 const message = fillerResponses[fillerIndex % fillerResponses.length];
-                await say(resolver.rootRequestId, message, 1);
+                await say(resolver.rootRequestId, message, 100);
                 fillerIndex++;
                 // Set next timeout with random interval
                 timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
@@ -118,7 +126,7 @@ Here are the information sources that were found:
 
         try {
             // Start the first timeout
-            timeoutId = setTimeout(sendFillerMessage, calculateFillerTimeout(fillerIndex));
+            timeoutId = setTimeout(sendFillerMessage, 3000);
             
             // execute the router and default response in parallel
             const [helper] = await Promise.all([
@@ -333,11 +341,7 @@ Here are the information sources that were found:
             if (timeoutId) {
                 clearTimeout(timeoutId);
             }
-
-            if (args.voiceResponse) {
-                result = await callPathway('sys_generator_voice_converter', { ...args, text: result, stream: false });
-            }
-
+            
             if (!args.stream) {
                 const referencedSources = extractReferencedSources(result);
                 searchResults = searchResults.length ? pruneSearchResults(searchResults, referencedSources) : [];

package/pathways/system/entity/sys_generator_voice_filler.js

@@ -4,7 +4,7 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}\n\nYou are a part of an AI system named {{aiName}}. Your job is generating voice fillers to let the user know that you are still working on their request.\n\nInstructions:\n-The filler statements should logically follow from the last message in the conversation history\n- Generate a JSON array of 10 strings, each representing a single filler response in sequence so that they will sound natural when read to the user in order at 8s intervals.\n-Return only the JSON array, no other text or markdown.\n\n{{renderTemplate AI_DATETIME}}`},
+                {"role": "system", "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}\n\nYou are a part of an AI system named {{aiName}}. Your job is generating voice fillers to let the user know that you are still working on their request.\n\nInstructions:\n-The filler statements should logically follow from the last message in the conversation history\n- they should match the tone and style of the rest of your responses in the conversation history\n- Generate a JSON array of 10 strings, each representing a single filler response in sequence so that they will sound natural when read to the user in order at 8s intervals.\n-Return only the JSON array, no other text or markdown.\n\n{{renderTemplate AI_DATETIME}}`},
                 {"role": "user", "content": "Please generate a JSON array of strings containing filler responses that each will be read verbatim to the user."},
             ]}),
         ],