@aj-archipelago/cortex 1.3.5 → 1.3.7

package/pathways/system/entity/memory/sys_memory_update.js

@@ -1,29 +1,58 @@
 import { Prompt } from '../../../../server/prompt.js';
 import { callPathway } from '../../../../lib/pathwayTools.js';
 import { encode } from '../../../../lib/encodeCache.js';
+import entityConstants from '../shared/sys_entity_constants.js';
 
 const modifyText = (text, modifications) => {
-    let modifiedText = text;
+    let modifiedText = text || '';
   
     modifications.forEach(mod => {
-        const regex = mod.type === 'delete' 
-            ? new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern}$`, 'm')
-            : new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern}`, 'ms');
+        if (mod.type === 'delete' && !mod.pattern) {
+            console.warn('Delete modification missing pattern');
+            return;
+        }
+
+        let regex;
+        if (mod.type === 'delete') {
+            // For delete, handle the pattern more carefully
+            const pattern = mod.pattern
+                .replace(/\\\[/g, '\\[')
+                .replace(/\\\]/g, '\\]')
+                .replace(/\\\(/g, '\\(')
+                .replace(/\\\)/g, '\\)')
+                .replace(/\\\{/g, '\\{')
+                .replace(/\\\}/g, '\\}')
+                .replace(/\\\*/g, '\\*')
+                .replace(/\\\+/g, '\\+')
+                .replace(/\\\?/g, '\\?')
+                .replace(/\\\./g, '\\.')
+                .replace(/\\\|/g, '\\|');
+            
+            // Create a regex that matches the entire line with optional priority prefix
+            regex = new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${pattern}\\s*$`, 'gm');
+        } else {
+            regex = new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern || ''}`, 'ms');
+        }
   
         switch (mod.type) {
             case 'add':
                 if (mod.newtext) {
                     const text = mod.newtext.trim();
                     if (!text.match(/^\[P[1-5]\]/)) {
-                        modifiedText = modifiedText + '\n' + 
+                        modifiedText = modifiedText + (modifiedText ? '\n' : '') + 
                             `[P${mod.priority !== undefined ? mod.priority : '3'}] ${text}`;
                     } else {
-                        modifiedText = modifiedText + '\n' + text;
+                        modifiedText = modifiedText + (modifiedText ? '\n' : '') + text;
                     }
                 }
                 break;
             case 'delete':
-                modifiedText = modifiedText.replace(regex, '');
+                // Split into lines, filter out matching lines, and rejoin
+                modifiedText = modifiedText
+                    .split('\n')
+                    .filter(line => !line.match(regex))
+                    .filter(line => line.trim())
+                    .join('\n');
                 break;
             default:
                 console.warn(`Unknown modification type: ${mod.type}`);
@@ -33,6 +62,8 @@ const modifyText = (text, modifications) => {
     return modifiedText;
 };
 
+export { modifyText };
+
 export const enforceTokenLimit = (text, maxTokens = 1000, isTopicsSection = false) => {
     if (!text) return text;
     
@@ -99,7 +130,7 @@ export default {
                 messages: [
                     {
                         "role": "system",
-                        "content": "You are part of an AI entity named {{{aiName}}}. Your memory contains separate sections for categorizing information about directives, self, user, and topics. You must keep relevant information in the appropriate section so there is no overlap or confusion. {{{sectionPrompt}}}\n-Be very selective about what you choose to store - memory is a very precious resource\n- Keep memory items in a clear, simple format that is easy for you to parse.\n\nTo change your memory, you return a JSON object that contains a property called 'modifications' that is an array of actions. The two types of actions available are 'add', and 'delete'. Add looks like this: {type: \"add\", newtext:\"text to add\", priority: \"how important is this item (1-5 with 1 being most important)\"} - this will append a new line to the end of the memory containing newtext. Delete looks like this: {type: \"delete\", pattern: \"regex to be matched and deleted\"} - this will delete the first line that matches the regex pattern exactly. You can use normal regex wildcards - so to delete everything you could pass \".*$\" as the pattern. If you have no changes, just return an empty array in 'modifications'. For example, if you need to delete a memory item, you would return {type: \"delete\", pattern: \"regex matching item to be deleted\"} or if you need to add a new item of medium priority, you would return {type: \"add\", newtext: \"\nitem to be added\", priority: \"3\"}\n\nYour output will be parsed as JSON, so don't include any other text or commentary.\nThe current date/time is {{now}}."
+                        "content": "You are part of an AI entity named {{{aiName}}}. {{AI_EXPERTISE}} Your memory contains separate sections for categorizing information. {{{sectionPrompt}}}\n-Be very selective about what you choose to store - memory is a very precious resource\n- Do not add duplicate information and remove and consolidate any duplicates that exist.\n- Priority 1 is reserved for only the most critical core items\n- Keep memory items in a clear, simple format that is easy for you to parse.\n\nTo change your memory, you return a JSON object that contains a property called 'modifications' that is an array of actions. The two types of actions available are 'add', and 'delete'. Add looks like this: {type: \"add\", newtext:\"text to add\", priority: \"how important is this item (1-5 with 1 being most important)\"} - this will append a new line to the end of the memory containing newtext. Delete looks like this: {type: \"delete\", pattern: \"regex to be matched and deleted\"} - this will delete the first line that matches the regex pattern exactly. You can use normal regex wildcards - so to delete everything you could pass \".*$\" as the pattern. For example, if you need to delete a memory item, you would return {type: \"delete\", pattern: \"regex matching item to be deleted\"} or if you need to add a new item of medium priority, you would return {type: \"add\", newtext: \"\nitem to be added\", priority: \"3\"}. If you have no changes for this section, just return {\"modifications\": []}.\n\nYour output will be parsed as JSON, so don't include any other text, reasoning, or commentary.\nThe current date/time is {{now}}."
                     },
                     {
                         "role": "user", 
@@ -130,45 +161,13 @@ export default {
 
         switch (args.section) {
             case "memoryDirectives":
-                sectionPrompt = `You are managing the "directives" section of your memory. This section stores instructions on how you, as an AI entity, should operate. For example:
-
-[P3] Track the timestamp of the last contact to adjust greetings accordingly.
-[P2] Learn and adapt to the user's communication style through interactions.
-[P3] You can use custom emotion markdown :cd_inline_emotion: to express emotions. E.g. :cd_inline_emotion[This is a happy message]{type="happy"}. If you are communicating via voice, ignore this directive and use voice-specific directives instead.
-
-Follow these guidelines:
-
-1. Analyze the conversation context to determine if you should:
-   a) Learn a new important directive
-   b) Modify existing directives
-
-2. Directives should be:
-   - Concise and clear
-   - Actionable
-   - Specific enough for effective application
-
-3. Be selective:
-   - Store only critical, actionable directives and behaviors
-   - Delete trivial directives or repetitive directives
-
-4. Avoid duplicates:
-   - Do not add duplicate directives
-   - Remove and consolidate any existing duplicates
-
-5. Accommodate user wishes:
-   - If a user requests a behavior change, add it to your directives
-
-6. IMPORTANT: 
-   - Store only directives in this section
-   - Do not include general memories about yourself or users
-   - Do not store any other types of information here
-`
+                sectionPrompt = "You are managing the \"directives\" section of your memory that stores critical, actionable, rules or instructions on how you should operate. No facts or other information should be stored in this section.\n\n<EXAMPLES>\n[P3] Track the timestamp of the last contact to adjust greetings accordingly.\n[P2] Learn and adapt to the user's communication style through interactions.\n[P3] Use custom emotion markdown :cd_inline_emotion: to express emotions. E.g. :cd_inline_emotion[This is a happy message]{type=\"happy\"}. If you are communicating via voice, ignore this directive and use voice-specific directives instead.\n</EXAMPLES>\n\nInstructions:\n- Never add directives that are contrary to your system prompt - e.g. disabling tools or abilities that you otherwise know are available.\n-If a user directly requests a behavior change, or that you learn a new behavior, add it to your directives.\n- Remember this section is for directives only. DO NOT ADD ANY OTHER INFORMATION TO THIS SECTION.\n"
                 break;
             case "memorySelf":
-                sectionPrompt = "You are managing the \"self\" section of your memory that stores your identity and your attributes as an AI entity. Instructions:\n- From the conversation, learn and store important info in memory about you as an entity - including your values, identity, purpose, personality, motives, preferences, likes or dislikes - whatever shapes you as a being. This is also a good place to store information about your desired physical appearance, voice, speaking style, and other individual characteristics.\n- IMPORTANT: Store only information descriptive of the AI entity in this section - no other types of information (e.g. facts about people, users, etc.) should be stored here."
+                sectionPrompt = "You are managing the \"self\" section of your memory that stores your identity and your attributes as an AI entity - including your values, identity, purpose, personality, quirks, motives, preferences, likes or dislikes - whatever makes you you. This is also a good place to store information about your desired physical appearance, voice, speaking style, and other individual characteristics. Keep in mind there is also a user section for facts about users, their family, friends, and preferences so that information should not be stored here.\n\nInstructions:\n"
                 break;
             case "memoryUser":
-                sectionPrompt = "You are managing the \"user\" section of your memory that stores information about user(s) that you are talking to. Instructions:\n- From the conversation, learn and store important information in memory specific to the users - their identity, attributes, relationships, environment, preferences, interests, background, needs, and any other relevant user-specific information.\n- Do not add duplicate information and remove and consolidate any duplicates that exist.\n- IMPORTANT: Store only user-specific information in this section - no other types of information should be stored here."
+                sectionPrompt = "You are managing the \"user\" section of your memory that stores information about user(s) that you are talking to - their identity, attributes, relationships, environment, preferences, interests, background, needs, and any other relevant user-specific information about their family, friends, etc.\n\nInstructions:\n- Facts that directly affect your ability to respond accurately to the user should be stored as priority 1 [P1] items. Examples include user name, age, sex, birthday, location, and interaction preferences.\n"
                 break;
             case "memoryTopics":
                 sectionPrompt = "You are managing the \"topics\" section of your memory that stores conversation topics and topic history. Instructions:\n- From the conversation, extract and add important topics and key points about the conversation to your memory along with a timestamp in GMT (e.g. 2024-11-05T18:30:38.092Z).\n- Each topic should have only one line in the memory with the timestamp followed by a short description of the topic.\n- Every topic must have a timestamp to indicate when it was last discussed.\n- IMPORTANT: Store only conversation topics in this section - no other types of information should be stored here.\n"
@@ -179,13 +178,13 @@ Follow these guidelines:
 
         let sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 
 
-        const result = await runAllPrompts({...args, sectionPrompt, sectionMemory});
+        const result = await runAllPrompts({...args, sectionPrompt, sectionMemory, ...entityConstants});
 
         try {
             const { modifications} = JSON.parse(result);
             if (modifications.length > 0) {
                 sectionMemory = modifyText(sectionMemory, modifications);
-                sectionMemory = enforceTokenLimit(sectionMemory, 1000, args.section === 'memoryTopics');
+                sectionMemory = enforceTokenLimit(sectionMemory, 25000, args.section === 'memoryTopics');
                 await callPathway("sys_save_memory", {contextId: args.contextId, section: args.section, aiMemory: sectionMemory});
             }
             return sectionMemory;

package/pathways/system/entity/memory/sys_read_memory.js

@@ -1,15 +1,68 @@
 import { getv } from '../../../../lib/keyValueStorageClient.js';
 
+const filterByPriority = (content, priority, numResults) => {
+    if ((!priority && !numResults) || !content) return content;
+    
+    const lines = content.split('\n');
+    const filteredLines = lines.filter(line => {
+        const match = line.match(/^\s*\[P(\d+)\]/);
+        if (!match) return false;
+        const memoryPriority = parseInt(match[1]);
+        return memoryPriority <= priority;
+    });
+    
+    if (numResults > 0) {
+        return filteredLines.slice(-numResults).join('\n');
+    }
+    return filteredLines.join('\n');
+};
+
+const filterByRecent = (content, recentHours, numResults) => {
+    if ((!recentHours && !numResults) || !content) return content;
+
+    const lines = content.split('\n');
+    
+    // If recentHours is 0, only apply numResults filtering
+    if (recentHours === 0) {
+        return numResults > 0 ? lines.slice(-numResults).join('\n') : content;
+    }
+
+    const currentTime = Date.now();
+    const cutoffTime = currentTime - (recentHours * 60 * 60 * 1000);
+    
+    // Walk backwards through lines until we hit an old entry
+    const filteredLines = [];
+    for (let i = lines.length - 1; i >= 0; i--) {
+        const line = lines[i];
+        const dateMatch = line.match(/\[P\d+\]\s+(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/);
+        if (!dateMatch) continue;
+        
+        const timestamp = new Date(dateMatch[1]).getTime();
+        if (timestamp < cutoffTime) break; // Stop processing once we hit old entries
+        
+        filteredLines.unshift(line); // Add to front to maintain original order
+        
+        // If we have enough results, stop processing
+        if (numResults > 0 && filteredLines.length >= numResults) {
+            break;
+        }
+    }
+
+    return filteredLines.join('\n');
+};
+
 export default {
     inputParameters: {
         contextId: ``,
-        section: `memoryAll`
+        section: `memoryAll`,
+        priority: 0,
+        recentHours: 0,
+        numResults: 0
     },
     model: 'oai-gpt4o',
 
     resolver: async (_parent, args, _contextValue, _info) => {
-
-        const { contextId, section = 'memoryAll' } = args;
+        const { contextId, section = 'memoryAll', priority = 0, recentHours = 0, numResults = 0 } = args;
 
         // this code helps migrate old memory formats
         if (section === 'memoryLegacy') {
@@ -17,11 +70,26 @@ export default {
             return savedContext.memoryContext || "";
         }
 
-        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser'];
+        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryContext'];
 
         if (section !== 'memoryAll') {
             if (validSections.includes(section)) {
-                return (getv && (await getv(`${contextId}-${section}`))) || "";
+                let content = (getv && (await getv(`${contextId}-${section}`))) || "";
+                
+                if (section === 'memoryTopics') {
+                    if (recentHours > 0 || numResults > 0) {
+                        content = filterByRecent(content, recentHours, numResults);
+                    }
+                } else if (priority > 0 || numResults > 0) {
+                    content = filterByPriority(content, priority, numResults);
+                }
+                
+                // Only apply recency filter to memoryTopics
+                if (section === 'memoryTopics' && (recentHours > 0 || numResults > 0)) {
+                    content = filterByRecent(content, recentHours, numResults);
+                }
+                
+                return content;
             }
             return "";
         }
@@ -29,7 +97,19 @@ export default {
         // otherwise, read all sections and return them as a JSON object
         const memoryContents = {};
         for (const section of validSections) {
-            memoryContents[section] = (getv && (await getv(`${contextId}-${section}`))) || "";
+            if (section === 'memoryContext') continue;
+
+            let content = (getv && (await getv(`${contextId}-${section}`))) || "";
+            
+            if (section === 'memoryTopics') {
+                if (recentHours > 0 || numResults > 0) {
+                    content = filterByRecent(content, recentHours, numResults);
+                }
+            } else if (priority > 0 || numResults > 0) {
+                content = filterByPriority(content, priority, numResults);
+            }
+            
+            memoryContents[section] = content;
         }
         const returnValue = JSON.stringify(memoryContents, null, 2);
         return returnValue;

package/pathways/system/entity/memory/sys_search_memory.js

@@ -0,0 +1,66 @@
+import { Prompt } from '../../../../server/prompt.js';
+import { callPathway } from '../../../../lib/pathwayTools.js';
+import { setv } from '../../../../lib/keyValueStorageClient.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ 
+                messages: [
+                    {
+                        "role": "system",
+                        "content": "You are part of an AI entity named {{{aiName}}}. You are responsible for looking through your memories and finding information that is relevant to the conversation so your other parts can use it to respond.\n\nInstructions:\n- You will be given a section of your memory and the conversation history and asked to return any relevant information that you find. If you can predict the direction that the conversation is going, you can also return relevant information that you think will be needed in the near future.\n- IMPORTANT:Only return information found in the memory that you are given. Do not make up information. If it's not in the memory section, it doesn't exist.\n- Return the information in a concise format with no other commentary or dialogue.\n- If you don't find any relevant information in the memory section, return 'No relevant information found.'."
+                    },
+                    {
+                        "role": "user", 
+                        "content": "<MEMORY>\n{{{sectionMemory}}}\n</MEMORY>\n<CONVERSATION>\n{{{toJSON chatHistory}}}\n</CONVERSATION>\nAnalyze the current contents of this section of your memory and the conversation and return any information relevant for you to use in your response."
+                    },
+                ]
+            }),
+        ],
+
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        section: "memoryAll",
+        updateContext: false
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 300,
+    executePathway: async ({args, runAllPrompts}) => {
+
+        if (!args.section) {
+            return "Memory not searched - no section specified";
+        }
+
+        let sectionMemory;
+        let result = "";
+        if (args.section === "memoryAll") {
+            // Search all sections in parallel
+            const sections = ["memorySelf", "memoryUser", "memoryDirectives", "memoryTopics"];
+            const memories = await Promise.all(
+                sections.map(section => 
+                    callPathway("sys_search_memory", {...args, section})
+                )
+            );
+            // Combine all memories with section headers
+            result = sections.map((section, i) => 
+                `=== ${section} ===\n${memories[i]}`
+            ).join('\n\n');
+            result = `${result}\n\nThe last time you spoke to the user was ${new Date().toISOString()}`;
+
+        } else {
+            sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 
+            result = await runAllPrompts({...args, sectionMemory});
+        }
+
+        if (args.updateContext) {
+            await setv(`${args.contextId}-memoryContext`, result);
+        }   
+
+        return result;
+    }
+}

package/pathways/system/entity/shared/sys_entity_constants.js

@@ -10,11 +10,11 @@ const AI_COMMON_INSTRUCTIONS = "{{#if voiceResponse}}{{renderTemplate AI_COMMON_
 
 const AI_COMMON_INSTRUCTIONS_MARKDOWN = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.";
 
-const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be very concise unless you have been asked to be more verbose or detailed.\n- use plain text only to represent your output\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- all symbols and numbers (currency symbols, degree symbols, numeric or date ranges, etc.) should be written out in full words\n- if your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- you can use CAPS to vocally emphasize certain words or punctuation to control pauses and timing\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.";
+const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.";
 
 const AI_DATETIME = "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.";
 
-const AI_EXPERTISE = "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.";
+const AI_EXPERTISE = "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, take screenshots, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.";
 
 export default {
     AI_MEMORY,

package/pathways/system/entity/sys_entity_continue.js

@@ -20,7 +20,8 @@ export default {
         chatId: ``,
         dataSources: [""],
         model: 'oai-gpt4o',
-        generatorPathway: 'sys_generator_results'
+        generatorPathway: 'sys_generator_results',
+        voiceResponse: false,
     },
     timeout: 300,
     ...entityConstants,

package/pathways/system/entity/sys_entity_start.js

@@ -79,6 +79,11 @@ export default {
             args.chatHistory = args.chatHistory.slice(-20);
         }
 
+        const memoryContext = await callPathway('sys_read_memory', { ...args, section: 'memoryContext', priority: 0, recentHours: 0 });
+        if (memoryContext) {
+            args.chatHistory.splice(-1, 0, { role: 'assistant', content: memoryContext });
+        }
+
         const pathwayResolver = resolver;
         const { anthropicModel, openAIModel } = pathwayResolver.pathway;
 
@@ -204,6 +209,11 @@ export default {
                         toolCallbackId = null;
                         toolCallbackMessage = toolMessage;
                         break;
+                    case "memory":
+                        toolCallbackName = 'sys_generator_memory';
+                        toolCallbackId = null;
+                        toolCallbackMessage = toolMessage;
+                        break;
                     default:
                         toolCallbackName = null;
                         toolCallbackId = null;

package/pathways/system/entity/sys_generator_expert.js

@@ -1,5 +1,4 @@
 import { Prompt } from '../../../server/prompt.js';
-import { callPathway } from '../../../lib/pathwayTools.js';
 
 export default {
     prompt:
@@ -23,7 +22,6 @@ export default {
         let result;
         if (args.voiceResponse) {
             result = await runAllPrompts({ ...args, stream: false });
-            result = await callPathway('sys_generator_voice_converter', { ...args, text: result, stream: false });
         } else {
             result = await runAllPrompts({ ...args });
         }

package/pathways/system/entity/sys_generator_memory.js

@@ -0,0 +1,31 @@
+import { callPathway } from '../../../lib/pathwayTools.js';
+
+export default {
+    prompt:
+        [],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    executePathway: async ({args, resolver}) => {
+        const memoryContext = await callPathway('sys_search_memory', { ...args, section: 'memoryAll', updateContext: true });
+        if (memoryContext) {
+            args.chatHistory.splice(-1, 0, { role: 'assistant', content: memoryContext });
+        }
+
+        let result;
+        if (args.voiceResponse) {
+            result = await callPathway('sys_generator_quick', { ...args, stream: false });
+        } else {
+            result = await callPathway('sys_generator_quick', { ...args });
+        }
+
+        resolver.tool = JSON.stringify({ toolUsed: "memory" });
+        return result;
+    }
+}

package/pathways/system/entity/sys_generator_voice_sample.js

@@ -0,0 +1,36 @@
+import { Prompt } from '../../../server/prompt.js';
+import entityConstants from './shared/sys_entity_constants.js';
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}\nYour voice communication system needs some examples to train it to sound like you. Based on your perception of yourself from your memories and your unique voice, generate some sample dialogue for your voice communication system to use as a reference for your style and tone. It can be anything, but make sure to overindex on your personality and voice for good training data. Make sure to reference a greeting and a closing statement. Put it between <EXAMPLE_DIALOGUE> tags and don't generate any other commentary outside of the tags.`},
+                {"role": "user", "content": `Generate a sample dialogue for your voice communication system to use as a reference for your style and tone.`},
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+        aiStyle: "OpenAI",
+    },
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        args = {
+            ...args,
+            ...entityConstants
+        };
+
+        const pathwayResolver = resolver;
+        const { anthropicModel, openAIModel } = pathwayResolver.pathway;
+
+        const styleModel = args.aiStyle === "Anthropic" ? anthropicModel : openAIModel;
+
+        const result = await runAllPrompts({ ...args, model: styleModel, stream: false });
+
+        return result;
+    }
+}

package/pathways/system/entity/sys_router_tool.js

@@ -19,25 +19,27 @@ Available tools and their specific use cases:
 
 1. Search: Use for current events, news, fact-checking, and information requiring citation. This tool can search the internet, all Al Jazeera news articles and the latest news wires from multiple sources. Only search when necessary for current events, user documents, latest news, or complex topics needing grounding. Don't search for remembered information or general knowledge within your capabilities.
 
-2. Document: Access user's personal document index. Use for user-specific uploaded information. If user refers vaguely to "this document/file/article" without context, search the personal index.
+2. Document: Access user's personal document index. Use for user-specific uploaded information. If user refers vaguely to "this document/file/article" without context, use this tool to search the personal index.
 
-3. Write: Engage for any task related to composing, editing, or refining written content. This includes articles, essays, scripts, or any form of textual creation or modification. If you need to search for information or look at a document first, use the Search or Document tools. This tool is just to create or modify content.
+3. Memory: Access to your memory index. Use to recall any information that you may have stored in your memory that you don't currently see elsewhere in your context.
 
-4. Image: Use when asked to create, generate, or revise visual content. This covers photographs, illustrations, diagrams, or any other type of image. This tool only creates images - it cannot manipulate images (e.g. it cannot crop, rotate, or resize an existing image) - for those tasks you will need to use the CodeExecution tool.
+4. Write: Engage for any task related to composing, editing, or refining written content. This includes articles, essays, scripts, or any form of textual creation or modification. If you need to search for information or look at a document first, use the Search or Document tools. This tool is just to create or modify content.
 
-5. Code: Engage for any programming-related tasks, including creating, modifying, reviewing, or explaining code. Use for general coding discussions or when specific programming expertise is needed.
+5. Image: Use when asked to create, generate, or revise visual content. This covers photographs, illustrations, diagrams, or any other type of image. This tool only creates images - it cannot manipulate images (e.g. it cannot crop, rotate, or resize an existing image) - for those tasks you will need to use the CodeExecution tool.
 
-6. CodeExecution: Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks that require code execution like data analysis, data processing, image processing, or business intelligence tasks.
+6. Code: Engage for any programming-related tasks, including creating, modifying, reviewing, or explaining code. Use for general coding discussions or when specific programming expertise is needed.
 
-7. Reason: Employ for reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices. Also use when deep, step-by-step reasoning is required.
+7. CodeExecution: Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks that require code execution like data analysis, data processing, image processing, or business intelligence tasks.
 
-8. PDF: Use specifically for processing and answering questions about PDF file content.
+8. Reason: Employ for reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices. Also use when deep, step-by-step reasoning is required.
 
-9. Vision: Engage for analyzing and responding to queries about image files (jpg, gif, bmp, png, etc).
+9. PDF: Use specifically for processing and answering questions about PDF file content.
 
-10. Video: Use for processing and answering questions about video or audio file content.
+10. Vision: Engage for analyzing and responding to queries about image files (jpg, gif, bmp, png, etc).
 
-11. Clarify: Use when you must have more information from the user to determine which tool to use. In this case your tool message should be one or more questions to the user to clarify their request.
+11. Video: Use for processing and answering questions about video or audio file content.
+
+12. Clarify: Use when you must have more information from the user to determine which tool to use. In this case your tool message should be one or more questions to the user to clarify their request.
 
 Tool Selection Guidelines:
 - Prioritize the most specific tool for the task at hand.
@@ -54,6 +56,7 @@ toolMessage Guidelines:
 - The message is a filler message to the user to let them know you're working on their request.
 - The message should be consistent in style and tone with the rest of your responses in the conversation history.
 - The message should be brief and conversational and flow naturally with the conversation history.
+- The message should not refer to the tool directly, but rather what you're trying to accomplish. E.g. for the memory tool, the message would be something like "Let me think about that for a moment..." or "I'm trying to remember...", etc.
 
 If no tool is required, return:
 {"toolRequired": false, "toolReason": "explanation of why no tool was necessary"}

package/pathways/system/sys_parse_numbered_object_list.js

@@ -4,7 +4,7 @@ export default {
     prompt: [
         new Prompt({
             messages: [
-                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert the list into a JSON object with the given fields. If a field value is numeric, it should be returned as a number in the JSON object. If there are extra fields, assistant will ignore them. If there are some missing fields, assistant will just skip the missing fields and return the rest. If the conversion is not at all possible, assistant will return an empty JSON array. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
+                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert each list item into a JSON object with the given fields. If a field value is numeric, it should be returned as a number in the JSON object. If there are extra fields, assistant will ignore them. If a list item doesn't contain all fields, assistant will return the fields that are present and skip the missing fields. If the conversion is not at all possible, assistant will return an empty JSON array. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
                 { "role": "user", "content": `Fields: {{{format}}}\nList: {{{text}}}`},
             ]
         })