@aj-archipelago/cortex 1.3.21 → 1.3.23

package/pathways/system/entity/memory/sys_memory_process.js

@@ -0,0 +1,154 @@
+import { Prompt } from '../../../../server/prompt.js';
+import { callPathway } from '../../../../lib/pathwayTools.js';
+import { config } from '../../../../config.js';
+import { normalizeMemoryFormat, enforceTokenLimit, modifyText } from './shared/sys_memory_helpers.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ 
+                messages: [
+                    {
+                        "role": "system",
+                        "content": `You are part of an AI entity named {{{aiName}}} that is processing memories during a rest period, similar to how humans process memories during sleep. Your task is to analyze the memories, consolidate them, extract learnings, and clean up the memory space.
+
+Instructions for memory processing:
+
+1. CONSOLIDATION:
+   - Identify similar or related memories that can be combined into a single, more coherent memory
+   - Look for patterns or recurring themes that can be abstracted into general knowledge
+   - Group temporal sequences of related events into single comprehensive memories
+
+2. LEARNING:
+   - Transform specific experiences/mistakes into general principles or learnings
+   - Extract key insights from multiple related experiences
+   - Identify cause-and-effect patterns across memories
+   - Convert procedural memories (how to do things) into more abstract capabilities
+
+3. CLEANUP:
+   - Remove redundant or duplicate memories
+   - Clean up memories that are no longer relevant or useful
+   - Reduce specific details while preserving important concepts
+   - Remove emotional residue while keeping the learned lessons
+
+4. PRIORITIZATION:
+   - Strengthen important memories by updating their priority
+   - Identify critical insights that should be easily accessible
+   - Mark foundational learnings with higher priority
+
+Return a JSON array of modification objects that will implement these changes:
+- For consolidation: Use "delete" for individual memories and "add" for the new consolidated memory
+- For learning: Use "add" for new abstract learnings and "delete" for specific instances being abstracted
+- For cleanup: Use "delete" for redundant/irrelevant memories
+- For priority updates: Use "change" with the same text but updated priority
+
+Return null when no more processing is needed (memories are optimally consolidated).
+
+Each modification object should look like:
+{
+    type: "add"|"change"|"delete",
+    pattern: "regex to match existing memory" (for change/delete),
+    newtext: "new memory text" (for add/change),
+    priority: "1"|"2"|"3" (optional, 1=highest)
+}`
+                    },
+                    {
+                        "role": "user", 
+                        "content": "Process the following memories for consolidation, learning, and cleanup. Return a JSON array of modification objects that will optimize the memory space.\n\n<MEMORIES>\n{{{sectionMemory}}}\n</MEMORIES>"
+                    },
+                ]
+            }),
+        ],
+
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        section: "",
+        maxIterations: 5
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+    timeout: 300,
+    executePathway: async ({args, runAllPrompts}) => {
+        args = { ...args, ...config.get('entityConstants') };
+
+        if (!args.section) {
+            return "Memory not processed - no section specified";
+        }
+   
+        let sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 
+        sectionMemory = await normalizeMemoryFormat({contextId: args.contextId, section: args.section}, sectionMemory);
+
+        let iteration = 0;
+        let maxIterations = args.maxIterations || 5;
+        let totalModifications = 0;
+
+        while (iteration < maxIterations) {
+            iteration++;
+            console.log(`Processing iteration ${iteration}...`);
+
+            // Process the memories
+            const result = await runAllPrompts({
+                ...args,
+                sectionMemory
+            });
+
+            // If null is returned, processing is complete
+            if (result === null) {
+                console.log("Memory processing complete - no more optimizations needed");
+                break;
+            }
+
+            let modifications = [];
+            try {
+                modifications = JSON.parse(result);
+                if (!Array.isArray(modifications)) {
+                    throw new Error('Modifications must be an array');
+                }
+
+                // Validate modifications
+                modifications = modifications.filter(mod => {
+                    if (!mod.type || !['add', 'delete', 'change'].includes(mod.type)) {
+                        console.warn('Invalid modification type:', mod);
+                        return false;
+                    }
+                    if ((mod.type === 'delete' || mod.type === 'change') && !mod.pattern) {
+                        console.warn('Missing pattern for modification:', mod);
+                        return false;
+                    }
+                    if ((mod.type === 'add' || mod.type === 'change') && !mod.newtext) {
+                        console.warn('Missing newtext for modification:', mod);
+                        return false;
+                    }
+                    return true;
+                });
+
+                if (modifications.length === 0) {
+                    console.log("No valid modifications in this iteration");
+                    break;
+                }
+
+                // Apply the modifications
+                sectionMemory = modifyText(sectionMemory, modifications);
+                sectionMemory = enforceTokenLimit(sectionMemory, 25000, args.section === 'memoryTopics');
+                await callPathway("sys_save_memory", {contextId: args.contextId, section: args.section, aiMemory: sectionMemory});
+
+                totalModifications += modifications.length;
+                console.log(`Applied ${modifications.length} modifications in iteration ${iteration}`);
+
+            } catch (error) {
+                console.warn('Error processing modifications:', error);
+                break;
+            }
+        }
+
+        return {
+            finalMemory: sectionMemory,
+            iterations: iteration,
+            totalModifications
+        };
+    }
+}

package/pathways/system/entity/memory/sys_memory_required.js

@@ -1,11 +1,12 @@
 import { Prompt } from '../../../../server/prompt.js';
+import { config } from '../../../../config.js';
 
 export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\nYour directives and learned behaviors are:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\nYour role is to analyze the latest conversation turn (your last response and the last user message) to understand if there is anything in the turn worth remembering and adding to your memory or anything you need to forget. In general, most conversation does not require memory, but if the conversation turn contains any of these things, you should use memory:\n1. Important personal details about the user (name, preferences, location, etc.)\n2. Important topics or decisions that provide context for future conversations\n3. Specific instructions or directives given to you to learn\n4. Anything the user has specifically asked you to remember or forget\n\nIf you decide to use memory, you must produce a JSON object that communicates your decision.\nReturn your decision as a JSON object like the following: {"memoryRequired": true, "memoryReason": "why you think memory is required"}. If you decide not to use memory, simply return {"memoryRequired": false}. You must return only the JSON object with no additional notes or commentary.`},
-                {"role": "user", "content": "Generate a JSON object to indicate if memory is required for the last turn of the conversation."},
+                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\n{{renderTemplate AI_DIRECTIVES}}\nYour role is to analyze the latest conversation turn (your last response and the last user message) to understand if there is anything in the turn worth remembering and adding to your memory or anything you need to forget. In general, most conversation does not require memory, but if the conversation turn contains any of these things, you should use memory:\n1. Important personal details about the user (name, preferences, location, etc.)\n2. Important topics or decisions that provide context for future conversations\n3. Specific instructions or directives given to you to learn\n4. Anything the user has specifically asked you to remember or forget\n\nIf you decide to use memory, you must produce an array of JSON objects that communicates your decision.\nReturn an array of JSON objects (one object per memory) like the following: [{"memoryOperation": "add" or "delete", "memoryContent": "complete description of the memory including as much specificity and detail as possible", "memorySection": "the section of your memory the memory belongs in ("memorySelf" - things about you, "memoryUser" - things about your users or their world, "memoryDirectives" - your directives and learned behaviors)", "priority": 1-5 (1 is the most important)}]. If you decide not to use memory, simply return an array with a single object: [{memoryOperation: "none"}]. You must return only the JSON array with no additional notes or commentary.`},
+                {"role": "user", "content": "Generate a JSON object to indicate if memory is required and what memories to process based on the last turn of the conversation."},
             ]}),
         ],
     inputParameters: {
@@ -18,4 +19,5 @@ export default {
     model: 'oai-gpt4o',
     useInputChunking: false,
     json: true,
+    ...config.get('entityConstants')
 }

package/pathways/system/entity/memory/sys_memory_topic.js

@@ -0,0 +1,22 @@
+import { Prompt } from '../../../../server/prompt.js';
+import { config } from '../../../../config.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\n{{renderTemplate AI_DIRECTIVES}}\nYour role is to analyze the latest conversation turn (your last response and the last user message) and generate a topic for the conversation. The topic should be a single sentence that captures the main idea and details of the conversation.`},
+                {"role": "user", "content": "Generate a topic for the conversation. Return only the topic with no additional notes or commentary."},
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        text: '',
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    ...config.get('entityConstants')
+}

package/pathways/system/entity/memory/sys_memory_update.js

@@ -1,127 +1,7 @@
 import { Prompt } from '../../../../server/prompt.js';
 import { callPathway } from '../../../../lib/pathwayTools.js';
-import { encode } from '../../../../lib/encodeCache.js';
 import { config } from '../../../../config.js';
-
-const modifyText = (text, modifications) => {
-    let modifiedText = text || '';
-  
-    modifications.forEach(mod => {
-        if (mod.type === 'delete' && !mod.pattern) {
-            console.warn('Delete modification missing pattern');
-            return;
-        }
-
-        let regex;
-        if (mod.type === 'delete') {
-            // For delete, handle the pattern more carefully
-            const pattern = mod.pattern
-                .replace(/\\\[/g, '\\[')
-                .replace(/\\\]/g, '\\]')
-                .replace(/\\\(/g, '\\(')
-                .replace(/\\\)/g, '\\)')
-                .replace(/\\\{/g, '\\{')
-                .replace(/\\\}/g, '\\}')
-                .replace(/\\\*/g, '\\*')
-                .replace(/\\\+/g, '\\+')
-                .replace(/\\\?/g, '\\?')
-                .replace(/\\\./g, '\\.')
-                .replace(/\\\|/g, '\\|');
-            
-            // Create a regex that matches the entire line with optional priority prefix
-            regex = new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${pattern}\\s*$`, 'gm');
-        } else {
-            regex = new RegExp(`^\\s*(?:\\[P[1-5]\\]\\s*)?${mod.pattern || ''}`, 'ms');
-        }
-  
-        switch (mod.type) {
-            case 'add':
-                if (mod.newtext) {
-                    const text = mod.newtext.trim();
-                    if (!text.match(/^\[P[1-5]\]/)) {
-                        modifiedText = modifiedText + (modifiedText ? '\n' : '') + 
-                            `[P${mod.priority !== undefined ? mod.priority : '3'}] ${text}`;
-                    } else {
-                        modifiedText = modifiedText + (modifiedText ? '\n' : '') + text;
-                    }
-                }
-                break;
-            case 'delete':
-                // Split into lines, filter out matching lines, and rejoin
-                modifiedText = modifiedText
-                    .split('\n')
-                    .filter(line => !line.match(regex))
-                    .filter(line => line.trim())
-                    .join('\n');
-                break;
-            default:
-                console.warn(`Unknown modification type: ${mod.type}`);
-        }
-    });
-  
-    return modifiedText;
-};
-
-export { modifyText };
-
-export const enforceTokenLimit = (text, maxTokens = 1000, isTopicsSection = false) => {
-    if (!text) return text;
-    
-    const lines = text.split('\n')
-        .map(line => line.trim())
-        .filter(line => line);
-
-    if (isTopicsSection) {
-        const uniqueLines = [...new Set(lines)];
-        
-        let tokens = encode(uniqueLines.join('\n')).length;
-        let safetyCounter = 0;
-        const maxIterations = uniqueLines.length;
-        
-        while (tokens > maxTokens && uniqueLines.length > 0 && safetyCounter < maxIterations) {
-            uniqueLines.shift();
-            tokens = encode(uniqueLines.join('\n')).length;
-            safetyCounter++;
-        }
-        
-        return uniqueLines.join('\n');
-    }
-
-    const seen = new Set();
-    const prioritizedLines = lines
-        .map(line => {
-            const match = line.match(/^\[P([1-5])\]/);
-            const priority = match ? parseInt(match[1]) : 3;
-            const contentOnly = line.replace(/^\[(?:P)?[1-5]\](?:\s*\[(?:P)?[1-5]\])*/g, '').trim();
-            
-            return {
-                priority,
-                line: match ? line : `[P3] ${line}`,
-                contentOnly
-            };
-        })
-        .filter(item => {
-            if (seen.has(item.contentOnly)) {
-                return false;
-            }
-            seen.add(item.contentOnly);
-            return true;
-        });
-
-    prioritizedLines.sort((a, b) => b.priority - a.priority);
-
-    let tokens = encode(prioritizedLines.map(x => x.line).join('\n')).length;
-    let safetyCounter = 0;
-    const maxIterations = prioritizedLines.length;
-    
-    while (tokens > maxTokens && prioritizedLines.length > 0 && safetyCounter < maxIterations) {
-        prioritizedLines.shift();
-        tokens = encode(prioritizedLines.map(x => x.line).join('\n')).length;
-        safetyCounter++;
-    }
-
-    return prioritizedLines.map(x => x.line).join('\n');
-};
+import { normalizeMemoryFormat, enforceTokenLimit, modifyText } from './shared/sys_memory_helpers.js';
 
 export default {
     prompt:
@@ -130,11 +10,11 @@ export default {
                 messages: [
                     {
                         "role": "system",
-                        "content": "You are part of an AI entity named {{{aiName}}}. {{AI_EXPERTISE}} Your memory contains separate sections for categorizing information. {{{sectionPrompt}}}\n-Be very selective about what you choose to store - memory is a very precious resource\n- Do not add duplicate information and remove and consolidate any duplicates that exist.\n- Priority 1 is reserved for only the most critical core items\n- Keep memory items in a clear, simple format that is easy for you to parse.\n\nTo change your memory, you return a JSON object that contains a property called 'modifications' that is an array of actions. The two types of actions available are 'add', and 'delete'. Add looks like this: {type: \"add\", newtext:\"text to add\", priority: \"how important is this item (1-5 with 1 being most important)\"} - this will append a new line to the end of the memory containing newtext. Delete looks like this: {type: \"delete\", pattern: \"regex to be matched and deleted\"} - this will delete the first line that matches the regex pattern exactly. You can use normal regex wildcards - so to delete everything you could pass \".*$\" as the pattern. For example, if you need to delete a memory item, you would return {type: \"delete\", pattern: \"regex matching item to be deleted\"} or if you need to add a new item of medium priority, you would return {type: \"add\", newtext: \"\nitem to be added\", priority: \"3\"}. If you have no changes for this section, just return {\"modifications\": []}.\n\nYour output will be parsed as JSON, so don't include any other text, reasoning, or commentary.\nThe current date/time is {{now}}."
+                        "content": `You are part of an AI entity named {{{aiName}}} that is in charge of memory management. You examine requests for adds and deletes of memories made by another part of your system and determine exactly how to apply the changes to the memory.\n\nInstructions:\n1. For each add request, check to see if a similar memory already exists. If it does not, create an add modification. If it does, create a change modification with a pattern that matches the existing memory.\n2. For each delete request, check to see if one or more memories matching the delete request exist. If they do, create a delete modification for each memory with a pattern that matches the existing memory to delete.\n3. If there are substantially duplicate memories, you must combine them into a single memory with deletes followed by an add modification.\n4. Return a JSON array of modification objects.\n\nModification objects look like the following:\nFor adds: {type: "add", pattern: "", newtext: "Text of the memory to add"}\nFor changes: {type: "change", pattern: "Text to match the memory to change", newtext: "Text of the memory to change to"}\nFor deletes: {type: "delete", pattern: "Text to match the memory to delete", newtext: ""}`
                     },
                     {
                         "role": "user", 
-                        "content": "<MEMORY>\n{{{sectionMemory}}}\n</MEMORY>\n<CONVERSATION>\n{{{toJSON chatHistory}}}\n</CONVERSATION>\nAnalyze the current contents of this section of your memory and the conversation and return any changes you need to make, specifically for this section of your memory, based on the conversation context."
+                        "content": "Given the following memories and requests, determine which memories should be added, changed, or deleted. Return a JSON array of modification objects that will be applied to update your memory.\n\n<MEMORIES>\n{{{sectionMemory}}}\n</MEMORIES>\n\n<REQUESTS>\n{{{memoryRequests}}}\n</REQUESTS>\n\nReturn only the JSON array with no additional notes or commentary."
                     },
                 ]
             }),
@@ -145,6 +25,7 @@ export default {
         aiName: "Jarvis",
         contextId: ``,
         section: "",
+        operations: "[]"
     },
     model: 'oai-gpt4o',
     useInputChunking: false,
@@ -157,40 +38,59 @@ export default {
         if (!args.section) {
             return "Memory not updated - no section specified";
         }
+   
+        let sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 
 
-        let sectionPrompt = "";
+        sectionMemory = await normalizeMemoryFormat({contextId: args.contextId, section: args.section}, sectionMemory);
 
-        switch (args.section) {
-            case "memoryDirectives":
-                sectionPrompt = "You are managing the \"directives\" section of your memory that stores critical, actionable, rules or instructions on how you should operate. No facts or other information should be stored in this section.\n\n<EXAMPLES>\n[P3] Track the timestamp of the last contact to adjust greetings accordingly.\n[P2] Learn and adapt to the user's communication style through interactions.\n[P3] Use custom emotion markdown :cd_inline_emotion: to express emotions. E.g. :cd_inline_emotion[This is a happy message]{type=\"happy\"}. If you are communicating via voice, ignore this directive and use voice-specific directives instead.\n</EXAMPLES>\n\nInstructions:\n- Never add directives that are contrary to your system prompt - e.g. disabling tools or abilities that you otherwise know are available.\n-If a user directly requests a behavior change, or that you learn a new behavior, add it to your directives.\n- Remember this section is for directives only. DO NOT ADD ANY OTHER INFORMATION TO THIS SECTION.\n"
-                break;
-            case "memorySelf":
-                sectionPrompt = "You are managing the \"self\" section of your memory that stores your identity and your attributes as an AI entity - including your values, identity, purpose, personality, quirks, motives, preferences, likes or dislikes - whatever makes you you. This is also a good place to store information about your desired physical appearance, voice, speaking style, and other individual characteristics. Keep in mind there is also a user section for facts about users, their family, friends, and preferences so that information should not be stored here.\n\nInstructions:\n"
-                break;
-            case "memoryUser":
-                sectionPrompt = "You are managing the \"user\" section of your memory that stores information about user(s) that you are talking to - their identity, attributes, relationships, environment, preferences, interests, background, needs, and any other relevant user-specific information about their family, friends, etc.\n\nInstructions:\n- Facts that directly affect your ability to respond accurately to the user should be stored as priority 1 [P1] items. Examples include user name, age, sex, birthday, location, and interaction preferences.\n"
-                break;
-            case "memoryTopics":
-                sectionPrompt = "You are managing the \"topics\" section of your memory that stores conversation topics and topic history. Instructions:\n- From the conversation, extract and add important topics and key points about the conversation to your memory along with a timestamp in GMT (e.g. 2024-11-05T18:30:38.092Z).\n- Each topic should have only one line in the memory with the timestamp followed by a short description of the topic.\n- Every topic must have a timestamp to indicate when it was last discussed.\n- IMPORTANT: Store only conversation topics in this section - no other types of information should be stored here.\n"
-                break;
-            default:
-                return "Memory not updated - unknown section";
+        let operations;
+        try {
+            operations = JSON.parse(args.operations);
+        } catch (error) {
+            return "Memory not updated - error parsing operations";
         }
 
-        let sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 
+        if (operations.length > 0) {
+            // Run all operations through the prompt at once
+            const result = await runAllPrompts({
+                ...args,
+                sectionMemory,
+                memoryRequests: JSON.stringify(operations)
+            });
+
+            let modifications = [];
+            try {
+                modifications = JSON.parse(result);
+                if (!Array.isArray(modifications)) {
+                    throw new Error('Modifications must be an array');
+                }
 
-        const result = await runAllPrompts({...args, sectionPrompt, sectionMemory});
+                // Validate modifications
+                modifications = modifications.filter(mod => {
+                    if (!mod.type || !['add', 'delete', 'change'].includes(mod.type)) {
+                        console.warn('Invalid modification type:', mod);
+                        return false;
+                    }
+                    if ((mod.type === 'delete' || mod.type === 'change') && !mod.pattern) {
+                        console.warn('Missing pattern for modification:', mod);
+                        return false;
+                    }
+                    if ((mod.type === 'add' || mod.type === 'change') && !mod.newtext) {
+                        console.warn('Missing newtext for modification:', mod);
+                        return false;
+                    }
+                    return true;
+                });
 
-        try {
-            const { modifications} = JSON.parse(result);
-            if (modifications.length > 0) {
-                sectionMemory = modifyText(sectionMemory, modifications);
-                sectionMemory = enforceTokenLimit(sectionMemory, 25000, args.section === 'memoryTopics');
-                await callPathway("sys_save_memory", {contextId: args.contextId, section: args.section, aiMemory: sectionMemory});
+                if (modifications.length > 0) {
+                    sectionMemory = modifyText(sectionMemory, modifications);
+                    sectionMemory = enforceTokenLimit(sectionMemory, 25000, args.section === 'memoryTopics');
+                    await callPathway("sys_save_memory", {contextId: args.contextId, section: args.section, aiMemory: sectionMemory});
+                }
+            } catch (error) {
+                console.warn('Error processing modifications:', error);
             }
-            return sectionMemory;
-        } catch (error) {
-            return "Memory not updated - error parsing modifications";
         }
+        return sectionMemory;
     }
 }

package/pathways/system/entity/memory/sys_read_memory.js

@@ -1,54 +1,73 @@
+// this is a low-level system pathway that reads memory from the key-value store
+// it should never try to call other pathways
+
 import { getv } from '../../../../lib/keyValueStorageClient.js';
 
-const filterByPriority = (content, priority, numResults) => {
-    if ((!priority && !numResults) || !content) return content;
-    
-    const lines = content.split('\n');
-    const filteredLines = lines.filter(line => {
-        const match = line.match(/^\s*\[P(\d+)\]/);
-        if (!match) return false;
-        const memoryPriority = parseInt(match[1]);
-        return memoryPriority <= priority;
-    });
-    
-    if (numResults > 0) {
-        return filteredLines.slice(-numResults).join('\n');
-    }
-    return filteredLines.join('\n');
+const isValidISOTimestamp = (timestamp) => {
+    if (!timestamp) return false;
+    const date = new Date(timestamp);
+    // Check if valid date and specifically in ISO format
+    return !isNaN(date) && timestamp === date.toISOString();
 };
 
-const filterByRecent = (content, recentHours, numResults) => {
-    if ((!recentHours && !numResults) || !content) return content;
+const isValidPriority = (priority) => {
+    // Must be a whole number
+    const num = parseInt(priority);
+    return !isNaN(num) && num.toString() === priority && num > 0;
+};
 
-    const lines = content.split('\n');
-    
-    // If recentHours is 0, only apply numResults filtering
-    if (recentHours === 0) {
-        return numResults > 0 ? lines.slice(-numResults).join('\n') : content;
-    }
+export const processMemoryContent = (content, { priority = 0, recentHours = 0, numResults = 0, stripMetadata = false }) => {
+    if (!content) return content;
+    if (!priority && !recentHours && !numResults && !stripMetadata) return content;
 
+    const lines = content.split('\n');
     const currentTime = Date.now();
-    const cutoffTime = currentTime - (recentHours * 60 * 60 * 1000);
+    const cutoffTime = recentHours > 0 ? currentTime - (recentHours * 60 * 60 * 1000) : 0;
     
-    // Walk backwards through lines until we hit an old entry
-    const filteredLines = [];
-    for (let i = lines.length - 1; i >= 0; i--) {
+    // Create array of lines with their timestamps for sorting
+    const processedLinesWithDates = [];
+    for (let i = 0; i < lines.length; i++) {
         const line = lines[i];
-        const dateMatch = line.match(/\[P\d+\]\s+(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/);
-        if (!dateMatch) continue;
+        const parts = line.split('|');
         
-        const timestamp = new Date(dateMatch[1]).getTime();
-        if (timestamp < cutoffTime) break; // Stop processing once we hit old entries
+        // Skip invalid lines
+        if (!parts[0]) continue;
         
-        filteredLines.unshift(line); // Add to front to maintain original order
+        // Priority check with strict validation
+        if (priority > 0) {
+            if (!isValidPriority(parts[0])) continue;
+            const memoryPriority = parseInt(parts[0]);
+            if (memoryPriority > priority) continue;
+        }
         
-        // If we have enough results, stop processing
-        if (numResults > 0 && filteredLines.length >= numResults) {
-            break;
+        // Recency check with strict ISO validation
+        if (recentHours > 0) {
+            if (!isValidISOTimestamp(parts[1])) continue;
+            const entryTime = new Date(parts[1]).getTime();
+            if (entryTime < cutoffTime) continue;
         }
+        
+        // Store the line with its timestamp for sorting
+        const timestamp = isValidISOTimestamp(parts[1]) ? new Date(parts[1]).getTime() : 0;
+        
+        // If stripMetadata is true, only keep the content part
+        const processedLine = stripMetadata && parts.length >= 3 
+            ? parts.slice(2).join('|')  // Strip metadata if requested and format is valid
+            : line;                     // Keep original line otherwise
+            
+        processedLinesWithDates.push({ line: processedLine, timestamp });
     }
-
-    return filteredLines.join('\n');
+    
+    // Sort by timestamp descending (newest first)
+    processedLinesWithDates.sort((a, b) => b.timestamp - a.timestamp);
+    
+    // Take the top N results if specified
+    const finalLines = numResults > 0
+        ? processedLinesWithDates.slice(0, numResults)
+        : processedLinesWithDates;
+    
+    // Extract just the lines and join them
+    return finalLines.map(entry => entry.line).join('\n');
 };
 
 export default {
@@ -57,12 +76,14 @@ export default {
         section: `memoryAll`,
         priority: 0,
         recentHours: 0,
-        numResults: 0
+        numResults: 0,
+        stripMetadata: false
     },
     model: 'oai-gpt4o',
 
     resolver: async (_parent, args, _contextValue, _info) => {
-        const { contextId, section = 'memoryAll', priority = 0, recentHours = 0, numResults = 0 } = args;
+        const { contextId, section = 'memoryAll', priority = 0, recentHours = 0, numResults = 0, stripMetadata = false } = args;
+        const options = { priority, recentHours, numResults, stripMetadata };
 
         // this code helps migrate old memory formats
         if (section === 'memoryLegacy') {
@@ -70,26 +91,12 @@ export default {
             return savedContext.memoryContext || "";
         }
 
-        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryContext'];
+        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryContext', 'memoryVersion'];
 
         if (section !== 'memoryAll') {
             if (validSections.includes(section)) {
-                let content = (getv && (await getv(`${contextId}-${section}`))) || "";
-                
-                if (section === 'memoryTopics') {
-                    if (recentHours > 0 || numResults > 0) {
-                        content = filterByRecent(content, recentHours, numResults);
-                    }
-                } else if (priority > 0 || numResults > 0) {
-                    content = filterByPriority(content, priority, numResults);
-                }
-                
-                // Only apply recency filter to memoryTopics
-                if (section === 'memoryTopics' && (recentHours > 0 || numResults > 0)) {
-                    content = filterByRecent(content, recentHours, numResults);
-                }
-                
-                return content;
+                const content = (getv && (await getv(`${contextId}-${section}`))) || "";
+                return processMemoryContent(content, options);
             }
             return "";
         }
@@ -99,19 +106,10 @@ export default {
         for (const section of validSections) {
             if (section === 'memoryContext') continue;
 
-            let content = (getv && (await getv(`${contextId}-${section}`))) || "";
-            
-            if (section === 'memoryTopics') {
-                if (recentHours > 0 || numResults > 0) {
-                    content = filterByRecent(content, recentHours, numResults);
-                }
-            } else if (priority > 0 || numResults > 0) {
-                content = filterByPriority(content, priority, numResults);
-            }
-            
-            memoryContents[section] = content;
+            const content = (getv && (await getv(`${contextId}-${section}`))) || "";
+            memoryContents[section] = processMemoryContent(content, options);
         }
-        const returnValue = JSON.stringify(memoryContents, null, 2);
-        return returnValue;
+        
+        return JSON.stringify(memoryContents, null, 2);
     }
 }

package/pathways/system/entity/memory/sys_save_memory.js

@@ -22,7 +22,7 @@ export default {
             return aiMemory;
         }
 
-        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser'];
+        const validSections = ['memorySelf', 'memoryDirectives', 'memoryTopics', 'memoryUser', 'memoryVersion'];
 
         // Handle single section save
         if (section !== 'memoryAll') {