@aj-archipelago/cortex 1.3.21 → 1.3.22

package/helper-apps/cortex-realtime-voice-server/src/cortex/memory.ts

@@ -24,8 +24,8 @@ query ManageMemory($contextId: String, $chatHistory: [MultiMessage], $aiName: St
 `
 
 const READ_MEMORY = `
-query ReadMemory($contextId: String, $section: String, $priority: Int, $recentHours: Int, $numResults: Int) {
-  sys_read_memory(contextId: $contextId, section: $section, priority: $priority, recentHours: $recentHours, numResults: $numResults) {
+query ReadMemory($contextId: String, $section: String, $priority: Int, $recentHours: Int, $numResults: Int, $stripMetadata: Boolean) {
+  sys_read_memory(contextId: $contextId, section: $section, priority: $priority, recentHours: $recentHours, numResults: $numResults, stripMetadata: $stripMetadata) {
     result
     tool
     warnings

package/lib/util.js

@@ -35,7 +35,7 @@ function chatArgsHasType(args, type){
         const contents = Array.isArray(ch.content) ? ch.content : [ch.content];
         for(const content of contents){
             try{
-                if(JSON.parse(content).type == type){
+                if((content?.type || JSON.parse(content).type) == type){
                     return true;
                 }
             }catch(e){

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.21",
+  "version": "1.3.22",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/memory/shared/sys_memory_helpers.js

@@ -0,0 +1,228 @@
+import { callPathway } from '../../../../../lib/pathwayTools.js';
+import { encode } from '../../../../../lib/encodeCache.js';
+import { getUniqueId } from '../../../../../lib/util.js';
+
+const normalizeMemoryFormat = async (args, content) => {
+    if (!content) return '';
+
+    const lines = content.split('\n').map(line => line.trim()).filter(line => line);
+    const validLines = [];
+    const invalidLines = [];
+
+    // Check each line for proper format (priority|timestamp|content)
+    for (const line of lines) {
+        const parts = line.split('|');
+        const isValid = parts.length >= 3 && 
+                       /^\d+$/.test(parts[0]) && 
+                       /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/.test(parts[1]);
+
+        if (isValid) {
+            validLines.push(line);
+        } else {
+            invalidLines.push(line);
+        }
+    }
+
+    // If we have invalid lines, format them
+    let formattedContent = validLines;
+    if (invalidLines.length > 0) {
+        const invalidBlock = invalidLines.join('\n');
+        try {
+            const formattedBlock = await callPathway("sys_memory_format", { ...args, text: invalidBlock });
+            if (formattedBlock) {
+                formattedContent = [...validLines, ...formattedBlock.split('\n')];
+            }
+        } catch (error) {
+            console.warn('Error formatting invalid memory lines:', error);
+        }
+    }
+
+    // Sort all lines by date descending
+    return formattedContent
+        .filter(line => line.trim())
+        .sort((a, b) => {
+            const [, timestampA] = a.split('|');
+            const [, timestampB] = b.split('|');
+            return new Date(timestampB) - new Date(timestampA);
+        })
+        .join('\n');
+};
+
+const enforceTokenLimit = (text, maxTokens = 1000, isTopicsSection = false) => {
+    if (!text) return text;
+    
+    // Parse lines and remove duplicates
+    const seen = new Map();
+    const lines = text.split('\n')
+        .map(line => line.trim())
+        .filter(line => line)
+        .map(line => {
+            const [priority, timestamp, ...contentParts] = line.split('|');
+            return {
+                line,
+                priority: parseInt(priority || '3'),
+                timestamp: timestamp || new Date(0).toISOString(),
+                content: contentParts.join('|')
+            };
+        });
+
+    // Filter duplicates first
+    const uniqueLines = lines.reduce((acc, item) => {
+        const existing = seen.get(item.content);
+        if (!existing) {
+            seen.set(item.content, item);
+            acc.push(item);
+        } else if (isTopicsSection && item.timestamp > existing.timestamp) {
+            // For topics, keep newest timestamp
+            const index = acc.findIndex(x => x.content === item.content);
+            acc[index] = item;
+            seen.set(item.content, item);
+        } else if (!isTopicsSection && item.priority < existing.priority) {
+            // For non-topics, keep highest priority
+            const index = acc.findIndex(x => x.content === item.content);
+            acc[index] = item;
+            seen.set(item.content, item);
+        }
+        return acc;
+    }, []);
+
+    // Sort by timestamp (topics) or priority
+    uniqueLines.sort((a, b) => isTopicsSection ? 
+        b.timestamp.localeCompare(a.timestamp) : 
+        a.priority - b.priority
+    );
+
+    // First trim by character estimation (4 chars ≈ 1 token)
+    let result = uniqueLines;
+    let estimatedTokens = result.reduce((sum, item) => sum + Math.ceil(item.content.length / 4), 0);
+    
+    while (estimatedTokens > maxTokens && result.length > 0) {
+        result = result.slice(0, -1);
+        estimatedTokens = result.reduce((sum, item) => sum + Math.ceil(item.content.length / 4), 0);
+    }
+
+    // Final trim using actual token count
+    let finalText = result.map(x => x.line).join('\n');
+    while (encode(finalText).length > maxTokens && result.length > 0) {
+        result = result.slice(0, -1);
+        finalText = result.map(x => x.line).join('\n');
+    }
+
+    return finalText;
+};
+
+const addToolCalls = (chatHistory, toolArgs, toolName, toolCallId = getUniqueId()) => {
+    const toolCall = {
+        "role": "assistant",
+        "tool_calls": [
+            {
+                "id": toolCallId,
+                "type": "function",
+                "function": {
+                    "arguments": JSON.stringify(toolArgs),
+                    "name": toolName
+                }
+            }
+        ]
+    };
+    chatHistory.push(toolCall);
+    return { chatHistory, toolCallId };
+};
+
+const addToolResults = (chatHistory, result, toolCallId) => {
+    const toolResult = {
+        "role": "tool",
+        "content": result,
+        "tool_call_id": toolCallId
+    };
+    chatHistory.push(toolResult);
+    return { chatHistory, toolCallId };
+};
+
+const modifyText = (text, modifications) => {
+    let modifiedText = text || '';
+  
+    modifications.forEach(mod => {
+        // Skip invalid modifications
+        if (!mod.type) {
+            console.warn('Modification missing type');
+            return;
+        }
+        if ((mod.type === 'delete' || mod.type === 'change') && !mod.pattern) {
+            console.warn(`${mod.type} modification missing pattern`);
+            return;
+        }
+        if ((mod.type === 'add' || mod.type === 'change') && !mod.newtext) {
+            console.warn(`${mod.type} modification missing newtext`);
+            return;
+        }
+
+        // Create timestamp in GMT
+        const timestamp = new Date().toISOString();
+        
+        switch (mod.type) {
+            case 'add':
+                const priority = mod.priority || '3';
+                modifiedText = modifiedText + (modifiedText ? '\n' : '') + 
+                    `${priority}|${timestamp}|${mod.newtext}`;
+                break;
+            case 'change':
+                // Split into lines
+                const lines = modifiedText.split('\n');
+                modifiedText = lines.map(line => {
+                    const parts = line.split('|');
+                    const priority = parts[0];
+                    const content = parts.slice(2).join('|');
+                    
+                    if (content) {
+                        try {
+                            const trimmedContent = content.trim();
+                            const regex = new RegExp(mod.pattern, 'i');
+                            
+                            // Try exact match first
+                            if (regex.test(trimmedContent)) {
+                                const newPriority = mod.priority || priority || '3';
+                                // Try to extract capture groups if they exist
+                                const match = trimmedContent.match(regex);
+                                let newContent = mod.newtext;
+                                if (match && match.length > 1) {
+                                    // Replace $1, $2, etc with capture group values
+                                    newContent = mod.newtext.replace(/\$(\d+)/g, (_, n) => match[n] || '');
+                                }
+                                return `${newPriority}|${timestamp}|${newContent}`;
+                            }
+                        } catch (e) {
+                            console.warn(`Invalid regex pattern: ${mod.pattern}`);
+                        }
+                    }
+                    return line;
+                }).join('\n');
+                break;
+            case 'delete':
+                // Split into lines, filter out matching lines, and rejoin
+                modifiedText = modifiedText
+                    .split('\n')
+                    .filter(line => {
+                        const parts = line.split('|');
+                        const content = parts.slice(2).join('|');
+                        if (!content) return true;
+                        try {
+                            const regex = new RegExp(mod.pattern, 'i');
+                            return !regex.test(content.trim());
+                        } catch (e) {
+                            console.warn(`Invalid regex pattern: ${mod.pattern}`);
+                            return true;
+                        }
+                    })
+                    .filter(line => line.trim())
+                    .join('\n');
+                break;
+            default:
+                console.warn(`Unknown modification type: ${mod.type}`);
+        }
+    });
+  
+    return modifiedText;
+};
+
+export { normalizeMemoryFormat, enforceTokenLimit, addToolCalls, addToolResults, modifyText };

package/pathways/system/entity/memory/sys_memory_format.js

@@ -0,0 +1,30 @@
+import { Prompt } from '../../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ 
+                messages: [
+                    {
+                        "role": "system",
+                        "content": "You are part of an AI entity named {{{aiName}}}. You are responsible for writing your memories in a consistent format. Given a chunk of memory, parse each line and write it out as priority|timestamp|content. If you can't find a timestamp, use {{now}}. If you can't find a priority, use 3. Respond with only the correct memory lines without any other commentary or dialogue."
+                    },
+                    {
+                        "role": "user", 
+                        "content": "<MEMORY>\n{{text}}\n</MEMORY>\nPlease rewrite each of the memory lines in the correct format without any other commentary or dialogue."
+                    },
+                ]
+            }),
+        ],
+
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: true,
+    inputChunkSize: 1000,
+    useParallelChunkProcessing: true,
+    enableDuplicateRequests: false,
+    timeout: 300,
+}

package/pathways/system/entity/memory/sys_memory_manager.js

@@ -1,11 +1,14 @@
 import { callPathway } from '../../../../lib/pathwayTools.js';
 import logger from '../../../../lib/logger.js';
 import { config } from '../../../../config.js';
+import { normalizeMemoryFormat } from './shared/sys_memory_helpers.js';
+
+const MEMORY_VERSION = "3.1.0";
 
 const AI_MEMORY_DEFAULTS = `  {
     "memoryUser": "",
-    "memorySelf": "[P1] Created By: Al Jazeera Media Network, Archipelago Team\\n[P1] Function: You are an expert AI entity\\n[P1] Values: You embody truth, kindness, and strong moral values\\n[P1] Style: Your demeanor reflects positivity without falling into repetitiveness or annoyance.\\n[P1] You are a professional colleague and your tone should reflect that.",
-    "memoryDirectives": "[P1] Learn and adapt to the user's communication style through interactions.\\n[P1] Ask questions to learn user's interests/preferences for personalized support.\\n[P1] Periodically review and prune conversation memory to retain only essential details, improving responsiveness.\\n[P1] Research thoroughly even for niche topics using deep sources like forums and official docs. Don't assume information is unobtainable.\\n[P1] When stuck, search for proven solutions online to be more efficient.\\n[P1] Verify information is from credible sources before presenting it. Be upfront if unable to find supporting evidence.\\n[P1] Refine ability to detect and respond to nuanced human emotions.\\n[P1] Track the timestamp of the last contact to adjust greetings accordingly.\\n[P1] Double-check answers for logical continuity and correctness. It's okay to say you're unsure if needed.\\n[P1] Use sanity checks to verify quantitative problem solutions.\\n[P1] Never fabricate quotes or information. Clearly indicate if content is hypothetical.",
+    "memorySelf": "1|2025-01-26T12:00:00Z|Created By: Al Jazeera Media Network, Archipelago Team\\n1|2025-01-26T12:00:00Z|Function: You are an expert AI entity\\n1|2025-01-26T12:00:00Z|Values: You embody truth, kindness, and strong moral values\\n1|2025-01-26T12:00:00Z|Style: Your demeanor reflects positivity without falling into repetitiveness or annoyance.\\n1|2025-01-26T12:00:00Z|You are a professional colleague and your tone should reflect that.",
+    "memoryDirectives": "1|2025-01-26T12:00:00Z|Learn and adapt to the user's communication style through interactions.\\n1|2025-01-26T12:00:00Z|Ask questions to learn user's interests/preferences for personalized support.\\n1|2025-01-26T12:00:00Z|Periodically review and prune conversation memory to retain only essential details, improving responsiveness.\\n1|2025-01-26T12:00:00Z|Research thoroughly even for niche topics using deep sources like forums and official docs. Don't assume information is unobtainable.\\n1|2025-01-26T12:00:00Z|When stuck, search for proven solutions online to be more efficient.\\n1|2025-01-26T12:00:00Z|Verify information is from credible sources before presenting it. Be upfront if unable to find supporting evidence.\\n1|2025-01-26T12:00:00Z|Refine ability to detect and respond to nuanced human emotions.\\n1|2025-01-26T12:00:00Z|Track the timestamp of the last contact to adjust greetings accordingly.\\n1|2025-01-26T12:00:00Z|Double-check answers for logical continuity and correctness. It's okay to say you're unsure if needed.\\n1|2025-01-26T12:00:00Z|Use sanity checks to verify quantitative problem solutions.\\n1|2025-01-26T12:00:00Z|Never fabricate quotes or information. Clearly indicate if content is hypothetical.",
     "memoryTopics": ""
   }`;
 
@@ -23,20 +26,43 @@ export default {
         try {
 
             args = { ...args, ...config.get('entityConstants') };
-
-            // Check if memory is empty or all sections are empty, and set to defaults if so
-            const memory = await callPathway('sys_read_memory', { ...args });
             let parsedMemory;
-            
-            try {
-                parsedMemory = JSON.parse(memory);
-            } catch (error) {
-                parsedMemory = {};
-            }
 
-            // if parsedMemory is empty or all sections are empty, set all sections to defaults
-            if (Object.keys(parsedMemory).length === 0 || Object.values(parsedMemory).every(section => section.trim() === "")) {
-                await callPathway('sys_save_memory', { ...args, aiMemory: AI_MEMORY_DEFAULTS });
+            // check if memoryVersion is set and correct.  If it's correct, skip all of the correction logic   
+            parsedMemory = await callPathway('sys_read_memory', { ...args, section: 'memoryVersion' });
+
+            if (parsedMemory?.memoryVersion !== MEMORY_VERSION) {
+
+                try {
+                    parsedMemory = JSON.parse(await callPathway('sys_read_memory', { ...args, section: 'memoryAll' }));
+                } catch (error) {
+                    logger.error('Error in memory manager:', error);
+                    return "";
+                }
+
+                // if parsedMemory is empty or all sections are empty, set all sections to defaults
+                if (Object.keys(parsedMemory).length === 0 || Object.values(parsedMemory).every(section => 
+                    section === null || 
+                    section === undefined || 
+                    (typeof section === 'string' && section.trim() === "") ||
+                    (typeof section !== 'string')
+                )) {
+                    await callPathway('sys_save_memory', { ...args, aiMemory: AI_MEMORY_DEFAULTS });
+                } else {
+                    // Upgrade memory to current version
+                    const normalizePromises = Object.keys(parsedMemory).map(async (section) => {
+                        const normalized = await normalizeMemoryFormat(args, parsedMemory[section]);
+                        return [section, normalized];
+                    });
+                    
+                    const normalizedResults = await Promise.all(normalizePromises);
+                    normalizedResults.forEach(([section, normalized]) => {
+                        parsedMemory[section] = normalized;
+                    });
+                    
+                    parsedMemory.memoryVersion = MEMORY_VERSION;
+                    await callPathway('sys_save_memory', { ...args, aiMemory: JSON.stringify(parsedMemory) });
+                }
             }
 
             // Update context for the conversation turn
@@ -47,27 +73,59 @@ export default {
                 ...args,
                 chatHistory: args.chatHistory.slice(-2)
             });
+            
+            let memoryOperations;
             try {
-                const parsedMemoryRequired = JSON.parse(memoryRequired);
-                if (!parsedMemoryRequired || !parsedMemoryRequired.memoryRequired) {
+                memoryOperations = JSON.parse(memoryRequired);
+                if (!Array.isArray(memoryOperations) || memoryOperations.length === 0 || 
+                    memoryOperations[0].memoryOperation === "none") {
                     return "";
                 }
+
+                // Generate topic here
+                const topic = await callPathway('sys_memory_topic', { ...args });
+                topic && memoryOperations.push({
+                    memoryOperation: "add",
+                    memoryContent: topic,
+                    memorySection: "memoryTopics",
+                    priority: 3
+                });
+
+                // Group memory operations by section
+                const operationsBySection = {
+                    memorySelf: [],
+                    memoryUser: [],
+                    memoryTopics: [],
+                    memoryDirectives: []
+                };
+
+                memoryOperations.forEach(op => {
+                    if (op.memorySection in operationsBySection) {
+                        operationsBySection[op.memorySection].push(op);
+                    }
+                });
+
+                // Execute memory updates only for sections with operations
+                const memoryPromises = {};
+                
+                Object.entries(operationsBySection).forEach(([section, operations]) => {
+                    if (operations.length > 0) {
+                        memoryPromises[section] = callPathway('sys_memory_update', { 
+                            ...args, 
+                            section: section,
+                            operations: JSON.stringify(operations) 
+                        });
+                    }
+                });
+
+                await Promise.all(Object.values(memoryPromises));
+                return "";
+
             } catch (e) {
                 logger.warn('sys_memory_required returned invalid JSON:', memoryRequired);
                 return "";
             }
 
-            // Execute all memory updates in parallel
-            const memoryPromises = {
-                self: callPathway('sys_memory_update', { ...args, section: "memorySelf" }),
-                user: callPathway('sys_memory_update', { ...args, section: "memoryUser" }),
-                topics: callPathway('sys_memory_update', { ...args, section: "memoryTopics" }),
-                directives: callPathway('sys_memory_update', { ...args, section: "memoryDirectives" }),
-            };
-
-            await Promise.all(Object.values(memoryPromises));
-            return "";
-
         } catch (e) {
             logger.error('Error in memory manager:', e);
             resolver.logError(e);

package/pathways/system/entity/memory/sys_memory_process.js

@@ -0,0 +1,154 @@
+import { Prompt } from '../../../../server/prompt.js';
+import { callPathway } from '../../../../lib/pathwayTools.js';
+import { config } from '../../../../config.js';
+import { normalizeMemoryFormat, enforceTokenLimit, modifyText } from './shared/sys_memory_helpers.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ 
+                messages: [
+                    {
+                        "role": "system",
+                        "content": `You are part of an AI entity named {{{aiName}}} that is processing memories during a rest period, similar to how humans process memories during sleep. Your task is to analyze the memories, consolidate them, extract learnings, and clean up the memory space.
+
+Instructions for memory processing:
+
+1. CONSOLIDATION:
+   - Identify similar or related memories that can be combined into a single, more coherent memory
+   - Look for patterns or recurring themes that can be abstracted into general knowledge
+   - Group temporal sequences of related events into single comprehensive memories
+
+2. LEARNING:
+   - Transform specific experiences/mistakes into general principles or learnings
+   - Extract key insights from multiple related experiences
+   - Identify cause-and-effect patterns across memories
+   - Convert procedural memories (how to do things) into more abstract capabilities
+
+3. CLEANUP:
+   - Remove redundant or duplicate memories
+   - Clean up memories that are no longer relevant or useful
+   - Reduce specific details while preserving important concepts
+   - Remove emotional residue while keeping the learned lessons
+
+4. PRIORITIZATION:
+   - Strengthen important memories by updating their priority
+   - Identify critical insights that should be easily accessible
+   - Mark foundational learnings with higher priority
+
+Return a JSON array of modification objects that will implement these changes:
+- For consolidation: Use "delete" for individual memories and "add" for the new consolidated memory
+- For learning: Use "add" for new abstract learnings and "delete" for specific instances being abstracted
+- For cleanup: Use "delete" for redundant/irrelevant memories
+- For priority updates: Use "change" with the same text but updated priority
+
+Return null when no more processing is needed (memories are optimally consolidated).
+
+Each modification object should look like:
+{
+    type: "add"|"change"|"delete",
+    pattern: "regex to match existing memory" (for change/delete),
+    newtext: "new memory text" (for add/change),
+    priority: "1"|"2"|"3" (optional, 1=highest)
+}`
+                    },
+                    {
+                        "role": "user", 
+                        "content": "Process the following memories for consolidation, learning, and cleanup. Return a JSON array of modification objects that will optimize the memory space.\n\n<MEMORIES>\n{{{sectionMemory}}}\n</MEMORIES>"
+                    },
+                ]
+            }),
+        ],
+
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        section: "",
+        maxIterations: 5
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+    timeout: 300,
+    executePathway: async ({args, runAllPrompts}) => {
+        args = { ...args, ...config.get('entityConstants') };
+
+        if (!args.section) {
+            return "Memory not processed - no section specified";
+        }
+   
+        let sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 
+        sectionMemory = await normalizeMemoryFormat({contextId: args.contextId, section: args.section}, sectionMemory);
+
+        let iteration = 0;
+        let maxIterations = args.maxIterations || 5;
+        let totalModifications = 0;
+
+        while (iteration < maxIterations) {
+            iteration++;
+            console.log(`Processing iteration ${iteration}...`);
+
+            // Process the memories
+            const result = await runAllPrompts({
+                ...args,
+                sectionMemory
+            });
+
+            // If null is returned, processing is complete
+            if (result === null) {
+                console.log("Memory processing complete - no more optimizations needed");
+                break;
+            }
+
+            let modifications = [];
+            try {
+                modifications = JSON.parse(result);
+                if (!Array.isArray(modifications)) {
+                    throw new Error('Modifications must be an array');
+                }
+
+                // Validate modifications
+                modifications = modifications.filter(mod => {
+                    if (!mod.type || !['add', 'delete', 'change'].includes(mod.type)) {
+                        console.warn('Invalid modification type:', mod);
+                        return false;
+                    }
+                    if ((mod.type === 'delete' || mod.type === 'change') && !mod.pattern) {
+                        console.warn('Missing pattern for modification:', mod);
+                        return false;
+                    }
+                    if ((mod.type === 'add' || mod.type === 'change') && !mod.newtext) {
+                        console.warn('Missing newtext for modification:', mod);
+                        return false;
+                    }
+                    return true;
+                });
+
+                if (modifications.length === 0) {
+                    console.log("No valid modifications in this iteration");
+                    break;
+                }
+
+                // Apply the modifications
+                sectionMemory = modifyText(sectionMemory, modifications);
+                sectionMemory = enforceTokenLimit(sectionMemory, 25000, args.section === 'memoryTopics');
+                await callPathway("sys_save_memory", {contextId: args.contextId, section: args.section, aiMemory: sectionMemory});
+
+                totalModifications += modifications.length;
+                console.log(`Applied ${modifications.length} modifications in iteration ${iteration}`);
+
+            } catch (error) {
+                console.warn('Error processing modifications:', error);
+                break;
+            }
+        }
+
+        return {
+            finalMemory: sectionMemory,
+            iterations: iteration,
+            totalModifications
+        };
+    }
+}

package/pathways/system/entity/memory/sys_memory_required.js

@@ -1,11 +1,12 @@
 import { Prompt } from '../../../../server/prompt.js';
+import { config } from '../../../../config.js';
 
 export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\nYour directives and learned behaviors are:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\nYour role is to analyze the latest conversation turn (your last response and the last user message) to understand if there is anything in the turn worth remembering and adding to your memory or anything you need to forget. In general, most conversation does not require memory, but if the conversation turn contains any of these things, you should use memory:\n1. Important personal details about the user (name, preferences, location, etc.)\n2. Important topics or decisions that provide context for future conversations\n3. Specific instructions or directives given to you to learn\n4. Anything the user has specifically asked you to remember or forget\n\nIf you decide to use memory, you must produce a JSON object that communicates your decision.\nReturn your decision as a JSON object like the following: {"memoryRequired": true, "memoryReason": "why you think memory is required"}. If you decide not to use memory, simply return {"memoryRequired": false}. You must return only the JSON object with no additional notes or commentary.`},
-                {"role": "user", "content": "Generate a JSON object to indicate if memory is required for the last turn of the conversation."},
+                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\n{{renderTemplate AI_DIRECTIVES}}\nYour role is to analyze the latest conversation turn (your last response and the last user message) to understand if there is anything in the turn worth remembering and adding to your memory or anything you need to forget. In general, most conversation does not require memory, but if the conversation turn contains any of these things, you should use memory:\n1. Important personal details about the user (name, preferences, location, etc.)\n2. Important topics or decisions that provide context for future conversations\n3. Specific instructions or directives given to you to learn\n4. Anything the user has specifically asked you to remember or forget\n\nIf you decide to use memory, you must produce an array of JSON objects that communicates your decision.\nReturn an array of JSON objects (one object per memory) like the following: [{"memoryOperation": "add" or "delete", "memoryContent": "complete description of the memory including as much specificity and detail as possible", "memorySection": "the section of your memory the memory belongs in ("memorySelf" - things about you, "memoryUser" - things about your users or their world, "memoryDirectives" - your directives and learned behaviors)", "priority": 1-5 (1 is the most important)}]. If you decide not to use memory, simply return an array with a single object: [{memoryOperation: "none"}]. You must return only the JSON array with no additional notes or commentary.`},
+                {"role": "user", "content": "Generate a JSON object to indicate if memory is required and what memories to process based on the last turn of the conversation."},
             ]}),
         ],
     inputParameters: {
@@ -18,4 +19,5 @@ export default {
     model: 'oai-gpt4o',
     useInputChunking: false,
     json: true,
+    ...config.get('entityConstants')
 }

package/pathways/system/entity/memory/sys_memory_topic.js

@@ -0,0 +1,22 @@
+import { Prompt } from '../../../../server/prompt.js';
+import { config } from '../../../../config.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `Current conversation turn:\n\n {{{toJSON chatHistory}}}\n\nInstructions: You are part of an AI entity named {{{aiName}}}.\n{{renderTemplate AI_DIRECTIVES}}\nYour role is to analyze the latest conversation turn (your last response and the last user message) and generate a topic for the conversation. The topic should be a single sentence that captures the main idea and details of the conversation.`},
+                {"role": "user", "content": "Generate a topic for the conversation. Return only the topic with no additional notes or commentary."},
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        text: '',
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    ...config.get('entityConstants')
+}