@aj-archipelago/cortex 1.4.23 → 1.4.25

package/lib/fileUtils.js

@@ -702,23 +702,21 @@ function parseRawFileData(allFiles, contextKey = null) {
  * Filter and format file collection based on inCollection and chatId
  * @param {Array} rawFiles - Array of parsed file data objects
  * @param {string|null} chatId - Optional chat ID to filter by
- * @returns {Array} Filtered and sorted file collection
+ * @returns {Array} Filtered and sorted file collection (includes inCollection for reference counting)
  */
 function filterAndFormatFileCollection(rawFiles, chatId = null) {
     // Filter by inCollection and optional chatId
     const filtered = rawFiles.filter(file => isFileInCollection(file.inCollection, chatId));
     
-    // Remove inCollection from output (internal metadata)
-    const formatted = filtered.map(({ inCollection, ...file }) => file);
-    
+    // Keep inCollection in output (needed for reference counting display)
     // Sort by lastAccessed (most recent first)
-    formatted.sort((a, b) => {
+    filtered.sort((a, b) => {
         const aDate = new Date(a.lastAccessed || a.addedDate || 0);
         const bDate = new Date(b.lastAccessed || b.addedDate || 0);
         return bDate - aDate;
     });
     
-    return formatted;
+    return filtered;
 }
 
 async function loadFileCollection(contextId, contextKey = null, useCache = true, chatId = null) {
@@ -790,17 +788,15 @@ async function loadFileCollectionAll(contextId, contextKey = null) {
             // Parse raw file data
             const rawFiles = parseRawFileData(allFiles, contextKey);
             
-            // Return all files without inCollection filtering, just remove the internal metadata
-            const formatted = rawFiles.map(({ inCollection, ...file }) => file);
-            
+            // Return all files without inCollection filtering (keep inCollection for reference counting)
             // Sort by lastAccessed (most recent first)
-            formatted.sort((a, b) => {
+            rawFiles.sort((a, b) => {
                 const aDate = new Date(a.lastAccessed || a.addedDate || 0);
                 const bDate = new Date(b.lastAccessed || b.addedDate || 0);
                 return bDate - aDate;
             });
             
-            return formatted;
+            return rawFiles;
         }
     } catch (e) {
         // Collection doesn't exist yet or error reading
@@ -839,16 +835,94 @@ function normalizeInCollection(inCollection) {
     return ['*'];
 }
 
+/**
+ * Get the appropriate inCollection value based on chatId
+ * Centralized function to ensure consistent behavior across all file operations
+ * @param {string|null|undefined} chatId - Optional chat ID
+ * @returns {Array<string>} Array with chatId if provided, otherwise ['*'] for global
+ */
+function getInCollectionValue(chatId = null) {
+    if (chatId && typeof chatId === 'string' && chatId.trim() !== '') {
+        return [chatId];
+    }
+    return ['*'];
+}
+
+/**
+ * Add a chatId to an existing inCollection array (reference counting)
+ * If the chatId is already present, returns the array unchanged.
+ * 
+ * IMPORTANT: inCollection is either ['*'] (global) OR [chatId, ...] (chat-scoped), never mixed.
+ * If inCollection contains '*' (global), it stays global - no chatIds are added.
+ * 
+ * @param {Array<string>|undefined} existingInCollection - Current inCollection value
+ * @param {string|null} chatId - Chat ID to add
+ * @returns {Array<string>} Updated inCollection array
+ */
+function addChatIdToInCollection(existingInCollection, chatId) {
+    // Normalize existing to array
+    const existing = Array.isArray(existingInCollection) ? existingInCollection : [];
+    
+    // If already global, stay global
+    if (existing.includes('*')) {
+        return existing;
+    }
+    
+    // If no chatId provided, return existing or default to global
+    if (!chatId || typeof chatId !== 'string' || chatId.trim() === '') {
+        return existing.length > 0 ? existing : ['*'];
+    }
+    
+    // Add chatId if not already present
+    if (!existing.includes(chatId)) {
+        return [...existing, chatId];
+    }
+    
+    return existing;
+}
+
+/**
+ * Remove a chatId from an inCollection array (reference counting)
+ * Returns the updated array without the chatId.
+ * 
+ * IMPORTANT: Global files (['*']) are not reference-counted - they return unchanged.
+ * Only chat-scoped files have chatIds removed. When removing from collection,
+ * global files should be fully deleted, not reference-counted.
+ * 
+ * @param {Array<string>|undefined} existingInCollection - Current inCollection value
+ * @param {string|null} chatId - Chat ID to remove
+ * @returns {Array<string>} Updated inCollection array (may be empty for chat-scoped files)
+ */
+function removeChatIdFromInCollection(existingInCollection, chatId) {
+    // Normalize existing to array
+    const existing = Array.isArray(existingInCollection) ? existingInCollection : [];
+    
+    // If no chatId provided, can't remove anything
+    if (!chatId || typeof chatId !== 'string' || chatId.trim() === '') {
+        return existing;
+    }
+    
+    // If global, removing a specific chatId doesn't make sense - return as-is
+    // (global files aren't scoped to chats)
+    if (existing.includes('*')) {
+        return existing;
+    }
+    
+    // Remove the chatId
+    return existing.filter(id => id !== chatId);
+}
+
 /**
  * Update file metadata in Redis hash map (direct atomic operation)
  * @param {string} contextId - Context ID
  * @param {string} hash - File hash
  * @param {Object} metadata - Metadata to update (displayFilename, id, tags, notes, mimeType, addedDate, lastAccessed, permanent, inCollection)
  * @param {string} contextKey - Optional context key for encryption
+ * @param {string|null} chatId - Optional chat ID, used as default for inCollection if not provided in metadata and not already set
  * Note: Does NOT update CFH core fields (url, gcs, hash, filename) - those are managed by CFH
  * @returns {Promise<boolean>} True if successful
  */
-async function updateFileMetadata(contextId, hash, metadata, contextKey = null) {
+async function updateFileMetadata(contextId, hash, metadata, contextKey = null, chatId = null) {
     if (!contextId || !hash) {
         return false;
     }
@@ -872,12 +946,12 @@ async function updateFileMetadata(contextId, hash, metadata, contextKey = null)
         // Only update Cortex-managed fields, preserve CFH fields (url, gcs, hash, filename)
         const fileData = {
             ...existingData, // Preserve all CFH data (url, gcs, hash, filename, etc.)
-            // Handle inCollection: normalize if provided, otherwise preserve existing or default to global
+            // Handle inCollection: normalize if provided, otherwise preserve existing or default based on chatId
             inCollection: metadata.inCollection !== undefined
                 ? normalizeInCollection(metadata.inCollection)
                 : (existingData.inCollection !== undefined
                     ? normalizeInCollection(existingData.inCollection)
-                    : ['*']),
+                    : getInCollectionValue(chatId)),
             // Update only Cortex-managed metadata fields
             ...(metadata.displayFilename !== undefined && { displayFilename: metadata.displayFilename }),
             ...(metadata.id !== undefined && { id: metadata.id }),
@@ -914,9 +988,10 @@ async function updateFileMetadata(contextId, hash, metadata, contextKey = null)
  * @param {string} contextId - Context ID for the file collection
  * @param {string} contextKey - Optional context key for encryption (unused with hash maps)
  * @param {Array} collection - File collection array
+ * @param {string|null} chatId - Optional chat ID, used for inCollection value (chat-scoped if provided, global if not)
  * @returns {Promise<boolean>} True if successful
  */
-async function saveFileCollection(contextId, contextKey, collection) {
+async function saveFileCollection(contextId, contextKey, collection, chatId = null) {
     const cacheKey = getCollectionCacheKey(contextId, contextKey);
 
     try {
@@ -977,7 +1052,10 @@ async function saveFileCollection(contextId, contextKey, collection) {
                     addedDate: file.addedDate || existingData.timestamp || new Date().toISOString(),
                     lastAccessed: file.lastAccessed || new Date().toISOString(),
                     permanent: file.permanent !== undefined ? file.permanent : (existingData.permanent || false),
-                    inCollection: ['*'] // Mark as global chat file (available to all chats)
+                    // Add chatId to existing inCollection (reference counting) - file may be used in multiple chats
+                    inCollection: existingData.inCollection 
+                        ? addChatIdToInCollection(existingData.inCollection, chatId)
+                        : getInCollectionValue(chatId)
                 };
                 
                 // Write back to hash map (atomic operation) - encryption happens in helper
@@ -1013,9 +1091,11 @@ async function saveFileCollection(contextId, contextKey, collection) {
  * @param {string} hash - Optional file hash
  * @param {string} fileUrl - Optional: URL of file to upload (if not already in cloud storage)
  * @param {pathwayResolver} pathwayResolver - Optional pathway resolver for logging
+ * @param {boolean} permanent - If true, file is stored with permanent retention
+ * @param {string|null} chatId - Optional chat ID, used for inCollection value (chat-scoped if provided, global if not)
  * @returns {Promise<Object>} File entry object with id
  */
-async function addFileToCollection(contextId, contextKey, url, gcs, filename, tags = [], notes = '', hash = null, fileUrl = null, pathwayResolver = null, permanent = false) {
+async function addFileToCollection(contextId, contextKey, url, gcs, filename, tags = [], notes = '', hash = null, fileUrl = null, pathwayResolver = null, permanent = false, chatId = null) {
     if (!contextId || !filename) {
         throw new Error("contextId and filename are required");
     }
@@ -1123,7 +1203,10 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
                     tags: fileEntry.tags.length > 0 ? fileEntry.tags : (existingData.tags || []), // Merge tags if new ones provided
                     notes: fileEntry.notes || existingData.notes || '', // Keep existing notes if new ones empty
                     mimeType: fileEntry.mimeType || existingData.mimeType || null, // MIME type from URL (actual content type)
-                    inCollection: ['*'], // Mark as global chat file (available to all chats)
+                    // Add chatId to existing inCollection (reference counting) - file may be used in multiple chats
+                    inCollection: existingData.inCollection 
+                        ? addChatIdToInCollection(existingData.inCollection, chatId)
+                        : getInCollectionValue(chatId),
                     addedDate: existingData.addedDate || fileEntry.addedDate, // Keep earliest addedDate
                     lastAccessed: new Date().toISOString(), // Always update lastAccessed
                     permanent: fileEntry.permanent !== undefined ? fileEntry.permanent : (existingData.permanent || false),
@@ -1426,14 +1509,15 @@ async function getAvailableFiles(chatHistory, agentContext) {
 
 /**
  * Process files in chat history:
- * - Files IN collection (all agentContext contexts): update lastAccessed, strip from message (tools can access)
+ * - Files IN collection (all agentContext contexts): update lastAccessed, add chatId to inCollection (reference counting), strip from message (tools can access)
  * - Files NOT in collection: leave in message (model sees directly)
  * 
  * @param {Array} chatHistory - Chat history array
  * @param {Array} agentContext - Array of context objects { contextId, contextKey, default }
+ * @param {string|null} chatId - Optional chat ID, added to inCollection for reference counting when files are accessed
  * @returns {Promise<{chatHistory: Array, availableFiles: string}>}
  */
-async function syncAndStripFilesFromChatHistory(chatHistory, agentContext) {
+async function syncAndStripFilesFromChatHistory(chatHistory, agentContext, chatId = null) {
     if (!chatHistory || !Array.isArray(chatHistory)) {
         return { chatHistory: chatHistory || [], availableFiles: 'No files available.' };
     }
@@ -1490,9 +1574,12 @@ async function syncAndStripFilesFromChatHistory(chatHistory, agentContext) {
         const fileContextKey = contextKeyMap.get(file._contextId) || null;
         
         const now = new Date().toISOString();
+        // Update lastAccessed and add chatId to inCollection (reference counting)
+        // If this file is being used in a new chat, add that chat to the list
+        const updatedInCollection = addChatIdToInCollection(file.inCollection, chatId);
         updateFileMetadata(file._contextId, hash, {
             lastAccessed: now,
-            inCollection: ['*']
+            inCollection: updatedInCollection
         }, fileContextKey).catch((err) => {
             logger.warn(`Failed to update metadata for stripped file (hash=${hash}): ${err?.message || err}`);
         });
@@ -2438,6 +2525,9 @@ export {
     uploadFileToCloud,
     uploadImageToCloud,
     resolveFileHashesToContent,
+    getInCollectionValue,
+    addChatIdToInCollection,
+    removeChatIdFromInCollection,
     getMimeTypeFromFilename,
     getMimeTypeFromExtension,
     isTextMimeType,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.4.23",
+  "version": "1.4.25",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/files/sys_read_file_collection.js

@@ -16,7 +16,16 @@ export default {
     model: 'oai-gpt4o',
 
     resolver: async (_parent, args, _contextValue, _info) => {
-        const { agentContext } = args;
+        let { agentContext } = args;
+        
+        // Backward compatibility: if contextId is provided without agentContext, create agentContext
+        if ((!agentContext || !Array.isArray(agentContext) || agentContext.length === 0) && args.contextId) {
+            agentContext = [{ 
+                contextId: args.contextId, 
+                contextKey: args.contextKey || null, 
+                default: true 
+            }];
+        }
         
         // Validate that agentContext is provided
         if (!agentContext || !Array.isArray(agentContext) || agentContext.length === 0) {

package/pathways/system/entity/files/sys_update_file_metadata.js

@@ -19,7 +19,7 @@ export default {
     isMutation: true, // Declaratively mark this as a Mutation
 
     resolver: async (_parent, args, _contextValue, _info) => {
-        const { agentContext, hash, displayFilename, tags, notes, mimeType, permanent, inCollection } = args;
+        const { agentContext, hash, displayFilename, tags, notes, mimeType, permanent, inCollection, chatId } = args;
         
         const defaultCtx = getDefaultContext(agentContext);
         if (!defaultCtx) {
@@ -59,12 +59,13 @@ export default {
             }
             // inCollection can be: boolean true/false, or array of chat IDs (e.g., ['*'] for global, ['chat-123'] for specific chat)
             // Will be normalized by updateFileMetadata: true -> ['*'], false -> undefined (removed), array -> as-is
+            // If not provided, will default based on chatId
             if (inCollection !== undefined && inCollection !== null) {
                 metadata.inCollection = inCollection;
             }
             
             // Update metadata (only Cortex-managed fields)
-            const success = await updateFileMetadata(contextId, hash, metadata, contextKey);
+            const success = await updateFileMetadata(contextId, hash, metadata, contextKey, chatId);
             
             if (success) {
                 return JSON.stringify({ 

package/pathways/system/entity/memory/sys_memory_manager.js

@@ -26,6 +26,10 @@ export default {
     timeout: 300,
     executePathway: async ({args, resolver}) => {
         try {
+            // Skip if memory is disabled
+            if (args.useMemory === false) {
+                return "";
+            }
 
             args = { ...args, ...config.get('entityConstants') };
             let parsedMemory;

package/pathways/system/entity/sys_entity_agent.js

@@ -452,7 +452,11 @@ export default {
         
         const entityConfig = loadEntityConfig(entityId);
         const { entityTools, entityToolsOpenAiFormat } = getToolsForEntity(entityConfig);
-        const { useMemory: entityUseMemory = true, name: entityName, instructions: entityInstructions } = entityConfig || {};
+        const { name: entityName, instructions: entityInstructions } = entityConfig || {};
+        
+        // Determine useMemory: entityConfig.useMemory === false is a hard disable (entity can't use memory)
+        // Otherwise args.useMemory can disable it, default true
+        args.useMemory = entityConfig?.useMemory === false ? false : (args.useMemory ?? true);
 
         // Initialize chat history if needed
         if (!args.chatHistory || args.chatHistory.length === 0) {
@@ -488,7 +492,7 @@ export default {
 
         // Kick off the memory lookup required pathway in parallel - this takes like 500ms so we want to start it early
         let memoryLookupRequiredPromise = null;
-        if (entityUseMemory) {
+        if (args.useMemory) {
             const chatHistoryLastTurn = args.chatHistory.slice(-2);
             const chatHistorySizeOk = (JSON.stringify(chatHistoryLastTurn).length < 5000);
             if (chatHistorySizeOk) {
@@ -512,7 +516,6 @@ export default {
             entityId,
             entityTools,
             entityToolsOpenAiFormat,
-            entityUseMemory,
             entityInstructions,
             voiceResponse,
             aiMemorySelfModify,
@@ -524,7 +527,7 @@ export default {
 
         const promptPrefix = '';
 
-        const memoryTemplates = entityUseMemory ? 
+        const memoryTemplates = args.useMemory ? 
             `{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n\n{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_MEMORY_CONTEXT}}\n\n` : '';
 
         const instructionTemplates = entityInstructions ? (entityInstructions + '\n\n') : `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n\n{{renderTemplate AI_EXPERTISE}}\n\n`;
@@ -552,7 +555,7 @@ export default {
         // - Files in collection (all agentContext contexts): stripped, accessible via tools
         // - Files not in collection: left in message for model to see directly
         const { chatHistory: strippedHistory, availableFiles } = await syncAndStripFilesFromChatHistory(
-            args.chatHistory, args.agentContext
+            args.chatHistory, args.agentContext, chatId
         );
         args.chatHistory = strippedHistory;
 
@@ -560,7 +563,7 @@ export default {
         const truncatedChatHistory = resolver.modelExecutor.plugin.truncateMessagesToTargetLength(args.chatHistory, null, 1000);
       
         // Asynchronously manage memory for this context
-        if (args.aiMemorySelfModify && entityUseMemory) {
+        if (args.aiMemorySelfModify && args.useMemory) {
             callPathway('sys_memory_manager', {  ...args, chatHistory: truncatedChatHistory, stream: false })    
             .catch(error => logger.error(error?.message || "Error in sys_memory_manager pathway"));
         }

package/pathways/system/entity/tools/sys_tool_editfile.js

@@ -145,7 +145,7 @@ export default {
     ],
 
     executePathway: async ({args, runAllPrompts, resolver}) => {
-        const { file, startLine, endLine, content, oldString, newString, replaceAll = false, agentContext } = args;
+        const { file, startLine, endLine, content, oldString, newString, replaceAll = false, agentContext, chatId } = args;
         
         const defaultCtx = getDefaultContext(agentContext);
         if (!defaultCtx) {
@@ -514,7 +514,7 @@ export default {
                 
                 // Write new entry with CFH data (url, gcs, hash) + Cortex metadata
                 if (uploadResult.hash) {
-                    const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+                    const { getRedisClient, addChatIdToInCollection, getInCollectionValue } = await import('../../../../lib/fileUtils.js');
                     const redisClient = await getRedisClient();
                     if (redisClient) {
                         const contextMapKey = `FileStoreMap:ctx:${contextId}`;
@@ -529,6 +529,12 @@ export default {
                             }
                         }
                         
+                        // Merge chatId into existing inCollection (reference counting)
+                        const existingInCollection = fileToUpdate.inCollection || existingData.inCollection;
+                        const updatedInCollection = existingInCollection 
+                            ? addChatIdToInCollection(existingInCollection, chatId)
+                            : getInCollectionValue(chatId);
+                        
                         const fileData = {
                             ...existingData,
                             url: uploadResult.url,
@@ -540,7 +546,7 @@ export default {
                             tags: fileToUpdate.tags || [],
                             notes: fileToUpdate.notes || '',
                             mimeType: fileToUpdate.mimeType || mimeType || null,
-                            inCollection: ['*'],
+                            inCollection: updatedInCollection,
                             addedDate: fileToUpdate.addedDate,
                             lastAccessed: new Date().toISOString(),
                             permanent: fileToUpdate.permanent || false

package/pathways/system/entity/tools/sys_tool_file_collection.js

@@ -204,6 +204,7 @@ export default {
         }
         const contextId = defaultCtx.contextId;
         const contextKey = defaultCtx.contextKey || null;
+        const chatId = args.chatId || null;
 
         // Determine which function was called based on which parameters are present
         // Order matters: check most specific operations first
@@ -293,7 +294,7 @@ export default {
                 metadataUpdate.lastAccessed = new Date().toISOString();
 
                 // Perform the atomic update
-                const success = await updateFileMetadata(contextId, foundFile.hash, metadataUpdate, contextKey);
+                const success = await updateFileMetadata(contextId, foundFile.hash, metadataUpdate, contextKey, chatId);
                 
                 if (!success) {
                     throw new Error(`Failed to update file metadata for "${file}"`);
@@ -342,7 +343,8 @@ export default {
                     hash,
                     fileUrl,
                     resolver,
-                    permanent
+                    permanent,
+                    chatId
                 );
 
                 resolver.tool = JSON.stringify({ toolUsed: "AddFileToCollection" });
@@ -386,6 +388,7 @@ export default {
                     
                     if (matchesQuery && matchesTags) {
                         // Update lastAccessed directly (atomic operation)
+                        // Don't pass chatId - we're only updating access time, not changing inCollection
                         await updateFileMetadata(contextId, file.hash, {
                             lastAccessed: now
                         }, contextKey);
@@ -449,7 +452,8 @@ export default {
                 });
 
             } else if (isRemove) {
-                // Remove file(s) from collection and delete from cloud storage
+                // Remove file(s) from this chat's collection (reference counting)
+                // Only delete from cloud if no other chats reference the file
                 const { fileIds, fileId } = args;
                 
                 // Normalize input to array
@@ -465,9 +469,9 @@ export default {
                 }
 
                 let notFoundFiles = [];
-                let filesToRemove = [];
+                let filesToProcess = [];
 
-                // Load collection ONCE to find all files and their hashes
+                // Load collection ONCE to find all files and their data
                 // Use useCache: false to get fresh data
                 const collection = await loadFileCollection(contextId, contextKey, false);
                 
@@ -479,12 +483,13 @@ export default {
                     
                     if (foundFile) {
                         // Avoid duplicates (by hash since that's the unique key in Redis)
-                        if (!filesToRemove.some(f => f.hash === foundFile.hash)) {
-                            filesToRemove.push({
+                        if (!filesToProcess.some(f => f.hash === foundFile.hash)) {
+                            filesToProcess.push({
                                 id: foundFile.id,
                                 displayFilename: foundFile.displayFilename || foundFile.filename || null,
                                 hash: foundFile.hash || null,
-                                permanent: foundFile.permanent ?? false
+                                permanent: foundFile.permanent ?? false,
+                                inCollection: foundFile.inCollection || []
                             });
                         }
                     } else {
@@ -492,34 +497,76 @@ export default {
                     }
                 }
 
-                if (filesToRemove.length === 0 && notFoundFiles.length > 0) {
+                if (filesToProcess.length === 0 && notFoundFiles.length > 0) {
                     throw new Error(`No files found matching: ${notFoundFiles.join(', ')}`);
                 }
 
-                // Use the hashes collected from the single collection load
-                // No need to reload - we already have all the info we need
-                const hashesToDelete = filesToRemove.filter(f => f.hash);
-                
-                // Delete entries directly from hash map (atomic operations)
-                const { getRedisClient } = await import('../../../../lib/fileUtils.js');
+                // Import helpers for reference counting
+                const { getRedisClient, removeChatIdFromInCollection } = await import('../../../../lib/fileUtils.js');
                 const redisClient = await getRedisClient();
+                const contextMapKey = `FileStoreMap:ctx:${contextId}`;
+                
+                // Track files that will be fully deleted vs just updated
+                const filesToFullyDelete = [];
+                const filesToUpdate = [];
+                
+                for (const fileInfo of filesToProcess) {
+                    if (!fileInfo.hash) continue;
+                    
+                    // Check if file is global ('*') - global files can't be removed per-chat
+                    const isGlobal = Array.isArray(fileInfo.inCollection) && fileInfo.inCollection.includes('*');
+                    
+                    if (isGlobal) {
+                        // Global file - fully remove it (no reference counting for global files)
+                        filesToFullyDelete.push(fileInfo);
+                    } else if (!chatId) {
+                        // No chatId context - fully remove
+                        filesToFullyDelete.push(fileInfo);
+                    } else {
+                        // Remove this chatId from inCollection
+                        const updatedInCollection = removeChatIdFromInCollection(fileInfo.inCollection, chatId);
+                        
+                        if (updatedInCollection.length === 0) {
+                            // No more references - fully delete
+                            filesToFullyDelete.push(fileInfo);
+                        } else {
+                            // Still has references from other chats - just update inCollection
+                            filesToUpdate.push({ ...fileInfo, updatedInCollection });
+                        }
+                    }
+                }
+                
+                // Update files that still have references (remove this chatId only)
+                for (const fileInfo of filesToUpdate) {
+                    if (redisClient) {
+                        try {
+                            const existingDataStr = await redisClient.hget(contextMapKey, fileInfo.hash);
+                            if (existingDataStr) {
+                                const existingData = JSON.parse(existingDataStr);
+                                existingData.inCollection = fileInfo.updatedInCollection;
+                                await redisClient.hset(contextMapKey, fileInfo.hash, JSON.stringify(existingData));
+                                logger.info(`Removed chatId ${chatId} from file: ${fileInfo.displayFilename} (still referenced by: ${fileInfo.updatedInCollection.join(', ')})`);
+                            }
+                        } catch (e) {
+                            logger.warn(`Failed to update inCollection for file ${fileInfo.hash}: ${e.message}`);
+                        }
+                    }
+                }
+                
+                // Fully delete files with no remaining references
                 if (redisClient) {
-                    const contextMapKey = `FileStoreMap:ctx:${contextId}`;
-                    for (const fileInfo of hashesToDelete) {
+                    for (const fileInfo of filesToFullyDelete) {
                         await redisClient.hdel(contextMapKey, fileInfo.hash);
                     }
                 }
                 
-                // Always invalidate cache immediately so list operations reflect removals
-                // (even if Redis operations failed, cache might be stale)
+                // Always invalidate cache immediately so list operations reflect changes
                 invalidateFileCollectionCache(contextId, contextKey);
 
-                // Delete files from cloud storage ASYNC (fire and forget, but log errors)
-                // We do this after updating collection so user gets fast response and files are "gone" from UI immediately
-                // Use hashes captured inside the lock to ensure we delete the correct files
+                // Delete files from cloud storage ASYNC (only for files with no remaining references)
                 // IMPORTANT: Don't delete permanent files from cloud storage - they should persist
                 (async () => {
-                    for (const fileInfo of hashesToDelete) {
+                    for (const fileInfo of filesToFullyDelete) {
                         // Skip deletion if file is marked as permanent
                         if (fileInfo.permanent) {
                             logger.info(`Skipping cloud deletion for permanent file: ${fileInfo.displayFilename} (hash: ${fileInfo.hash})`);
@@ -527,7 +574,7 @@ export default {
                         }
                         
                         try {
-                            logger.info(`Deleting file from cloud storage: ${fileInfo.displayFilename} (hash: ${fileInfo.hash})`);
+                            logger.info(`Deleting file from cloud storage (no remaining references): ${fileInfo.displayFilename} (hash: ${fileInfo.hash})`);
                             await deleteFileByHash(fileInfo.hash, resolver, contextId);
                         } catch (error) {
                             logger.warn(`Failed to delete file ${fileInfo.displayFilename} (hash: ${fileInfo.hash}) from cloud storage: ${error?.message || String(error)}`);
@@ -535,8 +582,13 @@ export default {
                     }
                 })().catch(err => logger.error(`Async cloud deletion error: ${err}`));
 
-                const removedCount = filesToRemove.length;
-                const removedFiles = filesToRemove;
+                const removedCount = filesToProcess.length;
+                const removedFiles = filesToProcess.map(f => ({
+                    id: f.id,
+                    displayFilename: f.displayFilename,
+                    hash: f.hash,
+                    fullyDeleted: filesToFullyDelete.some(fd => fd.hash === f.hash)
+                }));
 
                 // Get remaining files count after deletion
                 const remainingCollection = await loadFileCollection(contextId, contextKey, false);

package/pathways/system/entity/tools/sys_tool_store_memory.js

@@ -52,8 +52,7 @@ export default {
 
     executePathway: async ({args, runAllPrompts, resolver}) => {
         // Check if memory is enabled for this entity
-        const useMemory = args.entityUseMemory !== undefined ? args.entityUseMemory : args.useMemory;
-        if (useMemory === false) {
+        if (args.useMemory === false) {
             return JSON.stringify({ 
                 error: 'Memory storage is disabled for this entity. Cannot store memories when useMemory is false.' 
             });