@aj-archipelago/cortex 1.4.23 → 1.4.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/fileUtils.js CHANGED
@@ -702,23 +702,21 @@ function parseRawFileData(allFiles, contextKey = null) {
702
702
  * Filter and format file collection based on inCollection and chatId
703
703
  * @param {Array} rawFiles - Array of parsed file data objects
704
704
  * @param {string|null} chatId - Optional chat ID to filter by
705
- * @returns {Array} Filtered and sorted file collection
705
+ * @returns {Array} Filtered and sorted file collection (includes inCollection for reference counting)
706
706
  */
707
707
  function filterAndFormatFileCollection(rawFiles, chatId = null) {
708
708
  // Filter by inCollection and optional chatId
709
709
  const filtered = rawFiles.filter(file => isFileInCollection(file.inCollection, chatId));
710
710
 
711
- // Remove inCollection from output (internal metadata)
712
- const formatted = filtered.map(({ inCollection, ...file }) => file);
713
-
711
+ // Keep inCollection in output (needed for reference counting display)
714
712
  // Sort by lastAccessed (most recent first)
715
- formatted.sort((a, b) => {
713
+ filtered.sort((a, b) => {
716
714
  const aDate = new Date(a.lastAccessed || a.addedDate || 0);
717
715
  const bDate = new Date(b.lastAccessed || b.addedDate || 0);
718
716
  return bDate - aDate;
719
717
  });
720
718
 
721
- return formatted;
719
+ return filtered;
722
720
  }
723
721
 
724
722
  async function loadFileCollection(contextId, contextKey = null, useCache = true, chatId = null) {
@@ -790,17 +788,15 @@ async function loadFileCollectionAll(contextId, contextKey = null) {
790
788
  // Parse raw file data
791
789
  const rawFiles = parseRawFileData(allFiles, contextKey);
792
790
 
793
- // Return all files without inCollection filtering, just remove the internal metadata
794
- const formatted = rawFiles.map(({ inCollection, ...file }) => file);
795
-
791
+ // Return all files without inCollection filtering (keep inCollection for reference counting)
796
792
  // Sort by lastAccessed (most recent first)
797
- formatted.sort((a, b) => {
793
+ rawFiles.sort((a, b) => {
798
794
  const aDate = new Date(a.lastAccessed || a.addedDate || 0);
799
795
  const bDate = new Date(b.lastAccessed || b.addedDate || 0);
800
796
  return bDate - aDate;
801
797
  });
802
798
 
803
- return formatted;
799
+ return rawFiles;
804
800
  }
805
801
  } catch (e) {
806
802
  // Collection doesn't exist yet or error reading
@@ -839,16 +835,94 @@ function normalizeInCollection(inCollection) {
839
835
  return ['*'];
840
836
  }
841
837
 
838
+ /**
839
+ * Get the appropriate inCollection value based on chatId
840
+ * Centralized function to ensure consistent behavior across all file operations
841
+ * @param {string|null|undefined} chatId - Optional chat ID
842
+ * @returns {Array<string>} Array with chatId if provided, otherwise ['*'] for global
843
+ */
844
+ function getInCollectionValue(chatId = null) {
845
+ if (chatId && typeof chatId === 'string' && chatId.trim() !== '') {
846
+ return [chatId];
847
+ }
848
+ return ['*'];
849
+ }
850
+
851
+ /**
852
+ * Add a chatId to an existing inCollection array (reference counting)
853
+ * If the chatId is already present, returns the array unchanged.
854
+ *
855
+ * IMPORTANT: inCollection is either ['*'] (global) OR [chatId, ...] (chat-scoped), never mixed.
856
+ * If inCollection contains '*' (global), it stays global - no chatIds are added.
857
+ *
858
+ * @param {Array<string>|undefined} existingInCollection - Current inCollection value
859
+ * @param {string|null} chatId - Chat ID to add
860
+ * @returns {Array<string>} Updated inCollection array
861
+ */
862
+ function addChatIdToInCollection(existingInCollection, chatId) {
863
+ // Normalize existing to array
864
+ const existing = Array.isArray(existingInCollection) ? existingInCollection : [];
865
+
866
+ // If already global, stay global
867
+ if (existing.includes('*')) {
868
+ return existing;
869
+ }
870
+
871
+ // If no chatId provided, return existing or default to global
872
+ if (!chatId || typeof chatId !== 'string' || chatId.trim() === '') {
873
+ return existing.length > 0 ? existing : ['*'];
874
+ }
875
+
876
+ // Add chatId if not already present
877
+ if (!existing.includes(chatId)) {
878
+ return [...existing, chatId];
879
+ }
880
+
881
+ return existing;
882
+ }
883
+
884
+ /**
885
+ * Remove a chatId from an inCollection array (reference counting)
886
+ * Returns the updated array without the chatId.
887
+ *
888
+ * IMPORTANT: Global files (['*']) are not reference-counted - they return unchanged.
889
+ * Only chat-scoped files have chatIds removed. When removing from collection,
890
+ * global files should be fully deleted, not reference-counted.
891
+ *
892
+ * @param {Array<string>|undefined} existingInCollection - Current inCollection value
893
+ * @param {string|null} chatId - Chat ID to remove
894
+ * @returns {Array<string>} Updated inCollection array (may be empty for chat-scoped files)
895
+ */
896
+ function removeChatIdFromInCollection(existingInCollection, chatId) {
897
+ // Normalize existing to array
898
+ const existing = Array.isArray(existingInCollection) ? existingInCollection : [];
899
+
900
+ // If no chatId provided, can't remove anything
901
+ if (!chatId || typeof chatId !== 'string' || chatId.trim() === '') {
902
+ return existing;
903
+ }
904
+
905
+ // If global, removing a specific chatId doesn't make sense - return as-is
906
+ // (global files aren't scoped to chats)
907
+ if (existing.includes('*')) {
908
+ return existing;
909
+ }
910
+
911
+ // Remove the chatId
912
+ return existing.filter(id => id !== chatId);
913
+ }
914
+
842
915
  /**
843
916
  * Update file metadata in Redis hash map (direct atomic operation)
844
917
  * @param {string} contextId - Context ID
845
918
  * @param {string} hash - File hash
846
919
  * @param {Object} metadata - Metadata to update (displayFilename, id, tags, notes, mimeType, addedDate, lastAccessed, permanent, inCollection)
847
920
  * @param {string} contextKey - Optional context key for encryption
921
+ * @param {string|null} chatId - Optional chat ID, used as default for inCollection if not provided in metadata and not already set
848
922
  * Note: Does NOT update CFH core fields (url, gcs, hash, filename) - those are managed by CFH
849
923
  * @returns {Promise<boolean>} True if successful
850
924
  */
851
- async function updateFileMetadata(contextId, hash, metadata, contextKey = null) {
925
+ async function updateFileMetadata(contextId, hash, metadata, contextKey = null, chatId = null) {
852
926
  if (!contextId || !hash) {
853
927
  return false;
854
928
  }
@@ -872,12 +946,12 @@ async function updateFileMetadata(contextId, hash, metadata, contextKey = null)
872
946
  // Only update Cortex-managed fields, preserve CFH fields (url, gcs, hash, filename)
873
947
  const fileData = {
874
948
  ...existingData, // Preserve all CFH data (url, gcs, hash, filename, etc.)
875
- // Handle inCollection: normalize if provided, otherwise preserve existing or default to global
949
+ // Handle inCollection: normalize if provided, otherwise preserve existing or default based on chatId
876
950
  inCollection: metadata.inCollection !== undefined
877
951
  ? normalizeInCollection(metadata.inCollection)
878
952
  : (existingData.inCollection !== undefined
879
953
  ? normalizeInCollection(existingData.inCollection)
880
- : ['*']),
954
+ : getInCollectionValue(chatId)),
881
955
  // Update only Cortex-managed metadata fields
882
956
  ...(metadata.displayFilename !== undefined && { displayFilename: metadata.displayFilename }),
883
957
  ...(metadata.id !== undefined && { id: metadata.id }),
@@ -914,9 +988,10 @@ async function updateFileMetadata(contextId, hash, metadata, contextKey = null)
914
988
  * @param {string} contextId - Context ID for the file collection
915
989
  * @param {string} contextKey - Optional context key for encryption (unused with hash maps)
916
990
  * @param {Array} collection - File collection array
991
+ * @param {string|null} chatId - Optional chat ID, used for inCollection value (chat-scoped if provided, global if not)
917
992
  * @returns {Promise<boolean>} True if successful
918
993
  */
919
- async function saveFileCollection(contextId, contextKey, collection) {
994
+ async function saveFileCollection(contextId, contextKey, collection, chatId = null) {
920
995
  const cacheKey = getCollectionCacheKey(contextId, contextKey);
921
996
 
922
997
  try {
@@ -977,7 +1052,10 @@ async function saveFileCollection(contextId, contextKey, collection) {
977
1052
  addedDate: file.addedDate || existingData.timestamp || new Date().toISOString(),
978
1053
  lastAccessed: file.lastAccessed || new Date().toISOString(),
979
1054
  permanent: file.permanent !== undefined ? file.permanent : (existingData.permanent || false),
980
- inCollection: ['*'] // Mark as global chat file (available to all chats)
1055
+ // Add chatId to existing inCollection (reference counting) - file may be used in multiple chats
1056
+ inCollection: existingData.inCollection
1057
+ ? addChatIdToInCollection(existingData.inCollection, chatId)
1058
+ : getInCollectionValue(chatId)
981
1059
  };
982
1060
 
983
1061
  // Write back to hash map (atomic operation) - encryption happens in helper
@@ -1013,9 +1091,11 @@ async function saveFileCollection(contextId, contextKey, collection) {
1013
1091
  * @param {string} hash - Optional file hash
1014
1092
  * @param {string} fileUrl - Optional: URL of file to upload (if not already in cloud storage)
1015
1093
  * @param {pathwayResolver} pathwayResolver - Optional pathway resolver for logging
1094
+ * @param {boolean} permanent - If true, file is stored with permanent retention
1095
+ * @param {string|null} chatId - Optional chat ID, used for inCollection value (chat-scoped if provided, global if not)
1016
1096
  * @returns {Promise<Object>} File entry object with id
1017
1097
  */
1018
- async function addFileToCollection(contextId, contextKey, url, gcs, filename, tags = [], notes = '', hash = null, fileUrl = null, pathwayResolver = null, permanent = false) {
1098
+ async function addFileToCollection(contextId, contextKey, url, gcs, filename, tags = [], notes = '', hash = null, fileUrl = null, pathwayResolver = null, permanent = false, chatId = null) {
1019
1099
  if (!contextId || !filename) {
1020
1100
  throw new Error("contextId and filename are required");
1021
1101
  }
@@ -1123,7 +1203,10 @@ async function addFileToCollection(contextId, contextKey, url, gcs, filename, ta
1123
1203
  tags: fileEntry.tags.length > 0 ? fileEntry.tags : (existingData.tags || []), // Merge tags if new ones provided
1124
1204
  notes: fileEntry.notes || existingData.notes || '', // Keep existing notes if new ones empty
1125
1205
  mimeType: fileEntry.mimeType || existingData.mimeType || null, // MIME type from URL (actual content type)
1126
- inCollection: ['*'], // Mark as global chat file (available to all chats)
1206
+ // Add chatId to existing inCollection (reference counting) - file may be used in multiple chats
1207
+ inCollection: existingData.inCollection
1208
+ ? addChatIdToInCollection(existingData.inCollection, chatId)
1209
+ : getInCollectionValue(chatId),
1127
1210
  addedDate: existingData.addedDate || fileEntry.addedDate, // Keep earliest addedDate
1128
1211
  lastAccessed: new Date().toISOString(), // Always update lastAccessed
1129
1212
  permanent: fileEntry.permanent !== undefined ? fileEntry.permanent : (existingData.permanent || false),
@@ -1426,14 +1509,15 @@ async function getAvailableFiles(chatHistory, agentContext) {
1426
1509
 
1427
1510
  /**
1428
1511
  * Process files in chat history:
1429
- * - Files IN collection (all agentContext contexts): update lastAccessed, strip from message (tools can access)
1512
+ * - Files IN collection (all agentContext contexts): update lastAccessed, add chatId to inCollection (reference counting), strip from message (tools can access)
1430
1513
  * - Files NOT in collection: leave in message (model sees directly)
1431
1514
  *
1432
1515
  * @param {Array} chatHistory - Chat history array
1433
1516
  * @param {Array} agentContext - Array of context objects { contextId, contextKey, default }
1517
+ * @param {string|null} chatId - Optional chat ID, added to inCollection for reference counting when files are accessed
1434
1518
  * @returns {Promise<{chatHistory: Array, availableFiles: string}>}
1435
1519
  */
1436
- async function syncAndStripFilesFromChatHistory(chatHistory, agentContext) {
1520
+ async function syncAndStripFilesFromChatHistory(chatHistory, agentContext, chatId = null) {
1437
1521
  if (!chatHistory || !Array.isArray(chatHistory)) {
1438
1522
  return { chatHistory: chatHistory || [], availableFiles: 'No files available.' };
1439
1523
  }
@@ -1490,9 +1574,12 @@ async function syncAndStripFilesFromChatHistory(chatHistory, agentContext) {
1490
1574
  const fileContextKey = contextKeyMap.get(file._contextId) || null;
1491
1575
 
1492
1576
  const now = new Date().toISOString();
1577
+ // Update lastAccessed and add chatId to inCollection (reference counting)
1578
+ // If this file is being used in a new chat, add that chat to the list
1579
+ const updatedInCollection = addChatIdToInCollection(file.inCollection, chatId);
1493
1580
  updateFileMetadata(file._contextId, hash, {
1494
1581
  lastAccessed: now,
1495
- inCollection: ['*']
1582
+ inCollection: updatedInCollection
1496
1583
  }, fileContextKey).catch((err) => {
1497
1584
  logger.warn(`Failed to update metadata for stripped file (hash=${hash}): ${err?.message || err}`);
1498
1585
  });
@@ -2438,6 +2525,9 @@ export {
2438
2525
  uploadFileToCloud,
2439
2526
  uploadImageToCloud,
2440
2527
  resolveFileHashesToContent,
2528
+ getInCollectionValue,
2529
+ addChatIdToInCollection,
2530
+ removeChatIdFromInCollection,
2441
2531
  getMimeTypeFromFilename,
2442
2532
  getMimeTypeFromExtension,
2443
2533
  isTextMimeType,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aj-archipelago/cortex",
3
- "version": "1.4.23",
3
+ "version": "1.4.24",
4
4
  "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
5
5
  "private": false,
6
6
  "repository": {
@@ -16,7 +16,16 @@ export default {
16
16
  model: 'oai-gpt4o',
17
17
 
18
18
  resolver: async (_parent, args, _contextValue, _info) => {
19
- const { agentContext } = args;
19
+ let { agentContext } = args;
20
+
21
+ // Backward compatibility: if contextId is provided without agentContext, create agentContext
22
+ if ((!agentContext || !Array.isArray(agentContext) || agentContext.length === 0) && args.contextId) {
23
+ agentContext = [{
24
+ contextId: args.contextId,
25
+ contextKey: args.contextKey || null,
26
+ default: true
27
+ }];
28
+ }
20
29
 
21
30
  // Validate that agentContext is provided
22
31
  if (!agentContext || !Array.isArray(agentContext) || agentContext.length === 0) {
@@ -19,7 +19,7 @@ export default {
19
19
  isMutation: true, // Declaratively mark this as a Mutation
20
20
 
21
21
  resolver: async (_parent, args, _contextValue, _info) => {
22
- const { agentContext, hash, displayFilename, tags, notes, mimeType, permanent, inCollection } = args;
22
+ const { agentContext, hash, displayFilename, tags, notes, mimeType, permanent, inCollection, chatId } = args;
23
23
 
24
24
  const defaultCtx = getDefaultContext(agentContext);
25
25
  if (!defaultCtx) {
@@ -59,12 +59,13 @@ export default {
59
59
  }
60
60
  // inCollection can be: boolean true/false, or array of chat IDs (e.g., ['*'] for global, ['chat-123'] for specific chat)
61
61
  // Will be normalized by updateFileMetadata: true -> ['*'], false -> undefined (removed), array -> as-is
62
+ // If not provided, will default based on chatId
62
63
  if (inCollection !== undefined && inCollection !== null) {
63
64
  metadata.inCollection = inCollection;
64
65
  }
65
66
 
66
67
  // Update metadata (only Cortex-managed fields)
67
- const success = await updateFileMetadata(contextId, hash, metadata, contextKey);
68
+ const success = await updateFileMetadata(contextId, hash, metadata, contextKey, chatId);
68
69
 
69
70
  if (success) {
70
71
  return JSON.stringify({
@@ -552,7 +552,7 @@ export default {
552
552
  // - Files in collection (all agentContext contexts): stripped, accessible via tools
553
553
  // - Files not in collection: left in message for model to see directly
554
554
  const { chatHistory: strippedHistory, availableFiles } = await syncAndStripFilesFromChatHistory(
555
- args.chatHistory, args.agentContext
555
+ args.chatHistory, args.agentContext, chatId
556
556
  );
557
557
  args.chatHistory = strippedHistory;
558
558
 
@@ -145,7 +145,7 @@ export default {
145
145
  ],
146
146
 
147
147
  executePathway: async ({args, runAllPrompts, resolver}) => {
148
- const { file, startLine, endLine, content, oldString, newString, replaceAll = false, agentContext } = args;
148
+ const { file, startLine, endLine, content, oldString, newString, replaceAll = false, agentContext, chatId } = args;
149
149
 
150
150
  const defaultCtx = getDefaultContext(agentContext);
151
151
  if (!defaultCtx) {
@@ -514,7 +514,7 @@ export default {
514
514
 
515
515
  // Write new entry with CFH data (url, gcs, hash) + Cortex metadata
516
516
  if (uploadResult.hash) {
517
- const { getRedisClient } = await import('../../../../lib/fileUtils.js');
517
+ const { getRedisClient, addChatIdToInCollection, getInCollectionValue } = await import('../../../../lib/fileUtils.js');
518
518
  const redisClient = await getRedisClient();
519
519
  if (redisClient) {
520
520
  const contextMapKey = `FileStoreMap:ctx:${contextId}`;
@@ -529,6 +529,12 @@ export default {
529
529
  }
530
530
  }
531
531
 
532
+ // Merge chatId into existing inCollection (reference counting)
533
+ const existingInCollection = fileToUpdate.inCollection || existingData.inCollection;
534
+ const updatedInCollection = existingInCollection
535
+ ? addChatIdToInCollection(existingInCollection, chatId)
536
+ : getInCollectionValue(chatId);
537
+
532
538
  const fileData = {
533
539
  ...existingData,
534
540
  url: uploadResult.url,
@@ -540,7 +546,7 @@ export default {
540
546
  tags: fileToUpdate.tags || [],
541
547
  notes: fileToUpdate.notes || '',
542
548
  mimeType: fileToUpdate.mimeType || mimeType || null,
543
- inCollection: ['*'],
549
+ inCollection: updatedInCollection,
544
550
  addedDate: fileToUpdate.addedDate,
545
551
  lastAccessed: new Date().toISOString(),
546
552
  permanent: fileToUpdate.permanent || false
@@ -204,6 +204,7 @@ export default {
204
204
  }
205
205
  const contextId = defaultCtx.contextId;
206
206
  const contextKey = defaultCtx.contextKey || null;
207
+ const chatId = args.chatId || null;
207
208
 
208
209
  // Determine which function was called based on which parameters are present
209
210
  // Order matters: check most specific operations first
@@ -293,7 +294,7 @@ export default {
293
294
  metadataUpdate.lastAccessed = new Date().toISOString();
294
295
 
295
296
  // Perform the atomic update
296
- const success = await updateFileMetadata(contextId, foundFile.hash, metadataUpdate, contextKey);
297
+ const success = await updateFileMetadata(contextId, foundFile.hash, metadataUpdate, contextKey, chatId);
297
298
 
298
299
  if (!success) {
299
300
  throw new Error(`Failed to update file metadata for "${file}"`);
@@ -342,7 +343,8 @@ export default {
342
343
  hash,
343
344
  fileUrl,
344
345
  resolver,
345
- permanent
346
+ permanent,
347
+ chatId
346
348
  );
347
349
 
348
350
  resolver.tool = JSON.stringify({ toolUsed: "AddFileToCollection" });
@@ -386,6 +388,7 @@ export default {
386
388
 
387
389
  if (matchesQuery && matchesTags) {
388
390
  // Update lastAccessed directly (atomic operation)
391
+ // Don't pass chatId - we're only updating access time, not changing inCollection
389
392
  await updateFileMetadata(contextId, file.hash, {
390
393
  lastAccessed: now
391
394
  }, contextKey);
@@ -449,7 +452,8 @@ export default {
449
452
  });
450
453
 
451
454
  } else if (isRemove) {
452
- // Remove file(s) from collection and delete from cloud storage
455
+ // Remove file(s) from this chat's collection (reference counting)
456
+ // Only delete from cloud if no other chats reference the file
453
457
  const { fileIds, fileId } = args;
454
458
 
455
459
  // Normalize input to array
@@ -465,9 +469,9 @@ export default {
465
469
  }
466
470
 
467
471
  let notFoundFiles = [];
468
- let filesToRemove = [];
472
+ let filesToProcess = [];
469
473
 
470
- // Load collection ONCE to find all files and their hashes
474
+ // Load collection ONCE to find all files and their data
471
475
  // Use useCache: false to get fresh data
472
476
  const collection = await loadFileCollection(contextId, contextKey, false);
473
477
 
@@ -479,12 +483,13 @@ export default {
479
483
 
480
484
  if (foundFile) {
481
485
  // Avoid duplicates (by hash since that's the unique key in Redis)
482
- if (!filesToRemove.some(f => f.hash === foundFile.hash)) {
483
- filesToRemove.push({
486
+ if (!filesToProcess.some(f => f.hash === foundFile.hash)) {
487
+ filesToProcess.push({
484
488
  id: foundFile.id,
485
489
  displayFilename: foundFile.displayFilename || foundFile.filename || null,
486
490
  hash: foundFile.hash || null,
487
- permanent: foundFile.permanent ?? false
491
+ permanent: foundFile.permanent ?? false,
492
+ inCollection: foundFile.inCollection || []
488
493
  });
489
494
  }
490
495
  } else {
@@ -492,34 +497,76 @@ export default {
492
497
  }
493
498
  }
494
499
 
495
- if (filesToRemove.length === 0 && notFoundFiles.length > 0) {
500
+ if (filesToProcess.length === 0 && notFoundFiles.length > 0) {
496
501
  throw new Error(`No files found matching: ${notFoundFiles.join(', ')}`);
497
502
  }
498
503
 
499
- // Use the hashes collected from the single collection load
500
- // No need to reload - we already have all the info we need
501
- const hashesToDelete = filesToRemove.filter(f => f.hash);
502
-
503
- // Delete entries directly from hash map (atomic operations)
504
- const { getRedisClient } = await import('../../../../lib/fileUtils.js');
504
+ // Import helpers for reference counting
505
+ const { getRedisClient, removeChatIdFromInCollection } = await import('../../../../lib/fileUtils.js');
505
506
  const redisClient = await getRedisClient();
507
+ const contextMapKey = `FileStoreMap:ctx:${contextId}`;
508
+
509
+ // Track files that will be fully deleted vs just updated
510
+ const filesToFullyDelete = [];
511
+ const filesToUpdate = [];
512
+
513
+ for (const fileInfo of filesToProcess) {
514
+ if (!fileInfo.hash) continue;
515
+
516
+ // Check if file is global ('*') - global files can't be removed per-chat
517
+ const isGlobal = Array.isArray(fileInfo.inCollection) && fileInfo.inCollection.includes('*');
518
+
519
+ if (isGlobal) {
520
+ // Global file - fully remove it (no reference counting for global files)
521
+ filesToFullyDelete.push(fileInfo);
522
+ } else if (!chatId) {
523
+ // No chatId context - fully remove
524
+ filesToFullyDelete.push(fileInfo);
525
+ } else {
526
+ // Remove this chatId from inCollection
527
+ const updatedInCollection = removeChatIdFromInCollection(fileInfo.inCollection, chatId);
528
+
529
+ if (updatedInCollection.length === 0) {
530
+ // No more references - fully delete
531
+ filesToFullyDelete.push(fileInfo);
532
+ } else {
533
+ // Still has references from other chats - just update inCollection
534
+ filesToUpdate.push({ ...fileInfo, updatedInCollection });
535
+ }
536
+ }
537
+ }
538
+
539
+ // Update files that still have references (remove this chatId only)
540
+ for (const fileInfo of filesToUpdate) {
541
+ if (redisClient) {
542
+ try {
543
+ const existingDataStr = await redisClient.hget(contextMapKey, fileInfo.hash);
544
+ if (existingDataStr) {
545
+ const existingData = JSON.parse(existingDataStr);
546
+ existingData.inCollection = fileInfo.updatedInCollection;
547
+ await redisClient.hset(contextMapKey, fileInfo.hash, JSON.stringify(existingData));
548
+ logger.info(`Removed chatId ${chatId} from file: ${fileInfo.displayFilename} (still referenced by: ${fileInfo.updatedInCollection.join(', ')})`);
549
+ }
550
+ } catch (e) {
551
+ logger.warn(`Failed to update inCollection for file ${fileInfo.hash}: ${e.message}`);
552
+ }
553
+ }
554
+ }
555
+
556
+ // Fully delete files with no remaining references
506
557
  if (redisClient) {
507
- const contextMapKey = `FileStoreMap:ctx:${contextId}`;
508
- for (const fileInfo of hashesToDelete) {
558
+ for (const fileInfo of filesToFullyDelete) {
509
559
  await redisClient.hdel(contextMapKey, fileInfo.hash);
510
560
  }
511
561
  }
512
562
 
513
- // Always invalidate cache immediately so list operations reflect removals
514
- // (even if Redis operations failed, cache might be stale)
563
+ // Always invalidate cache immediately so list operations reflect changes
515
564
  invalidateFileCollectionCache(contextId, contextKey);
516
565
 
517
- // Delete files from cloud storage ASYNC (fire and forget, but log errors)
518
- // We do this after updating collection so user gets fast response and files are "gone" from UI immediately
519
- // Use hashes captured inside the lock to ensure we delete the correct files
566
+ // Delete files from cloud storage ASYNC (only for files with no remaining references)
520
567
  // IMPORTANT: Don't delete permanent files from cloud storage - they should persist
521
568
  (async () => {
522
- for (const fileInfo of hashesToDelete) {
569
+ for (const fileInfo of filesToFullyDelete) {
523
570
  // Skip deletion if file is marked as permanent
524
571
  if (fileInfo.permanent) {
525
572
  logger.info(`Skipping cloud deletion for permanent file: ${fileInfo.displayFilename} (hash: ${fileInfo.hash})`);
@@ -527,7 +574,7 @@ export default {
527
574
  }
528
575
 
529
576
  try {
530
- logger.info(`Deleting file from cloud storage: ${fileInfo.displayFilename} (hash: ${fileInfo.hash})`);
577
+ logger.info(`Deleting file from cloud storage (no remaining references): ${fileInfo.displayFilename} (hash: ${fileInfo.hash})`);
531
578
  await deleteFileByHash(fileInfo.hash, resolver, contextId);
532
579
  } catch (error) {
533
580
  logger.warn(`Failed to delete file ${fileInfo.displayFilename} (hash: ${fileInfo.hash}) from cloud storage: ${error?.message || String(error)}`);
@@ -535,8 +582,13 @@ export default {
535
582
  }
536
583
  })().catch(err => logger.error(`Async cloud deletion error: ${err}`));
537
584
 
538
- const removedCount = filesToRemove.length;
539
- const removedFiles = filesToRemove;
585
+ const removedCount = filesToProcess.length;
586
+ const removedFiles = filesToProcess.map(f => ({
587
+ id: f.id,
588
+ displayFilename: f.displayFilename,
589
+ hash: f.hash,
590
+ fullyDeleted: filesToFullyDelete.some(fd => fd.hash === f.hash)
591
+ }));
540
592
 
541
593
  // Get remaining files count after deletion
542
594
  const remainingCollection = await loadFileCollection(contextId, contextKey, false);