@aj-archipelago/cortex 1.4.6 → 1.4.8

package/helper-apps/cortex-file-handler/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@aj-archipelago/cortex-file-handler",
-  "version": "2.6.2",
+  "version": "2.6.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@aj-archipelago/cortex-file-handler",
-      "version": "2.6.2",
+      "version": "2.6.3",
       "dependencies": {
         "@azure/storage-blob": "^12.13.0",
         "@distube/ytdl-core": "^4.14.3",

package/helper-apps/cortex-file-handler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex-file-handler",
-  "version": "2.6.2",
+  "version": "2.6.3",
   "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
   "type": "module",
   "main": "src/index.js",

package/helper-apps/cortex-file-handler/src/index.js

@@ -47,6 +47,27 @@ async function cleanupInactive(context) {
 }
 
 async function CortexFileHandler(context, req) {
+  // Parse body if it's a string (Azure Functions sometimes doesn't auto-parse DELETE bodies)
+  let parsedBody = req.body;
+  if (typeof req.body === 'string' && req.body.length > 0) {
+    try {
+      parsedBody = JSON.parse(req.body);
+    } catch (e) {
+      // If parsing fails, treat as empty object
+      parsedBody = {};
+    }
+  }
+  
+  // For GET requests, prioritize query string. For other methods, check body first, then query
+  // Also check if parsedBody actually has content (not just empty object)
+  const hasBodyContent = parsedBody && typeof parsedBody === 'object' && Object.keys(parsedBody).length > 0;
+  const bodySource = hasBodyContent ? (parsedBody.params || parsedBody) : {};
+  const querySource = req.query || {};
+  
+  // Merge sources: for GET, query takes priority; for others, body takes priority
+  const isGet = req.method?.toLowerCase() === 'get';
+  const source = isGet ? { ...bodySource, ...querySource } : { ...querySource, ...bodySource };
+  
   const {
     uri,
     requestId,
@@ -59,7 +80,7 @@ async function CortexFileHandler(context, req) {
     load,
     restore,
     container,
-  } = req.body?.params || req.query;
+  } = source;
 
   // Normalize boolean parameters
   const shouldSave = save === true || save === "true";
@@ -136,8 +157,10 @@ async function CortexFileHandler(context, req) {
   // 1. Delete multiple files by requestId (existing behavior)
   // 2. Delete single file by hash (new behavior)
   if (operation === "delete") {
-    const deleteRequestId = req.query.requestId || requestId;
-    const deleteHash = req.query.hash || hash;
+    // Check both query string and body params for delete parameters
+    // Handle both req.body.params.hash and req.body.hash formats
+    const deleteRequestId = req.query.requestId || parsedBody?.params?.requestId || parsedBody?.requestId || requestId;
+    const deleteHash = req.query.hash || parsedBody?.params?.hash || parsedBody?.hash || hash;
     
     // If only hash is provided, delete single file by hash
     if (deleteHash && !deleteRequestId) {
@@ -164,7 +187,7 @@ async function CortexFileHandler(context, req) {
     if (!deleteRequestId) {
       context.res = {
         status: 400,
-        body: "Please pass either a requestId or hash on the query string",
+        body: "Please pass either a requestId or hash in the query string or request body",
       };
       return;
     }

package/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js

@@ -119,6 +119,11 @@ export class AzureStorageProvider extends StorageProvider {
       blobName = generateBlobName(requestId, `${shortId}${fileExtension}`);
     }
 
+    // Validate blobName is not empty
+    if (!blobName || blobName.trim().length === 0) {
+      throw new Error(`Invalid blob name generated: blobName="${blobName}", requestId="${requestId}", filename="${filename}"`);
+    }
+
     // Create a read stream for the file
     const fileStream = fs.createReadStream(filePath);
 
@@ -134,8 +139,20 @@ export class AzureStorageProvider extends StorageProvider {
     // Generate SAS token after successful upload
     const sasToken = this.generateSASToken(containerClient, blobName);
 
+    const url = `${blockBlobClient.url}?${sasToken}`;
+    
+    // Validate that the URL contains a blob name (not just container)
+    // Azure blob URLs should be: https://account.blob.core.windows.net/container/blobname
+    // Container-only URLs end with /container/ or /container
+    const urlObj = new URL(url);
+    const pathParts = urlObj.pathname.split('/').filter(p => p.length > 0);
+    if (pathParts.length <= 1) {
+      // Only container name, no blob name - this is invalid
+      throw new Error(`Generated invalid Azure URL (container-only): ${url}, blobName: ${blobName}`);
+    }
+
     return {
-      url: `${blockBlobClient.url}?${sasToken}`,
+      url: url,
       blobName: blobName,
     };
   }
@@ -148,6 +165,11 @@ export class AzureStorageProvider extends StorageProvider {
     let blobName = sanitizeFilename(encodedFilename);
     blobName = encodeURIComponent(blobName);
 
+    // Validate blobName is not empty
+    if (!blobName || blobName.trim().length === 0) {
+      throw new Error(`Invalid blob name generated from encodedFilename: "${encodedFilename}"`);
+    }
+
     const options = {
       blobHTTPHeaders: {
         ...(contentType ? { blobContentType: contentType } : {}),
@@ -163,7 +185,16 @@ export class AzureStorageProvider extends StorageProvider {
     await blockBlobClient.uploadStream(stream, undefined, undefined, options);
     const sasToken = this.generateSASToken(containerClient, blobName);
     
-    return `${blockBlobClient.url}?${sasToken}`;
+    const url = `${blockBlobClient.url}?${sasToken}`;
+    
+    // Validate that the URL contains a blob name (not just container)
+    const urlObj = new URL(url);
+    const pathParts = urlObj.pathname.split('/').filter(p => p.length > 0);
+    if (pathParts.length <= 1) {
+      throw new Error(`Generated invalid Azure URL (container-only) from uploadStream: ${url}, blobName: ${blobName}`);
+    }
+    
+    return url;
   }
 
   async deleteFiles(requestId) {
@@ -204,19 +235,52 @@ export class AzureStorageProvider extends StorageProvider {
       const urlObj = new URL(url);
       let blobName = urlObj.pathname.substring(1); // Remove leading slash
       
-      // Handle Azurite URLs which include account name in path: /devstoreaccount1/container/blob
+      // Handle different URL formats:
+      // 1. Azurite: /devstoreaccount1/container/blobname (3 segments)
+      // 2. Standard Azure: /container/blobname (2 segments)
+      // 3. Container-only: /container or /container/ (invalid)
+      
       if (blobName.includes('/')) {
-        const pathSegments = blobName.split('/');
-        if (pathSegments.length >= 2) {
-          // For Azurite: devstoreaccount1/container/blobname -> blobname
+        const pathSegments = blobName.split('/').filter(segment => segment.length > 0);
+        
+        if (pathSegments.length === 1) {
+          // Only container name, no blob name - this is invalid
+          console.warn(`Invalid blob URL (container-only): ${url}`);
+          return null;
+        } else if (pathSegments.length === 2) {
+          // Standard Azure format: container/blobname
+          // Check if first segment matches container name
+          if (pathSegments[0] === this.containerName) {
+            blobName = pathSegments[1];
+          } else {
+            // Container name doesn't match, but assume second segment is blob name
+            blobName = pathSegments[1];
+          }
+        } else if (pathSegments.length >= 3) {
+          // Azurite format: devstoreaccount1/container/blobname
           // Skip the account and container segments to get the actual blob name
-          blobName = pathSegments.slice(2).join('/');
+          // Check if second segment matches container name
+          if (pathSegments[1] === this.containerName) {
+            blobName = pathSegments.slice(2).join('/');
+          } else {
+            // Container name doesn't match, but assume remaining segments are blob name
+            blobName = pathSegments.slice(2).join('/');
+          }
+        }
+      } else {
+        // No slashes - could be just container name or just blob name
+        if (blobName === this.containerName || blobName === this.containerName + '/') {
+          // URL is just the container name - invalid blob URL
+          console.warn(`Invalid blob URL (container-only): ${url}`);
+          return null;
         }
+        // Otherwise assume it's a blob name at root level (unlikely but possible)
       }
       
-      // Remove container name prefix if present (for non-Azurite URLs)
-      if (blobName.startsWith(this.containerName + '/')) {
-        blobName = blobName.substring(this.containerName.length + 1);
+      // Validate that we have a non-empty blob name
+      if (!blobName || blobName.trim().length === 0) {
+        console.warn(`Invalid blob URL (empty blob name): ${url}`);
+        return null;
       }
       
       const blockBlobClient = containerClient.getBlockBlobClient(blobName);

package/helper-apps/cortex-file-handler/src/services/storage/StorageService.js

@@ -191,8 +191,11 @@ export class StorageService {
       const effectiveContainer = containerName || defaultContainerName;
       if (effectiveContainer === defaultContainerName && scopedHash.includes(':')) {
         const [legacyHash] = scopedHash.split(':', 2);
-        // Try to remove legacy key - it's okay if it doesn't exist
-        await removeFromFileStoreMap(legacyHash);
+        // Try to remove legacy key - only attempt if it exists to avoid unnecessary "does not exist" logs
+        const legacyExists = await getFileStoreMap(legacyHash);
+        if (legacyExists) {
+          await removeFromFileStoreMap(legacyHash);
+        }
       }
     }
     
@@ -203,9 +206,16 @@ export class StorageService {
     // Delete from primary storage
     if (hashResult.url) {
       try {
+        // Log the URL being deleted for debugging
+        console.log(`Deleting file from primary storage - hash: ${hash}, url: ${hashResult.url}`);
         const primaryResult = await this.deleteFile(hashResult.url);
         if (primaryResult) {
+          console.log(`Successfully deleted from primary storage - hash: ${hash}, result: ${primaryResult}`);
           results.push({ provider: 'primary', result: primaryResult });
+        } else {
+          // deleteFile returned null, which means the URL was invalid
+          console.warn(`Invalid or empty URL for hash ${hash}: ${hashResult.url}`);
+          results.push({ provider: 'primary', error: 'Invalid URL (container-only or empty blob name)' });
         }
       } catch (error) {
         console.error(`Error deleting file from primary storage:`, error);
@@ -216,14 +226,25 @@ export class StorageService {
     // Delete from backup storage (GCS)
     if (hashResult.gcs && this.backupProvider) {
       try {
+        console.log(`Deleting file from backup storage - hash: ${hash}, gcs: ${hashResult.gcs}`);
         const backupResult = await this.deleteFileFromBackup(hashResult.gcs);
         if (backupResult) {
+          console.log(`Successfully deleted from backup storage - hash: ${hash}, result: ${backupResult}`);
           results.push({ provider: 'backup', result: backupResult });
+        } else {
+          console.warn(`Backup deletion returned null for hash ${hash}: ${hashResult.gcs}`);
+          results.push({ provider: 'backup', error: 'Deletion returned null' });
         }
       } catch (error) {
         console.error(`Error deleting file from backup storage:`, error);
         results.push({ provider: 'backup', error: error.message });
       }
+    } else {
+      if (!hashResult.gcs) {
+        console.log(`No GCS URL found for hash ${hash}, skipping backup deletion`);
+      } else if (!this.backupProvider) {
+        console.log(`Backup provider not configured, skipping backup deletion for hash ${hash}`);
+      }
     }
 
     // Note: Hash was already removed from Redis atomically at the beginning

package/helper-apps/cortex-file-handler/src/start.js

@@ -48,6 +48,8 @@ const packageJson = JSON.parse(
 const version = packageJson.version;
 
 app.use(cors());
+// Parse JSON bodies for all requests (including DELETE)
+app.use(express.json());
 // Serve static files from the public folder
 app.use("/files", express.static(publicFolder));
 

package/helper-apps/cortex-file-handler/tests/deleteOperations.test.js

@@ -345,4 +345,291 @@ test.serial("should delete file uploaded with different filename", async (t) =>
       // Ignore cleanup errors
     }
   }
+});
+
+// Tests for DELETE with hash in request body
+test.serial("should delete file by hash from request body params", async (t) => {
+  const testContent = "test content for body params deletion";
+  const testHash = `test-body-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file with hash
+    uploadResponse = await uploadFile(filePath, null, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+
+    // Delete file by hash using body params
+    const deleteResponse = await axios.delete(baseUrl, {
+      data: { params: { hash: testHash } },
+      validateStatus: (status) => true,
+      timeout: 10000,
+    });
+    
+    t.is(deleteResponse.status, 200, "Delete should succeed");
+    t.truthy(deleteResponse.data.message, "Should have success message");
+    t.true(deleteResponse.data.message.includes(testHash), "Message should include hash");
+    t.is(deleteResponse.data.deleted.hash, testHash, "Should include deleted hash");
+
+    // Verify hash is gone
+    const hashCheckAfter = await checkHashExists(testHash);
+    t.is(hashCheckAfter.status, 404, "Hash should not exist after deletion");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    try {
+      await removeFromFileStoreMap(testHash);
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should delete file by hash from request body (direct)", async (t) => {
+  const testContent = "test content for direct body deletion";
+  const testHash = `test-direct-body-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file with hash
+    uploadResponse = await uploadFile(filePath, null, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+
+    // Delete file by hash using direct body (not in params)
+    const deleteResponse = await axios.delete(baseUrl, {
+      data: { hash: testHash },
+      validateStatus: (status) => true,
+      timeout: 10000,
+    });
+    
+    t.is(deleteResponse.status, 200, "Delete should succeed");
+    t.truthy(deleteResponse.data.message, "Should have success message");
+    t.is(deleteResponse.data.deleted.hash, testHash, "Should include deleted hash");
+
+    // Verify hash is gone
+    const hashCheckAfter = await checkHashExists(testHash);
+    t.is(hashCheckAfter.status, 404, "Hash should not exist after deletion");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    try {
+      await removeFromFileStoreMap(testHash);
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should prioritize query string over body params for hash", async (t) => {
+  const testContent = "test content for priority test";
+  const queryHash = `test-query-${uuidv4()}`;
+  const bodyHash = `test-body-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file with query hash
+    uploadResponse = await uploadFile(filePath, null, queryHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+
+    // Try to delete with hash in both query and body - query should take priority
+    const deleteResponse = await axios.delete(`${baseUrl}?hash=${queryHash}`, {
+      data: { params: { hash: bodyHash } },
+      validateStatus: (status) => true,
+      timeout: 10000,
+    });
+    
+    t.is(deleteResponse.status, 200, "Delete should succeed");
+    t.is(deleteResponse.data.deleted.hash, queryHash, "Should use query hash, not body hash");
+
+    // Verify query hash is gone
+    const queryHashCheck = await checkHashExists(queryHash);
+    t.is(queryHashCheck.status, 404, "Query hash should not exist after deletion");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    try {
+      await removeFromFileStoreMap(queryHash);
+      await removeFromFileStoreMap(bodyHash);
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should delete file by requestId from body params", async (t) => {
+  const testContent = "test content for requestId body deletion";
+  const requestId = uuidv4();
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file with requestId
+    uploadResponse = await uploadFile(filePath, requestId, null);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+
+    // Delete file by requestId using body params
+    const deleteResponse = await axios.delete(baseUrl, {
+      data: { params: { requestId: requestId } },
+      validateStatus: (status) => true,
+      timeout: 10000,
+    });
+    
+    t.is(deleteResponse.status, 200, "Delete should succeed");
+    t.truthy(deleteResponse.data.body, "Should have deletion body");
+    t.true(Array.isArray(deleteResponse.data.body), "Deletion body should be array");
+
+  } finally {
+    fs.unlinkSync(filePath);
+  }
+});
+
+test.serial("should handle standard Azure URL format correctly", async (t) => {
+  const testContent = "test content for standard URL format";
+  const testHash = `test-standard-url-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file
+    uploadResponse = await uploadFile(filePath, null, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+    t.truthy(uploadResponse.data.url, "Should have file URL");
+    
+    // Verify URL format is standard Azure format (container/blob)
+    const url = uploadResponse.data.url;
+    const urlObj = new URL(url);
+    const pathParts = urlObj.pathname.split('/').filter(p => p.length > 0);
+    t.true(pathParts.length >= 2, "URL should have at least container and blob name");
+
+    // Delete file - should parse URL correctly
+    const deleteResponse = await deleteFileByHash(testHash);
+    t.is(deleteResponse.status, 200, "Delete should succeed");
+    
+    // Verify deletion was successful
+    const hashCheckAfter = await checkHashExists(testHash);
+    t.is(hashCheckAfter.status, 404, "Hash should not exist after deletion");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    try {
+      await removeFromFileStoreMap(testHash);
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should handle backwards compatibility key removal correctly", async (t) => {
+  const testContent = "test content for legacy key test";
+  const testHash = `test-legacy-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file
+    uploadResponse = await uploadFile(filePath, null, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+
+    // Manually create a legacy unscoped key to test backwards compatibility
+    const { setFileStoreMap, getFileStoreMap, getScopedHashKey } = await import("../src/redis.js");
+    const { getDefaultContainerName } = await import("../src/constants.js");
+    const defaultContainer = getDefaultContainerName();
+    const scopedHash = getScopedHashKey(testHash, defaultContainer);
+    const hashResult = await getFileStoreMap(scopedHash);
+    
+    if (hashResult) {
+      // Create legacy unscoped key
+      await setFileStoreMap(testHash, hashResult);
+      
+      // Verify both keys exist
+      const scopedExists = await getFileStoreMap(scopedHash);
+      const legacyExists = await getFileStoreMap(testHash);
+      t.truthy(scopedExists, "Scoped key should exist");
+      t.truthy(legacyExists, "Legacy key should exist");
+
+      // Delete file - should remove both keys
+      const deleteResponse = await deleteFileByHash(testHash);
+      t.is(deleteResponse.status, 200, "Delete should succeed");
+
+      // Verify both keys are removed
+      const scopedAfter = await getFileStoreMap(scopedHash);
+      const legacyAfter = await getFileStoreMap(testHash);
+      t.falsy(scopedAfter, "Scoped key should be removed");
+      t.falsy(legacyAfter, "Legacy key should be removed");
+    }
+
+  } finally {
+    fs.unlinkSync(filePath);
+    try {
+      await removeFromFileStoreMap(testHash);
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should not log 'does not exist' when legacy key doesn't exist", async (t) => {
+  const testContent = "test content for no legacy key test";
+  const testHash = `test-no-legacy-${uuidv4()}`;
+  const filePath = await createTestFile(testContent, "txt");
+  let uploadResponse;
+
+  try {
+    // Upload file (this creates only the scoped key, no legacy key)
+    uploadResponse = await uploadFile(filePath, null, testHash);
+    t.is(uploadResponse.status, 200, "Upload should succeed");
+
+    // Verify only scoped key exists
+    const { getFileStoreMap, getScopedHashKey } = await import("../src/redis.js");
+    const { getDefaultContainerName } = await import("../src/constants.js");
+    const defaultContainer = getDefaultContainerName();
+    const scopedHash = getScopedHashKey(testHash, defaultContainer);
+    const scopedExists = await getFileStoreMap(scopedHash);
+    const legacyExists = await getFileStoreMap(testHash);
+    t.truthy(scopedExists, "Scoped key should exist");
+    t.falsy(legacyExists, "Legacy key should not exist");
+
+    // Delete file - should not try to remove non-existent legacy key
+    // (This test verifies the fix doesn't log "does not exist" unnecessarily)
+    const deleteResponse = await deleteFileByHash(testHash);
+    t.is(deleteResponse.status, 200, "Delete should succeed");
+
+    // Verify scoped key is removed
+    const scopedAfter = await getFileStoreMap(scopedHash);
+    t.falsy(scopedAfter, "Scoped key should be removed");
+
+  } finally {
+    fs.unlinkSync(filePath);
+    try {
+      await removeFromFileStoreMap(testHash);
+    } catch (e) {
+      // Ignore cleanup errors
+    }
+  }
+});
+
+test.serial("should handle error message for missing hash/requestId correctly", async (t) => {
+  // Test with no parameters at all
+  const deleteResponse1 = await axios.delete(baseUrl, {
+    validateStatus: (status) => true,
+    timeout: 10000,
+  });
+  
+  t.is(deleteResponse1.status, 400, "Should return 400 for missing parameters");
+  t.truthy(deleteResponse1.data, "Should have error message");
+  t.true(
+    deleteResponse1.data.includes("query string or request body"),
+    "Error should mention both query string and request body"
+  );
+
+  // Test with empty body
+  const deleteResponse2 = await axios.delete(baseUrl, {
+    data: {},
+    validateStatus: (status) => true,
+    timeout: 10000,
+  });
+  
+  t.is(deleteResponse2.status, 400, "Should return 400 for missing parameters");
 });

package/helper-apps/cortex-file-handler/tests/start.test.js

@@ -361,7 +361,7 @@ test.serial("should validate requestId for delete operation", async (t) => {
   t.is(response.status, 400, "Should return 400 for missing requestId");
   t.is(
     response.data,
-    "Please pass either a requestId or hash on the query string",
+    "Please pass either a requestId or hash in the query string or request body",
     "Should return proper error message",
   );
 });

package/lib/entityConstants.js

@@ -99,7 +99,7 @@ term~N                 (Match terms similar to "term", edit distance N)
 
     AI_GROUNDING_INSTRUCTIONS: "# Grounding Responses\n\nIf you base part or all of your response on one or more search results, you MUST cite the source using a custom markdown directive of the form :cd_source[searchResultId]. There is NO other valid way to cite a source and a good UX depends on you using this directive correctly. Do not include other clickable links to the source when using the :cd_source[searchResultId] directive. Every search result has a unique searchResultId. You must include it verbatim, copied directly from the search results. Place the directives at the end of the phrase, sentence or paragraph that is grounded in that particular search result. If you are citing multiple search results, use multiple individual :cd_source[searchResultId] directives (e.g. :cd_source[searchResultId1] :cd_source[searchResultId2] :cd_source[searchResultId3] etc.)",
 
-    AI_AVAILABLE_FILES: "# Available Files\n\nThe following files are available for you to use in your tool calls or responses:\n{{{availableFiles}}}\n",
+    AI_AVAILABLE_FILES: "# Available Files (Last 10 Most Recently Used)\n\nThe following files are available for you to use in your tool calls or responses. This shows the last 10 most recently used files. More files may be available in your collection - use ListFileCollection or SearchFileCollection to see all files.\n\n{{{availableFiles}}}\n",
 
     AI_MEMORY_INSTRUCTIONS: `# Memory Instructions