@aj-archipelago/cortex 1.3.62 → 1.3.63

package/helper-apps/cortex-file-handler/src/index.js

@@ -3,7 +3,7 @@ import os from "os";
 import path from "path";
 import { v4 as uuidv4 } from "uuid";
 
-import { DOC_EXTENSIONS } from "./constants.js";
+import { DOC_EXTENSIONS, AZURITE_ACCOUNT_NAME } from "./constants.js";
 import { easyChunker } from "./docHelper.js";
 import { downloadFile, splitMediaFile } from "./fileChunker.js";
 import { ensureEncoded, ensureFileExtension, urlExists } from "./helper.js";
@@ -53,17 +53,22 @@ async function CortexFileHandler(context, req) {
     hash,
     checkHash,
     clearHash,
+    shortLivedMinutes,
     fetch,
     load,
     restore,
+    container,
   } = req.body?.params || req.query;
 
   // Normalize boolean parameters
   const shouldSave = save === true || save === "true";
   const shouldCheckHash = checkHash === true || checkHash === "true";
   const shouldClearHash = clearHash === true || clearHash === "true";
+  const shortLivedDuration = parseInt(shortLivedMinutes) || 5; // Default to 5 minutes
   const shouldFetchRemote = fetch || load || restore;
 
+
+
   const operation = shouldSave
     ? "save"
     : shouldCheckHash
@@ -90,9 +95,11 @@ async function CortexFileHandler(context, req) {
 
   // Initialize services
   const storageService = new StorageService();
+  await storageService._initialize(); // Ensure providers are initialized
   const conversionService = new FileConversionService(
     context,
     storageService.primaryProvider.constructor.name === "AzureStorageProvider",
+    null,
   );
 
   // Validate URL for document processing and media chunking operations
@@ -124,13 +131,39 @@ async function CortexFileHandler(context, req) {
   }
 
   // Clean up files when request delete which means processing marked completed
+  // Supports two modes:
+  // 1. Delete multiple files by requestId (existing behavior)
+  // 2. Delete single file by hash (new behavior)
   if (operation === "delete") {
     const deleteRequestId = req.query.requestId || requestId;
     const deleteHash = req.query.hash || hash;
+    
+    // If only hash is provided, delete single file by hash
+    if (deleteHash && !deleteRequestId) {
+      try {
+        const deleted = await storageService.deleteFileByHash(deleteHash);
+        context.res = {
+          status: 200,
+          body: { 
+            message: `File with hash ${deleteHash} deleted successfully`,
+            deleted 
+          },
+        };
+        return;
+      } catch (error) {
+        context.res = {
+          status: 404,
+          body: error.message,
+        };
+        return;
+      }
+    }
+    
+    // If requestId is provided, use the existing multi-file delete flow
     if (!deleteRequestId) {
       context.res = {
         status: 400,
-        body: "Please pass a requestId on the query string",
+        body: "Please pass either a requestId or hash on the query string",
       };
       return;
     }
@@ -167,15 +200,16 @@ async function CortexFileHandler(context, req) {
         return;
       }
 
-      // Check if file already exists (using hash as the key)
-      const exists = await getFileStoreMap(remoteUrl);
+      // Check if file already exists (using hash or URL as the key)
+      const cacheKey = hash || remoteUrl;
+      const exists = await getFileStoreMap(cacheKey);
       if (exists) {
         context.res = {
           status: 200,
           body: exists,
         };
         //update redis timestamp with current time
-        await setFileStoreMap(remoteUrl, exists);
+        await setFileStoreMap(cacheKey, exists);
         return;
       }
 
@@ -190,10 +224,10 @@ async function CortexFileHandler(context, req) {
 
       // For remote files, we don't need a requestId folder structure since it's just a single file
       // Pass empty string to store the file directly in the root
-      const res = await storageService.uploadFile(context, filename, '');
+      const res = await storageService.uploadFile(context, filename, '', null, null, container);
 
-      //Update Redis (using hash as the key)
-      await setFileStoreMap(remoteUrl, res);
+      //Update Redis (using hash or URL as the key)
+      await setFileStoreMap(cacheKey, res);
 
       // Return the file URL
       context.res = {
@@ -322,6 +356,9 @@ async function CortexFileHandler(context, req) {
               context,
               downloadedFile,
               hash,
+              null,
+              null,
+              container,
             );
 
             // Update the hash result with the new primary storage URL
@@ -382,6 +419,7 @@ async function CortexFileHandler(context, req) {
           hashResult = await conversionService.ensureConvertedVersion(
             hashResult,
             requestId,
+            container,
           );
         } catch (error) {
           context.log(`Error ensuring converted version: ${error}`);
@@ -395,6 +433,70 @@ async function CortexFileHandler(context, req) {
           };
         }
 
+        // Always generate short-lived URL for checkHash operations
+        // Use converted URL if available, otherwise use original URL
+        const urlForShortLived = hashResult.converted?.url || hashResult.url;
+        try {
+          // Extract blob name from the URL to generate new SAS token
+          let blobName;
+          try {
+            const url = new URL(urlForShortLived);
+            // Extract blob name from the URL path (remove leading slash)
+            let path = url.pathname.substring(1);
+            
+            // For Azurite URLs, the path includes account name: devstoreaccount1/container/blob
+            // For real Azure URLs, the path is: container/blob
+            const containerName = storageService.primaryProvider.containerName;
+            
+    // Check if this is an Azurite URL (contains devstoreaccount1)
+    if (path.startsWith(`${AZURITE_ACCOUNT_NAME}/`)) {
+      path = path.substring(`${AZURITE_ACCOUNT_NAME}/`.length); // Remove account prefix
+            }
+            
+            // Now remove container prefix if it exists
+            if (path.startsWith(containerName + '/')) {
+              blobName = path.substring(containerName.length + 1);
+            } else {
+              blobName = path;
+            }
+            
+          } catch (urlError) {
+            context.log(`Error parsing URL for short-lived generation: ${urlError}`);
+          }
+
+          // Generate short-lived SAS token
+          if (blobName && storageService.primaryProvider.generateShortLivedSASToken) {
+            const { containerClient } = await storageService.primaryProvider.getBlobClient();
+            const sasToken = storageService.primaryProvider.generateShortLivedSASToken(
+              containerClient, 
+              blobName, 
+              shortLivedDuration
+            );
+            
+            // Construct new URL with short-lived SAS token
+            const baseUrl = urlForShortLived.split('?')[0]; // Remove existing SAS token
+            const shortLivedUrl = `${baseUrl}?${sasToken}`;
+            
+            // Add short-lived URL to response
+            response.shortLivedUrl = shortLivedUrl;
+            response.expiresInMinutes = shortLivedDuration;
+            
+            const urlType = hashResult.converted?.url ? 'converted' : 'original';
+            context.log(`Generated short-lived URL for hash: ${hash} using ${urlType} URL (expires in ${shortLivedDuration} minutes)`);
+          } else {
+            // Fallback for storage providers that don't support short-lived tokens
+            response.shortLivedUrl = urlForShortLived;
+            response.expiresInMinutes = shortLivedDuration;
+            const urlType = hashResult.converted?.url ? 'converted' : 'original';
+            context.log(`Storage provider doesn't support short-lived tokens, using ${urlType} URL`);
+          }
+        } catch (error) {
+          context.log(`Error generating short-lived URL: ${error}`);
+          // Provide fallback even on error
+          response.shortLivedUrl = urlForShortLived;
+          response.expiresInMinutes = shortLivedDuration;
+        }
+
         //update redis timestamp with current time
         await setFileStoreMap(hash, hashResult);
 
@@ -428,7 +530,7 @@ async function CortexFileHandler(context, req) {
       storageService.primaryProvider.constructor.name ===
       "LocalStorageProvider";
     // Use uploadBlob to handle multipart/form-data
-    const result = await uploadBlob(context, req, saveToLocal, null, hash);
+    const result = await uploadBlob(context, req, saveToLocal, null, hash, container);
     if (result?.hash && context?.res?.body) {
       await setFileStoreMap(result.hash, context.res.body);
     }
@@ -496,6 +598,8 @@ async function CortexFileHandler(context, req) {
               await conversionService._saveConvertedFile(
                 conversion.convertedPath,
                 requestId,
+                null,
+                container,
               );
 
             // Return the converted file URL
@@ -511,6 +615,8 @@ async function CortexFileHandler(context, req) {
             const saveResult = await conversionService._saveConvertedFile(
               downloadedFile,
               requestId,
+              null,
+              container,
             );
 
             // Return the original file URL
@@ -592,6 +698,7 @@ async function CortexFileHandler(context, req) {
           requestId,
           null,
           chunkFilename,
+          container,
         );
 
         const chunkOffset = chunkOffsets[index];

package/helper-apps/cortex-file-handler/src/redis.js

@@ -1,6 +1,65 @@
 import redis from "ioredis";
+
 const connectionString = process.env["REDIS_CONNECTION_STRING"];
-const client = redis.createClient(connectionString);
+
+// Create a mock client for test environment when Redis is not configured
+const createMockClient = () => {
+  const store = new Map();
+  const hashMap = new Map();
+  
+  return {
+    connected: false,
+    async connect() { return Promise.resolve(); },
+    async publish() { return Promise.resolve(); },
+    async hgetall(hashName) { 
+      const hash = hashMap.get(hashName);
+      return hash ? Object.fromEntries(hash) : {};
+    },
+    async hset(hashName, key, value) { 
+      if (!hashMap.has(hashName)) {
+        hashMap.set(hashName, new Map());
+      }
+      hashMap.get(hashName).set(key, value);
+      return Promise.resolve();
+    },
+    async hget(hashName, key) { 
+      const hash = hashMap.get(hashName);
+      return hash ? hash.get(key) || null : null;
+    },
+    async hdel(hashName, key) { 
+      const hash = hashMap.get(hashName);
+      if (hash && hash.has(key)) {
+        hash.delete(key);
+        return 1;
+      }
+      return 0;
+    },
+    async eval(script, numKeys, ...args) {
+      // Mock implementation for atomic get-and-delete operation
+      if (script.includes('hget') && script.includes('hdel')) {
+        const hashName = args[0];
+        const key = args[1];
+        const hash = hashMap.get(hashName);
+        if (hash && hash.has(key)) {
+          const value = hash.get(key);
+          hash.delete(key);
+          return value;
+        }
+        return null;
+      }
+      throw new Error('Mock eval only supports atomic get-and-delete');
+    },
+  };
+};
+
+// Only create real Redis client if connection string is provided
+let client;
+if (connectionString && process.env.NODE_ENV !== 'test') {
+  client = redis.createClient(connectionString);
+} else {
+  console.log('Using mock Redis client for tests or missing connection string');
+  client = createMockClient();
+}
 
 const channel = "requestProgress";
 
@@ -222,4 +281,5 @@ export {
   removeFromFileStoreMap,
   cleanupRedisFileStoreMap,
   cleanupRedisFileStoreMapAge,
+  client,
 };

package/helper-apps/cortex-file-handler/src/services/ConversionService.js

@@ -9,11 +9,7 @@ import { CONVERTED_EXTENSIONS } from "../constants.js";
 import { v4 as uuidv4 } from "uuid";
 import { sanitizeFilename, generateShortId } from "../utils/filenameUtils.js";
 
-const MARKITDOWN_CONVERT_URL = process.env.MARKITDOWN_CONVERT_URL;
-
-if (!MARKITDOWN_CONVERT_URL) {
-  throw new Error("MARKITDOWN_CONVERT_URL is not set");
-}
+const MARKITDOWN_CONVERT_URL = process.env.MARKITDOWN_CONVERT_URL || null;
 
 export class ConversionService {
   constructor(context) {
@@ -100,9 +96,10 @@ export class ConversionService {
    * Ensures a file has both original and converted versions
    * @param {Object} fileInfo - Information about the file
    * @param {string} requestId - Request ID for storage
+   * @param {string} containerName - Optional container name for storage
    * @returns {Promise<Object>} - Updated file info with conversion if needed
    */
-  async ensureConvertedVersion(fileInfo, requestId) {
+  async ensureConvertedVersion(fileInfo, requestId, containerName = null) {
     const { url, gcs } = fileInfo;
     // Remove any query parameters before extension check
     const extension = path.extname(url.split("?")[0]).toLowerCase();
@@ -162,6 +159,8 @@ export class ConversionService {
         const convertedSaveResult = await this._saveConvertedFile(
           conversion.convertedPath,
           requestId,
+          null,
+          containerName,
         );
         if (!convertedSaveResult) {
           throw new Error("Failed to save converted file to primary storage");
@@ -257,7 +256,11 @@ export class ConversionService {
 
   async _convertToMarkdown(fileUrl) {
     try {
-      const apiUrl = `${MARKITDOWN_CONVERT_URL}${encodeURIComponent(fileUrl)}`;
+      const markitdownUrl = process.env.MARKITDOWN_CONVERT_URL;
+      if (!markitdownUrl) {
+        throw new Error("MARKITDOWN_CONVERT_URL is not set");
+      }
+      const apiUrl = `${markitdownUrl}${encodeURIComponent(fileUrl)}`;
       const response = await axios.get(apiUrl);
       return response.data.markdown || "";
     } catch (err) {
@@ -302,7 +305,7 @@ export class ConversionService {
     throw new Error("Method _downloadFile must be implemented");
   }
 
-  async _saveConvertedFile(filePath, requestId) {
+  async _saveConvertedFile(filePath, requestId, filename = null, containerName = null) {
     throw new Error("Method _saveConvertedFile must be implemented");
   }
 

package/helper-apps/cortex-file-handler/src/services/FileConversionService.js

@@ -33,13 +33,13 @@ export class FileConversionService extends ConversionService {
     return downloadFile(url, destination);
   }
 
-  async _saveConvertedFile(filePath, requestId, filename = null) {
+  async _saveConvertedFile(filePath, requestId, filename = null, containerName = null) {
     // Generate a fallback requestId if none supplied (e.g. during checkHash calls)
     const reqId = requestId || uuidv4();
 
     let fileUrl;
     if (this.useAzure) {
-      const savedBlob = await saveFileToBlob(filePath, reqId, filename);
+      const savedBlob = await saveFileToBlob(filePath, reqId, filename, containerName);
       fileUrl = savedBlob.url;
     } else {
       fileUrl = await moveFileToPublicFolder(filePath, reqId);

package/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js

@@ -7,6 +7,7 @@ import fs from "fs";
 import path from "path";
 
 import { StorageProvider } from "./StorageProvider.js";
+import { AZURITE_ACCOUNT_NAME } from "../../constants.js";
 import {
   generateShortId,
   generateBlobName,
@@ -43,21 +44,54 @@ export class AzureStorageProvider extends StorageProvider {
     return { blobServiceClient, containerClient };
   }
 
-  generateSASToken(containerClient, blobName) {
-    const { accountName, accountKey } = containerClient.credential;
+  generateSASToken(containerClient, blobName, options = {}) {
+    // Handle Azurite (development storage) credentials
+    let accountName, accountKey;
+    
+    // Note: Debug logging removed for production
+    
+    if (containerClient.credential && containerClient.credential.accountName) {
+      // Regular Azure Storage credentials
+      accountName = containerClient.credential.accountName;
+      
+      // Handle Buffer case (Azurite) vs string case (real Azure)
+      if (Buffer.isBuffer(containerClient.credential.accountKey)) {
+        accountKey = containerClient.credential.accountKey.toString('base64');
+      } else {
+        accountKey = containerClient.credential.accountKey;
+      }
+    } else {
+      // Azurite development storage fallback
+      accountName = AZURITE_ACCOUNT_NAME;
+      accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
+    }
+    
     const sharedKeyCredential = new StorageSharedKeyCredential(
       accountName,
       accountKey,
     );
 
+    // Support custom duration: minutes, hours, or fall back to default days
+    let expirationTime;
+    if (options.minutes) {
+      expirationTime = new Date(new Date().valueOf() + options.minutes * 60 * 1000);
+    } else if (options.hours) {
+      expirationTime = new Date(new Date().valueOf() + options.hours * 60 * 60 * 1000);
+    } else if (options.days) {
+      expirationTime = new Date(new Date().valueOf() + options.days * 24 * 60 * 60 * 1000);
+    } else {
+      // Default to configured sasTokenLifeDays
+      expirationTime = new Date(
+        new Date().valueOf() + this.sasTokenLifeDays * 24 * 60 * 60 * 1000,
+      );
+    }
+
     const sasOptions = {
       containerName: containerClient.containerName,
       blobName: blobName,
-      permissions: "r",
+      permissions: options.permissions || "r",
       startsOn: new Date(),
-      expiresOn: new Date(
-        new Date().valueOf() + this.sasTokenLifeDays * 24 * 60 * 60 * 1000,
-      ),
+      expiresOn: expirationTime,
     };
 
     return generateBlobSASQueryParameters(
@@ -66,6 +100,10 @@ export class AzureStorageProvider extends StorageProvider {
     ).toString();
   }
 
+  generateShortLivedSASToken(containerClient, blobName, minutes = 5) {
+    return this.generateSASToken(containerClient, blobName, { minutes });
+  }
+
   async uploadFile(context, filePath, requestId, hash = null, filename = null) {
     const { containerClient } = await this.getBlobClient();
 
@@ -123,6 +161,50 @@ export class AzureStorageProvider extends StorageProvider {
     return result;
   }
 
+  async deleteFile(url) {
+    if (!url) throw new Error("Missing URL parameter");
+
+    try {
+      const { containerClient } = await this.getBlobClient();
+      
+      // Extract blob name from URL
+      const urlObj = new URL(url);
+      let blobName = urlObj.pathname.substring(1); // Remove leading slash
+      
+      // Handle Azurite URLs which include account name in path: /devstoreaccount1/container/blob
+      if (blobName.includes('/')) {
+        const pathSegments = blobName.split('/');
+        if (pathSegments.length >= 2) {
+          // For Azurite: devstoreaccount1/container/blobname -> blobname
+          // Skip the account and container segments to get the actual blob name
+          blobName = pathSegments.slice(2).join('/');
+        }
+      }
+      
+      // Remove container name prefix if present (for non-Azurite URLs)
+      if (blobName.startsWith(this.containerName + '/')) {
+        blobName = blobName.substring(this.containerName.length + 1);
+      }
+      
+      const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+      
+      try {
+        await blockBlobClient.delete();
+        return blobName;
+      } catch (error) {
+        if (error.statusCode === 404) {
+          console.warn(`Azure blob not found during delete: ${blobName}`);
+          return null;
+        } else {
+          throw error;
+        }
+      }
+    } catch (error) {
+      console.error("Error deleting Azure blob:", error);
+      throw error;
+    }
+  }
+
   async fileExists(url) {
     try {
       // First attempt a lightweight HEAD request

package/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js

@@ -153,6 +153,64 @@ export class GCSStorageProvider extends StorageProvider {
     }
   }
 
+  async deleteFile(url) {
+    if (!url) throw new Error("Missing URL parameter");
+
+    try {
+      if (!url.startsWith("gs://")) {
+        throw new Error("Invalid GCS URL format");
+      }
+
+      const unencodedUrl = this.ensureUnencodedGcsUrl(url);
+      const urlParts = unencodedUrl.replace("gs://", "").split("/");
+      const bucketName = urlParts[0];
+      const fileName = urlParts.slice(1).join("/");
+
+      if (process.env.STORAGE_EMULATOR_HOST) {
+        // When using the emulator, use raw REST API
+        try {
+          const response = await axios.delete(
+            `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${bucketName}/o/${encodeURIComponent(fileName)}`,
+            {
+              validateStatus: (status) => status === 200 || status === 404,
+            }
+          );
+          
+          if (response.status === 200) {
+            return fileName;
+          } else if (response.status === 404) {
+            console.warn(`GCS file not found during delete: ${fileName}`);
+            return null;
+          }
+        } catch (error) {
+          if (error.response?.status === 404) {
+            console.warn(`GCS file not found during delete: ${fileName}`);
+            return null;
+          }
+          throw error;
+        }
+      } else {
+        // Real GCS - use client library
+        const bucket = this.storage.bucket(bucketName);
+        const file = bucket.file(fileName);
+        
+        try {
+          await file.delete();
+          return fileName;
+        } catch (error) {
+          if (error.code === 404) {
+            console.warn(`GCS file not found during delete: ${fileName}`);
+            return null;
+          }
+          throw error;
+        }
+      }
+    } catch (error) {
+      console.error("Error deleting GCS file:", error);
+      throw error;
+    }
+  }
+
   async fileExists(url) {
     try {
       if (!url || !url.startsWith("gs://")) {

package/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js

@@ -4,24 +4,44 @@ import { LocalStorageProvider } from "./LocalStorageProvider.js";
 import path from "path";
 import { fileURLToPath } from "url";
 
+// Lazy-load blob handler constants to avoid blocking module import
+let blobHandlerConstants = null;
+async function getBlobHandlerConstants() {
+  if (!blobHandlerConstants) {
+    blobHandlerConstants = await import("../../blobHandler.js");
+  }
+  return blobHandlerConstants;
+}
+
 export class StorageFactory {
   constructor() {
     this.providers = new Map();
   }
 
-  getPrimaryProvider() {
+  async getPrimaryProvider(containerName = null) {
     if (process.env.AZURE_STORAGE_CONNECTION_STRING) {
-      return this.getAzureProvider();
+      return await this.getAzureProvider(containerName);
     }
     return this.getLocalProvider();
   }
 
-  getAzureProvider() {
-    const key = "azure";
+  async getAzureProvider(containerName = null) {
+    const { AZURE_STORAGE_CONTAINER_NAMES, DEFAULT_AZURE_STORAGE_CONTAINER_NAME, isValidContainerName } = await getBlobHandlerConstants();
+    
+    // Use provided container name or default to first in whitelist
+    const finalContainerName = containerName || DEFAULT_AZURE_STORAGE_CONTAINER_NAME;
+    
+    // Validate container name
+    if (!isValidContainerName(finalContainerName)) {
+      throw new Error(`Invalid container name '${finalContainerName}'. Allowed containers: ${AZURE_STORAGE_CONTAINER_NAMES.join(', ')}`);
+    }
+    
+    // Create unique key for each container
+    const key = `azure-${finalContainerName}`;
     if (!this.providers.has(key)) {
       const provider = new AzureStorageProvider(
         process.env.AZURE_STORAGE_CONNECTION_STRING,
-        process.env.AZURE_STORAGE_CONTAINER_NAME || "whispertempfiles",
+        finalContainerName,
       );
       this.providers.set(key, provider);
     }

package/helper-apps/cortex-file-handler/src/services/storage/StorageProvider.js

@@ -24,6 +24,15 @@ export class StorageProvider {
     throw new Error("Method not implemented");
   }
 
+  /**
+   * Delete a single file by its URL
+   * @param {string} url - The URL of the file to delete
+   * @returns {Promise<string|null>} The deleted file path/name or null if not found
+   */
+  async deleteFile(url) {
+    throw new Error("Method not implemented");
+  }
+
   /**
    * Check if a file exists at the given URL
    * @param {string} url - The URL to check