@aj-archipelago/cortex 1.3.51 → 1.3.52

package/helper-apps/cortex-file-handler/{index.js → src/index.js}

@@ -1,32 +1,12 @@
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
-
 import { v4 as uuidv4 } from 'uuid';
 
-import {
-    saveFileToBlob,
-    deleteBlob,
-    deleteGCS,
-    uploadBlob,
-    cleanup,
-    cleanupGCS,
-    gcsUrlExists,
-    ensureGCSUpload,
-    gcs,
-    AZURE_STORAGE_CONTAINER_NAME,
-    uploadChunkToGCS,
-    downloadFromGCS,
-} from './blobHandler.js';
-import { DOC_EXTENSIONS, CONVERTED_EXTENSIONS } from './constants.js';
+import { DOC_EXTENSIONS } from './constants.js';
 import { easyChunker } from './docHelper.js';
 import { downloadFile, splitMediaFile } from './fileChunker.js';
 import { ensureEncoded, ensureFileExtension, urlExists } from './helper.js';
-import {
-    moveFileToPublicFolder,
-    deleteFolder,
-    cleanupLocal,
-} from './localFileHandler.js';
 import {
     cleanupRedisFileStoreMap,
     getFileStoreMap,
@@ -35,71 +15,32 @@ import {
     setFileStoreMap,
 } from './redis.js';
 import { FileConversionService } from './services/FileConversionService.js';
-
-const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
-const useGCS =
-  process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 ||
-  process.env.GCP_SERVICE_ACCOUNT_KEY
-      ? true
-      : false;
-
-console.log(
-    `Storage configuration - ${useAzure ? 'Azure' : 'Local'} Storage${useGCS ? ' and Google Cloud Storage' : ''}`,
-);
+import { StorageService } from './services/storage/StorageService.js';
+import { uploadBlob } from './blobHandler.js';
 
 let isCleanupRunning = false;
 async function cleanupInactive(context) {
     try {
         if (isCleanupRunning) {
             return;
-        } //no need to cleanup every call
+        }
         isCleanupRunning = true;
         const cleaned = await cleanupRedisFileStoreMap();
 
-        const cleanedAzure = [];
-        const cleanedLocal = [];
-        const cleanedGCS = [];
-
+        const urls = [];
         for (const key in cleaned) {
             const item = cleaned[key];
-            const { url, gcs } = item;
-            if (url) {
-                if (url.includes('.blob.core.windows.net/')) {
-                    cleanedAzure.push(url);
-                } else if (url.startsWith('gs://')) {
-                    cleanedGCS.push(url);
-                } else {
-                    cleanedLocal.push(url);
-                }
-            }
-
-            if (item && item.gcs) {
-                cleanedGCS.push(gcs);
-            }
-        }
-
-        try {
-            if (cleanedAzure && cleanedAzure.length > 0) {
-                await cleanup(context, cleanedAzure);
+            if (item.url) {
+                urls.push(item.url);
             }
-        } catch (error) {
-            console.log('Error occurred during azure cleanup:', error);
-        }
-
-        try {
-            if (cleanedLocal && cleanedLocal.length > 0) {
-                await cleanupLocal(cleanedLocal);
+            if (item.gcs) {
+                urls.push(item.gcs);
             }
-        } catch (err) {
-            console.log('Error occurred during local cleanup:', err);
         }
 
-        try {
-            if (cleanedGCS && cleanedGCS.length > 0) {
-                await cleanupGCS(cleanedGCS);
-            }
-        } catch (err) {
-            console.log('Error occurred during GCS cleanup:', err);
+        if (urls.length > 0) {
+            const storageService = new StorageService();
+            await storageService.cleanup(urls);
         }
     } catch (error) {
         console.log('Error occurred during cleanup:', error);
@@ -120,13 +61,20 @@ async function CortexFileHandler(context, req) {
         load,
         restore,
     } = req.body?.params || req.query;
-    const operation = save
+
+    // Normalize boolean parameters
+    const shouldSave = save === true || save === 'true';
+    const shouldCheckHash = checkHash === true || checkHash === 'true';
+    const shouldClearHash = clearHash === true || clearHash === 'true';
+    const shouldFetchRemote = fetch || load || restore;
+
+    const operation = shouldSave
         ? 'save'
-        : checkHash
+        : shouldCheckHash
             ? 'checkHash'
-            : clearHash
+            : shouldClearHash
                 ? 'clearHash'
-                : fetch || load || restore
+                : shouldFetchRemote
                     ? 'remoteFile'
                     : req.method.toLowerCase() === 'delete' ||
               req.query.operation === 'delete'
@@ -143,12 +91,42 @@ async function CortexFileHandler(context, req) {
 
     cleanupInactive(context); //trigger & no need to wait for it
 
-    // Initialize conversion service
-    const conversionService = new FileConversionService(context, useAzure);
+    // Initialize services
+    const storageService = new StorageService();
+    const conversionService = new FileConversionService(context, storageService.primaryProvider.constructor.name === 'AzureStorageProvider');
 
-    // Clean up blob when request delete which means processing marked completed
+    // Validate URL for document processing and media chunking operations
+    if (operation === 'document_processing' || operation === 'media_chunking') {
+        try {
+            const urlObj = new URL(uri);
+            if (!['http:', 'https:', 'gs:'].includes(urlObj.protocol)) {
+                context.res = {
+                    status: 400,
+                    body: 'Invalid URL protocol - only HTTP, HTTPS, and GCS URLs are supported',
+                };
+                return;
+            }
+            // Check if the pathname is too long (e.g., > 1024 characters)
+            if (urlObj.pathname.length > 1024) {
+                context.res = {
+                    status: 400,
+                    body: 'URL pathname is too long',
+                };
+                return;
+            }
+        } catch (error) {
+            context.res = {
+                status: 400,
+                body: 'Invalid URL format',
+            };
+            return;
+        }
+    }
+
+    // Clean up files when request delete which means processing marked completed
     if (operation === 'delete') {
         const deleteRequestId = req.query.requestId || requestId;
+        const deleteHash = req.query.hash || hash;
         if (!deleteRequestId) {
             context.res = {
                 status: 400,
@@ -157,26 +135,27 @@ async function CortexFileHandler(context, req) {
             return;
         }
 
-        // Delete from Azure/Local storage
-        const azureResult = useAzure
-            ? await deleteBlob(deleteRequestId)
-            : await deleteFolder(deleteRequestId);
-        const gcsResult = [];
-        if (gcs) {
-            gcsResult.push(...(await deleteGCS(deleteRequestId)));
+        // First, get the hash from the map if it exists
+        if (deleteHash) {
+            const hashResult = await getFileStoreMap(deleteHash);
+            if (hashResult) {
+                context.log(`Found hash in map for deletion: ${deleteHash}`);
+                await removeFromFileStoreMap(deleteHash);
+            }
         }
 
+        const deleted = await storageService.deleteFiles(deleteRequestId);
         context.res = {
             status: 200,
-            body: { body: [...azureResult, ...gcsResult] },
+            body: { body: deleted },
         };
         return;
     }
 
-    const remoteUrl = fetch || restore || load;
+    const remoteUrl = shouldFetchRemote;
     if (req.method.toLowerCase() === 'get' && remoteUrl) {
         context.log(`Remote file: ${remoteUrl}`);
-        let filename; // Declare filename outside try block
+        let filename;
         try {
             // Validate URL format and accessibility
             const urlCheck = await urlExists(remoteUrl);
@@ -227,10 +206,8 @@ async function CortexFileHandler(context, req) {
             await downloadFile(remoteUrl, filename);
 
             // Now upload the downloaded file
-            const res = await uploadBlob(
+            const res = await storageService.uploadFile(
                 context,
-                null,
-                !useAzure,
                 filename,
                 remoteUrl,
             );
@@ -296,147 +273,140 @@ async function CortexFileHandler(context, req) {
             // Log the URL retrieved from Redis before checking existence
             context.log(`Checking existence of URL from Redis: ${hashResult?.url}`);
 
-            // Detect double-encoding in the blob name
-            if (hashResult.url) {
-                const urlPath = hashResult.url.split('?')[0];
-                const blobName = urlPath.substring(urlPath.lastIndexOf('/') + 1);
-                if (/%25[0-9A-Fa-f]{2}/.test(blobName)) {
-                    context.log(
-                        `Double-encoded blob detected for hash ${hash}. Invalidating cache entry.`,
-                    );
+            try {
+                // Check primary storage first
+                const primaryExists = hashResult?.url ? await storageService.fileExists(hashResult.url) : false;
+                const gcsExists = hashResult?.gcs ? await storageService.fileExists(hashResult.gcs) : false;
+
+                // If neither storage has the file, remove from map and return not found
+                if (!primaryExists && !gcsExists) {
+                    context.log(`File not found in any storage. Removing from map: ${hash}`);
                     await removeFromFileStoreMap(hash);
                     context.res = {
                         status: 404,
-                        body: `Hash ${hash} is double-encoded and has been invalidated. Please re-upload.`,
+                        body: `Hash ${hash} not found in storage`,
                     };
                     return;
                 }
-            }
-
-            // Check primary storage (Azure/Local) first
-            const primaryExists = await urlExists(hashResult?.url);
-            const gcsExists = gcs ? await gcsUrlExists(hashResult?.gcs) : false;
-
-            // If neither storage has the file, remove from map and return not found
-            if (!primaryExists.valid && !gcsExists) {
-                context.log(
-                    `File not found in any storage. Removing from map: ${hash}`,
-                );
-                await removeFromFileStoreMap(hash);
-                context.res = {
-                    status: 404,
-                    body: `Hash ${hash} not found in storage`,
-                };
-                return;
-            }
 
-            // If GCS is missing but primary exists, restore to GCS
-            else if (primaryExists.valid && gcs && !gcsExists) {
-                context.log(`GCS file missing, restoring from primary: ${hash}`);
-                const { gcs: _, ...fileInfo } = hashResult;
-                hashResult = await ensureGCSUpload(context, fileInfo);
-            }
-
-            // If primary is missing but GCS exists, restore from GCS
-            if (!primaryExists.valid && gcsExists) {
-                context.log(`Primary storage file missing, restoring from GCS: ${hash}`);
-                try {
-                    // Create a temporary file to store the downloaded content
-                    const tempDir = path.join(os.tmpdir(), `${uuidv4()}`);
-                    fs.mkdirSync(tempDir);
-                    const downloadedFile = path.join(tempDir, path.basename(hashResult.gcs));
-
-                    // Download from GCS using the new function
-                    await downloadFromGCS(hashResult.gcs, downloadedFile);
-
-                    // Upload to primary storage
-                    const res = await uploadBlob(
-                        context,
-                        null,
-                        !useAzure,
-                        downloadedFile,
-                        hash
-                    );
-
-                    // Update the hash result with the new primary storage URL
-                    hashResult.url = res.url;
+                // If GCS is missing but primary exists, restore to GCS
+                if (primaryExists && !gcsExists && hashResult?.url) {
+                    context.log(`GCS file missing, restoring from primary: ${hash}`);
+                    try {
+                        hashResult = await storageService.ensureGCSUpload(context, hashResult);
+                    } catch (error) {
+                        context.log(`Error restoring to GCS: ${error}`);
+                        // If restoration fails, remove the hash from the map
+                        await removeFromFileStoreMap(hash);
+                        context.res = {
+                            status: 404,
+                            body: `Hash ${hash} not found`,
+                        };
+                        return;
+                    }
+                }
 
-                    // Clean up temp file
+                // If primary is missing but GCS exists, restore from GCS
+                if (!primaryExists && gcsExists && hashResult?.gcs && storageService.backupProvider?.isConfigured()) {
+                    context.log(`Primary storage file missing, restoring from GCS: ${hash}`);
                     try {
-                        if (downloadedFile && fs.existsSync(downloadedFile)) {
-                            fs.unlinkSync(downloadedFile);
-                        }
-                        if (tempDir && fs.existsSync(tempDir)) {
-                            fs.rmSync(tempDir, { recursive: true });
+                        // Create a temporary file to store the downloaded content
+                        const tempDir = path.join(os.tmpdir(), `${uuidv4()}`);
+                        fs.mkdirSync(tempDir);
+                        const downloadedFile = path.join(tempDir, path.basename(hashResult.gcs));
+
+                        // Download from GCS
+                        await storageService.downloadFile(hashResult.gcs, downloadedFile);
+
+                        // Upload to primary storage
+                        const res = await storageService.uploadFile(
+                            context,
+                            downloadedFile,
+                            hash
+                        );
+
+                        // Update the hash result with the new primary storage URL
+                        hashResult.url = res.url;
+
+                        // Clean up temp file
+                        try {
+                            if (downloadedFile && fs.existsSync(downloadedFile)) {
+                                fs.unlinkSync(downloadedFile);
+                            }
+                            if (tempDir && fs.existsSync(tempDir)) {
+                                fs.rmSync(tempDir, { recursive: true });
+                            }
+                        } catch (err) {
+                            console.log('Error cleaning up temp files:', err);
                         }
-                    } catch (err) {
-                        console.log('Error cleaning up temp files:', err);
+                    } catch (error) {
+                        console.error('Error restoring from GCS:', error);
+                        // If restoration fails, remove the hash from the map
+                        await removeFromFileStoreMap(hash);
+                        context.res = {
+                            status: 404,
+                            body: `Hash ${hash} not found`,
+                        };
+                        return;
                     }
-                } catch (error) {
-                    console.error('Error restoring from GCS:', error);
                 }
-            }
 
-            // Ensure converted version exists if needed
-            hashResult = await conversionService.ensureConvertedVersion(hashResult, requestId);
+                // Final check to ensure we have at least one valid storage location
+                const finalPrimaryCheck = hashResult?.url ? await storageService.fileExists(hashResult.url) : false;
+                const finalGCSCheck = hashResult?.gcs ? await storageService.fileExists(hashResult.gcs) : false;
+                if (!finalPrimaryCheck && !finalGCSCheck) {
+                    context.log(`Failed to restore file. Removing from map: ${hash}`);
+                    await removeFromFileStoreMap(hash);
+                    context.res = {
+                        status: 404,
+                        body: `Hash ${hash} not found`,
+                    };
+                    return;
+                }
 
-            // Final check to ensure we have at least one valid storage location
-            const finalPrimaryCheck = await urlExists(hashResult?.url);
-            if (!finalPrimaryCheck.valid && !(await gcsUrlExists(hashResult?.gcs))) {
-                context.log(`Failed to restore file. Removing from map: ${hash}`);
-                await removeFromFileStoreMap(hash);
-                context.res = {
-                    status: 404,
-                    body: `Hash ${hash} not found and restoration failed`,
+                // Create the response object
+                const response = {
+                    message: `File '${hashResult.filename}' uploaded successfully.`,
+                    filename: hashResult.filename,
+                    url: hashResult.url,
+                    gcs: hashResult.gcs,
+                    hash: hashResult.hash,
+                    timestamp: new Date().toISOString()
                 };
-                return;
-            }
 
-            // Create the response object
-            const response = {
-                message: `File '${hashResult.filename}' ${useAzure ? 'uploaded' : 'saved'} successfully.`,
-                filename: hashResult.filename,
-                url: hashResult.url,
-                gcs: hashResult.gcs,
-                hash: hashResult.hash,
-                timestamp: new Date().toISOString()
-            };
+                // Ensure converted version exists and is synced across storage providers
+                try {
+                    hashResult = await conversionService.ensureConvertedVersion(hashResult, requestId);
+                } catch (error) {
+                    context.log(`Error ensuring converted version: ${error}`);
+                }
 
-            // Add converted info if it exists and has a valid URL
-            if (hashResult.converted?.url) {
-                context.log(`Adding converted info to final response`);
-                response.converted = {
-                    url: hashResult.converted.url,
-                    gcs: hashResult.converted.gcs
-                };
-            } else if (hashResult.converted?.gcs) {
-                // If we only have GCS URL, trigger conversion
-                context.log(`Only GCS URL exists for converted file, triggering conversion`);
-                const convertedResult = await conversionService.convertFile(
-                    await downloadFile(hashResult.url, path.join(os.tmpdir(), path.basename(hashResult.url))),
-                    hashResult.url
-                );
-                if (convertedResult.converted) {
-                    const convertedSaveResult = await conversionService._saveConvertedFile(convertedResult.convertedPath, requestId);
+                // Attach converted info to response if present
+                if (hashResult.converted) {
                     response.converted = {
-                        url: convertedSaveResult.url,
+                        url: hashResult.converted.url,
                         gcs: hashResult.converted.gcs
                     };
-                    // Update the hash map with the new converted info
-                    await setFileStoreMap(`${hashResult.hash}_converted`, response.converted);
                 }
-            } else {
-                context.log(`No converted info to add to final response`);
-            }
 
-            //update redis timestamp with current time
-            await setFileStoreMap(hash, hashResult);
+                //update redis timestamp with current time
+                await setFileStoreMap(hash, hashResult);
 
-            context.res = {
-                status: 200,
-                body: response
-            };
-            return;
+                context.res = {
+                    status: 200,
+                    body: response
+                };
+                return;
+            } catch (error) {
+                context.log(`Error checking file existence: ${error}`);
+                // If there's an error checking file existence, remove the hash from the map
+                await removeFromFileStoreMap(hash);
+                context.res = {
+                    status: 404,
+                    body: `Hash ${hash} not found`,
+                };
+                return;
+            }
         }
 
         context.res = {
@@ -447,7 +417,10 @@ async function CortexFileHandler(context, req) {
     }
 
     if (req.method.toLowerCase() === 'post') {
-        const result = await uploadBlob(context, req, !useAzure, null, hash);
+        // Determine if we should save to local storage based on primary provider
+        const saveToLocal = storageService.primaryProvider.constructor.name === 'LocalStorageProvider';
+        // Use uploadBlob to handle multipart/form-data
+        const result = await uploadBlob(context, req, saveToLocal, null, hash);
         if (result?.hash && context?.res?.body) {
             await setFileStoreMap(result.hash, context.res.body);
         }
@@ -484,7 +457,7 @@ async function CortexFileHandler(context, req) {
     };
 
     try {
-    // Parse URL and get pathname without query parameters for extension check
+        // Parse URL and get pathname without query parameters for extension check
         const urlObj = new URL(uri);
         const pathWithoutQuery = urlObj.pathname;
 
@@ -498,22 +471,48 @@ async function CortexFileHandler(context, req) {
             await downloadFile(uri, downloadedFile);
 
             try {
-                if (save) {
-                    const saveResults = [];
-                    const originalFileName = `${uuidv4()}_${encodeURIComponent(path.basename(downloadedFile))}`;
-                    const originalFilePath = path.join(tempDir, originalFileName);
-                    await fs.promises.copyFile(downloadedFile, originalFilePath);
-                    let fileUrl;
-                    if (useAzure) {
-                        const savedBlob = await saveFileToBlob(originalFilePath, requestId);
-                        fileUrl = savedBlob?.url;
+                if (shouldSave) {
+                    // Check if file needs conversion first
+                    if (conversionService.needsConversion(downloadedFile)) {
+                        // Convert the file
+                        const conversion = await conversionService.convertFile(downloadedFile, uri);
+                        if (!conversion.converted) {
+                            throw new Error('File conversion failed');
+                        }
+                        
+                        // Save the converted file
+                        const convertedSaveResult = await conversionService._saveConvertedFile(conversion.convertedPath, requestId);
+                        
+                        // Return the converted file URL
+                        context.res = {
+                            status: 200,
+                            body: {
+                                url: convertedSaveResult.url,
+                                blobName: path.basename(convertedSaveResult.url)
+                            }
+                        };
                     } else {
-                        fileUrl = await moveFileToPublicFolder(originalFilePath, requestId);
+                        // File doesn't need conversion, save the original file
+                        const saveResult = await conversionService._saveConvertedFile(downloadedFile, requestId);
+                        
+                        // Return the original file URL
+                        context.res = {
+                            status: 200,
+                            body: {
+                                url: saveResult.url,
+                                blobName: path.basename(saveResult.url)
+                            }
+                        };
                     }
-                    saveResults.push(fileUrl);
-                    result.push(fileUrl);
+                    return;
                 } else {
-                    const text = await conversionService.convertFile(downloadedFile, uri, true);
+                    let text;
+                    if (conversionService.needsConversion(downloadedFile)) {
+                        text = await conversionService.convertFile(downloadedFile, uri, true);
+                    } else {
+                        // For files that don't need conversion, read the file contents directly
+                        text = await fs.promises.readFile(downloadedFile, 'utf-8');
+                    }
                     result.push(...easyChunker(text));
                 }
             } catch (err) {
@@ -533,24 +532,16 @@ async function CortexFileHandler(context, req) {
                     console.log(`Error cleaning temp file ${downloadedFile}:`, err);
                 }
 
-                try {
-                    //delete uploaded prev nontext file
-                    //check cleanup for uploaded files url
-                    const regex = new RegExp(
-                        `${AZURE_STORAGE_CONTAINER_NAME}/([a-z0-9-]+)`,
+                // Delete uploaded files only if we're NOT saving the converted version.
+                // When save=true we need to keep the converted file (which is stored under the same requestId prefix),
+                // so skip the cleanup in that case.
+                if (!shouldSave) {
+                    await storageService.deleteFiles(requestId);
+                    console.log(
+                        `Cleaned temp files for request id ${requestId}`,
                     );
-                    const match = uri.match(regex);
-                    if (match && match[1]) {
-                        const extractedValue = match[1];
-                        useAzure
-                            ? await deleteBlob(extractedValue)
-                            : await deleteFolder(extractedValue);
-                        console.log(
-                            `Cleaned temp file ${uri} with request id ${extractedValue}`,
-                        );
-                    }
-                } catch (err) {
-                    console.log(`Error cleaning temp file ${uri}:`, err);
+                } else {
+                    console.log(`Skip cleanup for request id ${requestId} because save flag is set`);
                 }
             }
         } else {
@@ -571,23 +562,12 @@ async function CortexFileHandler(context, req) {
             // sequential processing of chunks
             for (let index = 0; index < chunks.length; index++) {
                 const chunkPath = chunks[index];
-                let chunkUrl;
-                let chunkGcsUrl;
-
-                if (useAzure) {
-                    const savedBlob = await saveFileToBlob(chunkPath, requestId);
-                    chunkUrl = savedBlob.url;
-                } else {
-                    chunkUrl = await moveFileToPublicFolder(chunkPath, requestId);
-                }
-
-                // If GCS is configured, save to GCS
-                chunkGcsUrl = await uploadChunkToGCS(chunkPath, requestId);
+                const chunkResult = await storageService.uploadFile(context, chunkPath, requestId);
 
                 const chunkOffset = chunkOffsets[index];
-                result.push({ uri: chunkUrl, offset: chunkOffset, gcs: chunkGcsUrl });
+                result.push({ uri: chunkResult.url, offset: chunkOffset, gcs: chunkResult.gcs });
                 console.log(
-                    `Saved chunk as: ${chunkUrl}${chunkGcsUrl ? ` and ${chunkGcsUrl}` : ''}`,
+                    `Saved chunk as: ${chunkResult.url}${chunkResult.gcs ? ` and ${chunkResult.gcs}` : ''}`,
                 );
                 await sendProgress();
             }
@@ -625,4 +605,4 @@ async function CortexFileHandler(context, req) {
     };
 }
 
-export default CortexFileHandler;
+export default CortexFileHandler;